diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md new file mode 100644 index 0000000000000000000000000000000000000000..25750e654ca170049eedd6dabb1342f28769b8e5 --- /dev/null +++ b/ATTRIBUTION.md @@ -0,0 +1,48 @@ +# ATTRIBUTION + +This repository is **CC BY 4.0 without qualification**. It contains only what this work produced. + +## What is here + +| what | size | licence | +|---|---|---| +| `results/lenses/**` — 382 fitted operators + sidecars | 16.2 GB | CC BY 4.0 | +| `results/**.json`, `results/**.npz` — experimental records | ~27 MB | CC BY 4.0 | +| `corpora/manifest.json`, `corpora/manifest_ood.json`, `corpora/dataset_revisions.json` | 12 KB | CC BY 4.0 | + +Derived from **Pythia** (EleutherAI, Apache-2.0) via the **Jacobian-lens** reference +implementation (Apache-2.0, pinned `581d398`). Neither restricts these derivatives. + +## What is deliberately NOT here, and how to get it + +The eleven corpus plaintext files are **not distributed**. They are third-party text under +upstream terms; `Wikipedia_en.jsonl` and `OOD_Wikipedia_2023.jsonl` are Wikipedia-derived and +carry **CC BY-SA**, which is share-alike and cannot be relicensed as CC BY 4.0. Shipping them +would have forced a mixed-licence repository and made the CC BY 4.0 claim above false. + +Rebuild the pool from its public sources: + +```bash +bash repro/06_build_corpora.sh # builds, then hash-verifies against the manifests +bash repro/06_build_corpora.sh --verify # verify a pool you already have +``` + +`corpora/manifest.json` and `corpora/manifest_ood.json` ship here and carry, per corpus: the +SHA-256 of the exact file used, the document count, the source dataset id and config, and the +three disjoint seed-block index ranges. A rebuild that matches those hashes is the same pool. + +**Measured 2026-08-31, rebuilding from scratch on a clean machine:** + +| corpus | rebuild | +|---|---| +| Pile-CC, StackExchange, Wikipedia_en, Github, USPTO_Backgrounds | **5/5 bit-identical** | +| OOD_CommonPile, TRAP_FineWeb, CONTROL_PubMed_2023 | **3/3 bit-identical** | +| `corpora/manifest.json` regenerated | **byte-identical** | +| OOD_Wikipedia_2023, OOD_arXiv_2023 | not re-run (53 MB / 110 MB of streaming); both carry SHA-256 in the manifest and are checked by the script | + +**One caveat, stated plainly.** `src/build_corpora.py` calls `load_dataset( +"monology/pile-uncopyrighted", ...)` with **no `revision=` pin**, so an upstream change to that +dataset would produce a different pool. The result above is evidence the upstream had not moved +as of that date — not a guarantee it never will. The manifest is the check: a mismatch means the +upstream moved, which is a finding to report, not a failure to work around. The OOD panel is more +stable by construction because its configs are date-stamped (`20231101.en`, `2024-06`). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..b2e45968f569bee158bef73c497c23a62afef3da --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +Creative Commons Attribution 4.0 International (CC BY 4.0) + +Copyright (c) 2026 The Authors + +Everything in this repository is licensed under CC BY 4.0: +https://creativecommons.org/licenses/by/4.0/ + +That claim is unqualified, and it is unqualified BY CONSTRUCTION rather than by assertion: this +repository contains only artifacts this work produced. + + results/lenses/** fitted Jacobian operators (.pt) and their provenance sidecars + results/**.json, **.npz every experimental record and derived statistic + corpora/manifest*.json the build record of the corpus pool: per-file SHA-256, document + counts, source dataset id and config, disjoint seed-block ranges + +NO THIRD-PARTY CORPUS TEXT IS DISTRIBUTED HERE. The eleven corpus files this work used are +third-party text under upstream terms, and two are Wikipedia-derived and carry CC BY-SA, a +share-alike licence that cannot be relicensed as CC BY 4.0. Rather than qualify this licence, +the pool is withheld in full and rebuilt from its public sources by the code repository's +`repro/06_build_corpora.sh`. The manifests above are this work's own record of that pool and +are what make the rebuild verifiable. + +The operators are derived from EleutherAI's Pythia suite (Apache-2.0) using the Jacobian-lens +reference implementation (Apache-2.0, pinned at 581d398). Neither upstream licence restricts the +licensing of the derived operators recorded here. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5a8b08825cb82ff5f5dc6a2c0acb5e342ca8c1ef --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +--- +license: cc-by-4.0 +tags: + - interpretability + - jacobian-lens + - pythia + - reproducibility +library_name: pytorch +--- + +# Corpus and context dependence of the Jacobian lens — artifact mirror + +Fitted operators, experimental records and the corpus pool for the accompanying paper. + +**Licence.** Everything here is **CC BY 4.0**, without qualification — see `LICENSE`. That is +true by construction, not by assertion: this repository contains only artifacts this work +produced. **No third-party corpus text is distributed here.** The corpus pool is rebuilt from its +public sources by `repro/06_build_corpora.sh` and verified against the manifests that ship here; +8 of 11 corpora rebuilt bit-identically when measured. `ATTRIBUTION.md` has the detail. + +## Layout + +``` +results/lenses/e28/ 166 operators + sidecars corpus x N ladder +results/lenses/plad/ 53 operators + sidecars per-layer activation dumps +results/lenses/ladder/ 15 operators + sidecars canonical model ladder (70m ... 2.8b) +results/lenses/misc/ 8 operators smokes and one-offs +results/<17 experiment dirs> per-experiment operators and records +results/*.json 190 experimental records +corpora/ 3 provenance manifests (NO corpus text — rebuild it, see ATTRIBUTION.md) +``` + +Every `.pt` is a corpus-averaged Jacobian `J_l = E_x[dh_final/dh_l]` fitted with the reference +implementation at pin `581d398`, fp32, TF32 off, uncentered. + +## Reproducing + +Every statistic in the paper recomputes from `results/**.json` alone — no operators, no GPU: + +```bash +git clone +uv sync && bash repro/01_setup_local.sh +bash repro/04_fetch_results.sh # pulls results/**.json from here, ~7 MB +bash repro/RUN_ALL.sh --tier T0 # 12 modules +``` + +`--tier T1` re-scores the stored operators on CPU and needs the `.pt` from this repo +(`bash repro/03_fetch_artifacts.sh --all`). `--tier T2` refits on GPU. + +## Provenance + +Each results file carries a provenance block: the producing script and its SHA-256, the commit, +the argv, and the SHA-256 of every declared input. `payload_sha256` covers the record **excluding** +the provenance block, so it is stable across reruns that change only environment metadata — rerun +and compare it, and a match means no number moved. + +No hostname is recorded. Input hashes are the values at write time; where a record was later +redacted, `payload_sha256_before_redaction` retains the original so both remain verifiable. + +No corpus plaintext is distributed — see `ATTRIBUTION.md` for the rebuild. diff --git a/corpora/dataset_revisions.json b/corpora/dataset_revisions.json new file mode 100644 index 0000000000000000000000000000000000000000..8248c17a0f8506c70d9e4442b2d9e7bf9e103a5c --- /dev/null +++ b/corpora/dataset_revisions.json @@ -0,0 +1,111 @@ +{ + "_README": [ + "Dataset revisions pinned 2026-09-01. THE SOURCE OF TRUTH for every corpus build.", + "", + "WHY. src/build_corpora.py called load_dataset() with no revision=, so the pool depended on", + "whatever the upstream dataset was on the day you ran it. docs/reproducibility/RELEASE.md", + "finding B0 recorded that and concluded the pool was not regenerable. A rebuild on 2026-08-31", + "in fact reproduced 8 of 11 corpora BIT-IDENTICALLY -- evidence the upstream had not moved, not", + "a guarantee it never would. These pins turn that evidence into a guarantee.", + "", + "`sha` is the dataset repo commit on the Hub. Pass it as revision= to load_dataset().", + "`config` is load_dataset()'s SECOND POSITIONAL argument (the config/subset name), NOT a locale;", + "null means the dataset declares no configs and none may be passed." + ], + "monology/pile-uncopyrighted": { + "used_by": "src/build_corpora.py", + "config": null, + "split": "train", + "supplies": [ + "Pile-CC", + "StackExchange", + "Wikipedia (en)", + "Github", + "USPTO Backgrounds" + ], + "sha": "3be90335b66f24456a5d6659d9c8d208c0357119", + "pinned_on": "2026-09-01" + }, + "wikimedia/wikipedia": { + "used_by": "src/build_ood_corpora.py", + "config": "20231101.en", + "split": "train", + "supplies": [ + "OOD_Wikipedia_2023" + ], + "sha": "b04c8d1ceb2f5cd4588862100d08de323dccfbaa", + "pinned_on": "2026-09-01" + }, + "neuralwork/arxiver": { + "used_by": "src/build_ood_corpora.py", + "config": null, + "split": "train", + "supplies": [ + "OOD_arXiv_2023" + ], + "sha": "698a6662e77fd5dd45dbbec988abc8123e5fa086", + "pinned_on": "2026-09-01" + }, + "RealTimeData/bbc_news_alltime": { + "used_by": "src/build_ood_corpora.py", + "config": "2024-06", + "split": "train", + "supplies": [ + "OOD_News_2024" + ], + "sha": "8dd1ecdc92ac43f9c04a3da3e945537dbb08179b", + "pinned_on": "2026-09-01" + }, + "common-pile/comma_v0.1_training_dataset": { + "used_by": "src/build_ood_corpora.py", + "config": null, + "split": "train", + "supplies": [ + "OOD_CommonPile" + ], + "sha": "5afc546db324e7f39f297ba757c9a60547151e7c", + "pinned_on": "2026-09-01" + }, + "HuggingFaceFW/fineweb": { + "used_by": "src/build_ood_corpora.py", + "config": "sample-10BT", + "split": "train", + "supplies": [ + "TRAP_FineWeb" + ], + "sha": "9bb295ddab0e05d785b879661af7260fed5140fc", + "pinned_on": "2026-09-01" + }, + "MedRAG/pubmed": { + "used_by": "src/build_ood_corpora.py", + "config": null, + "split": "train", + "supplies": [ + "CONTROL_PubMed_2023" + ], + "sha": "33da3593d5756bc04c8909f170003c0b14197957", + "pinned_on": "2026-09-01" + }, + "EleutherAI/pile-deduped-pythia-preshuffled": { + "used_by": "experiments/t35_containment.py", + "config": null, + "split": null, + "supplies": [ + "the Pythia training stream itself, for E35/M1 containment" + ], + "sha": "4647773ea142ab1ff5694602fa104bbf49088408", + "pinned_on": "2026-09-01" + }, + "_DERIVED_CORPORA": { + "_note": "Corpora produced from another corpus in this tree, not loaded from the Hub. They have NO dataset revision of their own by construction. Their full derivation rule and parent hash live in corpora/manifest_ood.json. Recorded here so this file's source-of-truth claim covers every corpus on disk.", + "OOD_News_2024_dedup": { + "derived_from": "OOD_News_2024", + "parent_upstream_repo": "RealTimeData/bbc_news_alltime", + "parent_sha": "8dd1ecdc92ac43f9c04a3da3e945537dbb08179b", + "rule": "exact-text dedup, first occurrence kept, parent order preserved (2400 -> 1598)", + "full_record": "corpora/manifest_ood.json -> OOD_News_2024_dedup.derived_from", + "revision": null, + "revision_note": "INTENTIONALLY null \u2014 derived, not loaded. Never fabricate one." + } + } +} \ No newline at end of file diff --git a/corpora/manifest.json b/corpora/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..aa93dbeb8dc6f01a7882235d43a656977a5bcc02 --- /dev/null +++ b/corpora/manifest.json @@ -0,0 +1,98 @@ +{ + "Pile-CC": { + "file": "Pile-CC.jsonl", + "n": 2400, + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ] + }, + "StackExchange": { + "file": "StackExchange.jsonl", + "n": 2400, + "sha256": "bc25e2d5775e9719ebcbdaa3bd115d827f72ebf11f47fedb377f8b0a69a28971", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ] + }, + "Wikipedia (en)": { + "file": "Wikipedia_en.jsonl", + "n": 2400, + "sha256": "b30f4cef11add560f7df8ed07a613e3fcc6a4dda865975bec15f2952253bda77", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ] + }, + "Github": { + "file": "Github.jsonl", + "n": 2400, + "sha256": "f2bb459d10cad6c13a8b66c86191bc43df4393ba5b722a5315ea878fad2d1a26", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ] + }, + "USPTO Backgrounds": { + "file": "USPTO_Backgrounds.jsonl", + "n": 2400, + "sha256": "b09ac74bdc8b169ee5c9bea6be481262f743947a92107164f30eca2346e3ccb9", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ] + }, + "_note": "seed_blocks are DISJOINT index ranges; seed k draws its N prompts from block k, so seeds share no prompts and the seed variance is a genuine resampling variance rather than an overlap artifact" +} \ No newline at end of file diff --git a/corpora/manifest_ood.json b/corpora/manifest_ood.json new file mode 100644 index 0000000000000000000000000000000000000000..377bd61c6df28e1eeecf934c26fffe6f0275d639 --- /dev/null +++ b/corpora/manifest_ood.json @@ -0,0 +1,218 @@ +{ + "OOD_Wikipedia_2023": { + "file": "OOD_Wikipedia_2023.jsonl", + "n": 2400, + "sha256": "57fb26cf3997b571543281efe3330efcbab1f37e97f99eb6df4deaa027d98550", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ], + "hf": { + "path": "wikimedia/wikipedia", + "name": "20231101.en", + "split": "train" + }, + "text_field": "text", + "claim": "TEMPORAL. Same source family as the Pile's Wikipedia component, but a 2023-11 snapshot. Revisions written after the ~2020 Pile snapshot cannot have been in it. Confounds: three years of new entities, events and editorial convention move with the exclusion. This is external review's PRIMARY recommended panel because it holds language and register nearly fixed.", + "scanned": 2436, + "status": "CANDIDATE \u2014 outside-Pile is a HYPOTHESIS to be measured by M1, not a property asserted here" + }, + "OOD_arXiv_2023": { + "file": "OOD_arXiv_2023.jsonl", + "n": 2400, + "sha256": "565f3541ef189ba624ff56373d7d3df0b0341020578b0c6cffc6391b332ff4af", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ], + "hf": { + "path": "neuralwork/arxiver", + "split": "train" + }, + "text_field": "markdown", + "claim": "TEMPORAL. 2023 arXiv papers; the Pile's arXiv component predates the snapshot. Stable LaTeX/scientific register. Confounds: new terminology, fields, authors; heavy template reuse means short-k overlap will be high regardless of exposure.", + "scanned": 2401, + "status": "CANDIDATE \u2014 outside-Pile is a HYPOTHESIS to be measured by M1, not a property asserted here" + }, + "OOD_News_2024": { + "file": "OOD_News_2024.jsonl", + "n": 2400, + "sha256": "45aaab463d8987725fe9b7edb5bd316a29c2d547a5a1ffd2c54f41df3be1df80", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ], + "hf": { + "path": "RealTimeData/bbc_news_alltime", + "name": "2024-06", + "split": "train" + }, + "text_field": "content", + "claim": "TEMPORAL. June 2024 news, four years past the snapshot. Ordinary English, so it tests transport without immediately confounding on script or competence. Confounds: news register, named entities, and possible syndication into web crawls.", + "scanned": 2930, + "status": "CANDIDATE \u2014 outside-Pile is a HYPOTHESIS to be measured by M1, not a property asserted here" + }, + "OOD_CommonPile": { + "file": "OOD_CommonPile.jsonl", + "n": 2400, + "sha256": "0d7ca5e94b4b1891e1d99a310b263806f370bae861decdfc3cf0f1ebb7a755e4", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ], + "hf": { + "path": "common-pile/comma_v0.1_training_dataset", + "split": "train" + }, + "text_field": "text", + "claim": "PROVENANCE. Assembled later from openly-licensed and public-domain sources, covering domains not deliberately represented in the Pile. Tests whether the effect is about corpus construction/licensing rather than topic. Confounds: licence filtering changes the source population; some documents may still be older web material.", + "scanned": 3069, + "status": "CANDIDATE \u2014 outside-Pile is a HYPOTHESIS to be measured by M1, not a property asserted here" + }, + "TRAP_FineWeb": { + "file": "TRAP_FineWeb.jsonl", + "n": 2400, + "sha256": "6164a3cc1113f6c3b90fd5f67e3eba2e80324a69d9d4c0ae671aefdefc4b9260", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ], + "hf": { + "path": "HuggingFaceFW/fineweb", + "name": "sample-10BT", + "split": "train" + }, + "text_field": "text", + "claim": "DELIBERATE POSITIVE CONTROL, NOT A CLEAN Q. FineWeb is Common-Crawl-derived and Pile-CC is too, so it can contain the same pages, later versions of them, or mirrors. Both external review reviews name CC derivatives as a trap. Included so M1 has something it MUST flag as overlapping: if FineWeb does not show high containment, M1 is broken.", + "scanned": 2708, + "status": "CANDIDATE \u2014 outside-Pile is a HYPOTHESIS to be measured by M1, not a property asserted here" + }, + "CONTROL_PubMed_2023": { + "file": "CONTROL_PubMed_2023.jsonl", + "n": 2400, + "sha256": "cffa12d20484b15c2430fc402a994632a9a27595454d02b457b94ec80c27856a", + "seed_blocks": [ + [ + 0, + 800 + ], + [ + 800, + 1600 + ], + [ + 1600, + 2400 + ] + ], + "hf": { + "path": "MedRAG/pubmed", + "split": "train" + }, + "text_field": "content", + "claim": "AMBIGUOUS BY DESIGN. The Pile contains PubMed Central, so this is source-INCLUDED; only post-snapshot articles would be temporally excluded and this dump is not date-filtered. Expected to land BETWEEN the Pile components and the temporal panels, which makes it a useful mid-scale reference point for whatever divergence metric we fit.", + "scanned": 3161, + "status": "CANDIDATE \u2014 outside-Pile is a HYPOTHESIS to be measured by M1, not a property asserted here" + }, + "OOD_News_2024_dedup": { + "file": "OOD_News_2024_dedup.jsonl", + "n": 1598, + "sha256": "713cdbcf4512208f4cb3cac831d2a4eeb76ae0fe577d215473fd71919d022529", + "seed_blocks": [ + [ + 0, + 532 + ], + [ + 532, + 1064 + ], + [ + 1064, + 1596 + ] + ], + "seed_blocks_note": "computed as b = len(texts) // 3 at load time (experiments/trainval.py:256), so the blocks are NOT the 800/1600/2400 of the n=2400 corpora. Recorded explicitly because this corpus is shorter than every other entry here.", + "DERIVED": true, + "hf": null, + "hf_note": "INTENTIONALLY null. This corpus has NO independent upstream. It is produced deterministically from OOD_News_2024 by the rule below and must never be given a fabricated dataset revision. To regenerate it, rebuild the parent at the revision pinned in corpora/dataset_revisions.json and re-apply the rule.", + "derived_from": { + "parent": "OOD_News_2024", + "parent_file": "OOD_News_2024.jsonl", + "parent_sha256": "45aaab463d8987725fe9b7edb5bd316a29c2d547a5a1ffd2c54f41df3be1df80", + "parent_n": 2400, + "parent_upstream": { + "path": "RealTimeData/bbc_news_alltime", + "name": "2024-06", + "split": "train", + "sha": "8dd1ecdc92ac43f9c04a3da3e945537dbb08179b" + }, + "rule": "exact-text deduplication: keep the FIRST occurrence of each distinct `text` value, preserving the parent's record order; drop every later exact duplicate", + "rule_verified": "verified 2026-09-01 by recomputation: first-occurrence dedup of the parent reproduces this file's text sequence exactly (order-sensitive equality True); sorted(distinct) does NOT (False), so the order is first-occurrence, not sorted", + "records_in": 2400, + "records_out": 1598, + "exact_duplicates_removed": 802, + "text_field": "text", + "is_strict_subset_of_parent": true + }, + "claim": "DERIVED CONTROL. The 2024-06 BBC News panel with its 802 exact-duplicate records removed. It exists because the parent contains syndicated/repeated articles, which inflate any recurrence-based statistic. It is NOT an independent OOD rung: it shares the parent's upstream, register and time window entirely.", + "status": "DERIVED \u2014 provenance is the parent's plus the recorded rule. Excluded BY DESIGN from the E48b containment index (experiments/t48b_exposure_growth.py:77-81 --exclude default): it was created after the k-gram shards were built and is a strict subset of OOD_News_2024, so indexing it would double-count the parent's k-grams.", + "used_by": [ + "results/e48/lens_OOD_News_2024_dedup_410m_n200_s{0,1,2}.pt", + "results/e48/tv_OOD_News_2024_dedup_s{0,1,2}.json" + ], + "scanned": null, + "recorded_on": "2026-09-01", + "recorded_why": "this corpus had operators fitted on it but NO entry in this manifest and NO row in corpora/dataset_revisions.json, whose README claims to be the source of truth for every corpus build. Gap closed by recording the derivation, not by inventing an upstream pin." + } +} \ No newline at end of file diff --git a/results/ctrlbf1_control_backfill.json b/results/ctrlbf1_control_backfill.json new file mode 100644 index 0000000000000000000000000000000000000000..84507545ceedee0198ea7a1309c7a44f1542b295 --- /dev/null +++ b/results/ctrlbf1_control_backfill.json @@ -0,0 +1,305 @@ +{ + "experiment": "CTRLBF1 \u2014 backfill of three already-completed, previously unrecorded controls", + "prereg": [ + "docs/experiments/preregs/superseded/PREREG_E36_QLADDER.md (section E35)", + "docs/experiments/preregs/P0_t52_parallel_port.md" + ], + "status": "RECORDED FROM EXISTING OUTPUTS \u2014 no new experiment, no new threshold", + "recomputes_not_remeasures": true, + "why": "CLAUDE.md rule 10 requires a control to be named with the number it produced. These three were satisfied by data already on disk but were recorded as 'NOT RECORDED' (E35 M1-C3, M1-C4) or '(full-grid arms outstanding)' (P0 C2).", + "E35_M1_C3": { + "requirement_verbatim": "post-2020 text | must score below M1-C1. If not, either the date argument or the index is wrong", + "m1_c1_operationalised": "M1-C1 is 'a known Pile component ... high containment'. The bar used is the LOWEST of the five in-stream Pile components at k=32, 20/20 coverage \u2014 the most demanding reading of 'below M1-C1'.", + "k": 32, + "coverage": "20/20", + "in_stream_final": { + "Pile-CC": 0.9276060773863086, + "Github": 0.9340004374453194, + "StackExchange": 0.9179914195561899, + "USPTO_Backgrounds": 0.9267616923398619, + "Wikipedia_en": 0.9215951727072293 + }, + "lowest_in_stream": 0.9179914195561899, + "post_2020": { + "CONTROL_PubMed_2023": { + "containment_20of20": 0.7927273508312213, + "below_lowest_in_stream": true + }, + "OOD_Wikipedia_2023": { + "containment_20of20": 0.26957534299550123, + "below_lowest_in_stream": true + }, + "OOD_arXiv_2023": { + "containment_20of20": 8.598932872430531e-05, + "below_lowest_in_stream": true + }, + "OOD_News_2024": { + "containment_20of20": 0.0007487928942136383, + "below_lowest_in_stream": true + } + }, + "fires": true, + "source": "results/e48b_exposure_growth.json -> containment_by_coverage" + }, + "E35_M1_C4": { + "requirement_verbatim": "one shard vs twenty | containment on 1/20 of the stream must be a strict lower bound on containment over 20/20", + "k": 32, + "by_corpus": { + "CONTROL_PubMed_2023": { + "containment_1of20": 0.061166716783366985, + "containment_20of20": 0.7927273508312213, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "Github": { + "containment_1of20": 0.3742096158434741, + "containment_20of20": 0.9340004374453194, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "OOD_CommonPile": { + "containment_1of20": 0.0, + "containment_20of20": 0.00125, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "OOD_News_2024": { + "containment_1of20": 1.936533347104237e-05, + "containment_20of20": 0.0007487928942136383, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "OOD_Wikipedia_2023": { + "containment_1of20": 0.022133038856352236, + "containment_20of20": 0.26957534299550123, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "OOD_arXiv_2023": { + "containment_1of20": 0.0, + "containment_20of20": 8.598932872430531e-05, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "Pile-CC": { + "containment_1of20": 0.09955354052744514, + "containment_20of20": 0.9276060773863086, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "StackExchange": { + "containment_1of20": 0.07686719226900293, + "containment_20of20": 0.9179914195561899, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "TRAP_FineWeb": { + "containment_1of20": 0.007237811980212063, + "containment_20of20": 0.06309446548070848, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "USPTO_Backgrounds": { + "containment_1of20": 0.12033439046490556, + "containment_20of20": 0.9267616923398619, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "Wikipedia_en": { + "containment_1of20": 0.07066448338254318, + "containment_20of20": 0.9215951727072293, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + }, + "_CONTROL_random": { + "containment_1of20": 0.0, + "containment_20of20": 0.0, + "is_lower_bound": true, + "monotone_nondecreasing": true, + "n_coverage_points": 20 + } + }, + "strict_increase_for_in_stream_corpora": { + "Pile-CC": true, + "Github": true, + "StackExchange": true, + "USPTO_Backgrounds": true, + "Wikipedia_en": true + }, + "fires": true, + "note": "'strict' is checked as strict only for corpora actually present in the stream; a genuinely absent corpus pinned at 0.0 at every coverage satisfies the lower-bound requirement with equality, which is the intended behaviour and is what E48b's growth-rate reading rests on.", + "source": "results/e48b_exposure_growth.json -> containment_by_coverage" + }, + "P0_C2": { + "requirement_verbatim": "C2 - worker-count invariance. --workers 1, 2 and 8 must give identical output. Required: max_abs_diff = 0.0 between every pair, on the same 128 values.", + "arms_found": { + "w1": { + "file": "e52_factorial_410m_rstrip_w1.json", + "workers": 1, + "D": 0.000845372218579525 + }, + "w2": { + "file": "e52_factorial_410m_rstrip_w2.json", + "workers": 2, + "D": 0.000845372218579525 + }, + "w8": { + "file": "e52_factorial_410m_rstrip_w8.json", + "workers": 8, + "D": 0.000845372218579525 + }, + "w6": { + "file": "e52_factorial_410m_rstrip.json", + "workers": 6, + "D": 0.000845372218579525 + } + }, + "n_values_compared": 128, + "same_key_set_across_arms": true, + "pairwise_max_abs_diff": { + "w1_vs_w2": 0.0, + "w1_vs_w6": 0.0, + "w1_vs_w8": 0.0, + "w2_vs_w6": 0.0, + "w2_vs_w8": 0.0, + "w6_vs_w8": 0.0 + }, + "pairwise_max_abs_diff_registered_arms_only": { + "w1_vs_w2": 0.0, + "w1_vs_w8": 0.0, + "w2_vs_w8": 0.0 + }, + "max_abs_diff_registered": 0.0, + "max_abs_diff_including_w6_reference": 0.0, + "fires": true, + "note": "The 128 values are the 64 fit x read cells under each of the two aggregations. w6 is the reference run (--workers 6) and is reported alongside but is not one of the registered 1/2/8 arms.", + "source": "results/e52_factorial_410m_rstrip{,_w1,_w2,_w8}.json -> by_aggregation.{persist,min}.matrix" + }, + "controls_fired": { + "E35_M1_C3": true, + "E35_M1_C4": true, + "P0_C2": true + }, + "VERDICT": "ALL THREE RECORDED. E35 M1-C3 FIRES: all 4 post-2020 panels sit below the lowest in-stream Pile component (0.9180) at k=32, 20/20 \u2014 CONTROL_PubMed_2023 0.79273, OOD_Wikipedia_2023 0.26958, OOD_arXiv_2023 0.00009, OOD_News_2024 0.00075. E35 M1-C4 FIRES: containment at 1/20 coverage is a lower bound on 20/20 for 12/12 corpora and the series is monotone non-decreasing in all of them. P0 C2 FIRES: max_abs_diff = 0.0 across every pair of --workers 1/2/8 on all 128 values. No threshold was introduced here; each requirement is quoted from its own registration.", + "provenance": { + "schema_version": 1, + "experiment": "CTRLBF1 \u2014 backfill of three already-completed, previously unrecorded controls", + "utc": "2026-09-01T20:35:05+00:00", + "script": { + "path": "experiments/ctrlbf1_control_backfill.py", + "sha256": "6b944b4756c5d93fc012771d2bf57553b52192cc5832ee7bff360489515fdfb4", + "n_lines": 156 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "corpora/dataset_revisions.json", + "corpora/dataset_revisions.json.bak", + "corpora/manifest_ood.json", + "corpora/manifest_ood.json.bak", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E65R_phase1_rstrip_rescore.md", + "docs/experiments/preregs/E65T_threshold_defect_adjudication.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/EFFRANK1_effective_rank_adjudication.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py" + ], + "dirty_diff_sha256": "719949b81d9871a26c26430713cafa9ad3b949cc5eea0ad080ae38307d75f63d", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/ctrlbf1_control_backfill.py" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "244eafca442da2a20faa694df90c22fe3bdd41e299594c22382a37b5825e4962" + } +} \ No newline at end of file diff --git a/results/cv1_answer_competence.json b/results/cv1_answer_competence.json new file mode 100644 index 0000000000000000000000000000000000000000..468e4d0d6de204639a1ba9dfde24d5d6e349089d --- /dev/null +++ b/results/cv1_answer_competence.json @@ -0,0 +1,288 @@ +{ + "experiment": "CV1 \u2014 answer competence: can the model do the task at all?", + "status": "MEASUREMENT ONLY \u2014 no registered decision rule", + "why": "existing capability_* fields measure the INTERMEDIATE's rank, not the ANSWER's; t36_qladder.py:483-489 records the registered competence gate as degenerate and unable to fire. This supplies the missing number.", + "readout_convention": "STRIPPED (corrected) \u2014 rank of the target's first token at the final position of the rstripped prompt", + "sets_with_target": [ + "multihop", + "order-ops", + "multilingual" + ], + "sets_without_target": [ + "poetry", + "typo", + "association" + ], + "note_on_undefined": "poetry/typo/association carry target: null. The annotated intermediate is the object of interest and there is no separate answer, so answer competence is UNDEFINED for them \u2014 not zero.", + "by_model": { + "70m": { + "multihop": { + "n_scored": 93, + "n_multitoken_answers_scored_on_first_token": 3, + "top1_rate": 0.010752688172043012, + "top10_rate": 0.13978494623655913, + "median_rank": 80, + "mean_rank": 1106.5806451612902, + "median_prob": 0.0015416988171637058 + }, + "order-ops": { + "n_scored": 55, + "n_multitoken_answers_scored_on_first_token": 0, + "top1_rate": 0.03636363636363636, + "top10_rate": 0.38181818181818183, + "median_rank": 18, + "mean_rank": 22.09090909090909, + "median_prob": 0.006405120715498924 + }, + "multilingual": { + "n_scored": 107, + "n_multitoken_answers_scored_on_first_token": 92, + "top1_rate": 0.009345794392523364, + "top10_rate": 0.08411214953271028, + "median_rank": 3436, + "mean_rank": 4615.915887850468, + "median_prob": 2.3302003683056682e-05 + }, + "_pooled": { + "n": 255, + "top1_rate": 0.01568627450980392, + "top10_rate": 0.16862745098039217 + } + }, + "160m": { + "multihop": { + "n_scored": 93, + "n_multitoken_answers_scored_on_first_token": 3, + "top1_rate": 0.03225806451612903, + "top10_rate": 0.25806451612903225, + "median_rank": 42, + "mean_rank": 628.3655913978495, + "median_prob": 0.0031900564208626747 + }, + "order-ops": { + "n_scored": 55, + "n_multitoken_answers_scored_on_first_token": 0, + "top1_rate": 0.0, + "top10_rate": 0.43636363636363634, + "median_rank": 13, + "mean_rank": 14.49090909090909, + "median_prob": 0.01120622269809246 + }, + "multilingual": { + "n_scored": 107, + "n_multitoken_answers_scored_on_first_token": 92, + "top1_rate": 0.009345794392523364, + "top10_rate": 0.08411214953271028, + "median_rank": 3788, + "mean_rank": 4630.429906542056, + "median_prob": 1.6619400412309915e-05 + }, + "_pooled": { + "n": 255, + "top1_rate": 0.01568627450980392, + "top10_rate": 0.2235294117647059 + } + }, + "410m": { + "multihop": { + "n_scored": 93, + "n_multitoken_answers_scored_on_first_token": 3, + "top1_rate": 0.06451612903225806, + "top10_rate": 0.3333333333333333, + "median_rank": 31, + "mean_rank": 460.9139784946237, + "median_prob": 0.005166209302842617 + }, + "order-ops": { + "n_scored": 55, + "n_multitoken_answers_scored_on_first_token": 0, + "top1_rate": 0.09090909090909091, + "top10_rate": 0.5454545454545454, + "median_rank": 9, + "mean_rank": 11.909090909090908, + "median_prob": 0.026608118787407875 + }, + "multilingual": { + "n_scored": 107, + "n_multitoken_answers_scored_on_first_token": 92, + "top1_rate": 0.009345794392523364, + "top10_rate": 0.056074766355140186, + "median_rank": 3044, + "mean_rank": 4031.233644859813, + "median_prob": 1.0294868843629956e-05 + }, + "_pooled": { + "n": 255, + "top1_rate": 0.047058823529411764, + "top10_rate": 0.2627450980392157 + } + }, + "1b": { + "multihop": { + "n_scored": 93, + "n_multitoken_answers_scored_on_first_token": 3, + "top1_rate": 0.07526881720430108, + "top10_rate": 0.3333333333333333, + "median_rank": 18, + "mean_rank": 316.0967741935484, + "median_prob": 0.011097713373601437 + }, + "order-ops": { + "n_scored": 55, + "n_multitoken_answers_scored_on_first_token": 0, + "top1_rate": 0.07272727272727272, + "top10_rate": 0.6727272727272727, + "median_rank": 7, + "mean_rank": 9.163636363636364, + "median_prob": 0.029681021347641945 + }, + "multilingual": { + "n_scored": 107, + "n_multitoken_answers_scored_on_first_token": 92, + "top1_rate": 0.0, + "top10_rate": 0.06542056074766354, + "median_rank": 1649, + "mean_rank": 2892.6168224299067, + "median_prob": 3.0693336157128215e-05 + }, + "_pooled": { + "n": 255, + "top1_rate": 0.043137254901960784, + "top10_rate": 0.29411764705882354 + } + } + }, + "VERDICT": "Answer competence on the three sets carrying a target. 70m: top1 1.6%, top10 16.9% (n=255 items with a target) | 160m: top1 1.6%, top10 22.4% (n=255 items with a target) | 410m: top1 4.7%, top10 26.3% (n=255 items with a target) | 1b: top1 4.3%, top10 29.4% (n=255 items with a target)", + "provenance": { + "schema_version": 1, + "experiment": "CV1", + "utc": "2026-08-30T06:32:53+00:00", + "script": { + "path": "experiments/cv1_answer_competence.py", + "sha256": "29279dcb364dec36ae9f32438ab9ac607cc8b89983cf20c5796662e9a56a7a21", + "n_lines": 138 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/cv1_answer_competence.py", + "--models", + "70m,160m,410m,1b" + ], + "cwd": ".", + "inputs": [ + { + "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", + "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", + "bytes": 21869, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", + "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", + "bytes": 9589, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", + "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", + "bytes": 24284, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "8809019f865fe0939cee28bd5d201f7a842b85b9cb5929440541804f3786e74d" + } +} \ No newline at end of file diff --git a/results/cv2_position_support.json b/results/cv2_position_support.json new file mode 100644 index 0000000000000000000000000000000000000000..36e112a49c4318995ffb9d11b6dfa2cee7a81880 --- /dev/null +++ b/results/cv2_position_support.json @@ -0,0 +1,299 @@ +{ + "experiment": "CV2 \u2014 positional support audit", + "status": "MEASUREMENT ONLY \u2014 no registered decision rule; this establishes or refutes the mismatch flagged in docs/validity/CONSTRUCT_VALIDITY.md section 4.2", + "model": "EleutherAI/pythia-410m-deduped", + "fitting_support": { + "source": "src/fastfit.py:116 -> jlens.fitting.valid_position_mask(seq_len, skip_first)", + "skip_first": 16, + "max_seq_len": 128, + "excludes": "first skip_first positions (attention sink) and the final position (no next-token target)", + "inclusive_range": [ + 16, + 126 + ] + }, + "readout_convention": "STRIPPED (corrected). Position computed on the item alone, then shifted by the prefix length \u2014 matches t36_qladder.py:27-31", + "by_set": { + "association": { + "n_items": 102, + "readout_rule": "last", + "median_prompt_tokens": 28, + "Q0": { + "median_read_pos": 28, + "min": 19, + "max": 49, + "n_in_support": 102, + "frac_in_support": 1.0 + }, + "prefixed": { + "median_read_pos": 156, + "min": 147, + "max": 177, + "n_in_support": 0, + "frac_in_support": 0.0 + } + }, + "multihop": { + "n_items": 93, + "readout_rule": "last", + "median_prompt_tokens": 15, + "Q0": { + "median_read_pos": 15, + "min": 8, + "max": 40, + "n_in_support": 45, + "frac_in_support": 0.4838709677419355 + }, + "prefixed": { + "median_read_pos": 143, + "min": 136, + "max": 168, + "n_in_support": 0, + "frac_in_support": 0.0 + } + }, + "multilingual": { + "n_items": 107, + "readout_rule": "last", + "median_prompt_tokens": 10, + "Q0": { + "median_read_pos": 10, + "min": 5, + "max": 26, + "n_in_support": 6, + "frac_in_support": 0.056074766355140186 + }, + "prefixed": { + "median_read_pos": 138, + "min": 133, + "max": 154, + "n_in_support": 0, + "frac_in_support": 0.0 + } + }, + "order-ops": { + "n_items": 55, + "readout_rule": "last", + "median_prompt_tokens": 8, + "Q0": { + "median_read_pos": 8, + "min": 6, + "max": 18, + "n_in_support": 1, + "frac_in_support": 0.01818181818181818 + }, + "prefixed": { + "median_read_pos": 136, + "min": 134, + "max": 146, + "n_in_support": 0, + "frac_in_support": 0.0 + } + }, + "poetry": { + "n_items": 98, + "readout_rule": "last_newline", + "median_prompt_tokens": 26, + "Q0": { + "median_read_pos": 18, + "min": 16, + "max": 22, + "n_in_support": 98, + "frac_in_support": 1.0 + }, + "prefixed": { + "median_read_pos": 146, + "min": 144, + "max": 150, + "n_in_support": 0, + "frac_in_support": 0.0 + } + }, + "typo": { + "n_items": 96, + "readout_rule": "last", + "median_prompt_tokens": 8, + "Q0": { + "median_read_pos": 8, + "min": 6, + "max": 14, + "n_in_support": 0, + "frac_in_support": 0.0 + }, + "prefixed": { + "median_read_pos": 136, + "min": 134, + "max": 142, + "n_in_support": 0, + "frac_in_support": 0.0 + } + } + }, + "overall": { + "Q0": { + "n": 551, + "median_read_pos": 14, + "min": 5, + "max": 49, + "n_in_support": 252, + "frac_in_support": 0.4573502722323049 + }, + "prefixed": { + "n": 551, + "median_read_pos": 142, + "min": 133, + "max": 177, + "n_in_support": 0, + "frac_in_support": 0.0 + } + }, + "VERDICT": "Q0 reads inside the fitting support on 45.7% of items (median position 14); prefixed rungs on 0.0% (median 142). Fitting support is positions 16..126.", + "provenance": { + "schema_version": 1, + "experiment": "CV2", + "utc": "2026-08-30T00:19:04+00:00", + "script": { + "path": "experiments/cv2_position_support.py", + "sha256": "fee6460d8d930d3c9ebc138a29def2533ca5bd8fc2392daceed2b0644e20abbc", + "n_lines": 145 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reviews/REVIEW_INSIGHTS_2026-08-25.md", + "docs/validity/CONSTRUCT_VALIDITY.md" + ], + "dirty_diff_sha256": "6b143c292229b3d2480d1b73243675f0db5e23b27f66e4215e0a9c1947a761a8", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/cv2_position_support.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "jacobian-lens/data/evaluations/lens-eval-association.json", + "sha256": "d1a98cd4911b594282e74168091c77d849dae18ffe2acb5761074853f327d71c", + "bytes": 24228, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", + "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", + "bytes": 21869, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", + "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", + "bytes": 24284, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", + "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", + "bytes": 9589, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-poetry.json", + "sha256": "6aeb3415c5a5c3f3827c9efe63f006de02f5ef39a816bbac68e15e733aba60cc", + "bytes": 21533, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-typo.json", + "sha256": "9d05e16b7234a57d0773d120a4e1c4e94fd3bc2235a8125d4200a70e60ab17aa", + "bytes": 14646, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "1eb1ab9b76e77d6060876485b34291cf9e1c525a755ae0166d5f7ec98f6d0d92" + } +} \ No newline at end of file diff --git a/results/cv3_margins_410m.json b/results/cv3_margins_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..8c2bcd2e84ad4b766fd0c58dae6396bb90af5ac9 --- /dev/null +++ b/results/cv3_margins_410m.json @@ -0,0 +1,473 @@ +{ + "experiment": "CV3 \u2014 margins: score effect or ranking artifact?", + "prereg": "docs/experiments/preregs/CV3_margins.md", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "ACCEPT (SCORE EFFECT) if PRIMARY >= 0.50 | REJECT (RANKING ARTIFACT) if PRIMARY < 0.25 \u2014 stop and alert, do not re-cut | UNCLEAR if 0.25 <= PRIMARY < 0.50 \u2014 report and stop", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "readout_convention": "STRIPPED (corrected); no read-context prefix", + "n_items": 449, + "n_pairs": 801, + "per_arm": { + "logit_I": { + "rank_min_auc": 0.1981085192666858, + "margin_mean": -4.698809347247424, + "z_mean": 3.398138034577217 + }, + "J|Wikipedia_en|s0": { + "rank_min_auc": 0.2411940539381176, + "margin_mean": -3.5185320037810324, + "z_mean": 3.414767909333594 + }, + "J|Wikipedia_en|s1": { + "rank_min_auc": 0.24376219820086684, + "margin_mean": -3.5576964909083792, + "z_mean": 3.4289134198996036 + }, + "J|Wikipedia_en|s2": { + "rank_min_auc": 0.2409968710017892, + "margin_mean": -3.550353089992781, + "z_mean": 3.418068973837049 + }, + "J|USPTO_Backgrounds|s0": { + "rank_min_auc": 0.2588173671008463, + "margin_mean": -3.5918301268675705, + "z_mean": 3.5110595907230495 + }, + "J|USPTO_Backgrounds|s1": { + "rank_min_auc": 0.260959847421275, + "margin_mean": -3.6230063218555544, + "z_mean": 3.5260322405353177 + }, + "J|USPTO_Backgrounds|s2": { + "rank_min_auc": 0.26360051696399417, + "margin_mean": -3.573313506298794, + "z_mean": 3.5303729015355394 + }, + "J|Pile-CC|s0": { + "rank_min_auc": 0.2619082412030062, + "margin_mean": -3.723244701992529, + "z_mean": 3.564116526229665 + }, + "J|Pile-CC|s1": { + "rank_min_auc": 0.2652928121057275, + "margin_mean": -3.721754557559919, + "z_mean": 3.57523909996598 + }, + "J|Pile-CC|s2": { + "rank_min_auc": 0.26001365602369086, + "margin_mean": -3.710869514133819, + "z_mean": 3.5523003176418593 + }, + "J|StackExchange|s0": { + "rank_min_auc": 0.26678756139973614, + "margin_mean": -3.8218988278155637, + "z_mean": 3.5992482651800937 + }, + "J|StackExchange|s1": { + "rank_min_auc": 0.263895495234026, + "margin_mean": -3.8108850501164118, + "z_mean": 3.5846069769433595 + }, + "J|StackExchange|s2": { + "rank_min_auc": 0.26310253043676124, + "margin_mean": -3.8273039062383045, + "z_mean": 3.591593915826658 + }, + "J|Github|s0": { + "rank_min_auc": 0.2361789172766393, + "margin_mean": -3.407463207637055, + "z_mean": 3.2020849497495396 + }, + "J|Github|s1": { + "rank_min_auc": 0.24343462540305366, + "margin_mean": -3.4403224028664545, + "z_mean": 3.2485827239444456 + }, + "J|Github|s2": { + "rank_min_auc": 0.2416357198287925, + "margin_mean": -3.410493464114918, + "z_mean": 3.232178426457889 + }, + "shuf|Pile-CC|s0|d0": { + "rank_min_auc": 0.27188047356388173, + "margin_mean": -3.078709140214322, + "z_mean": 3.5705154663621523 + }, + "shuf|Pile-CC|s0|d1": { + "rank_min_auc": 0.25473107820874097, + "margin_mean": -3.5329598556097404, + "z_mean": 3.4034280227130598 + }, + "shuf|Pile-CC|s0|d2": { + "rank_min_auc": 0.27573013496239485, + "margin_mean": -3.3755934340423375, + "z_mean": 3.461937350128892 + } + }, + "rank_space": { + "spread": 0.024178774854012652, + "pooled_seed_sd": 0.0025811352265854045, + "spread_over_sd": 9.36749636554256, + "per_corpus": { + "Wikipedia_en": { + "mean": 0.24198437438025788, + "seed_sd": 0.001542794033705807 + }, + "USPTO_Backgrounds": { + "mean": 0.2611259104953718, + "seed_sd": 0.002395895106586836 + }, + "Pile-CC": { + "mean": 0.2624049031108082, + "seed_sd": 0.0026743928325177486 + }, + "StackExchange": { + "mean": 0.2645951956901745, + "seed_sd": 0.0019396001767817256 + }, + "Github": { + "mean": 0.24041642083616183, + "seed_sd": 0.0037784047730492635 + } + } + }, + "margin_space": { + "spread": 0.40060290318395086, + "pooled_seed_sd": 0.01737544449501888, + "spread_over_sd": 23.055692376606196, + "per_corpus": { + "Wikipedia_en": { + "mean": -3.542193861560731, + "seed_sd": 0.020818117576808237 + }, + "USPTO_Backgrounds": { + "mean": -3.596049985007306, + "seed_sd": 0.025113728945124182 + }, + "Pile-CC": { + "mean": -3.718622924562089, + "seed_sd": 0.006755861348445229 + }, + "StackExchange": { + "mean": -3.8200292613900935, + "seed_sd": 0.008367566435486981 + }, + "Github": { + "mean": -3.4194263582061426, + "seed_sd": 0.018159821734111948 + } + } + }, + "z_space": { + "spread": 0.36420101926607895, + "pooled_seed_sd": 0.013407308018930473, + "spread_over_sd": 27.164365788556857, + "per_corpus": { + "Wikipedia_en": { + "mean": 3.420583434356749, + "seed_sd": 0.00740038857575187 + }, + "USPTO_Backgrounds": { + "mean": 3.5224882442646357, + "seed_sd": 0.010132666273489743 + }, + "Pile-CC": { + "mean": 3.5638853146125014, + "seed_sd": 0.011471138903233818 + }, + "StackExchange": { + "mean": 3.5918163859833703, + "seed_sd": 0.007323178956966246 + }, + "Github": { + "mean": 3.2276153667172913, + "seed_sd": 0.0235823426935772 + } + } + }, + "PRIMARY": 2.461243802710653, + "VERDICT": "ACCEPT \u2014 SCORE EFFECT", + "SECONDARY_derangement": { + "real_minus_shuf_rank_min_auc": -0.0055389877086662875, + "real_minus_shuf_margin": -0.394157225370396, + "real_minus_shuf_z": 0.08548957982829686, + "note": "if the derangement's advantage is present in rank space and absent in margin space, `min` rewards rank reshuffling rather than better scoring" + }, + "controls": { + "C1_logit_reproduces_stored": { + "required": "|logit_I rank_min_auc - 0.19810852520167826| <= 1e-6", + "observed": 0.1981085192666858, + "abs_diff": 5.934992469702749e-09, + "fires": true + }, + "C2_margin_rank_consistency": { + "required": "margin > 0 iff rank == 1 on 100% of triples", + "violations": 0, + "fires": true + }, + "C4_seed_sd_nondegenerate": { + "required": "pooled seed SD > 0 in both spaces", + "rank": 0.0025811352265854045, + "margin": 0.01737544449501888, + "fires": true + } + }, + "declared_bias": "margin is measured against the top competitor, which is itself operator-dependent; z and the raw logit are recorded so a competitor-suppression effect is separable from a target-score effect", + "provenance": { + "schema_version": 1, + "experiment": "CV3", + "utc": "2026-08-30T06:24:45+00:00", + "script": { + "path": "experiments/cv3_margins.py", + "sha256": "44938026f765b4f605189d29781ddb59a76d2a1bf91a230fabedb941a85e9521", + "n_lines": 247 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/t1_module_audit.json" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/cv3_margins.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48_crossover_410m_rstrip.json", + "sha256": "fe0b5ae29baf7f946884618723610d19120158474538e540eb1be8583430d97d", + "bytes": 111333, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "38efe87c84be1b8bee5fa680a5a3d354912fedf8c7e29e686fd97fbecb9a8b5c" + } +} \ No newline at end of file diff --git a/results/cv4_phase1_capability.json b/results/cv4_phase1_capability.json new file mode 100644 index 0000000000000000000000000000000000000000..c164da093dfc7f5974b5cec995241b37556e6032 --- /dev/null +++ b/results/cv4_phase1_capability.json @@ -0,0 +1,719 @@ +{ + "experiment": "CV4 Phase 1 \u2014 capability calibration across the ladder, all six families", + "prereg": "docs/experiments/preregs/CV4_capability_ladder.md", + "status": "MEASUREMENT ONLY \u2014 Phase 1 gates which model later phases target; CV4's registered rule is adjudicated in Phase 3, not here", + "readout_convention": "STRIPPED (corrected)", + "n_items_total": 551, + "n_items_with_target": 255, + "measures": { + "answer_rank": "rank of the expected answer's first token; defined only where `target` is non-null (multihop, order-ops, multilingual)", + "prompt_surprisal": "mean per-token NLL of the prompt; defined for all six families. NOT a task-competence measure \u2014 it separates 'cannot do the task' from 'cannot parse the prompt'" + }, + "note_on_undefined": "poetry / typo / association carry target: null. The annotated intermediate is the object of interest and there is no separate answer, so ANSWER COMPETENCE IS UNDEFINED for them, not zero.", + "interval": "95% Wilson score interval on every rate", + "by_model": { + "70m": { + "multihop": { + "n_items": 93, + "mean_prompt_surprisal": 4.742897197764407, + "median_prompt_surprisal": 4.761600017547607, + "n_scored_for_answer": 93, + "top1_count": 1, + "top1_rate": 0.010752688172043012, + "top1_wilson95": [ + 0.001900593097955601, + 0.05842059118648415 + ], + "top10_count": 13, + "top10_rate": 0.13978494623655913, + "top10_wilson95": [ + 0.08354643741721157, + 0.22460212683616715 + ], + "median_rank": 80 + }, + "multilingual": { + "n_items": 107, + "mean_prompt_surprisal": 5.743064506031643, + "median_prompt_surprisal": 5.629157066345215, + "n_scored_for_answer": 107, + "top1_count": 1, + "top1_rate": 0.009345794392523364, + "top1_wilson95": [ + 0.0016516291511190044, + 0.05105060538898146 + ], + "top10_count": 9, + "top10_rate": 0.08411214953271028, + "top10_wilson95": [ + 0.044881436613206184, + 0.15217093361602183 + ], + "median_rank": 3436 + }, + "order-ops": { + "n_items": 55, + "mean_prompt_surprisal": 4.685458686135032, + "median_prompt_surprisal": 4.4597907066345215, + "n_scored_for_answer": 55, + "top1_count": 2, + "top1_rate": 0.03636363636363636, + "top1_wilson95": [ + 0.010029330855724924, + 0.1232369297490173 + ], + "top10_count": 21, + "top10_rate": 0.38181818181818183, + "top10_wilson95": [ + 0.26515766317415235, + 0.513910207176076 + ], + "median_rank": 18 + }, + "poetry": { + "n_items": 98, + "mean_prompt_surprisal": 5.154343254712163, + "median_prompt_surprisal": 5.116718769073486, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "typo": { + "n_items": 96, + "mean_prompt_surprisal": 6.320249448219935, + "median_prompt_surprisal": 6.34007453918457, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "association": { + "n_items": 102, + "mean_prompt_surprisal": 4.577553527027953, + "median_prompt_surprisal": 4.60029411315918, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "_pooled_answer": { + "n": 255, + "top1_count": 4, + "top1_rate": 0.01568627450980392, + "top1_wilson95": [ + 0.006116456526536161, + 0.03963198576403846 + ], + "top10_rate": 0.16862745098039217, + "top10_wilson95": [ + 0.12766685656458393, + 0.2194241828973882 + ] + }, + "_pooled_surprisal_all_six": 5.203927769981855 + }, + "160m": { + "multihop": { + "n_items": 93, + "mean_prompt_surprisal": 4.323448388807235, + "median_prompt_surprisal": 4.265374183654785, + "n_scored_for_answer": 93, + "top1_count": 3, + "top1_rate": 0.03225806451612903, + "top1_wilson95": [ + 0.011030412935116203, + 0.09059533467748004 + ], + "top10_count": 24, + "top10_rate": 0.25806451612903225, + "top10_wilson95": [ + 0.17998399469266027, + 0.35533966786557925 + ], + "median_rank": 42 + }, + "multilingual": { + "n_items": 107, + "mean_prompt_surprisal": 6.095417096236042, + "median_prompt_surprisal": 5.124428749084473, + "n_scored_for_answer": 107, + "top1_count": 1, + "top1_rate": 0.009345794392523364, + "top1_wilson95": [ + 0.0016516291511190044, + 0.05105060538898146 + ], + "top10_count": 9, + "top10_rate": 0.08411214953271028, + "top10_wilson95": [ + 0.044881436613206184, + 0.15217093361602183 + ], + "median_rank": 3788 + }, + "order-ops": { + "n_items": 55, + "mean_prompt_surprisal": 4.33481783433394, + "median_prompt_surprisal": 4.353231430053711, + "n_scored_for_answer": 55, + "top1_count": 0, + "top1_rate": 0.0, + "top1_wilson95": [ + 0.0, + 0.06528714378942788 + ], + "top10_count": 24, + "top10_rate": 0.43636363636363634, + "top10_wilson95": [ + 0.31373238636001305, + 0.5673041592131869 + ], + "median_rank": 13 + }, + "poetry": { + "n_items": 98, + "mean_prompt_surprisal": 4.700447340400851, + "median_prompt_surprisal": 4.699031829833984, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "typo": { + "n_items": 96, + "mean_prompt_surprisal": 5.784147493541241, + "median_prompt_surprisal": 5.752850532531738, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "association": { + "n_items": 102, + "mean_prompt_surprisal": 4.123158244525685, + "median_prompt_surprisal": 4.120728969573975, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "_pooled_answer": { + "n": 255, + "top1_count": 4, + "top1_rate": 0.01568627450980392, + "top1_wilson95": [ + 0.006116456526536161, + 0.03963198576403846 + ], + "top10_rate": 0.2235294117647059, + "top10_wilson95": [ + 0.1767132210847063, + 0.27855208402853665 + ] + }, + "_pooled_surprisal_all_six": 4.893572732974166 + }, + "410m": { + "multihop": { + "n_items": 93, + "mean_prompt_surprisal": 3.8603221395964264, + "median_prompt_surprisal": 3.814849376678467, + "n_scored_for_answer": 93, + "top1_count": 6, + "top1_rate": 0.06451612903225806, + "top1_wilson95": [ + 0.029901024142029796, + 0.13368156846280116 + ], + "top10_count": 31, + "top10_rate": 0.3333333333333333, + "top10_wilson95": [ + 0.24582243182201943, + 0.4340672023847677 + ], + "median_rank": 31 + }, + "multilingual": { + "n_items": 107, + "mean_prompt_surprisal": 4.675777234763743, + "median_prompt_surprisal": 4.748653411865234, + "n_scored_for_answer": 107, + "top1_count": 1, + "top1_rate": 0.009345794392523364, + "top1_wilson95": [ + 0.0016516291511190044, + 0.05105060538898146 + ], + "top10_count": 6, + "top10_rate": 0.056074766355140186, + "top10_wilson95": [ + 0.025950008789329404, + 0.11697106055647577 + ], + "median_rank": 3044 + }, + "order-ops": { + "n_items": 55, + "mean_prompt_surprisal": 4.2336343418468125, + "median_prompt_surprisal": 4.275657653808594, + "n_scored_for_answer": 55, + "top1_count": 5, + "top1_rate": 0.09090909090909091, + "top1_wilson95": [ + 0.039457573336373924, + 0.19577736249133978 + ], + "top10_count": 30, + "top10_rate": 0.5454545454545454, + "top10_wilson95": [ + 0.41522461348360484, + 0.669749282535538 + ], + "median_rank": 9 + }, + "poetry": { + "n_items": 98, + "mean_prompt_surprisal": 4.148344592172272, + "median_prompt_surprisal": 4.126254558563232, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "typo": { + "n_items": 96, + "mean_prompt_surprisal": 5.49502004434665, + "median_prompt_surprisal": 5.293105602264404, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "association": { + "n_items": 102, + "mean_prompt_surprisal": 3.8331370657565547, + "median_prompt_surprisal": 3.8432066440582275, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "_pooled_answer": { + "n": 255, + "top1_count": 12, + "top1_rate": 0.047058823529411764, + "top1_wilson95": [ + 0.02712132140633029, + 0.08044098697068483 + ], + "top10_rate": 0.2627450980392157, + "top10_wilson95": [ + 0.21253225697118083, + 0.3200003807501127 + ] + }, + "_pooled_surprisal_all_six": 4.374372569747076 + }, + "1b": { + "multihop": { + "n_items": 93, + "mean_prompt_surprisal": 3.7122991469598587, + "median_prompt_surprisal": 3.713099241256714, + "n_scored_for_answer": 93, + "top1_count": 7, + "top1_rate": 0.07526881720430108, + "top1_wilson95": [ + 0.036936214085016475, + 0.14729866018389276 + ], + "top10_count": 31, + "top10_rate": 0.3333333333333333, + "top10_wilson95": [ + 0.24582243182201943, + 0.4340672023847677 + ], + "median_rank": 18 + }, + "multilingual": { + "n_items": 107, + "mean_prompt_surprisal": 4.4941605663745205, + "median_prompt_surprisal": 4.559407711029053, + "n_scored_for_answer": 107, + "top1_count": 0, + "top1_rate": 0.0, + "top1_wilson95": [ + 0.0, + 0.03465846757895952 + ], + "top10_count": 7, + "top10_rate": 0.06542056074766354, + "top10_wilson95": [ + 0.032047896397421095, + 0.12891693990952502 + ], + "median_rank": 1649 + }, + "order-ops": { + "n_items": 55, + "mean_prompt_surprisal": 4.145381216569381, + "median_prompt_surprisal": 4.083983898162842, + "n_scored_for_answer": 55, + "top1_count": 4, + "top1_rate": 0.07272727272727272, + "top1_wilson95": [ + 0.028643596503688543, + 0.17260178091636796 + ], + "top10_count": 37, + "top10_rate": 0.6727272727272727, + "top10_wilson95": [ + 0.5410297617609947, + 0.7818710431117484 + ], + "median_rank": 7 + }, + "poetry": { + "n_items": 98, + "mean_prompt_surprisal": 3.935050759996687, + "median_prompt_surprisal": 3.910858154296875, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "typo": { + "n_items": 96, + "mean_prompt_surprisal": 5.290679027636846, + "median_prompt_surprisal": 5.2639923095703125, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "association": { + "n_items": 102, + "mean_prompt_surprisal": 3.6671034354789582, + "median_prompt_surprisal": 3.6480932235717773, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "_pooled_answer": { + "n": 255, + "top1_count": 11, + "top1_rate": 0.043137254901960784, + "top1_wilson95": [ + 0.024254970635041635, + 0.07558060448116843 + ], + "top10_rate": 0.29411764705882354, + "top10_wilson95": [ + 0.24157991671293233, + 0.3527665870948018 + ] + }, + "_pooled_surprisal_all_six": 4.207445692169375 + }, + "1.4b": { + "multihop": { + "n_items": 93, + "mean_prompt_surprisal": 3.5891012683991463, + "median_prompt_surprisal": 3.6213531494140625, + "n_scored_for_answer": 93, + "top1_count": 5, + "top1_rate": 0.053763440860215055, + "top1_wilson95": [ + 0.023180332422992958, + 0.11974997851775977 + ], + "top10_count": 33, + "top10_rate": 0.3548387096774194, + "top10_wilson95": [ + 0.26512726316736557, + 0.45606693436757817 + ], + "median_rank": 24 + }, + "multilingual": { + "n_items": 107, + "mean_prompt_surprisal": 4.305595877014588, + "median_prompt_surprisal": 4.307002067565918, + "n_scored_for_answer": 107, + "top1_count": 0, + "top1_rate": 0.0, + "top1_wilson95": [ + 0.0, + 0.03465846757895952 + ], + "top10_count": 8, + "top10_rate": 0.07476635514018691, + "top10_wilson95": [ + 0.03836964356083345, + 0.1406389597072536 + ], + "median_rank": 1345 + }, + "order-ops": { + "n_items": 55, + "mean_prompt_surprisal": 4.166462430087003, + "median_prompt_surprisal": 4.244797229766846, + "n_scored_for_answer": 55, + "top1_count": 3, + "top1_rate": 0.05454545454545454, + "top1_wilson95": [ + 0.01872287596750008, + 0.14853294304489928 + ], + "top10_count": 29, + "top10_rate": 0.5272727272727272, + "top10_wilson95": [ + 0.39791319071629383, + 0.6530711468951917 + ], + "median_rank": 7 + }, + "poetry": { + "n_items": 98, + "mean_prompt_surprisal": 3.6910850977411074, + "median_prompt_surprisal": 3.6510846614837646, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "typo": { + "n_items": 96, + "mean_prompt_surprisal": 5.277833836774032, + "median_prompt_surprisal": 5.16269588470459, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "association": { + "n_items": 102, + "mean_prompt_surprisal": 3.6086690916734585, + "median_prompt_surprisal": 3.6544148921966553, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "_pooled_answer": { + "n": 255, + "top1_count": 8, + "top1_rate": 0.03137254901960784, + "top1_wilson95": [ + 0.01598082278729794, + 0.060674552546496924 + ], + "top10_rate": 0.27450980392156865, + "top10_wilson95": [ + 0.22338671792303816, + 0.33232611958067076 + ] + }, + "_pooled_surprisal_all_six": 4.106457933614889 + }, + "2.8b": { + "multihop": { + "n_items": 93, + "mean_prompt_surprisal": 3.5109470018776516, + "median_prompt_surprisal": 3.531287670135498, + "n_scored_for_answer": 93, + "top1_count": 6, + "top1_rate": 0.06451612903225806, + "top1_wilson95": [ + 0.029901024142029796, + 0.13368156846280116 + ], + "top10_count": 40, + "top10_rate": 0.43010752688172044, + "top10_wilson95": [ + 0.33423358533237524, + 0.5315265838511162 + ], + "median_rank": 14 + }, + "multilingual": { + "n_items": 107, + "mean_prompt_surprisal": 4.153321266174316, + "median_prompt_surprisal": 4.131369590759277, + "n_scored_for_answer": 107, + "top1_count": 1, + "top1_rate": 0.009345794392523364, + "top1_wilson95": [ + 0.0016516291511190044, + 0.05105060538898146 + ], + "top10_count": 10, + "top10_rate": 0.09345794392523364, + "top10_wilson95": [ + 0.05155771001191554, + 0.1635384271784534 + ], + "median_rank": 578 + }, + "order-ops": { + "n_items": 55, + "mean_prompt_surprisal": 4.071762145649303, + "median_prompt_surprisal": 4.097884178161621, + "n_scored_for_answer": 55, + "top1_count": 8, + "top1_rate": 0.14545454545454545, + "top1_wilson95": [ + 0.07559199016771778, + 0.2616116208829674 + ], + "top10_count": 34, + "top10_rate": 0.6181818181818182, + "top10_wilson95": [ + 0.4860897928239239, + 0.7348423368258477 + ], + "median_rank": 7 + }, + "poetry": { + "n_items": 98, + "mean_prompt_surprisal": 3.6883874936979644, + "median_prompt_surprisal": 3.6747243404388428, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "typo": { + "n_items": 96, + "mean_prompt_surprisal": 5.161082645257314, + "median_prompt_surprisal": 5.04867696762085, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "association": { + "n_items": 102, + "mean_prompt_surprisal": 3.4943248594508454, + "median_prompt_surprisal": 3.5100271701812744, + "answer_competence": "UNDEFINED \u2014 target is null for this family" + }, + "_pooled_answer": { + "n": 255, + "top1_count": 15, + "top1_rate": 0.058823529411764705, + "top1_wilson95": [ + 0.03596805781169063, + 0.09477445034773968 + ], + "top10_rate": 0.32941176470588235, + "top10_wilson95": [ + 0.2746295550144033, + 0.3892575481405764 + ] + }, + "_pooled_surprisal_all_six": 4.0133042353512325 + } + }, + "VERDICT": "Answer competence, 255 items with a target, 95% Wilson: 70m: top1 1.6% [0.6,4.0] | 160m: top1 1.6% [0.6,4.0] | 410m: top1 4.7% [2.7,8.0] | 1b: top1 4.3% [2.4,7.6] | 1.4b: top1 3.1% [1.6,6.1] | 2.8b: top1 5.9% [3.6,9.5]. Prompt surprisal covers all 551 items in all six families.", + "provenance": { + "schema_version": 1, + "experiment": "CV4P1", + "utc": "2026-08-30T06:30:11+00:00", + "script": { + "path": "experiments/cv4_phase1_capability.py", + "sha256": "3aaadc49c285d30fe1aafb92714633b4d5e694caac40f82b0afbfa964a424cfb", + "n_lines": 173 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/cv4_phase1_capability.py", + "--models", + "70m,160m,410m,1b,1.4b,2.8b" + ], + "cwd": ".", + "inputs": [ + { + "path": "jacobian-lens/data/evaluations/lens-eval-multihop.json", + "sha256": "50b7e4c9255291c0ca2a8e94615be9f44531fa57bb1a844e4f9616056d987416", + "bytes": 21869, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-multilingual.json", + "sha256": "fa70b9bd89416a6d8d985a80dc628b109ae6fd3b25b9275c0fc5065d7ff4a0ef", + "bytes": 24284, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-order-ops.json", + "sha256": "b203206d16ff628152cc86f3838604e06cb54776f3e14fa1c34f150db8bc7560", + "bytes": 9589, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-poetry.json", + "sha256": "6aeb3415c5a5c3f3827c9efe63f006de02f5ef39a816bbac68e15e733aba60cc", + "bytes": 21533, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-typo.json", + "sha256": "9d05e16b7234a57d0773d120a4e1c4e94fd3bc2235a8125d4200a70e60ab17aa", + "bytes": 14646, + "exists": true + }, + { + "path": "jacobian-lens/data/evaluations/lens-eval-association.json", + "sha256": "d1a98cd4911b594282e74168091c77d849dae18ffe2acb5761074853f327d71c", + "bytes": 24228, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "43b21658bcc6a1acf619b614a5eb74db9252372b3ff7f151fbbde0ce7fbd9bad" + } +} \ No newline at end of file diff --git a/results/cv5_metric_sensitivity_410m.json b/results/cv5_metric_sensitivity_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..150b89e3e704d06912f02fdeebab2bb2699a1ccf --- /dev/null +++ b/results/cv5_metric_sensitivity_410m.json @@ -0,0 +1,296 @@ +{ + "experiment": "CV5 \u2014 metric sensitivity: is rank blind where z is sighted?", + "prereg": "docs/experiments/preregs/CV5_metric_sensitivity.md", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "at r=2.535e-03: ACCEPT (RANK WAS BLIND) if S_z >= 1.0 and S_rank < 0.5 | REJECT (FUNCTIONALLY IDENTICAL) if S_z < 0.5 | UNCLEAR otherwise", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "base_operator": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "readout_convention": "STRIPPED (corrected), no prefix; flat-mean-7 k-summary", + "denominators_from_cv3": { + "pooled_seed_sd_rank": 0.002581, + "pooled_seed_sd_z": 0.013407 + }, + "unperturbed": { + "rank": 0.2619082412030062, + "z": 3.564116526229665, + "logit_I_rank": 0.1981085192666858 + }, + "by_magnitude": { + "0.0": { + "r": 0.0, + "n_seeds": 1, + "rank": [ + 0.2619082412030062 + ], + "z": [ + 3.564116526229665 + ], + "S_rank": 0.0, + "S_z": 0.0 + }, + "1.06e-07": { + "r": 1.06e-07, + "n_seeds": 3, + "rank": [ + 0.2619082412030062, + 0.2619082412030062, + 0.2619082412030062 + ], + "z": [ + 3.5641165554510494, + 3.5641163831333875, + 3.5641164395128424 + ], + "S_rank": 0.0, + "S_z": 4.9872383888545455e-06 + }, + "0.002535": { + "r": 0.002535, + "n_seeds": 3, + "rank": [ + 0.2617184725746441, + 0.2626325979167178, + 0.2620876530516675 + ], + "z": [ + 3.562412863826774, + 3.5641458966086614, + 3.5612677342495367 + ], + "S_rank": 0.09221231228343889, + "S_z": 0.11245578190554997 + }, + "0.01": { + "r": 0.01, + "n_seeds": 3, + "rank": [ + 0.2663647765893257, + 0.2655696648942096, + 0.26799035067682875 + ], + "z": [ + 3.573528114914198, + 3.562376512847641, + 3.5774573911741827 + ], + "S_rank": 1.8339233567538897, + "S_z": 0.5224246102042763 + }, + "0.05": { + "r": 0.05, + "n_seeds": 3, + "rank": [ + 0.28435305275900635, + 0.27584753020009956, + 0.2805677163156452 + ], + "z": [ + 3.495605627184469, + 3.473200516113597, + 3.4979920391660375 + ], + "S_rank": 7.108817727719558, + "S_z": 5.607801800673574 + } + }, + "PRIMARY_at_e66_magnitude": { + "r": 0.002535, + "S_rank": 0.09221231228343889, + "S_z": 0.11245578190554997 + }, + "VERDICT": "REJECT \u2014 THE OPERATORS ARE FUNCTIONALLY IDENTICAL", + "controls": { + "C1_zero_perturbation_is_noop": { + "required": "r=0 reproduces the unperturbed read exactly in both spaces", + "S_rank": 0.0, + "S_z": 0.0, + "fires": true + }, + "C2_logit_anchor": { + "required": "logit_I flat-mean-7 min == 0.19810852520167826 within 1e-6", + "observed": 0.1981085192666858, + "abs_diff": 5.934992469702749e-09, + "fires": true + }, + "C3_harness_responds": { + "required": "at r=5e-2 at least one space must move >= 1 seed SD, else the scoring path is not responding to the operator and every number is void", + "S_rank": 7.108817727719558, + "S_z": 5.607801800673574, + "fires": true + }, + "C4_monotone_in_r": { + "required": "S_z non-decreasing across magnitudes", + "S_z_by_r": [ + 0.0, + 4.9872383888545455e-06, + 0.11245578190554997, + 0.5224246102042763, + 5.607801800673574 + ], + "fires": true + } + }, + "declared_bias": "Gaussian noise is not a fitter difference. This bounds sensitivity to a perturbation of that size; it does not reproduce E66's specific operator delta, which is structural (accumulation order, batching, kernels).", + "e66_context": "results/e66_fitter_equivalence_cuda.json: trainval vs stored operator max_rel 2.535e-03 with read difference exactly 0.0; its C1 did not fire and its verdict flags the stored lens's provenance. The flagged file is the base operator used here and in CV3.", + "provenance": { + "schema_version": 1, + "experiment": "CV5", + "utc": "2026-08-30T00:18:45+00:00", + "script": { + "path": "experiments/cv5_metric_sensitivity.py", + "sha256": "f153b87305050c47829552d76859bc6d11c9b343419628b8122d2cc3d9c06ef6", + "n_lines": 198 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reviews/REVIEW_INSIGHTS_2026-08-25.md", + "docs/validity/CONSTRUCT_VALIDITY.md" + ], + "dirty_diff_sha256": "6b143c292229b3d2480d1b73243675f0db5e23b27f66e4215e0a9c1947a761a8", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/cv5_metric_sensitivity.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e66_fitter_equivalence_cuda.json", + "sha256": "d6e9bd3cd0c9e9426fc69e9d8d711f9a46e934214aaa52f67e877d947ec55cd2", + "bytes": 3208, + "exists": true + }, + { + "path": "results/cv3_margins_410m.json", + "sha256": "6cc200df5827b3c358d533934a37d892bfe1f7e9893bc4e4480421700afe7e99", + "bytes": 14659, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "cde8769ac592f81096a80090b47a40f14c66626684029dad622f967b054ed1d9" + } +} \ No newline at end of file diff --git a/results/cv6/cv6_1.4b_n25.json b/results/cv6/cv6_1.4b_n25.json new file mode 100644 index 0000000000000000000000000000000000000000..058bf088da2e7534c0e99da9b4f251c72f25005b --- /dev/null +++ b/results/cv6/cv6_1.4b_n25.json @@ -0,0 +1,1336 @@ +{ + "experiment": "CV6 \u2014 the per-family corpus contrast, one model", + "prereg": "docs/experiments/preregs/CV6_per_family_ladder.md", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "Evaluated PER FAMILY; the verdict is the count across the five. REPLICATES if R(f,2.8B) >= 10 on >= 4 of 5 families AND ordering tau >= 0.6 on >= 3 of 5. ATTENUATES if R(f,2.8B) < 5 on >= 4 of 5 families. UNCLEAR otherwise, including a split where some families replicate and others do not \u2014 a split is an informative outcome, not a failure. Do not pool to force a verdict. Do not add models to chase a threshold. Do not drop a family.", + "model": "EleutherAI/pythia-1.4b-deduped", + "short": "1.4b", + "n_layers": 24, + "d_model": 2048, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "band_rule": "floor(0.38*L)..floor(0.92*L), layers < L-2", + "band_matches_rule": true, + "N": 25, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first": 16, + "target_layer_arg": -2, + "target_layer_effective": 22, + "device": "cuda", + "dtype": "float32", + "tf32": { + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "gpu": "NVIDIA H100 NVL", + "gpu_total_gb": 93.08551025390625, + "peak_vram_gb": 26.350069046020508, + "torch": "2.11.0+cu128", + "git_commit_at_launch": "518d192e9e215603957c85ac4a1308153bf4cf06", + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "seeds": [ + 0, + 1, + 2 + ], + "readout_convention": "STRIPPED (corrected), no prefix, flat-mean-7", + "pooling_convention": "flat pool over (item, intermediate) pairs WITHIN a family; rank = min over band then flat-mean-7 pass@k; z = max over band then mean over pairs", + "n_items": 449, + "n_pairs": 801, + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "per_arm": { + "logit_I": { + "multihop": { + "rank": 0.2496532593619972, + "z": 3.8530033965712613, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1899927483683829, + "z": 3.2260746860564664, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5597402597402598, + "z": 5.515925762870095, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.008746355685131196, + "z": 2.4992136784962247, + "n_pairs": 98 + }, + "typo": { + "rank": 0.1889880952380952, + "z": 3.7020364006360373, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s0": { + "multihop": { + "rank": 0.3106796116504854, + "z": 3.8606277831549782, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1892675852066715, + "z": 3.0605827324886614, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5740259740259741, + "z": 5.3812003374099735, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.026239067055393583, + "z": 2.3095518247205384, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3392857142857143, + "z": 3.6630871382852397, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s1": { + "multihop": { + "rank": 0.3092926490984743, + "z": 3.8443020744231142, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1907179115300943, + "z": 3.077618185033653, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5545454545454546, + "z": 5.271360293301669, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784258, + "z": 2.327480596547224, + "n_pairs": 98 + }, + "typo": { + "rank": 0.33779761904761907, + "z": 3.6704573507110276, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s2": { + "multihop": { + "rank": 0.3065187239944521, + "z": 3.843342467419152, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1787527193618564, + "z": 3.0286048066797595, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5610389610389611, + "z": 5.311658326062289, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.32845254516115, + "n_pairs": 98 + }, + "typo": { + "rank": 0.33482142857142855, + "z": 3.67272778848807, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s0": { + "multihop": { + "rank": 0.32177531206657417, + "z": 3.874361871515663, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1783901377810007, + "z": 3.081772235746916, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5727272727272729, + "z": 5.26822690855373, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.352261223355118, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4404761904761905, + "z": 3.9461715444922447, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s1": { + "multihop": { + "rank": 0.319001386962552, + "z": 3.9217262459032742, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1787527193618564, + "z": 3.1000644785498603, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.596103896103896, + "z": 5.3440335826440295, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.3861058938259982, + "n_pairs": 98 + }, + "typo": { + "rank": 0.45238095238095244, + "z": 3.9819413671890893, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s2": { + "multihop": { + "rank": 0.32593619972260746, + "z": 3.934496842541741, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.18419144307469182, + "z": 3.1254321258080187, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5805194805194805, + "z": 5.343620314381339, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.026239067055393583, + "z": 2.369717459897606, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4568452380952381, + "z": 4.044803236921628, + "n_pairs": 96 + } + }, + "J|Pile-CC|s0": { + "multihop": { + "rank": 0.3203883495145631, + "z": 3.9831742198721876, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.20123277737490936, + "z": 3.2589381551984604, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5870129870129871, + "z": 5.469181368567726, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784254, + "z": 2.397173393137601, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3988095238095238, + "z": 4.014354603985946, + "n_pairs": 96 + } + }, + "J|Pile-CC|s1": { + "multihop": { + "rank": 0.32593619972260746, + "z": 4.029289348032868, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.19361856417693982, + "z": 3.188464311627567, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5922077922077923, + "z": 5.454147653146223, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.01749271137026239, + "z": 2.3707593989615536, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4107142857142857, + "z": 4.036523262659709, + "n_pairs": 96 + } + }, + "J|Pile-CC|s2": { + "multihop": { + "rank": 0.32177531206657417, + "z": 3.953363766947996, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1943437273386512, + "z": 3.190489783202331, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5870129870129871, + "z": 5.500762580741536, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.013119533527696792, + "z": 2.357462505296785, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4047619047619047, + "z": 4.010696103175481, + "n_pairs": 96 + } + }, + "J|StackExchange|s0": { + "multihop": { + "rank": 0.3883495145631068, + "z": 4.0767282235969615, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.21537345902828137, + "z": 3.1823133939716417, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5935064935064934, + "z": 5.240476467392662, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.3227810050759996, + "n_pairs": 98 + }, + "typo": { + "rank": 0.46428571428571425, + "z": 4.006702713668346, + "n_pairs": 96 + } + }, + "J|StackExchange|s1": { + "multihop": { + "rank": 0.391123439667129, + "z": 4.067388656069931, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.22588832487309643, + "z": 3.2193740812049905, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5844155844155844, + "z": 5.164196674390273, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784254, + "z": 2.2825519923044713, + "n_pairs": 98 + }, + "typo": { + "rank": 0.45982142857142855, + "z": 3.988087110221386, + "n_pairs": 96 + } + }, + "J|StackExchange|s2": { + "multihop": { + "rank": 0.4105409153952843, + "z": 4.105738782188268, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.2411167512690355, + "z": 3.290932434767031, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5909090909090909, + "z": 5.2581051761453805, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.309936959524544, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4613095238095238, + "z": 3.990582413971424, + "n_pairs": 96 + } + }, + "J|Github|s0": { + "multihop": { + "rank": 0.30374479889042993, + "z": 3.6554290958978597, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.13379260333575055, + "z": 2.789235497035351, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5571428571428572, + "z": 5.291046177257191, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.011661807580174929, + "z": 2.0492812370767397, + "n_pairs": 98 + }, + "typo": { + "rank": 0.20833333333333334, + "z": 3.194101763268312, + "n_pairs": 96 + } + }, + "J|Github|s1": { + "multihop": { + "rank": 0.32871012482662965, + "z": 3.81565054412027, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.16678752719361858, + "z": 2.9543404418804924, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5844155844155844, + "z": 5.422733649340543, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.013119533527696792, + "z": 2.1037646602002944, + "n_pairs": 98 + }, + "typo": { + "rank": 0.2574404761904762, + "z": 3.366413981964191, + "n_pairs": 96 + } + }, + "J|Github|s2": { + "multihop": { + "rank": 0.3398058252427184, + "z": 3.787343767082807, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.14176939811457578, + "z": 2.774672096755904, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5701298701298702, + "z": 5.435313402522694, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.01603498542274052, + "z": 2.0549783214014403, + "n_pairs": 98 + }, + "typo": { + "rank": 0.22470238095238093, + "z": 3.2467485206822553, + "n_pairs": 96 + } + } + }, + "per_arm_fp16_roundtrip": { + "J|Wikipedia_en|s0": { + "multihop": { + "rank": 0.3106796116504854, + "z": 3.8606872315545684, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1892675852066715, + "z": 3.060567912413989, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5740259740259741, + "z": 5.381180027398196, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.026239067055393583, + "z": 2.309572807988342, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3392857142857143, + "z": 3.6631390204032264, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s1": { + "multihop": { + "rank": 0.3092926490984743, + "z": 3.844225617288386, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1907179115300943, + "z": 3.0775656076857283, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5545454545454546, + "z": 5.271341731331566, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784258, + "z": 2.3274755514397913, + "n_pairs": 98 + }, + "typo": { + "rank": 0.33779761904761907, + "z": 3.6704695199926696, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s2": { + "multihop": { + "rank": 0.30513176144244103, + "z": 3.8434583180159043, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1787527193618564, + "z": 3.0286072580342367, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5610389610389611, + "z": 5.311677533929998, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.328465061528342, + "n_pairs": 98 + }, + "typo": { + "rank": 0.33482142857142855, + "z": 3.6727247970799604, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s0": { + "multihop": { + "rank": 0.32177531206657417, + "z": 3.8744102160907485, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1783901377810007, + "z": 3.081757281486153, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5727272727272729, + "z": 5.268081293322823, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.35228185568537, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4404761904761905, + "z": 3.946172351638476, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s1": { + "multihop": { + "rank": 0.3203883495145631, + "z": 3.921612312492815, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1783901377810007, + "z": 3.100029178379756, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.596103896103896, + "z": 5.34396392323754, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.386144852151676, + "n_pairs": 98 + }, + "typo": { + "rank": 0.45238095238095244, + "z": 3.9819253981113434, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s2": { + "multihop": { + "rank": 0.32871012482662965, + "z": 3.934536116794475, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.18419144307469182, + "z": 3.125436220374809, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5805194805194805, + "z": 5.34366934299469, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.026239067055393583, + "z": 2.3696852350721556, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4568452380952381, + "z": 4.044789165258408, + "n_pairs": 96 + } + }, + "J|Pile-CC|s0": { + "multihop": { + "rank": 0.3203883495145631, + "z": 3.9830801978851986, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.20123277737490936, + "z": 3.258956805249761, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5870129870129871, + "z": 5.469027082486586, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784254, + "z": 2.3971886932849884, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3988095238095238, + "z": 4.014335438609123, + "n_pairs": 96 + } + }, + "J|Pile-CC|s1": { + "multihop": { + "rank": 0.32593619972260746, + "z": 4.029135812833471, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.19361856417693982, + "z": 3.188446533105095, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5922077922077923, + "z": 5.454229792681607, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.01749271137026239, + "z": 2.370773496068254, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4092261904761905, + "z": 4.0365254531304045, + "n_pairs": 96 + } + }, + "J|Pile-CC|s2": { + "multihop": { + "rank": 0.32177531206657417, + "z": 3.9533760107836677, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.19398114575779551, + "z": 3.1904335273098825, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5870129870129871, + "z": 5.500757904486223, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.013119533527696792, + "z": 2.357441063438143, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4047619047619047, + "z": 4.010686029990514, + "n_pairs": 96 + } + }, + "J|StackExchange|s0": { + "multihop": { + "rank": 0.3883495145631068, + "z": 4.076668833065959, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.21537345902828137, + "z": 3.1823146536870657, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5935064935064934, + "z": 5.24048402634534, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.3227494511069082, + "n_pairs": 98 + }, + "typo": { + "rank": 0.46428571428571425, + "z": 4.006694952646892, + "n_pairs": 96 + } + }, + "J|StackExchange|s1": { + "multihop": { + "rank": 0.391123439667129, + "z": 4.067319025113744, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.22588832487309643, + "z": 3.21935731233074, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5844155844155844, + "z": 5.164137942140752, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784254, + "z": 2.2825479994014817, + "n_pairs": 98 + }, + "typo": { + "rank": 0.45982142857142855, + "z": 3.9880845795075097, + "n_pairs": 96 + } + }, + "J|StackExchange|s2": { + "multihop": { + "rank": 0.4091539528432732, + "z": 4.10576043140541, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.2411167512690355, + "z": 3.2908637373883107, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5909090909090909, + "z": 5.258090955560857, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.3098818407374986, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4613095238095238, + "z": 3.9905417760213218, + "n_pairs": 96 + } + }, + "J|Github|s0": { + "multihop": { + "rank": 0.30374479889042993, + "z": 3.6555041063179092, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.13379260333575055, + "z": 2.7891979518578136, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5571428571428572, + "z": 5.291175675392151, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.011661807580174929, + "z": 2.0492619884257413, + "n_pairs": 98 + }, + "typo": { + "rank": 0.20833333333333334, + "z": 3.1940517015755177, + "n_pairs": 96 + } + }, + "J|Github|s1": { + "multihop": { + "rank": 0.32871012482662965, + "z": 3.815705211417189, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.16678752719361858, + "z": 2.954350498122007, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5844155844155844, + "z": 5.422900062257593, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.013119533527696792, + "z": 2.103769786199745, + "n_pairs": 98 + }, + "typo": { + "rank": 0.2604166666666667, + "z": 3.3664332094291844, + "n_pairs": 96 + } + }, + "J|Github|s2": { + "multihop": { + "rank": 0.3398058252427184, + "z": 3.787404161055111, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.14176939811457578, + "z": 2.774661488793223, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5701298701298702, + "z": 5.435368727553975, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.01603498542274052, + "z": 2.054967626625178, + "n_pairs": 98 + }, + "typo": { + "rank": 0.22470238095238093, + "z": 3.2467462296287217, + "n_pairs": 96 + } + } + }, + "fits": { + "J|Wikipedia_en|s0": { + "n_prompts": 25, + "fit_seconds": 272.6, + "prompts_sha256": "9e54a0769bd230f55ba825a256e03c0f34730227e20dfc1b75efc797f711db71", + "corpus": "Wikipedia_en", + "seed_block": 0, + "fro_norm_per_layer": { + "9": 23.294700622558594, + "10": 25.415515899658203, + "11": 28.33647918701172, + "12": 30.57115936279297, + "13": 33.302181243896484, + "14": 36.702049255371094, + "15": 40.84243392944336, + "16": 42.165000915527344, + "17": 44.4361457824707, + "18": 44.770687103271484, + "19": 44.682952880859375, + "20": 45.84259033203125, + "21": 46.23477554321289 + } + }, + "J|Wikipedia_en|s1": { + "n_prompts": 25, + "fit_seconds": 275.1, + "prompts_sha256": "2ee5bb179eac0bd1859f75821abcde75138e042cf6c0d93e091acb78931a9fad", + "corpus": "Wikipedia_en", + "seed_block": 1, + "fro_norm_per_layer": { + "9": 23.690460205078125, + "10": 25.781587600708008, + "11": 28.686138153076172, + "12": 30.9707088470459, + "13": 33.714210510253906, + "14": 37.1748161315918, + "15": 41.22645950317383, + "16": 42.623661041259766, + "17": 44.789859771728516, + "18": 45.092281341552734, + "19": 44.922061920166016, + "20": 46.048622131347656, + "21": 46.331031799316406 + } + }, + "J|Wikipedia_en|s2": { + "n_prompts": 25, + "fit_seconds": 275.9, + "prompts_sha256": "dcd7a38d916ab5bc0adf8bff4400eb59938b7991e201e082cdc0164d7642659b", + "corpus": "Wikipedia_en", + "seed_block": 2, + "fro_norm_per_layer": { + "9": 23.420244216918945, + "10": 25.618276596069336, + "11": 28.51605796813965, + "12": 30.880279541015625, + "13": 33.711265563964844, + "14": 37.20738983154297, + "15": 41.2188720703125, + "16": 42.609657287597656, + "17": 44.832557678222656, + "18": 45.15241622924805, + "19": 44.87080764770508, + "20": 46.04928970336914, + "21": 46.32510757446289 + } + }, + "J|USPTO_Backgrounds|s0": { + "n_prompts": 25, + "fit_seconds": 276.3, + "prompts_sha256": "5a8f7c90631d3924019d98c6c3a93febb4702b0267d73e3bc369a8a1e7b824e4", + "corpus": "USPTO_Backgrounds", + "seed_block": 0, + "fro_norm_per_layer": { + "9": 27.90863037109375, + "10": 30.208759307861328, + "11": 32.88587951660156, + "12": 35.370208740234375, + "13": 38.3754768371582, + "14": 41.75584411621094, + "15": 45.2714958190918, + "16": 46.36663818359375, + "17": 47.852264404296875, + "18": 47.41462707519531, + "19": 46.47177505493164, + "20": 47.27696228027344, + "21": 46.95705032348633 + } + }, + "J|USPTO_Backgrounds|s1": { + "n_prompts": 25, + "fit_seconds": 276.5, + "prompts_sha256": "2ec94c2fa4eb9cb721a09d8f1ecd51be4392b6c77ff95f9b66fb173812f04aa3", + "corpus": "USPTO_Backgrounds", + "seed_block": 1, + "fro_norm_per_layer": { + "9": 28.49028778076172, + "10": 30.822561264038086, + "11": 33.30125045776367, + "12": 35.712852478027344, + "13": 38.649085998535156, + "14": 41.987709045410156, + "15": 45.43867111206055, + "16": 46.37141036987305, + "17": 47.787593841552734, + "18": 47.34893035888672, + "19": 46.38490676879883, + "20": 47.20484924316406, + "21": 46.929969787597656 + } + }, + "J|USPTO_Backgrounds|s2": { + "n_prompts": 25, + "fit_seconds": 276.4, + "prompts_sha256": "97174f19e4276737d5949b098c9ee4c669f51f178d9d499b1a5c86b6faf4afad", + "corpus": "USPTO_Backgrounds", + "seed_block": 2, + "fro_norm_per_layer": { + "9": 28.800617218017578, + "10": 31.102834701538086, + "11": 33.716331481933594, + "12": 36.08961486816406, + "13": 39.015830993652344, + "14": 42.280330657958984, + "15": 45.75420379638672, + "16": 46.61201477050781, + "17": 47.96596145629883, + "18": 47.46554946899414, + "19": 46.47589111328125, + "20": 47.297393798828125, + "21": 46.97040939331055 + } + }, + "J|Pile-CC|s0": { + "n_prompts": 25, + "fit_seconds": 276.6, + "prompts_sha256": "e0757ef16e1a8a552d467cf839e49c86eb7e90c7f090efa560e063141bf88c8e", + "corpus": "Pile-CC", + "seed_block": 0, + "fro_norm_per_layer": { + "9": 24.955005645751953, + "10": 27.270200729370117, + "11": 29.907175064086914, + "12": 32.539894104003906, + "13": 35.55636215209961, + "14": 39.129798889160156, + "15": 43.26397705078125, + "16": 44.49295425415039, + "17": 46.42964553833008, + "18": 46.149452209472656, + "19": 45.46807861328125, + "20": 46.5279541015625, + "21": 46.48268508911133 + } + }, + "J|Pile-CC|s1": { + "n_prompts": 25, + "fit_seconds": 276.7, + "prompts_sha256": "3dfe7843f22cef9b3d308d1f1cec3cade456f380b53ba1b8db8776ebe7fed77c", + "corpus": "Pile-CC", + "seed_block": 1, + "fro_norm_per_layer": { + "9": 25.38190269470215, + "10": 27.569387435913086, + "11": 30.241737365722656, + "12": 32.736114501953125, + "13": 35.672340393066406, + "14": 39.08313751220703, + "15": 43.28975296020508, + "16": 44.42156219482422, + "17": 46.440616607666016, + "18": 46.22185516357422, + "19": 45.58104705810547, + "20": 46.608673095703125, + "21": 46.55390930175781 + } + }, + "J|Pile-CC|s2": { + "n_prompts": 25, + "fit_seconds": 276.6, + "prompts_sha256": "782515fa0a56b1398a7237cda29cfac2a4e75c3bf91d1ffba99c86a19d3ab5aa", + "corpus": "Pile-CC", + "seed_block": 2, + "fro_norm_per_layer": { + "9": 25.042362213134766, + "10": 27.282052993774414, + "11": 29.89794158935547, + "12": 32.4775505065918, + "13": 35.40144348144531, + "14": 38.882041931152344, + "15": 43.07960510253906, + "16": 44.226409912109375, + "17": 46.159019470214844, + "18": 45.96290588378906, + "19": 45.31592559814453, + "20": 46.45462417602539, + "21": 46.471435546875 + } + }, + "J|StackExchange|s0": { + "n_prompts": 25, + "fit_seconds": 277.0, + "prompts_sha256": "fb838aed6ad4bb5655506149821e49a7be3de1e40422abf3e744afa0fd80df39", + "corpus": "StackExchange", + "seed_block": 0, + "fro_norm_per_layer": { + "9": 26.929075241088867, + "10": 29.292173385620117, + "11": 31.924766540527344, + "12": 34.485313415527344, + "13": 37.15853500366211, + "14": 40.80641555786133, + "15": 44.48529815673828, + "16": 45.89080810546875, + "17": 47.45210266113281, + "18": 47.164772033691406, + "19": 46.41997528076172, + "20": 47.250492095947266, + "21": 46.89768600463867 + } + }, + "J|StackExchange|s1": { + "n_prompts": 25, + "fit_seconds": 276.4, + "prompts_sha256": "99d6fe9822aacd471eebbf96e115750a6247e2e1bef7353640f9461dfe1aad6b", + "corpus": "StackExchange", + "seed_block": 1, + "fro_norm_per_layer": { + "9": 27.617984771728516, + "10": 30.13306427001953, + "11": 32.612632751464844, + "12": 35.15741729736328, + "13": 37.79157257080078, + "14": 41.37997055053711, + "15": 45.01513671875, + "16": 46.328369140625, + "17": 47.70050811767578, + "18": 47.40119552612305, + "19": 46.614158630371094, + "20": 47.42560958862305, + "21": 47.04204559326172 + } + }, + "J|StackExchange|s2": { + "n_prompts": 25, + "fit_seconds": 276.3, + "prompts_sha256": "094be840f776043c464c6dc2ada90ee60b7a6349a4faf5edbee6f7646e048e58", + "corpus": "StackExchange", + "seed_block": 2, + "fro_norm_per_layer": { + "9": 27.36570930480957, + "10": 29.822795867919922, + "11": 32.43924331665039, + "12": 35.0574836730957, + "13": 37.744300842285156, + "14": 41.3569221496582, + "15": 45.00175476074219, + "16": 46.438682556152344, + "17": 47.874332427978516, + "18": 47.57386779785156, + "19": 46.77056884765625, + "20": 47.53171157836914, + "21": 47.10435104370117 + } + }, + "J|Github|s0": { + "n_prompts": 25, + "fit_seconds": 275.9, + "prompts_sha256": "fb6856c864ad3d738b939e1ca21b65a129f645bf18dd0221c4c9e2c1dadb7fe5", + "corpus": "Github", + "seed_block": 0, + "fro_norm_per_layer": { + "9": 23.499574661254883, + "10": 25.101781845092773, + "11": 27.236875534057617, + "12": 29.104747772216797, + "13": 31.28536033630371, + "14": 34.06266403198242, + "15": 37.80937957763672, + "16": 39.80789566040039, + "17": 42.34746551513672, + "18": 43.70899200439453, + "19": 44.678932189941406, + "20": 45.65796661376953, + "21": 46.37763214111328 + } + }, + "J|Github|s1": { + "n_prompts": 25, + "fit_seconds": 275.3, + "prompts_sha256": "8acd4ae02d98b1cb29ad26750a7f98b3f9bb842226f217fbc2ab18547092f0a1", + "corpus": "Github", + "seed_block": 1, + "fro_norm_per_layer": { + "9": 24.057697296142578, + "10": 25.858362197875977, + "11": 28.058307647705078, + "12": 30.10573387145996, + "13": 32.42531204223633, + "14": 35.27041244506836, + "15": 39.16661071777344, + "16": 41.07899856567383, + "17": 43.373783111572266, + "18": 44.43169403076172, + "19": 45.091068267822266, + "20": 46.02471160888672, + "21": 46.53116226196289 + } + }, + "J|Github|s2": { + "n_prompts": 25, + "fit_seconds": 275.3, + "prompts_sha256": "fb8e687f84bbaca1821949876dabd6beb24389556d54c1fa6d9234184037fc68", + "corpus": "Github", + "seed_block": 2, + "fro_norm_per_layer": { + "9": 23.682188034057617, + "10": 25.320621490478516, + "11": 27.47364044189453, + "12": 29.382030487060547, + "13": 31.588592529296875, + "14": 34.38534927368164, + "15": 38.232765197753906, + "16": 40.23326873779297, + "17": 42.76146697998047, + "18": 44.06857681274414, + "19": 44.8614501953125, + "20": 45.87016677856445, + "21": 46.53526306152344 + } + } + }, + "logit_I_pooled_rank_flatmean7": 0.23942414367877324, + "provenance": { + "schema_version": 1, + "experiment": "CV6", + "utc": "2026-08-24T00:26:12+00:00", + "script": { + "path": "experiments/cv6_per_family_ladder.py", + "sha256": "f416c3cb55e95a02315504066479c2f2ba073c0d4676bbf09adb2f481cdb4a5b", + "n_lines": 750 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/cv6_per_family_ladder.py", + "--model", + "1.4b", + "--device", + "cuda", + "--cache-out", + "results/cv6/ht_cache_1.4b.pt", + "--save-lens-dir", + "results/cv6/lenses" + ], + "cwd": ".", + "inputs": [ + { + "path": "corpora/Wikipedia_en.jsonl", + "sha256": "b30f4cef11add560f7df8ed07a613e3fcc6a4dda865975bec15f2952253bda77", + "bytes": 10329196, + "exists": true + }, + { + "path": "corpora/USPTO_Backgrounds.jsonl", + "sha256": "b09ac74bdc8b169ee5c9bea6be481262f743947a92107164f30eca2346e3ccb9", + "bytes": 10221054, + "exists": true + }, + { + "path": "corpora/Pile-CC.jsonl", + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "bytes": 13512530, + "exists": true + }, + { + "path": "corpora/StackExchange.jsonl", + "sha256": "bc25e2d5775e9719ebcbdaa3bd115d827f72ebf11f47fedb377f8b0a69a28971", + "bytes": 5980156, + "exists": true + }, + { + "path": "corpora/Github.jsonl", + "sha256": "f2bb459d10cad6c13a8b66c86191bc43df4393ba5b722a5315ea878fad2d1a26", + "bytes": 17486378, + "exists": true + } + ], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.1", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-157-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "307871427cdc9dbafbe962115e903a649c5f232dc75ac91163e4dd3832b28687" + } +} \ No newline at end of file diff --git a/results/cv6/cv6_2.8b_n25.json b/results/cv6/cv6_2.8b_n25.json new file mode 100644 index 0000000000000000000000000000000000000000..73e062017b62783a01eaf754e6a811701567a2af --- /dev/null +++ b/results/cv6/cv6_2.8b_n25.json @@ -0,0 +1,1416 @@ +{ + "experiment": "CV6 \u2014 the per-family corpus contrast, one model", + "prereg": "docs/experiments/preregs/CV6_per_family_ladder.md", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "Evaluated PER FAMILY; the verdict is the count across the five. REPLICATES if R(f,2.8B) >= 10 on >= 4 of 5 families AND ordering tau >= 0.6 on >= 3 of 5. ATTENUATES if R(f,2.8B) < 5 on >= 4 of 5 families. UNCLEAR otherwise, including a split where some families replicate and others do not \u2014 a split is an informative outcome, not a failure. Do not pool to force a verdict. Do not add models to chase a threshold. Do not drop a family.", + "model": "EleutherAI/pythia-2.8b-deduped", + "short": "2.8b", + "n_layers": 32, + "d_model": 2560, + "band": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "band_rule": "floor(0.38*L)..floor(0.92*L), layers < L-2", + "band_matches_rule": true, + "N": 25, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first": 16, + "target_layer_arg": -2, + "target_layer_effective": 30, + "device": "cuda", + "dtype": "float32", + "tf32": { + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "gpu": "NVIDIA H100 NVL", + "gpu_total_gb": 93.08551025390625, + "peak_vram_gb": 46.3552303314209, + "torch": "2.11.0+cu128", + "git_commit_at_launch": "518d192e9e215603957c85ac4a1308153bf4cf06", + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "seeds": [ + 0, + 1, + 2 + ], + "readout_convention": "STRIPPED (corrected), no prefix, flat-mean-7", + "pooling_convention": "flat pool over (item, intermediate) pairs WITHIN a family; rank = min over band then flat-mean-7 pass@k; z = max over band then mean over pairs", + "n_items": 449, + "n_pairs": 801, + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "per_arm": { + "logit_I": { + "multihop": { + "rank": 0.27600554785020803, + "z": 3.693257671826094, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.22588832487309643, + "z": 3.0744364978603755, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.538961038961039, + "z": 5.5365345900708975, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.02769679300291545, + "z": 2.430429516398177, + "n_pairs": 98 + }, + "typo": { + "rank": 0.28125, + "z": 3.6758319437503815, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s0": { + "multihop": { + "rank": 0.37170596393897365, + "z": 3.748083519704134, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3176214648295867, + "z": 3.4327350325390773, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5688311688311688, + "z": 5.169967714222994, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06268221574344024, + "z": 2.520261741414362, + "n_pairs": 98 + }, + "typo": { + "rank": 0.2976190476190476, + "z": 3.459335678567489, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s1": { + "multihop": { + "rank": 0.37586685159500693, + "z": 3.7837847610121793, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.31725888324873097, + "z": 3.460326204445156, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5844155844155845, + "z": 5.257566994970495, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06268221574344023, + "z": 2.5308002853880125, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3199404761904762, + "z": 3.5066992342472076, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s2": { + "multihop": { + "rank": 0.36893203883495146, + "z": 3.7126920130646344, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3110949963741842, + "z": 3.4116927991966306, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5805194805194805, + "z": 5.221894264221191, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06268221574344023, + "z": 2.5136427721198724, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3095238095238096, + "z": 3.4875277963777385, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s0": { + "multihop": { + "rank": 0.420249653259362, + "z": 3.9431607584351474, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3328498912255258, + "z": 3.617218898637646, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.6246753246753247, + "z": 5.513402173735878, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.08454810495626822, + "z": 2.645613257982293, + "n_pairs": 98 + }, + "typo": { + "rank": 0.42708333333333337, + "z": 3.7516257812579474, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s1": { + "multihop": { + "rank": 0.4133148404993065, + "z": 3.9598770234191303, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3390137781000725, + "z": 3.662088529410096, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.6571428571428571, + "z": 5.61998973976482, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.09329446064139943, + "z": 2.6892670818737576, + "n_pairs": 98 + }, + "typo": { + "rank": 0.44345238095238093, + "z": 3.785833690315485, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s2": { + "multihop": { + "rank": 0.39944521497919555, + "z": 3.9532608187314375, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.34590282813633066, + "z": 3.7125451938452456, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.6376623376623376, + "z": 5.639909208904613, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.10787172011661807, + "z": 2.747300364533249, + "n_pairs": 98 + }, + "typo": { + "rank": 0.44642857142857145, + "z": 3.8546179458498955, + "n_pairs": 96 + } + }, + "J|Pile-CC|s0": { + "multihop": { + "rank": 0.3828016643550624, + "z": 3.9039036672092178, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3332124728063815, + "z": 3.6563512620889598, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5922077922077922, + "z": 5.404743753780018, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06851311953352769, + "z": 2.5789867603048986, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3630952380952381, + "z": 3.8045580573379993, + "n_pairs": 96 + } + }, + "J|Pile-CC|s1": { + "multihop": { + "rank": 0.3883495145631068, + "z": 3.8896664404174657, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3234227701232778, + "z": 3.567083712913058, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5818181818181819, + "z": 5.46329207257791, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05393586005830903, + "z": 2.606388902177616, + "n_pairs": 98 + }, + "typo": { + "rank": 0.40476190476190477, + "z": 3.885098062455654, + "n_pairs": 96 + } + }, + "J|Pile-CC|s2": { + "multihop": { + "rank": 0.391123439667129, + "z": 3.9153419918226966, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.33575054387237124, + "z": 3.6062580548264656, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5909090909090909, + "z": 5.418533691492947, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06705539358600583, + "z": 2.5475755419049944, + "n_pairs": 98 + }, + "typo": { + "rank": 0.38244047619047616, + "z": 3.770227551460266, + "n_pairs": 96 + } + }, + "J|StackExchange|s0": { + "multihop": { + "rank": 0.42718446601941745, + "z": 4.062357205789066, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.33575054387237124, + "z": 3.7145613397135953, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5675324675324674, + "z": 5.500116468559612, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05830903790087464, + "z": 2.4863040441153, + "n_pairs": 98 + }, + "typo": { + "rank": 0.40029761904761907, + "z": 3.8565272353589535, + "n_pairs": 96 + } + }, + "J|StackExchange|s1": { + "multihop": { + "rank": 0.40083217753120665, + "z": 3.945768759088609, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.34699057287889773, + "z": 3.717590422497183, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5597402597402598, + "z": 5.4544677734375, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05830903790087464, + "z": 2.4743533736589005, + "n_pairs": 98 + }, + "typo": { + "rank": 0.40625, + "z": 3.858109233280023, + "n_pairs": 96 + } + }, + "J|StackExchange|s2": { + "multihop": { + "rank": 0.42857142857142866, + "z": 4.065541328735722, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3328498912255258, + "z": 3.696884382043393, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5857142857142856, + "z": 5.543083320964467, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05393586005830903, + "z": 2.4796542653015683, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4032738095238095, + "z": 3.8242265147467456, + "n_pairs": 96 + } + }, + "J|Github|s0": { + "multihop": { + "rank": 0.32454923717059636, + "z": 3.357145009688961, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.23567802755620013, + "z": 2.9966616588195567, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5999999999999999, + "z": 5.133839065378362, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.030612244897959186, + "z": 2.274473075964013, + "n_pairs": 98 + }, + "typo": { + "rank": 0.2202380952380952, + "z": 3.1765695475041866, + "n_pairs": 96 + } + }, + "J|Github|s1": { + "multihop": { + "rank": 0.348127600554785, + "z": 3.5227397807593483, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.2646845540246555, + "z": 3.135737712764498, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5896103896103896, + "z": 5.305073720758611, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.039358600583090375, + "z": 2.350923566185698, + "n_pairs": 98 + }, + "typo": { + "rank": 0.27232142857142855, + "z": 3.336014381299416, + "n_pairs": 96 + } + }, + "J|Github|s2": { + "multihop": { + "rank": 0.32316227461858527, + "z": 3.391732178845452, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.2548948513415518, + "z": 3.034613796599625, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5935064935064934, + "z": 5.225355126641014, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.03498542274052478, + "z": 2.3192731969210567, + "n_pairs": 98 + }, + "typo": { + "rank": 0.25595238095238093, + "z": 3.23986042663455, + "n_pairs": 96 + } + } + }, + "per_arm_fp16_roundtrip": { + "J|Wikipedia_en|s0": { + "multihop": { + "rank": 0.37170596393897365, + "z": 3.7479759431579738, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3176214648295867, + "z": 3.4326538239336255, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5688311688311688, + "z": 5.170113795453852, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06268221574344024, + "z": 2.5202506519093806, + "n_pairs": 98 + }, + "typo": { + "rank": 0.2976190476190476, + "z": 3.459295087804397, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s1": { + "multihop": { + "rank": 0.37586685159500693, + "z": 3.78365947783572, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.31725888324873097, + "z": 3.4602457260722437, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5844155844155845, + "z": 5.2574653549627826, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.061224489795918366, + "z": 2.5307455500777887, + "n_pairs": 98 + }, + "typo": { + "rank": 0.32142857142857145, + "z": 3.5066481679677963, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s2": { + "multihop": { + "rank": 0.36893203883495146, + "z": 3.7126931484463146, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3110949963741842, + "z": 3.4117022766377114, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5805194805194805, + "z": 5.221943621201949, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06268221574344023, + "z": 2.513642885247055, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3095238095238096, + "z": 3.487534542878469, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s0": { + "multihop": { + "rank": 0.420249653259362, + "z": 3.943197340641207, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3328498912255258, + "z": 3.617224749272245, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.6246753246753247, + "z": 5.513433169234883, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.08454810495626822, + "z": 2.6456040983297386, + "n_pairs": 98 + }, + "typo": { + "rank": 0.42708333333333337, + "z": 3.7516461834311485, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s1": { + "multihop": { + "rank": 0.4133148404993065, + "z": 3.9599806452260435, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3390137781000725, + "z": 3.6621401048251214, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.6571428571428571, + "z": 5.619987377253445, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.09329446064139943, + "z": 2.6892731098496183, + "n_pairs": 98 + }, + "typo": { + "rank": 0.44345238095238093, + "z": 3.7858387008309364, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s2": { + "multihop": { + "rank": 0.39944521497919555, + "z": 3.9531577559350763, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.34590282813633066, + "z": 3.712495117471908, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.6376623376623376, + "z": 5.639850367199291, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.10787172011661807, + "z": 2.747182385045655, + "n_pairs": 98 + }, + "typo": { + "rank": 0.44642857142857145, + "z": 3.854549737026294, + "n_pairs": 96 + } + }, + "J|Pile-CC|s0": { + "multihop": { + "rank": 0.3828016643550624, + "z": 3.903921564805855, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3332124728063815, + "z": 3.6563817554621525, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5922077922077922, + "z": 5.404603629762476, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06851311953352769, + "z": 2.5789884632947495, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3630952380952381, + "z": 3.804555675635735, + "n_pairs": 96 + } + }, + "J|Pile-CC|s1": { + "multihop": { + "rank": 0.3883495145631068, + "z": 3.8897387078664836, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3234227701232778, + "z": 3.56714168538902, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5818181818181819, + "z": 5.463297904621471, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05393586005830903, + "z": 2.6064579340876364, + "n_pairs": 98 + }, + "typo": { + "rank": 0.40476190476190477, + "z": 3.8851715164879956, + "n_pairs": 96 + } + }, + "J|Pile-CC|s2": { + "multihop": { + "rank": 0.391123439667129, + "z": 3.9151969303205174, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.33575054387237124, + "z": 3.606160680805971, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5909090909090909, + "z": 5.418469522216103, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.06705539358600583, + "z": 2.5475342316286906, + "n_pairs": 98 + }, + "typo": { + "rank": 0.38244047619047616, + "z": 3.7702168126900992, + "n_pairs": 96 + } + }, + "J|StackExchange|s0": { + "multihop": { + "rank": 0.42718446601941745, + "z": 4.062260240027048, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3353879622915156, + "z": 3.7145396611714725, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5675324675324674, + "z": 5.500007843971252, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05830903790087464, + "z": 2.4862898527359474, + "n_pairs": 98 + }, + "typo": { + "rank": 0.40029761904761907, + "z": 3.856445398181677, + "n_pairs": 96 + } + }, + "J|StackExchange|s1": { + "multihop": { + "rank": 0.40083217753120665, + "z": 3.9458269378513964, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.34699057287889773, + "z": 3.717639919616244, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5597402597402598, + "z": 5.454329214312813, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05830903790087464, + "z": 2.474340495406365, + "n_pairs": 98 + }, + "typo": { + "rank": 0.40625, + "z": 3.8580934616426625, + "n_pairs": 96 + } + }, + "J|StackExchange|s2": { + "multihop": { + "rank": 0.42857142857142866, + "z": 4.065570627601401, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.3328498912255258, + "z": 3.6968645884603415, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5857142857142856, + "z": 5.5429936354810545, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.05393586005830903, + "z": 2.4796188942023685, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4032738095238095, + "z": 3.8242418008546033, + "n_pairs": 96 + } + }, + "J|Github|s0": { + "multihop": { + "rank": 0.32454923717059636, + "z": 3.357211357181512, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.23567802755620013, + "z": 2.9966800858829226, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5999999999999999, + "z": 5.133947640115564, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.030612244897959186, + "z": 2.2744970212177353, + "n_pairs": 98 + }, + "typo": { + "rank": 0.2202380952380952, + "z": 3.176635969430208, + "n_pairs": 96 + } + }, + "J|Github|s1": { + "multihop": { + "rank": 0.348127600554785, + "z": 3.522636915873555, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.2646845540246555, + "z": 3.135660663171468, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5896103896103896, + "z": 5.3051688779484145, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.039358600583090375, + "z": 2.3509351890914294, + "n_pairs": 98 + }, + "typo": { + "rank": 0.27232142857142855, + "z": 3.3360372707247734, + "n_pairs": 96 + } + }, + "J|Github|s2": { + "multihop": { + "rank": 0.32316227461858527, + "z": 3.3917247172698235, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.2548948513415518, + "z": 3.034596077681798, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5935064935064934, + "z": 5.225318701700731, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.03498542274052478, + "z": 2.3192666671714006, + "n_pairs": 98 + }, + "typo": { + "rank": 0.25595238095238093, + "z": 3.2398867520193257, + "n_pairs": 96 + } + } + }, + "fits": { + "J|Wikipedia_en|s0": { + "n_prompts": 25, + "fit_seconds": 717.8, + "prompts_sha256": "9e54a0769bd230f55ba825a256e03c0f34730227e20dfc1b75efc797f711db71", + "corpus": "Wikipedia_en", + "seed_block": 0, + "fro_norm_per_layer": { + "12": 25.47411346435547, + "13": 28.14723014831543, + "14": 30.691478729248047, + "15": 33.2955436706543, + "16": 35.893863677978516, + "17": 37.72172927856445, + "18": 39.467960357666016, + "19": 41.63821029663086, + "20": 43.01144027709961, + "21": 44.35385513305664, + "22": 45.25417709350586, + "23": 46.21498489379883, + "24": 47.24529266357422, + "25": 48.25273513793945, + "26": 49.452205657958984, + "27": 49.94655990600586, + "28": 50.0068359375, + "29": 50.67486572265625 + } + }, + "J|Wikipedia_en|s1": { + "n_prompts": 25, + "fit_seconds": 715.9, + "prompts_sha256": "2ee5bb179eac0bd1859f75821abcde75138e042cf6c0d93e091acb78931a9fad", + "corpus": "Wikipedia_en", + "seed_block": 1, + "fro_norm_per_layer": { + "12": 25.824872970581055, + "13": 28.51796531677246, + "14": 31.0869197845459, + "15": 33.67380142211914, + "16": 36.31361770629883, + "17": 38.17379379272461, + "18": 39.9384880065918, + "19": 42.0767707824707, + "20": 43.39258575439453, + "21": 44.7066650390625, + "22": 45.6107063293457, + "23": 46.55977249145508, + "24": 47.58478927612305, + "25": 48.54484939575195, + "26": 49.71868133544922, + "27": 50.12144088745117, + "28": 50.1476936340332, + "29": 50.75876235961914 + } + }, + "J|Wikipedia_en|s2": { + "n_prompts": 25, + "fit_seconds": 716.5, + "prompts_sha256": "dcd7a38d916ab5bc0adf8bff4400eb59938b7991e201e082cdc0164d7642659b", + "corpus": "Wikipedia_en", + "seed_block": 2, + "fro_norm_per_layer": { + "12": 25.764179229736328, + "13": 28.572370529174805, + "14": 31.217775344848633, + "15": 33.85537338256836, + "16": 36.57388687133789, + "17": 38.43079376220703, + "18": 40.218143463134766, + "19": 42.400333404541016, + "20": 43.739261627197266, + "21": 45.03773880004883, + "22": 45.866294860839844, + "23": 46.75215530395508, + "24": 47.74189376831055, + "25": 48.71947479248047, + "26": 49.884422302246094, + "27": 50.27415466308594, + "28": 50.24495315551758, + "29": 50.82538604736328 + } + }, + "J|USPTO_Backgrounds|s0": { + "n_prompts": 25, + "fit_seconds": 715.8, + "prompts_sha256": "5a8f7c90631d3924019d98c6c3a93febb4702b0267d73e3bc369a8a1e7b824e4", + "corpus": "USPTO_Backgrounds", + "seed_block": 0, + "fro_norm_per_layer": { + "12": 30.236732482910156, + "13": 32.75661087036133, + "14": 35.58098220825195, + "15": 38.3680305480957, + "16": 40.91820526123047, + "17": 42.26591873168945, + "18": 43.79016876220703, + "19": 45.786720275878906, + "20": 46.8199577331543, + "21": 47.73601531982422, + "22": 48.22069549560547, + "23": 48.858943939208984, + "24": 49.517452239990234, + "25": 50.26908493041992, + "26": 51.17799377441406, + "27": 51.0509033203125, + "28": 50.70587921142578, + "29": 51.15618133544922 + } + }, + "J|USPTO_Backgrounds|s1": { + "n_prompts": 25, + "fit_seconds": 714.0, + "prompts_sha256": "2ec94c2fa4eb9cb721a09d8f1ecd51be4392b6c77ff95f9b66fb173812f04aa3", + "corpus": "USPTO_Backgrounds", + "seed_block": 1, + "fro_norm_per_layer": { + "12": 30.12862777709961, + "13": 32.7627067565918, + "14": 35.53038024902344, + "15": 38.333805084228516, + "16": 40.84742736816406, + "17": 42.114013671875, + "18": 43.58375930786133, + "19": 45.52498245239258, + "20": 46.5215950012207, + "21": 47.474063873291016, + "22": 47.912315368652344, + "23": 48.55392837524414, + "24": 49.21796798706055, + "25": 49.9576416015625, + "26": 50.92783737182617, + "27": 50.82968521118164, + "28": 50.54206085205078, + "29": 51.04257583618164 + } + }, + "J|USPTO_Backgrounds|s2": { + "n_prompts": 25, + "fit_seconds": 713.8, + "prompts_sha256": "97174f19e4276737d5949b098c9ee4c669f51f178d9d499b1a5c86b6faf4afad", + "corpus": "USPTO_Backgrounds", + "seed_block": 2, + "fro_norm_per_layer": { + "12": 31.08220863342285, + "13": 33.626853942871094, + "14": 36.36945724487305, + "15": 39.220069885253906, + "16": 41.716712951660156, + "17": 42.856788635253906, + "18": 44.255191802978516, + "19": 46.21339416503906, + "20": 47.17402648925781, + "21": 48.00918960571289, + "22": 48.380096435546875, + "23": 48.957420349121094, + "24": 49.573543548583984, + "25": 50.27798843383789, + "26": 51.223289489746094, + "27": 51.076263427734375, + "28": 50.735076904296875, + "29": 51.1742057800293 + } + }, + "J|Pile-CC|s0": { + "n_prompts": 25, + "fit_seconds": 712.9, + "prompts_sha256": "e0757ef16e1a8a552d467cf839e49c86eb7e90c7f090efa560e063141bf88c8e", + "corpus": "Pile-CC", + "seed_block": 0, + "fro_norm_per_layer": { + "12": 27.326833724975586, + "13": 30.332229614257812, + "14": 33.242000579833984, + "15": 36.10725784301758, + "16": 38.962711334228516, + "17": 40.80857849121094, + "18": 42.63570022583008, + "19": 44.908409118652344, + "20": 46.073490142822266, + "21": 47.05865478515625, + "22": 47.51934051513672, + "23": 48.309513092041016, + "24": 49.009803771972656, + "25": 49.740360260009766, + "26": 50.79483413696289, + "27": 50.875579833984375, + "28": 50.6129264831543, + "29": 51.0517692565918 + } + }, + "J|Pile-CC|s1": { + "n_prompts": 25, + "fit_seconds": 712.5, + "prompts_sha256": "3dfe7843f22cef9b3d308d1f1cec3cade456f380b53ba1b8db8776ebe7fed77c", + "corpus": "Pile-CC", + "seed_block": 1, + "fro_norm_per_layer": { + "12": 28.01110076904297, + "13": 30.7965030670166, + "14": 33.55686569213867, + "15": 36.41100311279297, + "16": 39.08540344238281, + "17": 40.86274337768555, + "18": 42.52493667602539, + "19": 44.74737548828125, + "20": 45.96009063720703, + "21": 46.924781799316406, + "22": 47.40066146850586, + "23": 48.23330307006836, + "24": 48.95878982543945, + "25": 49.670536041259766, + "26": 50.714534759521484, + "27": 50.77128601074219, + "28": 50.543941497802734, + "29": 50.97682189941406 + } + }, + "J|Pile-CC|s2": { + "n_prompts": 25, + "fit_seconds": 712.7, + "prompts_sha256": "782515fa0a56b1398a7237cda29cfac2a4e75c3bf91d1ffba99c86a19d3ab5aa", + "corpus": "Pile-CC", + "seed_block": 2, + "fro_norm_per_layer": { + "12": 27.110673904418945, + "13": 30.076528549194336, + "14": 32.917598724365234, + "15": 35.8157844543457, + "16": 38.64012908935547, + "17": 40.50718307495117, + "18": 42.25164794921875, + "19": 44.48286437988281, + "20": 45.72807312011719, + "21": 46.73719787597656, + "22": 47.23435592651367, + "23": 48.03388214111328, + "24": 48.77348327636719, + "25": 49.5684928894043, + "26": 50.67505645751953, + "27": 50.80683898925781, + "28": 50.57590866088867, + "29": 51.04689025878906 + } + }, + "J|StackExchange|s0": { + "n_prompts": 25, + "fit_seconds": 714.2, + "prompts_sha256": "fb838aed6ad4bb5655506149821e49a7be3de1e40422abf3e744afa0fd80df39", + "corpus": "StackExchange", + "seed_block": 0, + "fro_norm_per_layer": { + "12": 29.31014633178711, + "13": 31.878799438476562, + "14": 34.66938400268555, + "15": 37.237892150878906, + "16": 39.83375930786133, + "17": 41.555850982666016, + "18": 43.22056198120117, + "19": 45.463809967041016, + "20": 46.5365104675293, + "21": 47.511634826660156, + "22": 48.09621047973633, + "23": 48.869789123535156, + "24": 49.59775161743164, + "25": 50.24106979370117, + "26": 51.18525695800781, + "27": 50.98370361328125, + "28": 50.5977897644043, + "29": 50.98619079589844 + } + }, + "J|StackExchange|s1": { + "n_prompts": 25, + "fit_seconds": 714.2, + "prompts_sha256": "99d6fe9822aacd471eebbf96e115750a6247e2e1bef7353640f9461dfe1aad6b", + "corpus": "StackExchange", + "seed_block": 1, + "fro_norm_per_layer": { + "12": 29.9813289642334, + "13": 32.756526947021484, + "14": 35.454124450683594, + "15": 37.913490295410156, + "16": 40.541751861572266, + "17": 42.225624084472656, + "18": 43.83034896850586, + "19": 46.03562545776367, + "20": 47.06637954711914, + "21": 47.918819427490234, + "22": 48.48906707763672, + "23": 49.20771408081055, + "24": 49.94163131713867, + "25": 50.54900360107422, + "26": 51.485008239746094, + "27": 51.22560501098633, + "28": 50.785186767578125, + "29": 51.12050247192383 + } + }, + "J|StackExchange|s2": { + "n_prompts": 25, + "fit_seconds": 712.4, + "prompts_sha256": "094be840f776043c464c6dc2ada90ee60b7a6349a4faf5edbee6f7646e048e58", + "corpus": "StackExchange", + "seed_block": 2, + "fro_norm_per_layer": { + "12": 29.62559700012207, + "13": 32.32237243652344, + "14": 35.15523910522461, + "15": 37.623905181884766, + "16": 40.21584701538086, + "17": 41.93696975708008, + "18": 43.60802459716797, + "19": 45.85477828979492, + "20": 46.900814056396484, + "21": 47.83250045776367, + "22": 48.398345947265625, + "23": 49.1341667175293, + "24": 49.83674621582031, + "25": 50.47686767578125, + "26": 51.41249084472656, + "27": 51.16120529174805, + "28": 50.70438003540039, + "29": 51.053829193115234 + } + }, + "J|Github|s0": { + "n_prompts": 25, + "fit_seconds": 712.5, + "prompts_sha256": "fb6856c864ad3d738b939e1ca21b65a129f645bf18dd0221c4c9e2c1dadb7fe5", + "corpus": "Github", + "seed_block": 0, + "fro_norm_per_layer": { + "12": 25.596206665039062, + "13": 27.109172821044922, + "14": 29.121456146240234, + "15": 30.757375717163086, + "16": 32.600433349609375, + "17": 34.267913818359375, + "18": 35.85880661010742, + "19": 37.88335037231445, + "20": 39.29946517944336, + "21": 40.87211608886719, + "22": 42.26652145385742, + "23": 43.78268051147461, + "24": 45.19640350341797, + "25": 46.571468353271484, + "26": 48.045780181884766, + "27": 48.60406494140625, + "28": 49.111106872558594, + "29": 50.13490676879883 + } + }, + "J|Github|s1": { + "n_prompts": 25, + "fit_seconds": 711.2, + "prompts_sha256": "8acd4ae02d98b1cb29ad26750a7f98b3f9bb842226f217fbc2ab18547092f0a1", + "corpus": "Github", + "seed_block": 1, + "fro_norm_per_layer": { + "12": 26.054004669189453, + "13": 27.861385345458984, + "14": 30.03280258178711, + "15": 31.84650421142578, + "16": 33.80059814453125, + "17": 35.55202102661133, + "18": 37.19427490234375, + "19": 39.25769805908203, + "20": 40.61973190307617, + "21": 42.10810470581055, + "22": 43.381038665771484, + "23": 44.776756286621094, + "24": 46.08184051513672, + "25": 47.326202392578125, + "26": 48.72494125366211, + "27": 49.15113067626953, + "28": 49.486331939697266, + "29": 50.350563049316406 + } + }, + "J|Github|s2": { + "n_prompts": 25, + "fit_seconds": 711.6, + "prompts_sha256": "fb8e687f84bbaca1821949876dabd6beb24389556d54c1fa6d9234184037fc68", + "corpus": "Github", + "seed_block": 2, + "fro_norm_per_layer": { + "12": 25.251083374023438, + "13": 26.807607650756836, + "14": 28.799816131591797, + "15": 30.52140235900879, + "16": 32.41239929199219, + "17": 34.15595245361328, + "18": 35.801361083984375, + "19": 37.893062591552734, + "20": 39.344974517822266, + "21": 40.995826721191406, + "22": 42.41323471069336, + "23": 43.94055938720703, + "24": 45.35512924194336, + "25": 46.73331069946289, + "26": 48.25156784057617, + "27": 48.75074768066406, + "28": 49.18072509765625, + "29": 50.145565032958984 + } + } + }, + "logit_I_pooled_rank_flatmean7": 0.2699603409374518, + "provenance": { + "schema_version": 1, + "experiment": "CV6", + "utc": "2026-08-24T03:25:21+00:00", + "script": { + "path": "experiments/cv6_per_family_ladder.py", + "sha256": "f416c3cb55e95a02315504066479c2f2ba073c0d4676bbf09adb2f481cdb4a5b", + "n_lines": 750 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/cv6_per_family_ladder.py", + "--model", + "2.8b", + "--device", + "cuda", + "--cache-out", + "results/cv6/ht_cache_2.8b.pt", + "--save-lens-dir", + "results/cv6/lenses" + ], + "cwd": ".", + "inputs": [ + { + "path": "corpora/Wikipedia_en.jsonl", + "sha256": "b30f4cef11add560f7df8ed07a613e3fcc6a4dda865975bec15f2952253bda77", + "bytes": 10329196, + "exists": true + }, + { + "path": "corpora/USPTO_Backgrounds.jsonl", + "sha256": "b09ac74bdc8b169ee5c9bea6be481262f743947a92107164f30eca2346e3ccb9", + "bytes": 10221054, + "exists": true + }, + { + "path": "corpora/Pile-CC.jsonl", + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "bytes": 13512530, + "exists": true + }, + { + "path": "corpora/StackExchange.jsonl", + "sha256": "bc25e2d5775e9719ebcbdaa3bd115d827f72ebf11f47fedb377f8b0a69a28971", + "bytes": 5980156, + "exists": true + }, + { + "path": "corpora/Github.jsonl", + "sha256": "f2bb459d10cad6c13a8b66c86191bc43df4393ba5b722a5315ea878fad2d1a26", + "bytes": 17486378, + "exists": true + } + ], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.1", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-157-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "38cc82045d0d27b889fe4cd716dc6f5b6bed8d00cace9a81a394e4ead1f6c012" + } +} \ No newline at end of file diff --git a/results/cv6/cv6_c0_scorer_equivalence.json b/results/cv6/cv6_c0_scorer_equivalence.json new file mode 100644 index 0000000000000000000000000000000000000000..df556cd9b20ad8561d8681cc9f0a7f3576b3c33f --- /dev/null +++ b/results/cv6/cv6_c0_scorer_equivalence.json @@ -0,0 +1,171 @@ +{ + "experiment": "CV6 control C0 \u2014 the scorer reproduces D3 on the stored 410M operators", + "status": "CONTROL", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "n_items": 449, + "n_pairs": 801, + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "compared_against": "results/d3_corpus_by_family_410m.json", + "per_arm_deviation": { + "logit_I": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Wikipedia_en|s0": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Wikipedia_en|s1": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Wikipedia_en|s2": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "USPTO_Backgrounds|s0": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "USPTO_Backgrounds|s1": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "USPTO_Backgrounds|s2": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Pile-CC|s0": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Pile-CC|s1": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Pile-CC|s2": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "StackExchange|s0": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "StackExchange|s1": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "StackExchange|s2": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Github|s0": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Github|s1": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + }, + "Github|s2": { + "max_abs_dz": 0.0, + "max_abs_drank": 0.0 + } + }, + "worst_abs_dz": 0.0, + "worst_abs_drank": 0.0, + "required": "max|dz| <= 1e-6 and max|drank| <= 1e-9 over all 16 arms x 5 families, AND the negative control separates", + "negative_control": { + "what": "same scorer, LEGACY unstripped readout, identity arm, same D3 reference", + "why": "proves C0 can fail; a zero that a wrong readout also produces is not a test", + "max_abs_dz": 1.996877357092771, + "max_abs_drank": 0.30389610389610383, + "required": "max|dz| > 1e-3", + "separates": true, + "per_family_z_legacy": { + "multihop": 2.4813545022195984, + "multilingual": 2.652753429515713, + "order-ops": 3.0892464117570357, + "poetry": 2.5611357868325952, + "typo": 3.4371277851363025 + } + }, + "fires": true, + "what_it_does_not_cover": "nothing about the FIT path \u2014 it scores stored operators. It also says nothing about GPU scoring; D2 measured a CUDA-vs-CPU divergence of 2.774e-04 at the cell level.", + "provenance": { + "schema_version": 1, + "experiment": "CV6-C0", + "utc": "2026-08-23T23:04:06+00:00", + "script": { + "path": "experiments/cv6_per_family_ladder.py", + "sha256": "b7e6b609a8474750c75dbff539919282e7edd585ec2fba00321ae998e829711f", + "n_lines": 742 + }, + "git": { + "commit": "434bc077b023a5246e826a07cc325f79e6eeabbd", + "commit_short": "434bc077b023", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/cv6_per_family_ladder.py", + "figures/fA_effect_and_null.png", + "figures/fB_validity.png", + "figures/make_audit_figures.py" + ], + "dirty_diff_sha256": null, + "describe": "434bc07" + }, + "argv": [ + "experiments/cv6_per_family_ladder.py", + "--c0-reproduce-d3", + "--device", + "cpu" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/d3_corpus_by_family_410m.json", + "sha256": "b4342917486e93290856203f3ec8e32b65d55bb2d389273674210b49c140ad1e", + "bytes": 19603, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "ebbd6596928b4d50b4bc0a0034a5708d330f3d8481cc6c8b4511d190428dab36" + } +} \ No newline at end of file diff --git a/results/cv6/ht_cache_1.4b.pt b/results/cv6/ht_cache_1.4b.pt new file mode 100644 index 0000000000000000000000000000000000000000..ccc0cfd8bad52bfda8773d913a61e6f958cef787 --- /dev/null +++ b/results/cv6/ht_cache_1.4b.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d5f7ecfb9c8bacbea1822275e28d3faf73940426c0456183b3c52fb4d83095 +size 47847470 diff --git a/results/cv6/ht_cache_2.8b.pt b/results/cv6/ht_cache_2.8b.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef8b5fdd34e296afcf0e8a059a62323e8380f19c --- /dev/null +++ b/results/cv6/ht_cache_2.8b.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4cabaca7f8ca0d38312ce257855d1e8870c1429a2f1c56e088608056881da37 +size 82791805 diff --git a/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..77bc7db6ab38fb24f481bed34c06c6ba5de5d3aa --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c3df30f285ca6a2632ddf5ccda1c0bbeb72b288167df1f5bf63e48aadfa3f68 +size 109057367 diff --git a/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bb28f991a6ff9bbf7cbe45829686c468a72795b --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aade78c84bf70764a74a54018f1fa39966756618d5b74522194dc89e684d834 +size 109057367 diff --git a/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..26db56654f874190239c97c1b99f11654307c3c1 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Github_1.4b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b406658637da8ba432429bb262353698d1118b7efb531cb955328a95d5dbf01 +size 109057367 diff --git a/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f55dc37fb324002c59b727e01600a63a3c861dc2 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84f242a1abcd7293a4472f5a98fd7b5f3d047cae90890a31aa562dfb94decf5c +size 235936453 diff --git a/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e25e635c0edd4ec8517eb2cdbafc28ed89d891f --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f2d16e6011c4f2bdc865b401983bd9a490e98e5316ff0ae507d4b2e4096fd77 +size 235936453 diff --git a/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..14fe0cf62eb6876048385360698a7d0b01b6000a --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Github_2.8b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6734844c920ecfb5f3f18a4135a88679c6b50d6fe59f676f828917fc484fbab4 +size 235936453 diff --git a/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a30e4568e8c57c1a9d81398acb6720499fac6b7 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28fb8e54aae2bea11a122fdba06f6c30b9601a006d282d7872afd0d43b2c3e70 +size 109057386 diff --git a/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..d75f6fc9c24e29566c0ae1b552d05a2ce9484e81 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af172234b25addf33a0a086a1ecfd995e7914e4d1fdf21d588d74830e7073863 +size 109057386 diff --git a/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d7a6be71cefaf4e5236a7a03e088fa61f87f9dd --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Pile-CC_1.4b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6fb07ab61cb06a78913afb87427c9ca324f27a03b60f473edb83291c5c0ac00 +size 109057386 diff --git a/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..63557172e00457cb650f1e8c17e97415e21b5380 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e955f5d339ac9043539b179e56a9a0c59808955179af8e3aec85afc5ed384d73 +size 235936477 diff --git a/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..81aca7219f5a536beb2df0b833f119f3aea16e9e --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca7836772ad75f173f11aebfc423b61297da58c356e054cc67b0a1ec71416b5f +size 235936477 diff --git a/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..86808180917efc6c0e218b53a4dab61ccf970b4c --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Pile-CC_2.8b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:545bdf63efc4706e446d87f8e813e992fcc3461a3a6ff074ed90564fb3ab06f1 +size 235936477 diff --git a/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..b45f5e8f14db18dd908747cab860ed28c36ff697 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:469b8052d4a296314823529130a6bb8610870b860ef2c4ecbc2970c3b480b90a +size 109057564 diff --git a/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..d69ccc5f2aa4fbc14f0ce212f0bafaac16fb6fd7 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a9f037db52482669dc656456f0643a2c290e246866875fae8e36de261ae8a52 +size 109057564 diff --git a/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..93030bd5a0283a4659a31fb827e57416dba093ac --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_StackExchange_1.4b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07992cfcbb3d32fbeb1871f4aa79f003c6f43c4ba24ce977ae0b874f1f08a8b3 +size 109057564 diff --git a/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..62b541b56e02c39e6c29b0444e43b6635dc1741d --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38d60e2daff2203100167c2b4862a31486e3fc8e70c972af4441438f7e6418e +size 235936685 diff --git a/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..c25fb3e97d3635a1a02d279905acaf5fcb175149 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45fc266499b5b97f6d5ef4d5671558a30f561430db0cd200870213950106b05d +size 235936685 diff --git a/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f048b6474b13ef7cee65aadd82f7bc172e4f556 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_StackExchange_2.8b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3438dec229cfcacf26aef81d273c2080373e5d8d5ca3dc43eb1d1280de351c1e +size 235936685 diff --git a/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..44550b3cfdcb01c17544ca2747b9b02120385efc --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d068456087b18d29fd38f37345ca881c14311fb97d80a280ec3adaf12cad3c5 +size 109057640 diff --git a/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..fefdc86b38ec06f29150e60a88257d07fbc99d29 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:046eeef5dfb737093c822bec3607c8b22c56d5abb39e450d5aa0f7b4f8ff4265 +size 109057640 diff --git a/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..6986a4510901ba604a16468fd4f4d46bafdf474d --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_1.4b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f0cd894627cc85caffa5e712da65069534ac996256fbd075a25e938b85e59c8 +size 109057640 diff --git a/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b3a522640cab97002f619cc63d8ae3a685dc473 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09013abe933c8754d1ce9dcf55df4f4fc597bde7502ffb6641341b25e24242ce +size 235936781 diff --git a/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..5832926ae0d6abc0609db4ad855da323fca3d7e8 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58fd8c7f4fb4d6e589cc559140d205d0fcda3a03ecaa95eaab83bc8e4ba2f743 +size 235936781 diff --git a/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..9377e7db5be3925b338ebf8f54c29fc889eaef65 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_USPTO_Backgrounds_2.8b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f632a13fb0b1d0eb26b639bfe046d27e7f54980f9df4a09595d9a0287527da +size 235936781 diff --git a/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..102fb31edffe0a5bc06bab0a12eac85f21a71b4b --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7db6b6e4605bd974a732b9b3501c060be612e0ba39c21c5a780658a1b64a6250 +size 109057545 diff --git a/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..caf5eed5a3529f6b76eb266742415d585807f1e2 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fa1e2a931ca8454368ae8bcd67a13bc053c89e617b0c005f970b8ebbc813f76 +size 109057545 diff --git a/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca295978127941c58e512f8e8ea2a628d1565573 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_1.4b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1763b5e8c3ca65bc26749be894aeaeb8961230491410e7853ffe45e41e77f4e +size 109057545 diff --git a/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s0.pt b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..125377a0eaddbf7921b153e3389b4491dbb26865 --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f3e2850461bf1e416585181c361f0a5018f163e3af388735e668f2c977a5a64 +size 235936661 diff --git a/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s1.pt b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..9dc969011ade66de5d83bf48fa930a455a470dad --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29f72e060566702cc51497528e6935786b949a095f1322a4832843ab633a861e +size 235936661 diff --git a/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s2.pt b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d3389a5e11cdaed3dc46d8e16e746c2609d3c0b --- /dev/null +++ b/results/cv6/lenses/lens_INSTREAM_Wikipedia_en_2.8b_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a319a3aebd47c030cc5d3861f3455e91c80812eac3c7711a5d64a970206c9d6c +size 235936661 diff --git a/results/cv6_per_family_ladder.json b/results/cv6_per_family_ladder.json new file mode 100644 index 0000000000000000000000000000000000000000..59c6752c534fc6a050bad37c68db502f40f3378f --- /dev/null +++ b/results/cv6_per_family_ladder.json @@ -0,0 +1,1238 @@ +{ + "experiment": "CV6 \u2014 does the per-family corpus effect replicate at 1.4B and 2.8B?", + "prereg": "docs/experiments/preregs/CV6_per_family_ladder.md", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "Evaluated PER FAMILY; the verdict is the count across the five. REPLICATES if R(f,2.8B) >= 10 on >= 4 of 5 families AND ordering tau >= 0.6 on >= 3 of 5. ATTENUATES if R(f,2.8B) < 5 on >= 4 of 5 families. UNCLEAR otherwise, including a split where some families replicate and others do not \u2014 a split is an informative outcome, not a failure. Do not pool to force a verdict. Do not add models to chase a threshold. Do not drop a family.", + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "seeds": [ + 0, + 1, + 2 + ], + "N": 25, + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "readout_convention": "STRIPPED (corrected), no prefix, flat-mean-7", + "pooling_convention": "flat pool over (item, intermediate) pairs WITHIN a family; seed SD is the RMS over the 5 corpora of each corpus's 3-seed sample SD \u2014 trap 4: the convention is named because it moves 'x seed SD' ratios ~37%", + "reference_410m": { + "source": "results/d3_corpus_by_family_410m.json (EXPLORATORY)", + "N": 200, + "order_by_z": { + "multihop": [ + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "multilingual": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order-ops": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "poetry": [ + "Pile-CC", + "USPTO_Backgrounds", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "typo": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ] + }, + "R": { + "multihop": 25.36100542891225, + "multilingual": 19.426787847831193, + "order-ops": 24.632511354365725, + "poetry": 42.686224720210525, + "typo": 27.014210896207846 + }, + "z_pooled_seed_sd": { + "multihop": 0.017453481001782855, + "multilingual": 0.019695881058294912, + "order-ops": 0.012061420042855183, + "poetry": 0.005578110253433368, + "typo": 0.019451275697770446 + } + }, + "by_model": { + "1.4b": { + "model": "EleutherAI/pythia-1.4b-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "device": "cuda", + "dtype": "float32", + "gpu": "NVIDIA H100 NVL", + "N": 25, + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "logit_I": { + "multihop": { + "rank": 0.2496532593619972, + "z": 3.8530033965712613, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1899927483683829, + "z": 3.2260746860564664, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5597402597402598, + "z": 5.515925762870095, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.008746355685131196, + "z": 2.4992136784962247, + "n_pairs": 98 + }, + "typo": { + "rank": 0.1889880952380952, + "z": 3.7020364006360373, + "n_pairs": 96 + } + }, + "logit_I_pooled_rank_flatmean7": 0.23942414367877324, + "by_family": { + "multihop": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.849424108332415, + "z_sd": 0.00971452307361606, + "rank_mean": 0.30883032824780393, + "rank_sd": 0.002118620293553323, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 3.9101949866535595, + "z_sd": 0.03168250526694356, + "rank_mean": 0.32223763291724455, + "rank_sd": 0.003490445878534788, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 3.988609111617684, + "z_sd": 0.038253457386003685, + "rank_mean": 0.3226999537679149, + "rank_sd": 0.0028871927870542744, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 4.083285220618387, + "z_sd": 0.019998217439994845, + "rank_mean": 0.39667128987517336, + "rank_sd": 0.012091259205383278, + "n_seeds": 3 + }, + "Github": { + "z_mean": 3.752807802366979, + "z_sd": 0.0855118574336993, + "rank_mean": 0.324086916319926, + "rank_sd": 0.018469703518666825, + "n_seeds": 3 + } + }, + "z_spread": 0.33047741825140786, + "z_pooled_seed_sd": 0.045329210368280526, + "R": 7.2906061139477085, + "rank_spread": 0.08784096162736943, + "rank_pooled_seed_sd": 0.010122609780171063, + "rank_spread_over_sd": 8.677699085016492, + "order_by_z": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "StackExchange", + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "n_pairs": 103, + "kendall_tau_vs_410m": 0.8, + "fp16_roundtrip_diagnostic": { + "R": 7.291143850680914, + "max_abs_dz_mean": 7.843778357141673e-05, + "order_unchanged": true + } + }, + "multilingual": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.055601908067358, + "z_sd": 0.024883413619364386, + "rank_mean": 0.18624607203287408, + "rank_sd": 0.0065298248292193, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 3.1024229467015982, + "z_sd": 0.021925288574017928, + "rank_mean": 0.18044476673918297, + "rank_sd": 0.0032497775343845316, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 3.2126307500094526, + "z_sd": 0.04011617460924262, + "rank_mean": 0.19639835629683347, + "rank_sd": 0.004202402381124895, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 3.2308733033145542, + "z_sd": 0.055215016867437655, + "rank_mean": 0.2274595117234711, + "rank_sd": 0.012943366831597983, + "n_seeds": 3 + }, + "Github": { + "z_mean": 2.8394160118905822, + "z_sd": 0.09979349484462473, + "rank_mean": 0.14744984288131496, + "rank_sd": 0.017215308294004156, + "n_seeds": 3 + } + }, + "z_spread": 0.391457291423972, + "z_pooled_seed_sd": 0.05606540772822604, + "R": 6.982153653845515, + "rank_spread": 0.08000966884215613, + "rank_pooled_seed_sd": 0.010341735218918366, + "rank_spread_over_sd": 7.736580675145566, + "order_by_z": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "USPTO_Backgrounds", + "Github" + ], + "n_pairs": 394, + "kendall_tau_vs_410m": 1.0, + "fp16_roundtrip_diagnostic": { + "R": 6.981554279672338, + "max_abs_dz_mean": 2.806884584893865e-05, + "order_unchanged": true + } + }, + "order-ops": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 5.321406318924644, + "z_sd": 0.05556506407862744, + "rank_mean": 0.5632034632034633, + "rank_sd": 0.009918995010726954, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 5.318626935193033, + "z_sd": 0.0436481925352706, + "rank_mean": 0.5831168831168831, + "rank_sd": 0.011902794012872206, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 5.474697200818495, + "z_sd": 0.0237919350357011, + "rank_mean": 0.5887445887445888, + "rank_sd": 0.0029992221776084464, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 5.220926105976105, + "z_sd": 0.04991356678409937, + "rank_mean": 0.5896103896103896, + "rank_sd": 0.004682534123979193, + "n_seeds": 3 + }, + "Github": { + "z_mean": 5.383031076373476, + "z_sd": 0.07990919093950227, + "rank_mean": 0.5705627705627706, + "rank_sd": 0.01364151623947074, + "n_seeds": 3 + } + }, + "z_spread": 0.25377109484239035, + "z_pooled_seed_sd": 0.05373181585637624, + "R": 4.722920504319339, + "rank_spread": 0.026406926406926323, + "rank_pooled_seed_sd": 0.009561123282756314, + "rank_spread_over_sd": 2.761906276697819, + "order_by_z": [ + "Pile-CC", + "Github", + "Wikipedia_en", + "USPTO_Backgrounds", + "StackExchange" + ], + "order_by_rank": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Github", + "Wikipedia_en" + ], + "n_pairs": 110, + "kendall_tau_vs_410m": -0.4, + "fp16_roundtrip_diagnostic": { + "R": 4.72187977619938, + "max_abs_dz_mean": 0.00011707869443000618, + "order_unchanged": true + } + }, + "poetry": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 2.321828322142971, + "z_sd": 0.010642859707299999, + "rank_mean": 0.02235179786200194, + "rank_sd": 0.0036685298519294196, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 2.3693615256929075, + "z_sd": 0.016925142442212295, + "rank_mean": 0.023323615160349854, + "rank_sd": 0.0025248554046193535, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 2.3751317657986464, + "z_sd": 0.020213283926543354, + "rank_mean": 0.01652089407191448, + "rank_sd": 0.00303449854149582, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 2.305089985635005, + "z_sd": 0.02054782796446748, + "rank_mean": 0.020894071914480075, + "rank_sd": 0.0016832369364129039, + "n_seeds": 3 + }, + "Github": { + "z_mean": 2.069341406226158, + "z_sd": 0.029947195245575223, + "rank_mean": 0.013605442176870748, + "rank_sd": 0.002226713165673389, + "n_seeds": 3 + } + }, + "z_spread": 0.30579035957248824, + "z_pooled_seed_sd": 0.020626950786131512, + "R": 14.824797069767857, + "rank_spread": 0.009718172983479106, + "rank_pooled_seed_sd": 0.002714138006563447, + "rank_spread_over_sd": 3.580574370197166, + "order_by_z": [ + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "Github" + ], + "order_by_rank": [ + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "Pile-CC", + "Github" + ], + "n_pairs": 98, + "kendall_tau_vs_410m": 0.8, + "fp16_roundtrip_diagnostic": { + "R": 14.82280843880369, + "max_abs_dz_mean": 3.022188637524792e-05, + "order_unchanged": true + } + }, + "typo": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.6687574258281126, + "z_sd": 0.005040123266250421, + "rank_mean": 0.3373015873015873, + "rank_sd": 0.0022731030232916094, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 3.990972049534321, + "z_sd": 0.049932129992485516, + "rank_mean": 0.44990079365079366, + "rank_sd": 0.008461667712912667, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 4.020524656607045, + "z_sd": 0.013975432149536227, + "rank_mean": 0.4047619047619047, + "rank_sd": 0.005952380952380959, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 3.995124079287052, + "z_sd": 0.010104712554199926, + "rank_mean": 0.4618055555555555, + "rank_sd": 0.0022731030232915794, + "n_seeds": 3 + }, + "Github": { + "z_mean": 3.2690880886382527, + "z_sd": 0.08830157076926536, + "rank_mean": 0.23015873015873017, + "rank_sd": 0.025004133256207473, + "n_seeds": 3 + } + }, + "z_spread": 0.7514365679687924, + "z_pooled_seed_sd": 0.04607212990087249, + "R": 16.31000280615553, + "rank_spread": 0.23164682539682535, + "rank_pooled_seed_sd": 0.012186642967521382, + "rank_spread_over_sd": 19.0082556791224, + "order_by_z": [ + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "n_pairs": 96, + "kendall_tau_vs_410m": 0.8, + "fp16_roundtrip_diagnostic": { + "R": 16.30612241371698, + "max_abs_dz_mean": 2.035333050631749e-05, + "order_unchanged": true + } + } + } + }, + "2.8b": { + "model": "EleutherAI/pythia-2.8b-deduped", + "band": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "n_layers": 32, + "device": "cuda", + "dtype": "float32", + "gpu": "NVIDIA H100 NVL", + "N": 25, + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "logit_I": { + "multihop": { + "rank": 0.27600554785020803, + "z": 3.693257671826094, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.22588832487309643, + "z": 3.0744364978603755, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.538961038961039, + "z": 5.5365345900708975, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.02769679300291545, + "z": 2.430429516398177, + "n_pairs": 98 + }, + "typo": { + "rank": 0.28125, + "z": 3.6758319437503815, + "n_pairs": 96 + } + }, + "logit_I_pooled_rank_flatmean7": 0.2699603409374518, + "by_family": { + "multihop": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.7481867645936493, + "z_sd": 0.03554648642712381, + "rank_mean": 0.37216828478964403, + "rank_sd": 0.003490445878534788, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 3.9520995335285716, + "z_sd": 0.008418421244240502, + "rank_mean": 0.4110032362459547, + "rank_sd": 0.010593101467766616, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 3.90297069981646, + "z_sd": 0.01286317636152229, + "rank_mean": 0.3874248728617661, + "rank_sd": 0.004237240587106646, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 4.024555764537799, + "z_sd": 0.06825011963705907, + "rank_mean": 0.41886269070735094, + "rank_sd": 0.0156302741603116, + "n_seeds": 3 + }, + "Github": { + "z_mean": 3.4238723230979202, + "z_sd": 0.08735072241195618, + "rank_mean": 0.3319463707813222, + "rank_sd": 0.014030504766984291, + "n_seeds": 3 + } + }, + "z_spread": 0.6006834414398785, + "z_pooled_seed_sd": 0.052513067623269356, + "R": 11.43874217650363, + "rank_spread": 0.08691631992602872, + "rank_pooled_seed_sd": 0.010802886218722078, + "rank_spread_over_sd": 8.045657259205164, + "order_by_z": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "n_pairs": 103, + "kendall_tau_vs_410m": 0.6, + "fp16_roundtrip_diagnostic": { + "R": 11.450690222634297, + "max_abs_dz_mean": 7.72414469798477e-05, + "order_unchanged": true + } + }, + "multilingual": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.434918012060288, + "z_sd": 0.02439008151297054, + "rank_mean": 0.31532511481750064, + "rank_sd": 0.003667873073602506, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 3.663950873964329, + "z_sd": 0.04769042760272363, + "rank_mean": 0.3392554991539763, + "rank_sd": 0.00652982482921927, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 3.6098976766094943, + "z_sd": 0.04474493232770196, + "rank_mean": 0.3307952622673435, + "rank_sd": 0.006509660628230673, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 3.7096787147513903, + "z_sd": 0.011183248531158265, + "rank_mean": 0.3385303359922649, + "rank_sd": 0.007468945881305459, + "n_seeds": 3 + }, + "Github": { + "z_mean": 3.0556710560612266, + "z_sd": 0.07188945212789528, + "rank_mean": 0.2517524776408025, + "rank_sd": 0.014756372480988393, + "n_seeds": 3 + } + }, + "z_spread": 0.6540076586901638, + "z_pooled_seed_sd": 0.04508773736862208, + "R": 14.505222414316746, + "rank_spread": 0.0875030215131738, + "rank_pooled_seed_sd": 0.00862558134915699, + "rank_spread_over_sd": 10.144594082545613, + "order_by_z": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "n_pairs": 394, + "kendall_tau_vs_410m": 0.8, + "fp16_roundtrip_diagnostic": { + "R": 14.514366209896588, + "max_abs_dz_mean": 5.073651242781807e-05, + "order_unchanged": true + } + }, + "order-ops": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 5.21647632447156, + "z_sd": 0.04405024461019303, + "rank_mean": 0.577922077922078, + "rank_sd": 0.008110387010907067, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 5.591100374135104, + "z_sd": 0.06802171764709751, + "rank_mean": 0.6398268398268399, + "rank_sd": 0.016341632976776498, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 5.428856505950292, + "z_sd": 0.03060876969522738, + "rank_mean": 0.5883116883116883, + "rank_sd": 0.005660907718883943, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 5.4992225209871926, + "z_sd": 0.04431453680873469, + "rank_mean": 0.570995670995671, + "rank_sd": 0.013328834907654201, + "n_seeds": 3 + }, + "Github": { + "z_mean": 5.221422637592663, + "z_sd": 0.08568503459405673, + "rank_mean": 0.5943722943722943, + "rank_sd": 0.005248638810814727, + "n_seeds": 3 + } + }, + "z_spread": 0.37462404966354423, + "z_pooled_seed_sd": 0.057982718511798266, + "R": 6.460960425429451, + "rank_spread": 0.06883116883116891, + "rank_pooled_seed_sd": 0.010677820670698406, + "rank_spread_over_sd": 6.446181384189407, + "order_by_z": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "order_by_rank": [ + "USPTO_Backgrounds", + "Github", + "Pile-CC", + "Wikipedia_en", + "StackExchange" + ], + "n_pairs": 110, + "kendall_tau_vs_410m": 0.8, + "fp16_roundtrip_diagnostic": { + "R": 6.4624203254229515, + "max_abs_dz_mean": 0.00011228973215260396, + "order_unchanged": true + } + }, + "poetry": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 2.5215682663074155, + "z_sd": 0.008653052690162434, + "rank_mean": 0.06268221574344023, + "rank_sd": 8.012344526598183e-18, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 2.6940602347964333, + "z_sd": 0.051012720432079404, + "rank_mean": 0.09523809523809523, + "rank_sd": 0.011782658554890317, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 2.577650401462503, + "z_sd": 0.029429444912468822, + "rank_mean": 0.06316812439261418, + "rank_sd": 0.008028528494586155, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 2.4801038943585896, + "z_sd": 0.005988009335835477, + "rank_mean": 0.05685131195335277, + "rank_sd": 0.0025248554046193596, + "n_seeds": 3 + }, + "Github": { + "z_mean": 2.3148899463569226, + "z_sd": 0.03841326630104487, + "rank_mean": 0.03498542274052478, + "rank_sd": 0.004373177842565594, + "n_seeds": 3 + } + }, + "z_spread": 0.3791702884395107, + "z_pooled_seed_sd": 0.031795284173340224, + "R": 11.925362464834903, + "rank_spread": 0.06025267249757045, + "rank_pooled_seed_sd": 0.006764434812553877, + "rank_spread_over_sd": 8.907273729026057, + "order_by_z": [ + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "StackExchange", + "Github" + ], + "order_by_rank": [ + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "StackExchange", + "Github" + ], + "n_pairs": 98, + "kendall_tau_vs_410m": 0.6, + "fp16_roundtrip_diagnostic": { + "R": 11.928150397522604, + "max_abs_dz_mean": 4.037038809601157e-05, + "order_unchanged": true + } + }, + "typo": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.484520903064145, + "z_sd": 0.023824517984755707, + "rank_mean": 0.3090277777777778, + "rank_sd": 0.011168978421823135, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 3.7973591391411095, + "z_sd": 0.05245448951050608, + "rank_mean": 0.43898809523809523, + "rank_sd": 0.010416666666666649, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 3.8199612237513065, + "z_sd": 0.058963986684690994, + "rank_mean": 0.38343253968253965, + "rank_sd": 0.02085104122716638, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 3.846287661128574, + "z_sd": 0.019121880506788938, + "rank_mean": 0.40327380952380953, + "rank_sd": 0.0029761904761904656, + "n_seeds": 3 + }, + "Github": { + "z_mean": 3.250814785146051, + "z_sd": 0.08028488172623756, + "rank_mean": 0.24950396825396823, + "rank_sd": 0.02663371777511667, + "n_seeds": 3 + } + }, + "z_spread": 0.595472875982523, + "z_pooled_seed_sd": 0.052167310455261205, + "R": 11.414674645594424, + "rank_spread": 0.189484126984127, + "rank_pooled_seed_sd": 0.016650715155594485, + "rank_spread_over_sd": 11.379939252667, + "order_by_z": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "n_pairs": 96, + "kendall_tau_vs_410m": 1.0, + "fp16_roundtrip_diagnostic": { + "R": 11.415194254884774, + "max_abs_dz_mean": 3.8545578718185425e-05, + "order_unchanged": true + } + } + } + } + }, + "controls": { + "C0_scorer_equivalence": { + "required": "the scorer reproduces D3 on the stored 410M operators", + "result": { + "worst_abs_dz": 0.0, + "worst_abs_drank": 0.0, + "fires": true + }, + "fires": true + }, + "C1_logit_identity_arm": { + "required": "logit_I emitted for every model and family; at 410M the pooled admitted-5 flat-mean-7 min-rank equals 0.19810852520167826", + "emitted_per_model": { + "1.4b": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "2.8b": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ] + }, + "410m_anchor": "C0 scored the identity arm at 410M; see cv6_c0_scorer_equivalence.json", + "fires": true + }, + "C2_band_matches_rule": { + "required": "emitted band == band_for(n_layers); a tuned band voids the run", + "per_model": { + "1.4b": { + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "rule": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "match": true + }, + "2.8b": { + "band": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "rule": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "match": true + } + }, + "fires": true + }, + "C3_seed_sd_nondegenerate": { + "required": "pooled_seed_sd_z(f, m) > 0 for every (family, model); else that cell is VOID", + "per_model": { + "1.4b": { + "multihop": 0.045329210368280526, + "multilingual": 0.05606540772822604, + "order-ops": 0.05373181585637624, + "poetry": 0.020626950786131512, + "typo": 0.04607212990087249 + }, + "2.8b": { + "multihop": 0.052513067623269356, + "multilingual": 0.04508773736862208, + "order-ops": 0.057982718511798266, + "poetry": 0.031795284173340224, + "typo": 0.052167310455261205 + } + }, + "void_cells": [], + "fires": true + }, + "C4_N_identical": { + "required": "N == 25 and identical across all 15 arms of every model", + "n_prompts_seen": { + "1.4b": [ + 25 + ], + "2.8b": [ + 25 + ] + }, + "n_arms": { + "1.4b": 15, + "2.8b": 15 + }, + "prompt_sha_identical_across_models": { + "Wikipedia_en|s0": true, + "Wikipedia_en|s1": true, + "Wikipedia_en|s2": true, + "USPTO_Backgrounds|s0": true, + "USPTO_Backgrounds|s1": true, + "USPTO_Backgrounds|s2": true, + "Pile-CC|s0": true, + "Pile-CC|s1": true, + "Pile-CC|s2": true, + "StackExchange|s0": true, + "StackExchange|s1": true, + "StackExchange|s2": true, + "Github|s0": true, + "Github|s1": true, + "Github|s2": true + }, + "fires": true + }, + "C5_device_precision": { + "required": "activations and operators on the same device at the same dtype", + "per_model": { + "1.4b": { + "device": "cuda", + "dtype": "float32", + "gpu": "NVIDIA H100 NVL", + "tf32": { + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "2.8b": { + "device": "cuda", + "dtype": "float32", + "gpu": "NVIDIA H100 NVL", + "tf32": { + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + } + }, + "fires": true + } + }, + "PRIMARY": { + "R_per_family_2.8b": { + "multihop": 11.43874217650363, + "multilingual": 14.505222414316746, + "order-ops": 6.460960425429451, + "poetry": 11.925362464834903, + "typo": 11.414674645594424 + }, + "kendall_tau_vs_410m_2.8b": { + "multihop": 0.6, + "multilingual": 0.8, + "order-ops": 0.8, + "poetry": 0.6, + "typo": 1.0 + }, + "n_families_R_ge_10": 4, + "n_families_R_lt_5": 0, + "n_families_tau_ge_0.6": 5 + }, + "VERDICT": "REPLICATES", + "verdict_note": "the corpus effect is not an artifact of the low-competence regime at 410M", + "declared_bias": [ + "Capability and scale are confounded; every larger model is also differently trained. No outcome licenses a causal reading \u2014 only 'the effect does / does not survive up the ladder'.", + "The 5 corpora are a fixed panel, not a sample. n=5 is the replication unit for any claim about KINDS of corpus.", + "N=25 here vs N=200 at 410M. S2 established reads are flat in N, so this is expected to be immaterial, but it is a difference and an attenuation must not be attributed to scale without checking N.", + "Per-family seed SDs are thin: 3 seeds, 96-394 pairs per family. Thin denominators are what inflated the retired 58x figure. The SD is reported beside every ratio.", + "The 410M reference is D3, which is EXPLORATORY and was computed BEFORE this pre-registration. The per-family estimand may not be described as chosen on capability grounds alone." + ], + "provenance": { + "schema_version": 1, + "experiment": "CV6", + "utc": "2026-08-30T06:31:18+00:00", + "script": { + "path": "experiments/cv6_per_family_ladder.py", + "sha256": "f416c3cb55e95a02315504066479c2f2ba073c0d4676bbf09adb2f481cdb4a5b", + "n_lines": 750 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/cv6_per_family_ladder.py", + "--adjudicate" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/d3_corpus_by_family_410m.json", + "sha256": "2ab2b53f90bd9d3435ebdb65aa0c9c1a82ac599a5e3c714f9dd9681d4bb9487c", + "bytes": 23095, + "exists": true + }, + { + "path": "results/cv6/cv6_1.4b_n25.json", + "sha256": "4e8eab1ebedf0785a9e61db7fa77212cd2f89699fe627b08f7b9ea7b960cedae", + "bytes": 30228, + "exists": true + }, + { + "path": "results/cv6/cv6_2.8b_n25.json", + "sha256": "7f1a6d31f1c8728c4c966d960e5502c6bfaaa8dddf4543cf5d9e5a8424f82b9f", + "bytes": 32402, + "exists": true + }, + { + "path": "results/cv6/cv6_c0_scorer_equivalence.json", + "sha256": "2650f8b565cf128bc4ac01ee775849beec904a9a2329d4e75081492a4546d332", + "bytes": 4013, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "37c137de5d7788ff265c533db4dabc53540c7c9e6ad33e12b06d72182bbdd228" + } +} \ No newline at end of file diff --git a/results/cv7/cv7_1b_n200.json b/results/cv7/cv7_1b_n200.json new file mode 100644 index 0000000000000000000000000000000000000000..3f86c69c89fbe994243c5652c0899e718a035e08 --- /dev/null +++ b/results/cv7/cv7_1b_n200.json @@ -0,0 +1,821 @@ +{ + "experiment": "CV7 \u2014 the 1B rung, scoring only", + "prereg": "docs/experiments/preregs/CV7_1b_rung.md", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "Evaluated PER FAMILY; the verdict is the count across the five. Thresholds inherited VERBATIM from CV6_per_family_ladder.md, which fixed them before any ladder number existed. REPLICATES AT 1B if R(f,1B) >= 10 on >= 4 of 5 families AND ordering tau >= 0.6 on >= 3 of 5. ATTENUATES AT 1B if R(f,1B) < 5 on >= 4 of 5 families. UNCLEAR otherwise, including a split -- report the table and stop. Do not pool. Do not drop a family. Do not re-cut R across mismatched N.", + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "n_layers": 16, + "d_model": 2048, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "band_rule": "floor(0.38*L)..floor(0.92*L), layers < L-2", + "band_read_off_artifacts": { + "Wikipedia_en|s0": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Wikipedia_en|s1": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Wikipedia_en|s2": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "USPTO_Backgrounds|s0": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "USPTO_Backgrounds|s1": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "USPTO_Backgrounds|s2": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Pile-CC|s0": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Pile-CC|s1": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Pile-CC|s2": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "StackExchange|s0": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "StackExchange|s1": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "StackExchange|s2": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Github|s0": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Github|s1": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "Github|s2": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ] + }, + "band_matches_rule": true, + "N": 200, + "n_prompts_per_arm": { + "Wikipedia_en|s0": 200, + "Wikipedia_en|s1": 200, + "Wikipedia_en|s2": 200, + "USPTO_Backgrounds|s0": 200, + "USPTO_Backgrounds|s1": 200, + "USPTO_Backgrounds|s2": 200, + "Pile-CC|s0": 200, + "Pile-CC|s1": 200, + "Pile-CC|s2": 200, + "StackExchange|s0": 200, + "StackExchange|s1": 200, + "StackExchange|s2": 200, + "Github|s0": 200, + "Github|s1": 200, + "Github|s2": 200 + }, + "operators_from": "results/ladder1b_b613/ (fitted for E62; NOT refitted here)", + "device": "cpu", + "dtype": "float32", + "operator_storage_dtype": "float16", + "device_caveat": "operators fitted on CUDA, scored on CPU. D2 measured a CUDA-vs-CPU cell-level divergence of 2.774e-04. This is D3's situation, which is what makes 1B-vs-410M the matched comparison.", + "torch": "2.13.0", + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "seeds": [ + 0, + 1, + 2 + ], + "readout_convention": "STRIPPED (corrected), no prefix, flat-mean-7", + "pooling_convention": "flat pool over (item, intermediate) pairs WITHIN a family; rank = min over band then flat-mean-7 pass@k; z = max over band then mean", + "n_items": 449, + "n_pairs": 801, + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "per_arm": { + "logit_I": { + "multihop": { + "rank": 0.24410540915395282, + "z": 3.898444828477878, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1472081218274112, + "z": 2.7347642624287434, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5207792207792208, + "z": 5.480567513812672, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.0058309037900874635, + "z": 2.021673796736464, + "n_pairs": 98 + }, + "typo": { + "rank": 0.17708333333333331, + "z": 3.567758576323589, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s0": { + "multihop": { + "rank": 0.2538141470180305, + "z": 3.714084236367235, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.13741841914430747, + "z": 2.7458525141701102, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4597402597402597, + "z": 5.137804371660406, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.016034985422740525, + "z": 1.9996374888079507, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3050595238095238, + "z": 3.7074354047266147, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s1": { + "multihop": { + "rank": 0.2538141470180305, + "z": 3.723332085655731, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.13778100072516317, + "z": 2.751266322687249, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4584415584415584, + "z": 5.1602459907531735, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.01749271137026239, + "z": 1.988615335858598, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3050595238095238, + "z": 3.7324859214325747, + "n_pairs": 96 + } + }, + "J|Wikipedia_en|s2": { + "multihop": { + "rank": 0.2524271844660194, + "z": 3.706331000166032, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.13741841914430747, + "z": 2.7407319243846056, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4610389610389611, + "z": 5.139317949251695, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.016034985422740525, + "z": 1.9798154046340866, + "n_pairs": 98 + }, + "typo": { + "rank": 0.30357142857142855, + "z": 3.710389399745812, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s0": { + "multihop": { + "rank": 0.29680998613037446, + "z": 4.109125902351824, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.15155910079767948, + "z": 2.907341442816964, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.48051948051948046, + "z": 5.347905703024431, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.033527696793002916, + "z": 2.0518707047995863, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4181547619047619, + "z": 4.076707179347674, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s1": { + "multihop": { + "rank": 0.30513176144244103, + "z": 4.1576821566785425, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1508339376359681, + "z": 2.907443309632995, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4857142857142857, + "z": 5.373728732629256, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.033527696793002916, + "z": 2.059163775085947, + "n_pairs": 98 + }, + "typo": { + "rank": 0.40625, + "z": 4.066875881205003, + "n_pairs": 96 + } + }, + "J|USPTO_Backgrounds|s2": { + "multihop": { + "rank": 0.30235783633841884, + "z": 4.154952114068188, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.15228426395939088, + "z": 2.9097203547655024, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4753246753246753, + "z": 5.355981678312475, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.03644314868804665, + "z": 2.0739537419713274, + "n_pairs": 98 + }, + "typo": { + "rank": 0.41517857142857145, + "z": 4.111865448455016, + "n_pairs": 96 + } + }, + "J|Pile-CC|s0": { + "multihop": { + "rank": 0.2787794729542302, + "z": 4.083725291548423, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.15445975344452503, + "z": 2.9522800377477316, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.43376623376623374, + "z": 5.306177348440344, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784254, + "z": 1.9803421716300809, + "n_pairs": 98 + }, + "typo": { + "rank": 0.39285714285714285, + "z": 4.1813286903003855, + "n_pairs": 96 + } + }, + "J|Pile-CC|s1": { + "multihop": { + "rank": 0.2815533980582524, + "z": 4.094569732841936, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.15772298767222623, + "z": 2.986847078749144, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.44025974025974024, + "z": 5.305445389314131, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.018950437317784258, + "z": 2.0033713484905205, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3988095238095238, + "z": 4.20682845885555, + "n_pairs": 96 + } + }, + "J|Pile-CC|s2": { + "multihop": { + "rank": 0.2787794729542302, + "z": 4.0395108611838335, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1548223350253807, + "z": 2.943907567330631, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.43506493506493504, + "z": 5.280768312107433, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.014577259475218656, + "z": 1.9708401384402294, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3898809523809524, + "z": 4.157780651624004, + "n_pairs": 96 + } + }, + "J|StackExchange|s0": { + "multihop": { + "rank": 0.3370319001386962, + "z": 4.34045670449155, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.16243654822335027, + "z": 3.0495984307324826, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4857142857142857, + "z": 5.51830342466181, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.02040816326530612, + "z": 2.010183577938956, + "n_pairs": 98 + }, + "typo": { + "rank": 0.49107142857142855, + "z": 4.537338141351938, + "n_pairs": 96 + } + }, + "J|StackExchange|s1": { + "multihop": { + "rank": 0.334257975034674, + "z": 4.3061738963266025, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.16026105873821608, + "z": 3.0347819554662037, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5012987012987012, + "z": 5.519914856824008, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.021865889212827987, + "z": 2.0059123179134057, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4672619047619047, + "z": 4.418834609289964, + "n_pairs": 96 + } + }, + "J|StackExchange|s2": { + "multihop": { + "rank": 0.334257975034674, + "z": 4.317730897838629, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.15917331399564902, + "z": 3.045205917266117, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4987012987012987, + "z": 5.524625140970404, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.02040816326530612, + "z": 2.000217623248392, + "n_pairs": 98 + }, + "typo": { + "rank": 0.4761904761904762, + "z": 4.479793511331081, + "n_pairs": 96 + } + }, + "J|Github|s0": { + "multihop": { + "rank": 0.2524271844660194, + "z": 3.8118835438802403, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.14213197969543148, + "z": 2.8611055270385775, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4896103896103896, + "z": 5.030006443370472, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.013119533527696792, + "z": 1.850219145303174, + "n_pairs": 98 + }, + "typo": { + "rank": 0.29910714285714285, + "z": 3.70514493167866, + "n_pairs": 96 + } + }, + "J|Github|s1": { + "multihop": { + "rank": 0.2621359223300971, + "z": 3.836685398828636, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.14648295866569977, + "z": 2.8760104446747543, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5, + "z": 5.072230815887451, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.014577259475218656, + "z": 1.8696253643352159, + "n_pairs": 98 + }, + "typo": { + "rank": 0.3080357142857143, + "z": 3.753457091632299, + "n_pairs": 96 + } + }, + "J|Github|s2": { + "multihop": { + "rank": 0.2579750346740638, + "z": 3.8195294026032234, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.145032632342277, + "z": 2.8609475317850785, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.4922077922077922, + "z": 5.031250395558097, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.01020408163265306, + "z": 1.8547754169878912, + "n_pairs": 98 + }, + "typo": { + "rank": 0.30357142857142855, + "z": 3.727808784286026, + "n_pairs": 96 + } + } + }, + "logit_I_pooled_rank_flatmean7": 0.2190013977768011, + "provenance": { + "schema_version": 1, + "experiment": "CV7", + "utc": "2026-08-24T18:42:15+00:00", + "script": { + "path": "experiments/cv7_1b_rung.py", + "sha256": "17eed1b79d679e0044f836a0906667bbcfa1bcfe5dfe5399c99557321ea6f73d", + "n_lines": 316 + }, + "git": { + "commit": "180c1374d89790d6e60346e4d49581f5bba6b34f", + "commit_short": "180c1374d897", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/cv7_1b_rung.py", + "figures/fA_effect_and_null.png", + "figures/fB_validity.png", + "figures/make_audit_figures.py", + "logs/", + "ocs/context/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/audit_paper_5pp.tex", + "paper/audit_paper_7pp.tex", + "paper/build_audit.sh", + "paper/refs.bib", + "paper/repro-paper.tex", + "paper/tables_audit.tex", + "repro/30_repo_health.sh", + "repro/exp/cv7_1b.sh", + "repro/lib/banned_claims.py" + ], + "dirty_diff_sha256": "d380ffb38dde5c7efcbe498d9454e038c0e71c01a846d485b439e379c597db29", + "describe": "180c137-dirty" + }, + "argv": [ + "experiments/cv7_1b_rung.py", + "--device", + "cpu" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s0.pt", + "sha256": "f60c8ee8186e4d53e9a0195b5f0535506160d2d313b3bc174d3129b80d9426b2", + "bytes": 67112755, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s1.pt", + "sha256": "25a07e77788297cc89a4603c0a1771d9cf2f8580a481c015bf77d039adcd0464", + "bytes": 67112755, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s2.pt", + "sha256": "89838efaec6ce3b2f991ed142b320fd28c8f608e5205a8c934e0ac44675dd0fa", + "bytes": 67112755, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s0.pt", + "sha256": "4acda201f3fbc46f77e96d3f6f6840ab6cc53b8d80f694ec5080dab7321b054a", + "bytes": 67112825, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s1.pt", + "sha256": "4be9599eae66c8895479b2d0151fde7975929aedc420916b1b29d7415234eac1", + "bytes": 67112825, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s2.pt", + "sha256": "0f730745e88e5dae78b5f73b98a00ab55c28ed32dae8b7800ba8823445e090ff", + "bytes": 67112825, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Pile-CC_1b_n200_s0.pt", + "sha256": "3bf4ce26087e107fa74da45922f6ecc486d6eab6fb29b7fbbf9b356a0ca4eabd", + "bytes": 67112685, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Pile-CC_1b_n200_s1.pt", + "sha256": "f41713739a1f278fd0d1c5b5c23db6ecfe1df0991945aa6b83ad716200e9a8f0", + "bytes": 67112685, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Pile-CC_1b_n200_s2.pt", + "sha256": "fe7a6a5b4579d5f47f0119d943efc700eb70220fb16acdec55c1f724dcde0ac2", + "bytes": 67112685, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_StackExchange_1b_n200_s0.pt", + "sha256": "5048cd087976e256a7ff9f30d8066dd95d442229f8c2b0ff5f4d273eef512c5b", + "bytes": 67112769, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_StackExchange_1b_n200_s1.pt", + "sha256": "905a7e26d7af75d7e9a64a49ba5319156b0e220236c0bf8ec5c15d8989db2330", + "bytes": 67112769, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_StackExchange_1b_n200_s2.pt", + "sha256": "78e10c625c615a13dc475fef35688ce8a90b8ab368ec84e1c8c7f88d70d3a040", + "bytes": 67112769, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Github_1b_n200_s0.pt", + "sha256": "459f997d6661ff07494e3ac0829864e02d71b15e0609875a705e032371a30158", + "bytes": 67112671, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Github_1b_n200_s1.pt", + "sha256": "187bd291fc66a6f3e5cf9a35c4640d9d963e1fd47434124dc3e7b7832f1a7950", + "bytes": 67112671, + "exists": true + }, + { + "path": "results/ladder1b_b613/lens_Github_1b_n200_s2.pt", + "sha256": "e37d288145a6438be3c571ad3fac769b7fd314c00675573dc8e4e9b3812a2754", + "bytes": 67112671, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "39a12fb962ca27e489684090b92af88ea55b4d2c0aab3e43061cd00eb5e627e0" + } +} \ No newline at end of file diff --git a/results/cv7_1b_rung.json b/results/cv7_1b_rung.json new file mode 100644 index 0000000000000000000000000000000000000000..d28e351509a4ce395d7b0efac05a4db74c0ce929 --- /dev/null +++ b/results/cv7_1b_rung.json @@ -0,0 +1,640 @@ +{ + "experiment": "CV7 \u2014 does the per-family corpus effect hold at the 1B rung?", + "prereg": "docs/experiments/preregs/CV7_1b_rung.md", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "Evaluated PER FAMILY; the verdict is the count across the five. Thresholds inherited VERBATIM from CV6_per_family_ladder.md, which fixed them before any ladder number existed. REPLICATES AT 1B if R(f,1B) >= 10 on >= 4 of 5 families AND ordering tau >= 0.6 on >= 3 of 5. ATTENUATES AT 1B if R(f,1B) < 5 on >= 4 of 5 families. UNCLEAR otherwise, including a split -- report the table and stop. Do not pool. Do not drop a family. Do not re-cut R across mismatched N.", + "model": "EleutherAI/pythia-1b-deduped", + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "N": 200, + "device": "cpu", + "dtype": "float32", + "readout_convention": "STRIPPED (corrected), no prefix, flat-mean-7", + "pooling_convention": "flat pool over (item, intermediate) pairs WITHIN a family; rank = min over band then flat-mean-7 pass@k; z = max over band then mean", + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "logit_I": { + "multihop": { + "rank": 0.24410540915395282, + "z": 3.898444828477878, + "n_pairs": 103 + }, + "multilingual": { + "rank": 0.1472081218274112, + "z": 2.7347642624287434, + "n_pairs": 394 + }, + "order-ops": { + "rank": 0.5207792207792208, + "z": 5.480567513812672, + "n_pairs": 110 + }, + "poetry": { + "rank": 0.0058309037900874635, + "z": 2.021673796736464, + "n_pairs": 98 + }, + "typo": { + "rank": 0.17708333333333331, + "z": 3.567758576323589, + "n_pairs": 96 + } + }, + "logit_I_pooled_rank_flatmean7": 0.2190013977768011, + "by_family": { + "multihop": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.714582440729666, + "z_sd": 0.008511485337299939, + "rank_mean": 0.25335182616736013, + "rank_sd": 0.0008007632027595361, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 4.140586724366185, + "z_sd": 0.027280043521374003, + "rank_mean": 0.30143319463707813, + "rank_sd": 0.004237240587106646, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 4.072601961858064, + "z_sd": 0.029166182208444292, + "rank_mean": 0.27970411465557093, + "rank_sd": 0.0016015264055190721, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 4.321453832885594, + "z_sd": 0.01744198721308977, + "rank_mean": 0.33518261673601474, + "rank_sd": 0.0016015264055190721, + "n_seeds": 3 + }, + "Github": { + "z_mean": 3.8226994484373664, + "z_sd": 0.012701176920397636, + "rank_mean": 0.2575127138233934, + "rank_sd": 0.004870852405387302, + "n_seeds": 3 + } + }, + "z_spread": 0.6068713921559281, + "z_pooled_seed_sd": 0.02065359607863063, + "R": 29.383328203258092, + "rank_spread": 0.08183079056865461, + "rank_pooled_seed_sd": 0.0030805975495067357, + "rank_spread_over_sd": 26.563284964554143, + "order_by_z": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "order_by_rank": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "n_pairs": 103, + "kendall_tau_vs_410m": 0.4 + }, + "multilingual": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 2.7459502537473215, + "z_sd": 0.005267879238213426, + "rank_mean": 0.13753927967125937, + "rank_sd": 0.00020933657331023706, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 2.9081683690718205, + "z_sd": 0.0013450237570753005, + "rank_mean": 0.15155910079767948, + "rank_sd": 0.0007251631617113813, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 2.9610115612758356, + "z_sd": 0.02276246993532394, + "rank_mean": 0.155668358714044, + "rank_sd": 0.001788572466394562, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 3.043195434488268, + "z_sd": 0.007610093007255903, + "rank_mean": 0.16062364031907178, + "rank_sd": 0.0016615575398679734, + "n_seeds": 3 + }, + "Github": { + "z_mean": 2.8660211678328036, + "z_sd": 0.00865132819247334, + "rank_mean": 0.1445491902344694, + "rank_sd": 0.0022154100531572717, + "n_seeds": 3 + } + }, + "z_spread": 0.2972451807409464, + "z_pooled_seed_sd": 0.011665750496260346, + "R": 25.4801592778993, + "rank_spread": 0.023084360647812413, + "rank_pooled_seed_sd": 0.0015124476877892779, + "rank_spread_over_sd": 15.26291509728477, + "order_by_z": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Github", + "Wikipedia_en" + ], + "order_by_rank": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Github", + "Wikipedia_en" + ], + "n_pairs": 394, + "kendall_tau_vs_410m": 0.8 + }, + "order-ops": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 5.145789437221758, + "z_sd": 0.012542594798773603, + "rank_mean": 0.4597402597402597, + "rank_sd": 0.0012987012987013269, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 5.359205371322054, + "z_sd": 0.013209896364389977, + "rank_mean": 0.4805194805194805, + "rank_sd": 0.0051948051948051965, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 5.297463683287303, + "z_sd": 0.014463246702100545, + "rank_mean": 0.43636363636363634, + "rank_sd": 0.0034360406637202485, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 5.520947807485407, + "z_sd": 0.0032850060233199263, + "rank_mean": 0.4952380952380952, + "rank_sd": 0.00834948117834886, + "n_seeds": 3 + }, + "Github": { + "z_mean": 5.044495884938674, + "z_sd": 0.024027206460830428, + "rank_mean": 0.49393939393939396, + "rank_sd": 0.005406924673938009, + "n_seeds": 3 + } + }, + "z_spread": 0.4764519225467332, + "z_pooled_seed_sd": 0.015027318232436472, + "R": 31.705718557174862, + "rank_spread": 0.05887445887445886, + "rank_pooled_seed_sd": 0.0052806755608578, + "rank_spread_over_sd": 11.149039208327205, + "order_by_z": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "StackExchange", + "Github", + "USPTO_Backgrounds", + "Wikipedia_en", + "Pile-CC" + ], + "n_pairs": 110, + "kendall_tau_vs_410m": 0.8 + }, + "poetry": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 1.9893560764335452, + "z_sd": 0.00993178119465326, + "rank_mean": 0.01652089407191448, + "rank_sd": 0.0008416184682064519, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 2.0616627406189534, + "z_sd": 0.011251611183892346, + "rank_mean": 0.034499514091350825, + "rank_sd": 0.0016832369364129039, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 1.984851219520277, + "z_sd": 0.016727777541894202, + "rank_mean": 0.01749271137026239, + "rank_sd": 0.0025248554046193557, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 2.0054378397002512, + "z_sd": 0.00499989103927885, + "rank_mean": 0.020894071914480075, + "rank_sd": 0.0008416184682064519, + "n_seeds": 3 + }, + "Github": { + "z_mean": 1.8582066422087604, + "z_sd": 0.010147921667373206, + "rank_mean": 0.012633624878522836, + "rank_sd": 0.0022267131656733908, + "n_seeds": 3 + } + }, + "z_spread": 0.20345609841019296, + "z_pooled_seed_sd": 0.011251993583315474, + "R": 18.08178229961657, + "rank_spread": 0.02186588921282799, + "rank_pooled_seed_sd": 0.0017653937924766718, + "rank_spread_over_sd": 12.385842357671557, + "order_by_z": [ + "USPTO_Backgrounds", + "StackExchange", + "Wikipedia_en", + "Pile-CC", + "Github" + ], + "order_by_rank": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "n_pairs": 98, + "kendall_tau_vs_410m": 0.4 + }, + "typo": { + "per_corpus": { + "Wikipedia_en": { + "z_mean": 3.7167702419683337, + "z_sd": 0.013690086099452861, + "rank_mean": 0.30456349206349204, + "rank_sd": 0.0008591521862940829, + "n_seeds": 3 + }, + "USPTO_Backgrounds": { + "z_mean": 4.085149503002564, + "z_sd": 0.023653116324414965, + "rank_mean": 0.4131944444444445, + "rank_sd": 0.006195434522220642, + "n_seeds": 3 + }, + "Pile-CC": { + "z_mean": 4.181979266926646, + "z_sd": 0.024530374763050165, + "rank_mean": 0.39384920634920634, + "rank_sd": 0.004546206046583159, + "n_seeds": 3 + }, + "StackExchange": { + "z_mean": 4.478655420657661, + "z_sd": 0.059259963006878004, + "rank_mean": 0.47817460317460314, + "rank_sd": 0.01202813060811721, + "n_seeds": 3 + }, + "Github": { + "z_mean": 3.7288036025323286, + "z_sd": 0.024171438669509934, + "rank_mean": 0.30357142857142855, + "rank_sd": 0.004464285714285726, + "n_seeds": 3 + } + }, + "z_spread": 0.7618851786893277, + "z_pooled_seed_sd": 0.03299887670273901, + "R": 23.088215564200972, + "rank_spread": 0.1746031746031746, + "rank_pooled_seed_sd": 0.006699183736561167, + "rank_spread_over_sd": 26.063350621400048, + "order_by_z": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Github", + "Wikipedia_en" + ], + "order_by_rank": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "n_pairs": 96, + "kendall_tau_vs_410m": 0.8 + } + }, + "PRIMARY": { + "R_per_family_1b": { + "multihop": 29.383328203258092, + "multilingual": 25.4801592778993, + "order-ops": 31.705718557174862, + "poetry": 18.08178229961657, + "typo": 23.088215564200972 + }, + "kendall_tau_vs_410m": { + "multihop": 0.4, + "multilingual": 0.8, + "order-ops": 0.8, + "poetry": 0.4, + "typo": 0.8 + }, + "n_families_R_ge_10": 5, + "n_families_R_lt_5": 0, + "n_families_tau_ge_0.6": 3 + }, + "VERDICT": "REPLICATES AT 1B", + "SECONDARY_order_ops_tau_sign": { + "what": "pre-specified, NO threshold and NO verdict: order-ops carries tau -0.40 at 1.4B, the only negative tau in CV6. Its sign at 1B says whether that is a property of the family or of the 1.4B checkpoint", + "tau_1b_order_ops": 0.8, + "tau_1.4b_order_ops": -0.4, + "tau_2.8b_order_ops": 0.8, + "reading": "negative at BOTH 1B and 1.4B -> a property of order-ops; positive at 1B -> specific to the 1.4B checkpoint. n=1 rung either way; a sign, not a result" + }, + "CROSS_LADDER_z_spread": { + "why_not_R": "R is a noise-scaled ratio and its denominator moves with N -- measured 2.29x-5.70x between N=200 and N=25. 1B and 410M are both N=200 and their R is comparable; CV6's 1.4B and 2.8B are N=25 and theirs is not comparable to either. Rank rungs by SPREAD, and state the N.", + "z_spread_by_rung": { + "410m_N200": { + "multihop": 0.4426378264396318, + "multilingual": 0.3826277027956122, + "order-ops": 0.2971030661554046, + "poetry": 0.2381084677921672, + "typo": 0.5254608638998532 + }, + "1b_N200": { + "multihop": 0.6068713921559281, + "multilingual": 0.2972451807409464, + "order-ops": 0.4764519225467332, + "poetry": 0.20345609841019296, + "typo": 0.7618851786893277 + }, + "1.4b_N25": { + "multihop": 0.33047741825140786, + "multilingual": 0.391457291423972, + "order-ops": 0.25377109484239035, + "poetry": 0.30579035957248824, + "typo": 0.7514365679687924 + }, + "2.8b_N25": { + "multihop": 0.6006834414398785, + "multilingual": 0.6540076586901638, + "order-ops": 0.37462404966354423, + "poetry": 0.3791702884395107, + "typo": 0.595472875982523 + } + } + }, + "controls": { + "C1_identity_arm": { + "required": "logit_I emitted for all five families before any J arm is graded", + "emitted": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "pooled_rank_flatmean7": 0.2190013977768011, + "scorer_validated_by": "CV6 control C0: 0.000e+00 against D3 on the stored 410M operators; negative control separates at 1.997. The same scorer is IMPORTED here, not reimplemented", + "fires": true + }, + "C2_band_read_off_artifacts": { + "required": "every one of the 15 lenses carries source_layers == band_for(16)", + "rule": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "observed_distinct": [ + [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ] + ], + "fires": true + }, + "C3_seed_sd_nondegenerate": { + "required": "pooled_seed_sd_z(f) > 0 for all five families", + "per_family": { + "multihop": 0.02065359607863063, + "multilingual": 0.011665750496260346, + "order-ops": 0.015027318232436472, + "poetry": 0.011251993583315474, + "typo": 0.03299887670273901 + }, + "void_cells": [], + "fires": true + }, + "C4_N_identical": { + "required": "all 15 lenses record n_prompts == 200", + "observed": [ + 200 + ], + "fires": true + }, + "C5_battery_identical_to_D3": { + "required": "pair counts per family equal D3's exactly", + "observed": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "expected": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "fires": true + } + }, + "declared_bias": [ + "ORDERING: written after CV6's 1.4B and 2.8B tables were seen. Thresholds are inherited verbatim from CV6, but the decision to run 1B is informed by results, so this is a disclosed follow-up and not a blind extension.", + "1B is 16 layers at d_model 2048 against 1.4B's 24 at the SAME width -- shallower, not narrower -- so its band is 8 layers against 13 and 18. min-over-band and max-over-band are both band-width sensitive (D1's union effect). A 1B-vs-1.4B difference may be a BAND-WIDTH effect and nothing in this design separates them.", + "N=200 here vs N=25 in CV6. The single easiest error to make with this table.", + "Operators fitted on CUDA, scored on CPU; D2 measured 2.774e-04 at the cell level.", + "The 5 corpora are a fixed panel, not a sample. n=5 is the replication unit." + ], + "provenance": { + "schema_version": 1, + "experiment": "CV7", + "utc": "2026-08-30T06:30:14+00:00", + "script": { + "path": "experiments/cv7_1b_rung.py", + "sha256": "17eed1b79d679e0044f836a0906667bbcfa1bcfe5dfe5399c99557321ea6f73d", + "n_lines": 316 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/cv7_1b_rung.py", + "--adjudicate" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/cv7/cv7_1b_n200.json", + "sha256": "6a9b060cc475b332a53aa2e8ebed8473178b4cef99ffdf476436047f7835596c", + "bytes": 16835, + "exists": true + }, + { + "path": "results/d3_corpus_by_family_410m.json", + "sha256": "021be9d711b0d2aa098ec421ba913d5e06a56b07eca88d1678a7c7683a5e24f5", + "bytes": 19610, + "exists": true + }, + { + "path": "results/cv6_per_family_ladder.json", + "sha256": "58559aa1e3026a72885b6e4b225be66be65efb1b5f2e56d4506959418dc1d4ee", + "bytes": 30030, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "a073b38c6bced9536c03b40ba87b15e4a66e33aaaf9d1e8d3200b68cd08f33a7" + } +} \ No newline at end of file diff --git a/results/cv8_c6_prefix_corpus_control.json b/results/cv8_c6_prefix_corpus_control.json new file mode 100644 index 0000000000000000000000000000000000000000..9488c39a19b130c98f47444d322110e3045574e8 --- /dev/null +++ b/results/cv8_c6_prefix_corpus_control.json @@ -0,0 +1,177 @@ +{ + "experiment": "CV8 C6 \u2014 does the prefix corpus drive the verdict?", + "prereg": "docs/experiments/preregs/CV8_positional_extrapolation.md", + "prereg_control": "C6", + "status": "PRE-REGISTERED \u2014 the control was registered 2026-08-29 and implemented now", + "adjudicates": "results/cv8_positional_extrapolation.json:VERDICT \u2014 and nothing else", + "recomputes_not_remeasures": true, + "C6_required_verbatim": "the prefix corpus does not drive the verdict: repeat the primary with the read prefix drawn from a second corpus (Github, the extreme arm). The PRIMARY branch must be the same under both prefixes; if it is not, the verdict is UNCLEAR regardless of either p-value.", + "registered_branch_thresholds": { + "EXPLAINS": "p <= 0.05", + "NOT_EXPLAINED": "p >= 0.2", + "UNCLEAR": "0.05 < p < 0.2" + }, + "arms": { + "primary": { + "prefix_corpus": "Pile-CC", + "n_sequences_for_G": 128, + "rho": 0.14285714285714285, + "exact_two_sided_p": 0.7520337301587302, + "n_permutations": 40320, + "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, + "abs_rho_below_critical": true, + "corpora_in_order": [ + "Github", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "alignment_at_p_read": [ + 0.8388732396731006, + 0.8369051828581651, + 0.8455001686641376, + 0.8390375985784466, + 0.8829077025264984, + 0.8570446480097353, + 0.8381376088564241, + 0.8731856807813917 + ], + "branch": "NOT_EXPLAINED", + "branch_reason": "exact two-sided p = 0.7520 >= 0.2", + "controls_fired": { + "C1_scale_invariance": true, + "C2_metric_is_not_vacuous": true, + "C3_no_leakage": true, + "C4_probe_matches_the_real_read": true, + "C5_band_rule": true + }, + "VERDICT_as_stored": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.7520 >= 0.20, rho = +0.143, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly." + }, + "second": { + "prefix_corpus": "Github", + "n_sequences_for_G": 128, + "rho": -0.047619047619047616, + "exact_two_sided_p": 0.9348710317460317, + "n_permutations": 40320, + "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, + "abs_rho_below_critical": true, + "corpora_in_order": [ + "Github", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "alignment_at_p_read": [ + 0.8928635661375124, + 0.8221690899817258, + 0.7984675936770528, + 0.8231678802397187, + 0.8410997638017006, + 0.8680869951100778, + 0.8175291862732885, + 0.8326451279763215 + ], + "branch": "NOT_EXPLAINED", + "branch_reason": "exact two-sided p = 0.9349 >= 0.2", + "controls_fired": { + "C1_scale_invariance": true, + "C2_metric_is_not_vacuous": true, + "C3_no_leakage": true, + "C4_probe_matches_the_real_read": true, + "C5_band_rule": true + }, + "VERDICT_as_stored": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.9349 >= 0.20, rho = -0.048, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly." + } + }, + "arms_differ_only_in_prefix_corpus": true, + "fields_asserted_identical": [ + "model", + "band", + "device", + "smoke", + "p_read", + "probe_positions", + "in_support_positions", + "out_of_support_positions", + "n_sequences_for_G", + "read_scores_min_from_r9" + ], + "same_script_sha256_both_arms": true, + "source_files": { + "primary": "results/cv8_positional_extrapolation.json", + "second": "results/cv8_positional_extrapolation_github.json" + }, + "C6_fires": true, + "CV8_VERDICT_AFTER_C6": "CV8's verdict STANDS as recorded \u2014 C6 fired.", + "VERDICT": "C6 FIRES \u2014 both prefixes land in the same PRIMARY branch (NOT_EXPLAINED): Pile-CC p = 0.7520, Github p = 0.9349. The prefix corpus does not drive the verdict, so the instrument is not corpus-dependent in the way that would make CV8's argument circular. CV8's verdict stands as recorded.", + "scope": "Adjudicates CV8's C6 only. It changes neither arm's rho, p, nor alignment profile, and it does not revisit CV8's stated power: n=8 detects only a strong monotone relationship, and both arms carry that bound.", + "provenance": { + "schema_version": 1, + "experiment": "CV8_C6", + "utc": "2026-08-31T05:59:02+00:00", + "script": { + "path": "experiments/cv8_c6_prefix_corpus_control.py", + "sha256": "cdeccfaa8bf8123dc2f78a765e1f5ce3e58531d8c1ddf14aa03e81f429f0c08a", + "n_lines": 176 + }, + "git": { + "commit": "9770d208af381e9579ae259d49d95c0c0a5b9a33", + "commit_short": "9770d208af38", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "research-paper-writing/" + ], + "dirty_diff_sha256": null, + "describe": "9770d20" + }, + "argv": [ + "experiments/cv8_c6_prefix_corpus_control.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/cv8_positional_extrapolation.json", + "sha256": "b23d7f66e92e280c337dc15def1d21fd9a5f682c2a3b068b0aa1bace2919e3cf", + "bytes": 10449, + "exists": true + }, + { + "path": "results/cv8_positional_extrapolation_github.json", + "sha256": "f82514e1558f0e86c9961d80f2ec6a4f714a2fd091f0df8324dc80cff70e2616", + "bytes": 10436, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "a8912a2cd5ac46ec06074c9c59fde191571d6ad590bf9f14dae8076055985358" + } +} \ No newline at end of file diff --git a/results/cv8_positional_extrapolation.json b/results/cv8_positional_extrapolation.json new file mode 100644 index 0000000000000000000000000000000000000000..28869b107a5baa941c6d39a128c765b0c8034b5b --- /dev/null +++ b/results/cv8_positional_extrapolation.json @@ -0,0 +1,399 @@ +{ + "experiment": "CV8 \u2014 positional extrapolation: is the corpus effect an extrapolation artifact?", + "prereg": "docs/experiments/preregs/CV8_positional_extrapolation.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "smoke": false, + "n_sequences_for_G": 128, + "prefix_corpus": "Pile-CC", + "probe_positions": [ + 16, + 24, + 32, + 48, + 64, + 80, + 96, + 112, + 126, + 133, + 142, + 150, + 160, + 177 + ], + "p_read": 142, + "in_support_positions": [ + 16, + 24, + 32, + 48, + 64, + 80, + 96, + 112, + 126 + ], + "out_of_support_positions": [ + 133, + 142, + 150, + 160, + 177 + ], + "alignment_by_corpus_by_position": { + "Github": { + "16": 0.6996252853536956, + "24": 0.7195726800527922, + "32": 0.7419920931750147, + "48": 0.7890212811960222, + "64": 0.7941302109133536, + "80": 0.8161464913360663, + "96": 0.8183671340138088, + "112": 0.8229096214682946, + "126": 0.8238566421789222, + "133": 0.7944473162024065, + "142": 0.8388732396731006, + "150": 0.8087969634902892, + "160": 0.7728870509309129, + "177": 0.7256898382376776 + }, + "Pile-CC": { + "16": 0.8655362229126606, + "24": 0.8794407644581727, + "32": 0.9045990717899588, + "48": 0.9609246582531124, + "64": 0.9600123937570153, + "80": 0.9735561251166504, + "96": 0.9736115131621349, + "112": 0.9587698643649143, + "126": 0.9315715191158161, + "133": 0.8844150766354874, + "142": 0.8829077025264984, + "150": 0.812117650005082, + "160": 0.750308456655454, + "177": 0.6929892962431711 + }, + "StackExchange": { + "16": 0.8036795767735414, + "24": 0.823674148552886, + "32": 0.8445555689011814, + "48": 0.8826739930722733, + "64": 0.8859453230341394, + "80": 0.898833215588331, + "96": 0.904113715176498, + "112": 0.8884164602671668, + "126": 0.8710085702476517, + "133": 0.8486331253630209, + "142": 0.8570446480097353, + "150": 0.7892426785265432, + "160": 0.7309038381482675, + "177": 0.6775588355322955 + }, + "USPTO_Backgrounds": { + "16": 0.7907249900285207, + "24": 0.8090482826892232, + "32": 0.8344965816173697, + "48": 0.8756122774890299, + "64": 0.8785620695065707, + "80": 0.8933423001911186, + "96": 0.8913479278432859, + "112": 0.8787880476980111, + "126": 0.8573401928967778, + "133": 0.827182917778604, + "142": 0.8381376088564241, + "150": 0.7732960251599772, + "160": 0.7201225439786406, + "177": 0.6684383565635394 + }, + "Wikipedia_en": { + "16": 0.8116737040343527, + "24": 0.8243563752414974, + "32": 0.8515287521634032, + "48": 0.9105007011285444, + "64": 0.9187448191728964, + "80": 0.929348254154488, + "96": 0.925303206745383, + "112": 0.9200323122113367, + "126": 0.9014905392400295, + "133": 0.8630461395728837, + "142": 0.8731856807813917, + "150": 0.8077957514475436, + "160": 0.7556025082473946, + "177": 0.6978742364606902 + }, + "OOD_arXiv_2023": { + "16": 0.7916525423537559, + "24": 0.8102770192869214, + "32": 0.8335794211271619, + "48": 0.8782709376178942, + "64": 0.8771631566419636, + "80": 0.8915520081502442, + "96": 0.8949200856470252, + "112": 0.8814222544321175, + "126": 0.8556494233519906, + "133": 0.8358392088399716, + "142": 0.8390375985784466, + "150": 0.7702991962986523, + "160": 0.7144830722075272, + "177": 0.6603754919946767 + }, + "OOD_CommonPile": { + "16": 0.781236677486555, + "24": 0.7988611712590944, + "32": 0.8232537507439353, + "48": 0.8654877791238893, + "64": 0.8656894410263634, + "80": 0.8812748843918979, + "96": 0.8850409193797165, + "112": 0.8723165202545101, + "126": 0.8502470394124511, + "133": 0.833852988611821, + "142": 0.8369051828581651, + "150": 0.7682897033917105, + "160": 0.7138111346577654, + "177": 0.6595057776158877 + }, + "OOD_News_2024": { + "16": 0.827480165969923, + "24": 0.837726723624863, + "32": 0.8632292155316605, + "48": 0.9210869498301496, + "64": 0.9247255673704081, + "80": 0.9362241489898391, + "96": 0.9349798506919959, + "112": 0.9225137382747429, + "126": 0.8975991050504516, + "133": 0.8515537919652221, + "142": 0.8455001686641376, + "150": 0.7801679271847001, + "160": 0.7259258340690871, + "177": 0.6713449714937855 + } + }, + "read_scores_min_from_r9": { + "Github": 0.24049734957516195, + "Pile-CC": 0.26239902824163436, + "StackExchange": 0.26430597975850106, + "USPTO_Backgrounds": 0.26112590581178663, + "Wikipedia_en": 0.24160844944417476, + "OOD_arXiv_2023": 0.2692730328068137, + "OOD_CommonPile": 0.2528811445459723, + "OOD_News_2024": 0.25006329715251924 + }, + "PRIMARY": { + "statistic": "Spearman rho between alignment at p_read and the stored min read score", + "rho": 0.14285714285714285, + "exact_two_sided_p": 0.7520337301587302, + "n_permutations": 40320, + "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, + "minimum_detectable": "n=8 detects only |rho| >= 0.714 at alpha=0.05 two-sided; a moderate relationship would NOT be detected by this design", + "corpora_in_order": [ + "Github", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "alignment_at_p_read": [ + 0.8388732396731006, + 0.8369051828581651, + 0.8455001686641376, + 0.8390375985784466, + 0.8829077025264984, + 0.8570446480097353, + 0.8381376088564241, + 0.8731856807813917 + ], + "read_score": [ + 0.24049734957516195, + 0.2528811445459723, + 0.25006329715251924, + 0.2692730328068137, + 0.26239902824163436, + 0.26430597975850106, + 0.26112590581178663, + 0.24160844944417476 + ] + }, + "SECONDARY_alignment_spread": { + "between_corpus_sd_at_p_read": 0.01784753488872414, + "between_corpus_sd_in_support_p64": 0.049280116927836765, + "ratio": 0.3621650272230308, + "note": "descriptive; no threshold is attached, per the registration" + }, + "controls": { + "C1_scale_invariance": { + "required": "cos_F unchanged to < 1e-9 when G is multiplied by alpha in {0.5, 2, 100}; the control for the target-count artifact that motivated a scale-free metric. AMENDED from '== 0.0', which is unreachable in floating point.", + "max_abs_difference": 4.8365533800165394e-11, + "tolerance": 1e-09, + "fires": true + }, + "C2_metric_is_not_vacuous": { + "required": "in-support alignment >= 0.30 for all 8 corpora AND |alignment| vs a Gaussian matrix <= 0.05. IF THIS FAILS THE INSTRUMENT IS BROKEN AND NO VERDICT ISSUES.", + "min_in_support_p64": 0.7941302109133536, + "max_abs_vs_random": 0.00020370669755364056, + "per_corpus_in_support": { + "Github": 0.7941302109133536, + "Pile-CC": 0.9600123937570153, + "StackExchange": 0.8859453230341394, + "USPTO_Backgrounds": 0.8785620695065707, + "Wikipedia_en": 0.9187448191728964, + "OOD_arXiv_2023": 0.8771631566419636, + "OOD_CommonPile": 0.8656894410263634, + "OOD_News_2024": 0.9247255673704081 + }, + "per_corpus_vs_random": { + "Github": 8.715056257069297e-05, + "Pile-CC": 8.34612596484493e-05, + "StackExchange": 0.00020370669755364056, + "USPTO_Backgrounds": 7.459009373812329e-05, + "Wikipedia_en": 5.197126198105234e-05, + "OOD_arXiv_2023": 3.00259731882602e-05, + "OOD_CommonPile": -2.5453768332120378e-05, + "OOD_News_2024": -0.00011232876610156356 + }, + "fires": true + }, + "C3_no_leakage": { + "required": "the items building G and the items scored are disjoint", + "n_overlap": 0, + "n_build": 128, + "n_score": 270, + "fires": true + }, + "C4_probe_matches_the_real_read": { + "required": "p_read inside CV2's measured prefixed read range, and CV2's n_in_support == 0", + "p_read": 142, + "cv2_min": 133, + "cv2_max": 177, + "cv2_median": 142, + "cv2_n_in_support": 0, + "fires": true + }, + "C5_band_rule": { + "required": "band == floor(0.38L)..floor(0.92L) intersected with layers strictly below the penultimate target layer, at L=24", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "rule_band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "target_layer": 22, + "fires": true + } + }, + "controls_fired": { + "C1_scale_invariance": true, + "C2_metric_is_not_vacuous": true, + "C3_no_leakage": true, + "C4_probe_matches_the_real_read": true, + "C5_band_rule": true + }, + "VERDICT": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.7520 >= 0.20, rho = +0.143, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly.", + "provenance": { + "schema_version": 1, + "experiment": "CV8", + "utc": "2026-08-31T03:39:53+00:00", + "script": { + "path": "experiments/cv8_positional_extrapolation.py", + "sha256": "7bc5d7908a74bbd6713744b4af830c963396d9402c51b1fb289d7c92d23edb8d", + "n_lines": 408 + }, + "git": { + "commit": "99a6dd48ac17cd06f63d77e1e2fa51296cd189f7", + "commit_short": "99a6dd48ac17", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "PAPER_REWRITE_SCOPE.md", + "xperiments/e66b_determinism_floor.py" + ], + "dirty_diff_sha256": "01b0d0f1f9e05dca6bc8dca13e923a4920f8bf53c42c0d1a3cca9d266754795c", + "describe": "99a6dd4-dirty" + }, + "argv": [ + "experiments/cv8_positional_extrapolation.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/r9_permutation_calibrated_min.json", + "sha256": "eea61cccda689ed7903ca0f24af1961beb18f104fc7226a21f61a2563e3a9ba2", + "bytes": 20471, + "exists": true + }, + { + "path": "results/cv2_position_support.json", + "sha256": "e61248b05d2e8fa4db0262b325e2748579f873326414b1992d5bd4476c136b9c", + "bytes": 9152, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "a950c3b88b5401cefca4116adf35303854131b5bd45dfd084c0efe00325e4501" + } +} \ No newline at end of file diff --git a/results/cv8_positional_extrapolation_github.json b/results/cv8_positional_extrapolation_github.json new file mode 100644 index 0000000000000000000000000000000000000000..2437d84f64c60fd9242e5ffbd295d8dddfd403d5 --- /dev/null +++ b/results/cv8_positional_extrapolation_github.json @@ -0,0 +1,402 @@ +{ + "experiment": "CV8 \u2014 positional extrapolation: is the corpus effect an extrapolation artifact?", + "prereg": "docs/experiments/preregs/CV8_positional_extrapolation.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "smoke": false, + "n_sequences_for_G": 128, + "prefix_corpus": "Github", + "probe_positions": [ + 16, + 24, + 32, + 48, + 64, + 80, + 96, + 112, + 126, + 133, + 142, + 150, + 160, + 177 + ], + "p_read": 142, + "in_support_positions": [ + 16, + 24, + 32, + 48, + 64, + 80, + 96, + 112, + 126 + ], + "out_of_support_positions": [ + 133, + 142, + 150, + 160, + 177 + ], + "alignment_by_corpus_by_position": { + "Github": { + "16": 0.8199622410112316, + "24": 0.8649228662025228, + "32": 0.8823327600490843, + "48": 0.9208955206003056, + "64": 0.9392306171621081, + "80": 0.9484260866168507, + "96": 0.9504478884880948, + "112": 0.9423929234741516, + "126": 0.9310935599306583, + "133": 0.8484558265334022, + "142": 0.8928635661375124, + "150": 0.8991461850577471, + "160": 0.863023156475018, + "177": 0.8091136683178206 + }, + "Pile-CC": { + "16": 0.7388377966330032, + "24": 0.7623900261264125, + "32": 0.7401483099189051, + "48": 0.8078829412811408, + "64": 0.8026415395543312, + "80": 0.8074120082320871, + "96": 0.8188838080903893, + "112": 0.8275872410947619, + "126": 0.8261497692178901, + "133": 0.8825382212439484, + "142": 0.8410997638017006, + "150": 0.784536015586303, + "160": 0.737921293108437, + "177": 0.7043631819379887 + }, + "StackExchange": { + "16": 0.7868950844126045, + "24": 0.8172987029771465, + "32": 0.7958271283933123, + "48": 0.8631015421781737, + "64": 0.8521835524590238, + "80": 0.8567536478592028, + "96": 0.865747722698461, + "112": 0.869334769490467, + "126": 0.8632586365641963, + "133": 0.8908658940965233, + "142": 0.8680869951100778, + "150": 0.8141380848365287, + "160": 0.7637898622813123, + "177": 0.72162493963662 + }, + "USPTO_Backgrounds": { + "16": 0.7250339105496532, + "24": 0.7466318620527843, + "32": 0.7217538165312588, + "48": 0.7893456531556848, + "64": 0.7835063101091788, + "80": 0.7881745586910742, + "96": 0.8035122217701587, + "112": 0.8106891547649236, + "126": 0.8059748422771726, + "133": 0.8468851304825057, + "142": 0.8175291862732885, + "150": 0.765009421906371, + "160": 0.7211020727707582, + "177": 0.6890500060955719 + }, + "Wikipedia_en": { + "16": 0.714737290764607, + "24": 0.7414217829325931, + "32": 0.7282756412878298, + "48": 0.7893573136174032, + "64": 0.7913875814589846, + "80": 0.8014941703581413, + "96": 0.8124232223799928, + "112": 0.8217904107612356, + "126": 0.820110692737911, + "133": 0.8634235496340769, + "142": 0.8326451279763215, + "150": 0.7839842876667025, + "160": 0.7441040846658752, + "177": 0.7115479365349467 + }, + "OOD_arXiv_2023": { + "16": 0.7314797724797859, + "24": 0.7538180293815879, + "32": 0.7296685194098056, + "48": 0.7965723238808289, + "64": 0.7904637129490234, + "80": 0.792464811600057, + "96": 0.8055670247052423, + "112": 0.8121151935399167, + "126": 0.8085203811055055, + "133": 0.8561239410946141, + "142": 0.8231678802397187, + "150": 0.7658832922857227, + "160": 0.7208183538688316, + "177": 0.6866824113541039 + }, + "OOD_CommonPile": { + "16": 0.7274252644273965, + "24": 0.7487318124617317, + "32": 0.7244637094978772, + "48": 0.7910418690218445, + "64": 0.786705942925807, + "80": 0.7893214492024896, + "96": 0.8042403273110403, + "112": 0.8103238690236166, + "126": 0.8063952401312486, + "133": 0.8562419725480844, + "142": 0.8221690899817258, + "150": 0.7646649434641972, + "160": 0.7195846531182958, + "177": 0.6856726030758462 + }, + "OOD_News_2024": { + "16": 0.6930364953152686, + "24": 0.714579362305328, + "32": 0.6958332436342336, + "48": 0.759202188142498, + "64": 0.7562811167705011, + "80": 0.7640614480227068, + "96": 0.7759056468667923, + "112": 0.7856216585197024, + "126": 0.7848264476594219, + "133": 0.8403997315239466, + "142": 0.7984675936770528, + "150": 0.7452955111728733, + "160": 0.7056315182522144, + "177": 0.675980543031208 + } + }, + "read_scores_min_from_r9": { + "Github": 0.24049734957516195, + "Pile-CC": 0.26239902824163436, + "StackExchange": 0.26430597975850106, + "USPTO_Backgrounds": 0.26112590581178663, + "Wikipedia_en": 0.24160844944417476, + "OOD_arXiv_2023": 0.2692730328068137, + "OOD_CommonPile": 0.2528811445459723, + "OOD_News_2024": 0.25006329715251924 + }, + "PRIMARY": { + "statistic": "Spearman rho between alignment at p_read and the stored min read score", + "rho": -0.047619047619047616, + "exact_two_sided_p": 0.9348710317460317, + "n_permutations": 40320, + "critical_abs_rho_at_alpha_0.05": 0.7142857142857143, + "minimum_detectable": "n=8 detects only |rho| >= 0.714 at alpha=0.05 two-sided; a moderate relationship would NOT be detected by this design", + "corpora_in_order": [ + "Github", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "alignment_at_p_read": [ + 0.8928635661375124, + 0.8221690899817258, + 0.7984675936770528, + 0.8231678802397187, + 0.8410997638017006, + 0.8680869951100778, + 0.8175291862732885, + 0.8326451279763215 + ], + "read_score": [ + 0.24049734957516195, + 0.2528811445459723, + 0.25006329715251924, + 0.2692730328068137, + 0.26239902824163436, + 0.26430597975850106, + 0.26112590581178663, + 0.24160844944417476 + ] + }, + "SECONDARY_alignment_spread": { + "between_corpus_sd_at_p_read": 0.030232561695707077, + "between_corpus_sd_in_support_p64": 0.05773061221794703, + "ratio": 0.5236833723774111, + "note": "descriptive; no threshold is attached, per the registration" + }, + "controls": { + "C1_scale_invariance": { + "required": "cos_F unchanged to < 1e-9 when G is multiplied by alpha in {0.5, 2, 100}; the control for the target-count artifact that motivated a scale-free metric. AMENDED from '== 0.0', which is unreachable in floating point.", + "max_abs_difference": 7.949996216893851e-11, + "tolerance": 1e-09, + "fires": true + }, + "C2_metric_is_not_vacuous": { + "required": "in-support alignment >= 0.30 for all 8 corpora AND |alignment| vs a Gaussian matrix <= 0.05. IF THIS FAILS THE INSTRUMENT IS BROKEN AND NO VERDICT ISSUES.", + "min_in_support_p64": 0.7562811167705011, + "max_abs_vs_random": 0.00020370669755364056, + "per_corpus_in_support": { + "Github": 0.9392306171621081, + "Pile-CC": 0.8026415395543312, + "StackExchange": 0.8521835524590238, + "USPTO_Backgrounds": 0.7835063101091788, + "Wikipedia_en": 0.7913875814589846, + "OOD_arXiv_2023": 0.7904637129490234, + "OOD_CommonPile": 0.786705942925807, + "OOD_News_2024": 0.7562811167705011 + }, + "per_corpus_vs_random": { + "Github": 8.715056257069297e-05, + "Pile-CC": 8.34612596484493e-05, + "StackExchange": 0.00020370669755364056, + "USPTO_Backgrounds": 7.459009373812329e-05, + "Wikipedia_en": 5.197126198105234e-05, + "OOD_arXiv_2023": 3.00259731882602e-05, + "OOD_CommonPile": -2.5453768332120378e-05, + "OOD_News_2024": -0.00011232876610156356 + }, + "fires": true + }, + "C3_no_leakage": { + "required": "the items building G and the items scored are disjoint", + "n_overlap": 0, + "n_build": 128, + "n_score": 270, + "fires": true + }, + "C4_probe_matches_the_real_read": { + "required": "p_read inside CV2's measured prefixed read range, and CV2's n_in_support == 0", + "p_read": 142, + "cv2_min": 133, + "cv2_max": 177, + "cv2_median": 142, + "cv2_n_in_support": 0, + "fires": true + }, + "C5_band_rule": { + "required": "band == floor(0.38L)..floor(0.92L) intersected with layers strictly below the penultimate target layer, at L=24", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "rule_band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "target_layer": 22, + "fires": true + } + }, + "controls_fired": { + "C1_scale_invariance": true, + "C2_metric_is_not_vacuous": true, + "C3_no_leakage": true, + "C4_probe_matches_the_real_read": true, + "C5_band_rule": true + }, + "VERDICT": "EXTRAPOLATION DOES NOT EXPLAIN THE ORDERING \u2014 exact two-sided p = 0.9349 >= 0.20, rho = -0.048, below the alpha=0.05 critical |rho| of 0.714 for n=8. No STRONG monotone relationship between alignment at position 142 and the read. The positional hedge is retired; this design could not have detected a moderate relationship and the claim is bounded accordingly.", + "provenance": { + "schema_version": 1, + "experiment": "CV8", + "utc": "2026-08-31T05:58:45+00:00", + "script": { + "path": "experiments/cv8_positional_extrapolation.py", + "sha256": "7bc5d7908a74bbd6713744b4af830c963396d9402c51b1fb289d7c92d23edb8d", + "n_lines": 408 + }, + "git": { + "commit": "9770d208af381e9579ae259d49d95c0c0a5b9a33", + "commit_short": "9770d208af38", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "research-paper-writing/" + ], + "dirty_diff_sha256": null, + "describe": "9770d20" + }, + "argv": [ + "experiments/cv8_positional_extrapolation.py", + "--prefix-corpus", + "Github", + "--out", + "results/cv8_positional_extrapolation_github.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/r9_permutation_calibrated_min.json", + "sha256": "eea61cccda689ed7903ca0f24af1961beb18f104fc7226a21f61a2563e3a9ba2", + "bytes": 20471, + "exists": true + }, + { + "path": "results/cv2_position_support.json", + "sha256": "e61248b05d2e8fa4db0262b325e2748579f873326414b1992d5bd4476c136b9c", + "bytes": 9152, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "db838d5ea323fb6ef2b73cec85b2d427d891d37340a801e138f2bf20f0b01d15" + } +} \ No newline at end of file diff --git a/results/d1_min_union_diagnostic_410m.json b/results/d1_min_union_diagnostic_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..ea552c67a46efa52e97c11a3ae0b27aae122818b --- /dev/null +++ b/results/d1_min_union_diagnostic_410m.json @@ -0,0 +1,1548 @@ +{ + "experiment": "D1 \u2014 diagnostic: is `min`'s preference for J^shuf a union-over-layers artifact?", + "status": "DIAGNOSTIC RECOMPUTATION, not a pre-registered experiment. Recomputes from results/*.pt; measures nothing new.", + "hypothesis": "min is an existential union over the band; a derangement decorrelates the layers; a union of decorrelated weaker events can exceed a union of correlated stronger ones.", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "half_threshold_for_persist": 6, + "n_items": 541, + "n_pairs": 893, + "corpora": [ + "USPTO_Backgrounds", + "Pile-CC", + "Github" + ], + "seed": 0, + "n_derangements": 3, + "GATE_reproduces_stored_e48_arms": { + "tolerance": 1e-06, + "fires": true, + "per_arm": { + "logit_I": { + "min": { + "recomputed": 0.1247396256774664, + "stored": 0.1247396256774664, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.028439503419212996, + "stored": 0.028439503419212996, + "abs_diff": 0.0 + } + }, + "J|USPTO_Backgrounds|s0": { + "min": { + "recomputed": 0.15465880744159222, + "stored": 0.15465880744159222, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.04964358932338655, + "stored": 0.04964358932338655, + "abs_diff": 0.0 + } + }, + "shuf|USPTO_Backgrounds|s0|d0": { + "min": { + "recomputed": 0.14029880315065385, + "stored": 0.14029880315065385, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.028034024871885777, + "stored": 0.028034024871885777, + "abs_diff": 0.0 + } + }, + "shuf|USPTO_Backgrounds|s0|d1": { + "min": { + "recomputed": 0.1608631432056427, + "stored": 0.1608631432056427, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.029616341809742153, + "stored": 0.029616341809742153, + "abs_diff": 0.0 + } + }, + "shuf|USPTO_Backgrounds|s0|d2": { + "min": { + "recomputed": 0.16253653764724732, + "stored": 0.16253653764724732, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.026408451795578002, + "stored": 0.026408451795578002, + "abs_diff": 0.0 + } + }, + "J|Pile-CC|s0": { + "min": { + "recomputed": 0.1452333066612482, + "stored": 0.1452333066612482, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.04509159799199551, + "stored": 0.04509159799199551, + "abs_diff": 0.0 + } + }, + "shuf|Pile-CC|s0|d0": { + "min": { + "recomputed": 0.1562277466058731, + "stored": 0.1562277466058731, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.028093868121504783, + "stored": 0.028093868121504783, + "abs_diff": 0.0 + } + }, + "shuf|Pile-CC|s0|d1": { + "min": { + "recomputed": 0.17145237997174262, + "stored": 0.17145237997174262, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.0278983935713768, + "stored": 0.0278983935713768, + "abs_diff": 0.0 + } + }, + "shuf|Pile-CC|s0|d2": { + "min": { + "recomputed": 0.16748275458812714, + "stored": 0.16748275458812714, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.025132926274091007, + "stored": 0.025132926274091007, + "abs_diff": 0.0 + } + }, + "J|Github|s0": { + "min": { + "recomputed": 0.10961288511753083, + "stored": 0.10961288511753083, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.02704486474394798, + "stored": 0.02704486474394798, + "abs_diff": 0.0 + } + }, + "shuf|Github|s0|d0": { + "min": { + "recomputed": 0.11639587208628654, + "stored": 0.11639587208628654, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.015263549471274018, + "stored": 0.015263549471274018, + "abs_diff": 0.0 + } + }, + "shuf|Github|s0|d1": { + "min": { + "recomputed": 0.11431133672595024, + "stored": 0.11431133672595024, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.019197244732640683, + "stored": 0.019197244732640683, + "abs_diff": 0.0 + } + }, + "shuf|Github|s0|d2": { + "min": { + "recomputed": 0.13957404792308808, + "stored": 0.13957404792308808, + "abs_diff": 0.0 + }, + "persist": { + "recomputed": 0.019098307471722364, + "stored": 0.019098307471722364, + "abs_diff": 0.0 + } + } + } + }, + "arms": { + "logit_I": { + "label": "logit_I", + "min": 0.1247396256774664, + "persist": 0.028439503419212996, + "per_layer_auc": [ + 0.0069555905647575855, + 0.005350454244762659, + 0.002675227588042617, + 0.005172106437385082, + 0.010344212874770164, + 0.015694668516516685, + 0.02211521379649639, + 0.03477795422077179, + 0.06866417080163956, + 0.06509719789028168, + 0.06955591589212418, + 0.06991260498762131, + 0.057963259518146515 + ], + "best_single_layer_auc": 0.06991260498762131, + "mean_single_layer_auc": 0.03340604528784752, + "union_observed_by_k": [ + 0.018726591020822525, + 0.0387016236782074, + 0.06741572916507721, + 0.08863919973373413, + 0.12484394758939743, + 0.20099875330924988, + 0.30836454033851624 + ], + "union_if_layers_independent_by_k": [ + 0.03931021690368652, + 0.10272163152694702, + 0.19602113962173462, + 0.2618407607078552, + 0.3644673824310303, + 0.5456682443618774, + 0.7235082387924194 + ], + "redundancy_ratio_by_k": [ + 0.47637974287204554, + 0.3767621590789743, + 0.34392070822142196, + 0.3385233051344208, + 0.3425380530808465, + 0.36835340041512676, + 0.426207365451976 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.09914106130599976, + 0.2166750282049179, + 0.30075791478157043, + 0.2948482036590576, + 0.29378190636634827, + 0.3175992965698242, + 0.35248231887817383 + ], + "mean_layers_recovering_by_k": [ + 0.039950061589479446, + 0.10736579447984695, + 0.2147315889596939, + 0.2971285879611969, + 0.4394506812095642, + 0.750312089920044, + 1.1910111904144287 + ], + "min_auc_by_band_width": { + "1": 0.02209216309711337, + "2": 0.025468118023127317, + "3": 0.04031772119924426, + "4": 0.04076037211343646, + "5": 0.07104449141770601, + "6": 0.07193734962493181, + "7": 0.09212993532419204, + "8": 0.09212993532419204, + "9": 0.10359705798327923, + "10": 0.10447628721594811, + "11": 0.1184635378420353, + "12": 0.1199016124010086, + "13": 0.1247396256774664 + } + }, + "J|USPTO_Backgrounds|s0": { + "label": "J|USPTO_Backgrounds|s0", + "min": 0.15465880744159222, + "persist": 0.04964358932338655, + "per_layer_auc": [ + 0.023720350116491318, + 0.017834847792983055, + 0.01801319792866707, + 0.037274833768606186, + 0.03477795794606209, + 0.04797574505209923, + 0.03887997195124626, + 0.060995183885097504, + 0.08114856481552124, + 0.05992509052157402, + 0.06117353215813637, + 0.06705903261899948, + 0.06384875625371933 + ], + "best_single_layer_auc": 0.08114856481552124, + "mean_single_layer_auc": 0.047125160694122314, + "union_observed_by_k": [ + 0.033707864582538605, + 0.05867665261030197, + 0.10486891120672226, + 0.13483145833015442, + 0.16853933036327362, + 0.24719101190567017, + 0.30586767196655273 + ], + "union_if_layers_independent_by_k": [ + 0.08298718929290771, + 0.1466045379638672, + 0.2742736339569092, + 0.39154767990112305, + 0.5149980783462524, + 0.6825590133666992, + 0.7894540429115295 + ], + "redundancy_ratio_by_k": [ + 0.40618154283506214, + 0.40023762855665274, + 0.3823514119596275, + 0.3443551456216091, + 0.32726205679151743, + 0.36215331870926276, + 0.3874420236528321 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.21555741131305695, + 0.28876787424087524, + 0.3172539174556732, + 0.354358047246933, + 0.4094706177711487, + 0.45368310809135437, + 0.46720415353775024 + ], + "mean_layers_recovering_by_k": [ + 0.08614232391119003, + 0.15730337798595428, + 0.3158551752567291, + 0.4856429398059845, + 0.7003745436668396, + 1.0898876190185547, + 1.4531835317611694 + ], + "min_auc_by_band_width": { + "1": 0.04558134144172073, + "2": 0.07073765937238932, + "3": 0.08617502506822347, + "4": 0.09440531227737665, + "5": 0.11388459671288728, + "6": 0.12268156222999096, + "7": 0.12817503698170185, + "8": 0.13055598624050618, + "9": 0.13708783574402333, + "10": 0.1382520105689764, + "11": 0.1457491260021925, + "12": 0.14909512959420682, + "13": 0.15465880744159222 + } + }, + "shuf|USPTO_Backgrounds|s0|d0": { + "label": "shuf|USPTO_Backgrounds|s0|d0", + "min": 0.14029880315065385, + "persist": 0.028034024871885777, + "per_layer_auc": [ + 0.01979668438434601, + 0.021401820704340935, + 0.01765650138258934, + 0.01854824461042881, + 0.016229713335633278, + 0.026752274483442307, + 0.02782236598432064, + 0.047440700232982635, + 0.06527554243803024, + 0.06402710825204849, + 0.021580170840024948, + 0.06420546025037766, + 0.022828608751296997 + ], + "best_single_layer_auc": 0.06527554243803024, + "mean_single_layer_auc": 0.03335116431117058, + "union_observed_by_k": [ + 0.027465667575597763, + 0.04619225859642029, + 0.08614232391119003, + 0.12234706431627274, + 0.15730337798595428, + 0.227215975522995, + 0.30337077379226685 + ], + "union_if_layers_independent_by_k": [ + 0.05483764410018921, + 0.09554314613342285, + 0.18931978940963745, + 0.2806728482246399, + 0.3864063620567322, + 0.5544065237045288, + 0.6859962940216064 + ], + "redundancy_ratio_by_k": [ + 0.500854258534841, + 0.483470143760122, + 0.4550096119365581, + 0.435906305473306, + 0.4070931367399665, + 0.4098364030869338, + 0.4422338377570182 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.12752822041511536, + 0.19030694663524628, + 0.2284211665391922, + 0.26853522658348083, + 0.3102381229400635, + 0.3685932755470276, + 0.37441137433052063 + ], + "mean_layers_recovering_by_k": [ + 0.056179776787757874, + 0.09987515956163406, + 0.20724095404148102, + 0.323345810174942, + 0.4756554365158081, + 0.7765293121337891, + 1.0961298942565918 + ], + "min_auc_by_band_width": { + "1": 0.02500363448634744, + "2": 0.05584375690668821, + "3": 0.07580512072890996, + "4": 0.07809844892472029, + "5": 0.10060342960059643, + "6": 0.1029571283608675, + "7": 0.11308615989983081, + "8": 0.11576473303139209, + "9": 0.11941296644508839, + "10": 0.12496742494404316, + "11": 0.1348981551826, + "12": 0.13644054234027864, + "13": 0.14029880315065385 + } + }, + "shuf|USPTO_Backgrounds|s0|d1": { + "label": "shuf|USPTO_Backgrounds|s0|d1", + "min": 0.1608631432056427, + "persist": 0.029616341809742153, + "per_layer_auc": [ + 0.020866775885224342, + 0.005172105971723795, + 0.015516319312155247, + 0.009809167124330997, + 0.03656143695116043, + 0.03834492713212967, + 0.02514713816344738, + 0.022828606888651848, + 0.05029427632689476, + 0.07704655081033707, + 0.06295701116323471, + 0.06491885334253311, + 0.05510968342423439 + ], + "best_single_layer_auc": 0.07704655081033707, + "mean_single_layer_auc": 0.037274833768606186, + "union_observed_by_k": [ + 0.033707864582538605, + 0.048689138144254684, + 0.09737827628850937, + 0.12983770668506622, + 0.1785268485546112, + 0.2534331977367401, + 0.33707866072654724 + ], + "union_if_layers_independent_by_k": [ + 0.06784975528717041, + 0.1069028377532959, + 0.20243579149246216, + 0.3024303913116455, + 0.41803091764450073, + 0.5972967147827148, + 0.7348015308380127 + ], + "redundancy_ratio_by_k": [ + 0.49680156457266345, + 0.45545225147920415, + 0.48103290218881734, + 0.42931434940105717, + 0.42706613558768614, + 0.42430033761182545, + 0.458734292975849 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.1436370313167572, + 0.2387312352657318, + 0.21464835107326508, + 0.22722767293453217, + 0.2721562385559082, + 0.3380246162414551, + 0.349947065114975 + ], + "mean_layers_recovering_by_k": [ + 0.0699126124382019, + 0.11235955357551575, + 0.22347065806388855, + 0.3533083498477936, + 0.5255929827690125, + 0.8676654100418091, + 1.2397003173828125 + ], + "min_auc_by_band_width": { + "1": 0.036114420741796494, + "2": 0.05712682344019413, + "3": 0.06281333640217782, + "4": 0.0872852060943842, + "5": 0.10629639737308025, + "6": 0.10791589394211769, + "7": 0.13400385417044164, + "8": 0.13786075375974177, + "9": 0.14413371607661246, + "10": 0.14583004862070084, + "11": 0.1522133857011795, + "12": 0.1558573305606842, + "13": 0.1608631432056427 + } + }, + "shuf|USPTO_Backgrounds|s0|d2": { + "label": "shuf|USPTO_Backgrounds|s0|d2", + "min": 0.16253653764724732, + "persist": 0.026408451795578002, + "per_layer_auc": [ + 0.011771000921726227, + 0.007312288973480463, + 0.018726591020822525, + 0.016943106427788734, + 0.03656143695116043, + 0.02425539307296276, + 0.031924378126859665, + 0.04280363768339157, + 0.08114856481552124, + 0.06081683561205864, + 0.07187443971633911, + 0.02603888139128685, + 0.01676475815474987 + ], + "best_single_layer_auc": 0.08114856481552124, + "mean_single_layer_auc": 0.034380100667476654, + "union_observed_by_k": [ + 0.02871410734951496, + 0.05243445560336113, + 0.10112359374761581, + 0.13358302414417267, + 0.18227216601371765, + 0.2721598148345947, + 0.33957552909851074 + ], + "union_if_layers_independent_by_k": [ + 0.05256330966949463, + 0.09117072820663452, + 0.192446768283844, + 0.27343469858169556, + 0.3994895815849304, + 0.5757759809494019, + 0.7038078308105469 + ], + "redundancy_ratio_by_k": [ + 0.5462766239428666, + 0.5751237994339663, + 0.5254626754680878, + 0.4885372077394243, + 0.45626262715180993, + 0.4726835155329476, + 0.4824833061425807 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.08346430212259293, + 0.13849858939647675, + 0.16893534362316132, + 0.200520321726799, + 0.2768979072570801, + 0.31897783279418945, + 0.33612746000289917 + ], + "mean_layers_recovering_by_k": [ + 0.05368289723992348, + 0.09488140046596527, + 0.21098627150058746, + 0.3133583068847656, + 0.4956304728984833, + 0.8177278637886047, + 1.142322063446045 + ], + "min_auc_by_band_width": { + "1": 0.030626533459872006, + "2": 0.034132934361696246, + "3": 0.05739830266684294, + "4": 0.08182297609746456, + "5": 0.10774274505674838, + "6": 0.1149121206253767, + "7": 0.13583680912852286, + "8": 0.13904994651675223, + "9": 0.1552973859012127, + "10": 0.15647276267409324, + "11": 0.15850809291005136, + "12": 0.1606118641793728, + "13": 0.16253653764724732 + } + }, + "J|Pile-CC|s0": { + "label": "J|Pile-CC|s0", + "min": 0.1452333066612482, + "persist": 0.04509159799199551, + "per_layer_auc": [ + 0.01854824461042881, + 0.014802925288677216, + 0.013019440695643425, + 0.021936863660812378, + 0.02425539493560791, + 0.03923666849732399, + 0.04048511013388634, + 0.05724986642599106, + 0.10094524174928665, + 0.07276619225740433, + 0.07133939117193222, + 0.07276618480682373, + 0.06634563952684402 + ], + "best_single_layer_auc": 0.10094524174928665, + "mean_single_layer_auc": 0.04720747470855713, + "union_observed_by_k": [ + 0.04244694113731384, + 0.062421973794698715, + 0.09987515956163406, + 0.1498127281665802, + 0.1910112351179123, + 0.23845192790031433, + 0.2946317195892334 + ], + "union_if_layers_independent_by_k": [ + 0.10024118423461914, + 0.16397207975387573, + 0.2786230444908142, + 0.3858959674835205, + 0.5166325569152832, + 0.6783620715141296, + 0.7903714776039124 + ], + "redundancy_ratio_by_k": [ + 0.42344812126286147, + 0.38068660157506645, + 0.3584597955426002, + 0.38822050705408806, + 0.36972357347822754, + 0.3515112915556742, + 0.37277625513820156 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.335532009601593, + 0.3729729652404785, + 0.4030463993549347, + 0.4020135700702667, + 0.41894805431365967, + 0.46197083592414856, + 0.4779736399650574 + ], + "mean_layers_recovering_by_k": [ + 0.10486891120672226, + 0.17727839946746826, + 0.3208489418029785, + 0.47440698742866516, + 0.6991260647773743, + 1.0711610317230225, + 1.4481897354125977 + ], + "min_auc_by_band_width": { + "1": 0.04562543174251914, + "2": 0.05826702453196049, + "3": 0.07221080381423235, + "4": 0.07488937694579363, + "5": 0.10553387273102999, + "6": 0.11015784181654453, + "7": 0.11694746688008309, + "8": 0.11868922971189022, + "9": 0.1247547548264265, + "10": 0.12571405060589314, + "11": 0.13431683368980885, + "12": 0.1349120695143938, + "13": 0.1452333066612482 + } + }, + "shuf|Pile-CC|s0|d0": { + "label": "shuf|Pile-CC|s0|d0", + "min": 0.1562277466058731, + "persist": 0.028093868121504783, + "per_layer_auc": [ + 0.01801319606602192, + 0.018726591020822525, + 0.016586409881711006, + 0.019083289429545403, + 0.017299804836511612, + 0.023720350116491318, + 0.027108969166874886, + 0.0406634546816349, + 0.08007847517728806, + 0.07793829590082169, + 0.022650256752967834, + 0.06973426043987274, + 0.02996254898607731 + ], + "best_single_layer_auc": 0.08007847517728806, + "mean_single_layer_auc": 0.03550507128238678, + "union_observed_by_k": [ + 0.031210986897349358, + 0.05742821469902992, + 0.10112359374761581, + 0.1473158597946167, + 0.17977528274059296, + 0.2534331977367401, + 0.34456929564476013 + ], + "union_if_layers_independent_by_k": [ + 0.06316721439361572, + 0.11149823665618896, + 0.19051074981689453, + 0.29407936334609985, + 0.3961603045463562, + 0.5750750303268433, + 0.7198234796524048 + ], + "redundancy_ratio_by_k": [ + 0.4941010490483784, + 0.5150593984379594, + 0.53080256019573, + 0.5009391278545504, + 0.4537942864983758, + 0.44069588205334115, + 0.47868582421227635 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.1836680918931961, + 0.22814513742923737, + 0.24371437728405, + 0.26739829778671265, + 0.31035545468330383, + 0.3594497740268707, + 0.380222350358963 + ], + "mean_layers_recovering_by_k": [ + 0.06491885334253311, + 0.11735330522060394, + 0.20848938822746277, + 0.3408239781856537, + 0.48938825726509094, + 0.8164793848991394, + 1.1935081481933594 + ], + "min_auc_by_band_width": { + "1": 0.023558421758934855, + "2": 0.04545115493237972, + "3": 0.06323257088661194, + "4": 0.06518027149140834, + "5": 0.09539598505944014, + "6": 0.09901875630021095, + "7": 0.1140231292694807, + "8": 0.11668955646455288, + "9": 0.12094095423817634, + "10": 0.12338834591209888, + "11": 0.13372831530869006, + "12": 0.1369838934391737, + "13": 0.1562277466058731 + } + }, + "shuf|Pile-CC|s0|d1": { + "label": "shuf|Pile-CC|s0|d1", + "min": 0.17145237997174262, + "persist": 0.0278983935713768, + "per_layer_auc": [ + 0.019261637702584267, + 0.004815409425646067, + 0.013019440695643425, + 0.009274122305214405, + 0.027108972892165184, + 0.03852327913045883, + 0.02907080389559269, + 0.021936865523457527, + 0.05831995606422424, + 0.07793829590082169, + 0.06866417080163956, + 0.0704476535320282, + 0.05546638369560242 + ], + "best_single_layer_auc": 0.07793829590082169, + "mean_single_layer_auc": 0.03798822686076164, + "union_observed_by_k": [ + 0.032459426671266556, + 0.04244694113731384, + 0.09238451719284058, + 0.13233457505702972, + 0.18726591765880585, + 0.26716604828834534, + 0.3895131051540375 + ], + "union_if_layers_independent_by_k": [ + 0.07479846477508545, + 0.10800778865814209, + 0.2057608962059021, + 0.28645724058151245, + 0.41760051250457764, + 0.6028366088867188, + 0.7554007172584534 + ], + "redundancy_ratio_by_k": [ + 0.4339584611645457, + 0.3929988907713278, + 0.448989671489342, + 0.46196973338285524, + 0.4484331605238465, + 0.4431815260551794, + 0.5156377221452507 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.2277929037809372, + 0.2773028612136841, + 0.2704378664493561, + 0.26649582386016846, + 0.2930046021938324, + 0.3529264032840729, + 0.35494714975357056 + ], + "mean_layers_recovering_by_k": [ + 0.0774032473564148, + 0.1136079877614975, + 0.227215975522995, + 0.3308364450931549, + 0.5243445634841919, + 0.8776529431343079, + 1.3058676719665527 + ], + "min_auc_by_band_width": { + "1": 0.03807681892067194, + "2": 0.0546174444258213, + "3": 0.057772739231586455, + "4": 0.06818075682967902, + "5": 0.10240421183407307, + "6": 0.10313692726194859, + "7": 0.1431004401296377, + "8": 0.14431340396404266, + "9": 0.15443276278674603, + "10": 0.15538734011352062, + "11": 0.1644597314298153, + "12": 0.16801901236176492, + "13": 0.17145237997174262 + } + }, + "shuf|Pile-CC|s0|d2": { + "label": "shuf|Pile-CC|s0|d2", + "min": 0.16748275458812714, + "persist": 0.025132926274091007, + "per_layer_auc": [ + 0.009809167124330997, + 0.005707151256501675, + 0.015337969176471233, + 0.015337971039116383, + 0.027108972892165184, + 0.02550383284687996, + 0.03406456485390663, + 0.04333868622779846, + 0.0971999317407608, + 0.06474050134420395, + 0.07651150226593018, + 0.03852327540516853, + 0.02068842574954033 + ], + "best_single_layer_auc": 0.0971999317407608, + "mean_single_layer_auc": 0.03645168989896774, + "union_observed_by_k": [ + 0.03495630621910095, + 0.05992509424686432, + 0.09862671792507172, + 0.14606741070747375, + 0.18976280093193054, + 0.2808988690376282, + 0.3745318353176117 + ], + "union_if_layers_independent_by_k": [ + 0.05966615676879883, + 0.10268044471740723, + 0.1929377317428589, + 0.30200332403182983, + 0.41209858655929565, + 0.5909911394119263, + 0.7311155200004578 + ], + "redundancy_ratio_by_k": [ + 0.5858648874361658, + 0.5836076617294328, + 0.5111841889823717, + 0.48366159934080355, + 0.4604791356269943, + 0.4753013206207125, + 0.512274497082728 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.09536772221326828, + 0.1537671834230423, + 0.21656356751918793, + 0.23421448469161987, + 0.28380483388900757, + 0.3129563331604004, + 0.3316711187362671 + ], + "mean_layers_recovering_by_k": [ + 0.06117353215813637, + 0.10736579447984695, + 0.21098627150058746, + 0.3508114814758301, + 0.5131086111068726, + 0.8489388227462769, + 1.2247190475463867 + ], + "min_auc_by_band_width": { + "1": 0.03426547311246395, + "2": 0.03821156956255436, + "3": 0.05615544328466058, + "4": 0.06481561828404665, + "5": 0.10058530326932669, + "6": 0.10528503321111202, + "7": 0.14050876311957836, + "8": 0.14138139747083187, + "9": 0.15469214580953122, + "10": 0.15572247765958308, + "11": 0.16432746611535548, + "12": 0.1652806892991066, + "13": 0.16748275458812714 + } + }, + "J|Github|s0": { + "label": "J|Github|s0", + "min": 0.10961288511753083, + "persist": 0.02704486474394798, + "per_layer_auc": [ + 0.011949350126087666, + 0.010344213806092739, + 0.012484394945204258, + 0.018369896337389946, + 0.018726591020822525, + 0.027644017711281776, + 0.02693062089383602, + 0.03531300276517868, + 0.04993757978081703, + 0.04851079359650612, + 0.053147852420806885, + 0.05065097287297249, + 0.054752983152866364 + ], + "best_single_layer_auc": 0.054752983152866364, + "mean_single_layer_auc": 0.032212480902671814, + "union_observed_by_k": [ + 0.026217227801680565, + 0.041198503226041794, + 0.07116104662418365, + 0.1061173528432846, + 0.12858925759792328, + 0.17103619873523712, + 0.22097378969192505 + ], + "union_if_layers_independent_by_k": [ + 0.06074631214141846, + 0.1068882942199707, + 0.19214248657226562, + 0.278173565864563, + 0.3888155221939087, + 0.5261503458023071, + 0.6597692966461182 + ], + "redundancy_ratio_by_k": [ + 0.43158550498747, + 0.3854351267058052, + 0.3703556037691834, + 0.3814789249060098, + 0.33072048377171964, + 0.32507096137023417, + 0.33492584576946327 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.17644602060317993, + 0.3105510175228119, + 0.29603996872901917, + 0.30662479996681213, + 0.3696111738681793, + 0.40837839245796204, + 0.4326549470424652 + ], + "mean_layers_recovering_by_k": [ + 0.062421973794698715, + 0.11235955357551575, + 0.21098627150058746, + 0.3208489418029785, + 0.4806492030620575, + 0.7203495502471924, + 1.023720383644104 + ], + "min_auc_by_band_width": { + "1": 0.023271939158439635, + "2": 0.0311403077095747, + "3": 0.038967814203351735, + "4": 0.04765287963673472, + "5": 0.06220552530139685, + "6": 0.06569254454225301, + "7": 0.076693313755095, + "8": 0.076693313755095, + "9": 0.09125153236091137, + "10": 0.092289425060153, + "11": 0.09766928777098656, + "12": 0.10093701779842376, + "13": 0.10961288511753083 + } + }, + "shuf|Github|s0|d0": { + "label": "shuf|Github|s0|d0", + "min": 0.11639587208628654, + "persist": 0.015263549471274018, + "per_layer_auc": [ + 0.011771000921726227, + 0.00945247057825327, + 0.0085607273504138, + 0.011592650786042213, + 0.014981272630393505, + 0.023185303434729576, + 0.023720350116491318, + 0.021045122295618057, + 0.06028179079294205, + 0.03566969931125641, + 0.019975030794739723, + 0.042981989681720734, + 0.02960585057735443 + ], + "best_single_layer_auc": 0.06028179079294205, + "mean_single_layer_auc": 0.02406332828104496, + "union_observed_by_k": [ + 0.016229713335633278, + 0.027465667575597763, + 0.06866417080163956, + 0.09862671792507172, + 0.1398252248764038, + 0.19225966930389404, + 0.2634207308292389 + ], + "union_if_layers_independent_by_k": [ + 0.044092774391174316, + 0.06314074993133545, + 0.12050366401672363, + 0.19038671255111694, + 0.2910010814666748, + 0.44608795642852783, + 0.5810457468032837 + ], + "redundancy_ratio_by_k": [ + 0.3680810191631272, + 0.4349911523931254, + 0.569809817501568, + 0.5180336201172202, + 0.48049726884749216, + 0.4309904953345179, + 0.45335626717601885 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.18539959192276, + 0.23198558390140533, + 0.2689589858055115, + 0.2807226777076721, + 0.3077431619167328, + 0.3051567077636719, + 0.3337453305721283 + ], + "mean_layers_recovering_by_k": [ + 0.04494382068514824, + 0.06491885334253311, + 0.12734082341194153, + 0.20848938822746277, + 0.33707866072654724, + 0.5730336904525757, + 0.8339575529098511 + ], + "min_auc_by_band_width": { + "1": 0.024540934851393102, + "2": 0.04289936041459441, + "3": 0.048549057357013224, + "4": 0.0501670790836215, + "5": 0.07548373360186815, + "6": 0.07733441162854433, + "7": 0.08416016213595867, + "8": 0.08459674008190632, + "9": 0.08906395584344864, + "10": 0.09069708585739136, + "11": 0.09774349592626094, + "12": 0.10079220496118069, + "13": 0.11639587208628654 + } + }, + "shuf|Github|s0|d1": { + "label": "shuf|Github|s0|d1", + "min": 0.11431133672595024, + "persist": 0.019197244732640683, + "per_layer_auc": [ + 0.010879258625209332, + 0.006420545745640993, + 0.010879257693886757, + 0.009630819782614708, + 0.018369896337389946, + 0.02907080389559269, + 0.01979668252170086, + 0.010879257693886757, + 0.03834492340683937, + 0.06260031461715698, + 0.039950065314769745, + 0.03477795794606209, + 0.036739788949489594 + ], + "best_single_layer_auc": 0.06260031461715698, + "mean_single_layer_auc": 0.025256890803575516, + "union_observed_by_k": [ + 0.02247191034257412, + 0.02996254712343216, + 0.06117353215813637, + 0.08614232391119003, + 0.12734082341194153, + 0.18726591765880585, + 0.2684144675731659 + ], + "union_if_layers_independent_by_k": [ + 0.053666651248931885, + 0.07014638185501099, + 0.12823021411895752, + 0.19150406122207642, + 0.2966098189353943, + 0.4576452374458313, + 0.6059248447418213 + ], + "redundancy_ratio_by_k": [ + 0.4187313689154654, + 0.4271431587927546, + 0.4770602044022672, + 0.4498198281617415, + 0.4293209977639955, + 0.4091945077456889, + 0.4429831024465331 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.1517566293478012, + 0.19225458800792694, + 0.22544017434120178, + 0.256803959608078, + 0.2849011719226837, + 0.3138657510280609, + 0.3436553478240967 + ], + "mean_layers_recovering_by_k": [ + 0.054931335151195526, + 0.072409488260746, + 0.13607990741729736, + 0.20973782241344452, + 0.34456929564476013, + 0.5917602777481079, + 0.8888888955116272 + ], + "min_auc_by_band_width": { + "1": 0.02119425896089524, + "2": 0.03668580120429397, + "3": 0.03995353747159243, + "4": 0.04229337070137262, + "5": 0.06108183395117521, + "6": 0.06590114813297987, + "7": 0.09751414246857167, + "8": 0.09787820614874362, + "9": 0.10247626602649688, + "10": 0.1058096319437027, + "11": 0.11000333651900292, + "12": 0.11059249937534332, + "13": 0.11431133672595024 + } + }, + "shuf|Github|s0|d2": { + "label": "shuf|Github|s0|d2", + "min": 0.13957404792308808, + "persist": 0.019098307471722364, + "per_layer_auc": [ + 0.005885499995201826, + 0.00606384826824069, + 0.010879257693886757, + 0.011949348263442516, + 0.018369896337389946, + 0.023720350116491318, + 0.027822362259030342, + 0.02907080575823784, + 0.051364362239837646, + 0.037274833768606186, + 0.06670233607292175, + 0.03656144067645073, + 0.02122347056865692 + ], + "best_single_layer_auc": 0.06670233607292175, + "mean_single_layer_auc": 0.026683678850531578, + "union_observed_by_k": [ + 0.031210986897349358, + 0.04993757978081703, + 0.07990012317895889, + 0.10237203538417816, + 0.1473158597946167, + 0.2172284573316574, + 0.2796504497528076 + ], + "union_if_layers_independent_by_k": [ + 0.05603557825088501, + 0.0818629264831543, + 0.14796596765518188, + 0.22080999612808228, + 0.32536154985427856, + 0.4724557399749756, + 0.6021925210952759 + ], + "redundancy_ratio_by_k": [ + 0.5569851846198521, + 0.6100145930049686, + 0.5399898668939683, + 0.4636204754280988, + 0.45277587305751354, + 0.4597858359031965, + 0.4643871186646018 + ], + "mean_pairwise_layer_corr_by_k": [ + 0.1593371033668518, + 0.1558503806591034, + 0.19714854657649994, + 0.2711406946182251, + 0.2995009422302246, + 0.3090117871761322, + 0.3324390947818756 + ], + "mean_layers_recovering_by_k": [ + 0.05742821469902992, + 0.08489388227462769, + 0.15855181217193604, + 0.24594257771968842, + 0.3845193386077881, + 0.6167290806770325, + 0.8801498413085938 + ], + "min_auc_by_band_width": { + "1": 0.025532490853220225, + "2": 0.02939487947151065, + "3": 0.040183744952082635, + "4": 0.047866654396057126, + "5": 0.06557567901909352, + "6": 0.06787763126194477, + "7": 0.09151607640087604, + "8": 0.09419464953243732, + "9": 0.12192872278392315, + "10": 0.12303912453353405, + "11": 0.13793258666992186, + "12": 0.13892652094364166, + "13": 0.13957404792308808 + } + } + }, + "adjudication": { + "USPTO_Backgrounds": { + "H1_shuf_worse_at_every_layer": { + "n_layers_where_shuf_lower": 12, + "n_layers": 13, + "jp_mean_single_layer_auc": 0.047125156968832016, + "shuf_mean_single_layer_auc": 0.03500203415751457, + "jp_best_single_layer_auc": 0.08114856481552124, + "shuf_best_single_layer_auc": 0.0672968327999115, + "fires": true + }, + "H2_shuf_layers_less_correlated": { + "jp_mean_pairwise_corr": 0.35804216137954165, + "shuf_mean_pairwise_corr": 0.24646801430554616, + "fires": true + }, + "H3_shuf_union_closer_to_independence": { + "jp_redundancy_ratio": 0.37285473258950913, + "shuf_redundancy_ratio": 0.46923501364369197, + "fires": true + }, + "min_gap_jp_minus_shuf": 9.264610707759857e-05, + "persist_gap_jp_minus_shuf": 0.021623983164317906, + "H4_artifact_scales_with_band_width": { + "min_gap_jp_minus_shuf_by_band_width": { + "1": 0.014999811879048749, + "2": 0.021703154469529792, + "3": 0.020836105135579902, + "4": 0.01200310190518697, + "5": 0.009003739369412261, + "6": 0.014086514587203669, + "7": 0.0005327625821034132, + "8": -0.0003358248621225246, + "9": -0.002526853730281192, + "10": -0.004171401510635997, + "11": -0.0027907519290844562, + "12": -0.001874782765905053, + "13": 9.264610707759857e-05 + }, + "narrowest_width_where_jp_still_wins": "13" + } + }, + "Pile-CC": { + "H1_shuf_worse_at_every_layer": { + "n_layers_where_shuf_lower": 11, + "n_layers": 13, + "jp_mean_single_layer_auc": 0.04720747470855713, + "shuf_mean_single_layer_auc": 0.036648329347372055, + "jp_best_single_layer_auc": 0.10094524174928665, + "shuf_best_single_layer_auc": 0.07853279262781143, + "fires": false + }, + "H2_shuf_layers_less_correlated": { + "jp_mean_pairwise_corr": 0.41035106778144836, + "shuf_mean_pairwise_corr": 0.268771730363369, + "fires": true + }, + "H3_shuf_union_closer_to_independence": { + "jp_redundancy_ratio": 0.37783230651524563, + "shuf_redundancy_ratio": 0.48436288498343655, + "fires": true + }, + "min_gap_jp_minus_shuf": -0.019820987060666095, + "persist_gap_jp_minus_shuf": 0.018049868669671315, + "H4_artifact_scales_with_band_width": { + "min_gap_jp_minus_shuf_by_band_width": { + "1": 0.01365852714516222, + "2": 0.012173634891708696, + "3": 0.013157219346612695, + "4": 0.00883049474408229, + "5": 0.006072039343416685, + "6": 0.007677602892120675, + "7": -0.015596643959482487, + "8": -0.01543888958791892, + "9": -0.01860053278505802, + "10": -0.019118670622507716, + "11": -0.019855003928144765, + "12": -0.02184912885228793, + "13": -0.019820987060666095 + }, + "narrowest_width_where_jp_still_wins": "6" + } + }, + "Github": { + "H1_shuf_worse_at_every_layer": { + "n_layers_where_shuf_lower": 12, + "n_layers": 13, + "jp_mean_single_layer_auc": 0.032212480902671814, + "shuf_mean_single_layer_auc": 0.025334633886814117, + "jp_best_single_layer_auc": 0.054752983152866364, + "shuf_best_single_layer_auc": 0.04999702796339989, + "fires": true + }, + "H2_shuf_layers_less_correlated": { + "jp_mean_pairwise_corr": 0.3286151885986328, + "shuf_mean_pairwise_corr": 0.2574675338608878, + "fires": true + }, + "H3_shuf_union_closer_to_independence": { + "jp_redundancy_ratio": 0.36565320732569795, + "shuf_redundancy_ratio": 0.4694081788730341, + "fires": true + }, + "min_gap_jp_minus_shuf": -0.013814200460910794, + "persist_gap_jp_minus_shuf": 0.009191830852068959, + "H4_artifact_scales_with_band_width": { + "min_gap_jp_minus_shuf_by_band_width": { + "1": -0.0004839557300632194, + "2": -0.00518637265389164, + "3": -0.0039276323902110305, + "4": 0.0008771782430509745, + "5": -0.005174890222648776, + "6": -0.004678519132236644, + "7": -0.01437014658004046, + "8": -0.01552988483260076, + "9": -0.01323811585704486, + "10": -0.014225855718056374, + "11": -0.01755718526740868, + "12": -0.015833390628298125, + "13": -0.013814200460910794 + }, + "narrowest_width_where_jp_still_wins": "4" + } + } + }, + "hypotheses_firing_out_of_n_corpora": { + "H1_shuf_worse_at_every_layer": 2, + "H2_shuf_layers_less_correlated": 3, + "H3_shuf_union_closer_to_independence": 3 + }, + "VERDICT": "MIXED \u2014 hypotheses firing: {'H1_shuf_worse_at_every_layer': 2, 'H2_shuf_layers_less_correlated': 3, 'H3_shuf_union_closer_to_independence': 3}. Do not restate the mechanism as established.", + "provenance": { + "schema_version": 1, + "experiment": "D1 \u2014 diagnostic: is `min`'s preference for J^shuf a union-over-layers artifact?", + "utc": "2026-08-30T02:58:45+00:00", + "script": { + "path": "experiments/d1_min_union_diagnostic.py", + "sha256": "22894e1e54db6a3adb12f497f61336054826f61795dd54dda490d17a8ff8281d", + "n_lines": 303 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/t1_module_audit.json" + ], + "dirty_diff_sha256": "6de268d981510d7cce49efe3da1c011b7dc2c81b40eae3e61c2f0fa57ef1b5ff", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/d1_min_union_diagnostic.py", + "--corpora", + "USPTO_Backgrounds,Pile-CC,Github", + "--n-derangements", + "3" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "cc21cc60dc9df63dd2e49d5296467b2ec0a1976d3adf9dfcddff67b5a276b7be", + "bytes": 115713, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "6f8f06ad3ce7ef6c3a0dc44afa63838df505ac8cd1617b25fda0aa9747a31524", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s0.pt", + "sha256": "666e119f8cdcc6b218c020862e3de834c6920dcbed894121a108defa90877325", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s0.pt", + "sha256": "3895668dce09dae91c6300f6adbba2de8359c63bf45fd5676d554f6ed90e9dc8", + "bytes": 46144865, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "c0cdf93b343e66caa7b679cb898e7ea76bfbb295c8e353254c20613b3cf91143" + } +} \ No newline at end of file diff --git a/results/d2_device_crossvalidation.json b/results/d2_device_crossvalidation.json new file mode 100644 index 0000000000000000000000000000000000000000..9ca8247f717e1b1866f4bdaafa516d55f89164a3 --- /dev/null +++ b/results/d2_device_crossvalidation.json @@ -0,0 +1,392 @@ +{ + "experiment": "D2 \u2014 cross-validating CUDA-scored reads against CPU-scored reads", + "status": "DIAGNOSTIC. Joins two stored results files; measures nothing new.", + "why": "CONTEXT.md \u00a77: E65's CUDA scoring branch is wrong by 37% on an arm with no operator, and ladder410/ladder1b are CUDA-scored and feed the asymptotes and the interaction. No CUDA-vs-CPU cross-check had ever been run.", + "inputs": [ + "results/ladder410/*.json (device=cuda)", + "results/e48_crossover_410m.json (device=cpu)" + ], + "shared_conditions": { + "lenses": "results/e28__410m_n200_s.pt", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "rung": 200 + }, + "ladder_devices_seen": [ + "cuda" + ], + "tolerance": 1e-06, + "tolerance_derivation": "one (item,intermediate) pair flipping at one k moves a set mean by >= 1/(893*7) = 1.6e-4, so 1e-6 separates fp accumulation from a single changed decision", + "n_cells_compared": 30, + "max_abs_diff": 0.0002773862864289822, + "worst_cell": { + "corpus": "USPTO_Backgrounds", + "seed": 2, + "aggregation": "min", + "cuda_ladder": 0.1574542057301317, + "cpu_e48": 0.1571768194437027, + "abs_diff": 0.0002773862864289822, + "rel_diff": 0.0017648040430563356 + }, + "per_cell": [ + { + "corpus": "Github", + "seed": 0, + "aggregation": "persist", + "cuda_ladder": 0.02704486567527056, + "cpu_e48": 0.02704486474394798, + "abs_diff": 9.313225780849255e-10, + "rel_diff": 3.443620764616077e-08 + }, + { + "corpus": "Github", + "seed": 0, + "aggregation": "min", + "cuda_ladder": 0.10961288312183959, + "cpu_e48": 0.10961288511753083, + "abs_diff": 1.995691237266506e-09, + "rel_diff": 1.8206721181790396e-08 + }, + { + "corpus": "Github", + "seed": 1, + "aggregation": "persist", + "cuda_ladder": 0.029035190998443535, + "cpu_e48": 0.02903519310057163, + "abs_diff": 2.1021280965927147e-09, + "rel_diff": 7.23993151797337e-08 + }, + { + "corpus": "Github", + "seed": 1, + "aggregation": "min", + "cuda_ladder": 0.11494814133537667, + "cpu_e48": 0.1149481400847435, + "abs_diff": 1.2506331803496806e-09, + "rel_diff": 1.0879977522278075e-08 + }, + { + "corpus": "Github", + "seed": 2, + "aggregation": "persist", + "cuda_ladder": 0.027610584854015283, + "cpu_e48": 0.027610587328672408, + "abs_diff": 2.4746571250511273e-09, + "rel_diff": 8.962710918073453e-08 + }, + { + "corpus": "Github", + "seed": 2, + "aggregation": "min", + "cuda_ladder": 0.11328674435083355, + "cpu_e48": 0.11328674778342247, + "abs_diff": 3.432588921437052e-09, + "rel_diff": 3.0300004092264633e-08 + }, + { + "corpus": "Pile-CC", + "seed": 0, + "aggregation": "persist", + "cuda_ladder": 0.04509159454277583, + "cpu_e48": 0.04509159799199551, + "abs_diff": 3.449219680706772e-09, + "rel_diff": 7.649362263273668e-08 + }, + { + "corpus": "Pile-CC", + "seed": 0, + "aggregation": "min", + "cuda_ladder": 0.14523330333509615, + "cpu_e48": 0.1452333066612482, + "abs_diff": 3.3261520482330553e-09, + "rel_diff": 2.2902129853664994e-08 + }, + { + "corpus": "Pile-CC", + "seed": 1, + "aggregation": "persist", + "cuda_ladder": 0.04611681202160461, + "cpu_e48": 0.046116814273409544, + "abs_diff": 2.2518049369590898e-09, + "rel_diff": 4.882828470346999e-08 + }, + { + "corpus": "Pile-CC", + "seed": 1, + "aggregation": "min", + "cuda_ladder": 0.1453225617696132, + "cpu_e48": 0.1453225638717413, + "abs_diff": 2.1021281104705025e-09, + "rel_diff": 1.4465256147873893e-08 + }, + { + "corpus": "Pile-CC", + "seed": 2, + "aggregation": "persist", + "cuda_ladder": 0.045189316916678636, + "cpu_e48": 0.04518931868951768, + "abs_diff": 1.7728390422355744e-09, + "rel_diff": 3.923137355568962e-08 + }, + { + "corpus": "Pile-CC", + "seed": 2, + "aggregation": "min", + "cuda_ladder": 0.14300646082098994, + "cpu_e48": 0.14300646334886552, + "abs_diff": 2.5278755755309135e-09, + "rel_diff": 1.767665262348415e-08 + }, + { + "corpus": "StackExchange", + "seed": 0, + "aggregation": "persist", + "cuda_ladder": 0.04637858061386006, + "cpu_e48": 0.04637858076021075, + "abs_diff": 1.4635068851243105e-10, + "rel_diff": 3.1555663436339706e-09 + }, + { + "corpus": "StackExchange", + "seed": 0, + "aggregation": "min", + "cuda_ladder": 0.14699955502791065, + "cpu_e48": 0.1469995591789484, + "abs_diff": 4.151037763522325e-09, + "rel_diff": 2.8238436813739703e-08 + }, + { + "corpus": "StackExchange", + "seed": 1, + "aggregation": "persist", + "cuda_ladder": 0.04555068582828556, + "cpu_e48": 0.04555068640038371, + "abs_diff": 5.72098153572842e-10, + "rel_diff": 1.2559594569973873e-08 + }, + { + "corpus": "StackExchange", + "seed": 1, + "aggregation": "min", + "cuda_ladder": 0.14416987733649356, + "cpu_e48": 0.14416988678276538, + "abs_diff": 9.446271820312546e-09, + "rel_diff": 6.552180924263437e-08 + }, + { + "corpus": "StackExchange", + "seed": 2, + "aggregation": "persist", + "cuda_ladder": 0.045260619930922986, + "cpu_e48": 0.04526062114164233, + "abs_diff": 1.2107193442245645e-09, + "rel_diff": 2.6749949816986367e-08 + }, + { + "corpus": "StackExchange", + "seed": 2, + "aggregation": "min", + "cuda_ladder": 0.14551294004278523, + "cpu_e48": 0.14551294930279254, + "abs_diff": 9.26000731649168e-09, + "rel_diff": 6.36369983624129e-08 + }, + { + "corpus": "USPTO_Backgrounds", + "seed": 0, + "aggregation": "persist", + "cuda_ladder": 0.04964359144547156, + "cpu_e48": 0.04964358932338655, + "abs_diff": 2.1220850146552728e-09, + "rel_diff": 4.274640580139482e-08 + }, + { + "corpus": "USPTO_Backgrounds", + "seed": 0, + "aggregation": "min", + "cuda_ladder": 0.15465880749481065, + "cpu_e48": 0.15465880744159222, + "abs_diff": 5.3218429663104416e-11, + "rel_diff": 3.4410215973767067e-10 + }, + { + "corpus": "USPTO_Backgrounds", + "seed": 1, + "aggregation": "persist", + "cuda_ladder": 0.04933665746024677, + "cpu_e48": 0.04933666000142693, + "abs_diff": 2.541180162130008e-09, + "rel_diff": 5.150693545238999e-08 + }, + { + "corpus": "USPTO_Backgrounds", + "seed": 1, + "aggregation": "min", + "cuda_ladder": 0.15324432841901267, + "cpu_e48": 0.153244324401021, + "abs_diff": 4.017991661608988e-09, + "rel_diff": 2.6219513690402082e-08 + }, + { + "corpus": "USPTO_Backgrounds", + "seed": 2, + "aggregation": "persist", + "cuda_ladder": 0.04986586142331362, + "cpu_e48": 0.04986586356535554, + "abs_diff": 2.142041918840043e-09, + "rel_diff": 4.295607787946208e-08 + }, + { + "corpus": "USPTO_Backgrounds", + "seed": 2, + "aggregation": "min", + "cuda_ladder": 0.1574542057301317, + "cpu_e48": 0.1571768194437027, + "abs_diff": 0.0002773862864289822, + "rel_diff": 0.0017648040430563356 + }, + { + "corpus": "Wikipedia_en", + "seed": 0, + "aggregation": "persist", + "cuda_ladder": 0.03809930235147477, + "cpu_e48": 0.038099304959177974, + "abs_diff": 2.6077032061477823e-09, + "rel_diff": 6.844490231362073e-08 + }, + { + "corpus": "Wikipedia_en", + "seed": 0, + "aggregation": "min", + "cuda_ladder": 0.12926935620073762, + "cpu_e48": 0.12926936261355876, + "abs_diff": 6.412821135226565e-09, + "rel_diff": 4.960820573082906e-08 + }, + { + "corpus": "Wikipedia_en", + "seed": 1, + "aggregation": "persist", + "cuda_ladder": 0.03906317305352006, + "cpu_e48": 0.039063173532485965, + "abs_diff": 4.789659016624093e-10, + "rel_diff": 1.2261315667660452e-08 + }, + { + "corpus": "Wikipedia_en", + "seed": 1, + "aggregation": "min", + "cuda_ladder": 0.13214441066873925, + "cpu_e48": 0.1321444157510996, + "abs_diff": 5.0823603381378035e-09, + "rel_diff": 3.846065162307483e-08 + }, + { + "corpus": "Wikipedia_en", + "seed": 2, + "aggregation": "persist", + "cuda_ladder": 0.037466185060994965, + "cpu_e48": 0.03746618703007698, + "abs_diff": 1.9690820154960598e-09, + "rel_diff": 5.2556242617251837e-08 + }, + { + "corpus": "Wikipedia_en", + "seed": 2, + "aggregation": "min", + "cuda_ladder": 0.1300129838021738, + "cpu_e48": 0.13001298978924752, + "abs_diff": 5.9870737256773054e-09, + "rel_diff": 4.6049811910197724e-08 + } + ], + "VERDICT": "DIVERGENT \u2014 max |CUDA - CPU| = 2.774e-04 over 30 cells (worst: USPTO_Backgrounds s2, min, 0.18% relative). STOP. Every ladder-derived number in the paper, meaning the per-corpus asymptotes, the flatness in N and the 1B replication, must be re-sourced from a CPU scoring pass before it is cited.", + "provenance": { + "schema_version": 1, + "experiment": "D2 \u2014 cross-validating CUDA-scored reads against CPU-scored reads", + "utc": "2026-08-20T00:11:35+00:00", + "script": { + "path": "experiments/d2_device_crossvalidation.py", + "sha256": "e1033efb2cbd9ea71d60d628b4f33ee8fcf866946445cec4a1d054a28046c916", + "n_lines": 125 + }, + "git": { + "commit": "2dfeda4b4d89650f970dc362f8183e2f6077c9fe", + "commit_short": "2dfeda4b4d89", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/d2_device_crossvalidation.py", + "figures/fB_band_width.png", + "figures/make_figures.py", + "igures/f0_grid_single.png", + "paper/paper.tex", + "paper/paper_5pp.bbl", + "paper/paper_5pp.blg", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/refs.bib", + "paper/tables_auto.tex", + "tools/make_paper_tables.py" + ], + "dirty_diff_sha256": "0b3cf18514db7d8a624429659f62d811e911c001c158012b785b6f8927d12df6", + "describe": "2dfeda4-dirty" + }, + "argv": [ + "experiments/d2_device_crossvalidation.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", + "bytes": 116036, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "e3c1b0af60b548841f62ecd52aba7dcc0e8eb5db7f7528a2aee67a2044f53858" + } +} \ No newline at end of file diff --git a/results/d3_corpus_by_family_410m.json b/results/d3_corpus_by_family_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..6effe4062cb578afca3bbbd3811a1df4c0df20c6 --- /dev/null +++ b/results/d3_corpus_by_family_410m.json @@ -0,0 +1,887 @@ +{ + "experiment": "D3 \u2014 EXPLORATORY: CV3's corpus effect decomposed by eval family", + "status": "EXPLORATORY \u2014 no pre-registration, no decision rule, none may be invented", + "ordering_cost_recorded": "computed BEFORE any restricted-contrast pre-registration; any later restriction to a family subset must be disclosed as informed by this file, not as chosen on capability grounds alone", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "n_pairs_per_family": { + "multihop": 103, + "multilingual": 394, + "order-ops": 110, + "poetry": 98, + "typo": 96 + }, + "readout_convention": "STRIPPED (corrected), no prefix, flat-mean-7", + "per_arm": { + "logit_I": { + "multihop": { + "rank": 0.1844660194174757, + "z": 3.2861543355636225 + }, + "multilingual": { + "rank": 0.10877447425670776, + "z": 2.620148496503757 + }, + "order-ops": { + "rank": 0.5012987012987012, + "z": 5.086123768849807 + }, + "poetry": { + "rank": 0.020408163265306124, + "z": 2.5611357868325952 + }, + "typo": { + "rank": 0.1755952380952381, + "z": 3.4371277851363025 + } + }, + "Wikipedia_en|s0": { + "multihop": { + "rank": 0.2482662968099861, + "z": 3.521077022969144 + }, + "multilingual": { + "rank": 0.13850616388687456, + "z": 2.6941119621928573 + }, + "order-ops": { + "rank": 0.4805194805194805, + "z": 5.0819312778386205 + }, + "poetry": { + "rank": 0.02915451895043732, + "z": 2.259214861052377 + }, + "typo": { + "rank": 0.3095238095238095, + "z": 3.5175044226149716 + } + }, + "Wikipedia_en|s1": { + "multihop": { + "rank": 0.24410540915395282, + "z": 3.5277483602171964 + }, + "multilingual": { + "rank": 0.14720812182741114, + "z": 2.7284246911932972 + }, + "order-ops": { + "rank": 0.48441558441558447, + "z": 5.096012249859896 + }, + "poetry": { + "rank": 0.03206997084548105, + "z": 2.26507106423378 + }, + "typo": { + "rank": 0.31101190476190477, + "z": 3.5273107339938483 + } + }, + "Wikipedia_en|s2": { + "multihop": { + "rank": 0.24271844660194172, + "z": 3.514451161171626 + }, + "multilingual": { + "rank": 0.14176939811457576, + "z": 2.707142815636802 + }, + "order-ops": { + "rank": 0.4818181818181818, + "z": 5.091976642608643 + }, + "poetry": { + "rank": 0.02915451895043732, + "z": 2.262889931396562 + }, + "typo": { + "rank": 0.3095238095238095, + "z": 3.513884318371614 + } + }, + "USPTO_Backgrounds|s0": { + "multihop": { + "rank": 0.2524271844660194, + "z": 3.547351451174727 + }, + "multilingual": { + "rank": 0.1508339376359681, + "z": 2.74641798126516 + }, + "order-ops": { + "rank": 0.5103896103896104, + "z": 5.326678815754977 + }, + "poetry": { + "rank": 0.02478134110787172, + "z": 2.2967441440845024 + }, + "typo": { + "rank": 0.3556547619047619, + "z": 3.6381055613358817 + } + }, + "USPTO_Backgrounds|s1": { + "multihop": { + "rank": 0.2552011095700416, + "z": 3.556549455355672 + }, + "multilingual": { + "rank": 0.15300942712110224, + "z": 2.7714402158248244 + }, + "order-ops": { + "rank": 0.5116883116883116, + "z": 5.3522507797588 + }, + "poetry": { + "rank": 0.02478134110787172, + "z": 2.3082475084431318 + }, + "typo": { + "rank": 0.3601190476190476, + "z": 3.6416732432941594 + } + }, + "USPTO_Backgrounds|s2": { + "multihop": { + "rank": 0.2579750346740638, + "z": 3.5740954065785826 + }, + "multilingual": { + "rank": 0.1508339376359681, + "z": 2.768562420219334 + }, + "order-ops": { + "rank": 0.5168831168831168, + "z": 5.34596418467435 + }, + "poetry": { + "rank": 0.026239067055393583, + "z": 2.309358976933421 + }, + "typo": { + "rank": 0.36607142857142855, + "z": 3.6538835192720094 + } + }, + "Pile-CC|s0": { + "multihop": { + "rank": 0.2635228848821082, + "z": 3.665571606853633 + }, + "multilingual": { + "rank": 0.18527918781725888, + "z": 2.858225473141307 + }, + "order-ops": { + "rank": 0.4922077922077922, + "z": 5.25143859278072 + }, + "poetry": { + "rank": 0.02478134110787172, + "z": 2.325896689478232 + }, + "typo": { + "rank": 0.34375, + "z": 3.719450268894434 + } + }, + "Pile-CC|s1": { + "multihop": { + "rank": 0.27184466019417475, + "z": 3.681773201354499 + }, + "multilingual": { + "rank": 0.18745467730239304, + "z": 2.8711710954959586 + }, + "order-ops": { + "rank": 0.5, + "z": 5.264400079033591 + }, + "poetry": { + "rank": 0.018950437317784258, + "z": 2.3257758690386403 + }, + "typo": { + "rank": 0.3482142857142857, + "z": 3.733075254907211 + } + }, + "Pile-CC|s2": { + "multihop": { + "rank": 0.2635228848821082, + "z": 3.656815766709522 + }, + "multilingual": { + "rank": 0.18201595358955763, + "z": 2.841417075763575 + }, + "order-ops": { + "rank": 0.49480519480519475, + "z": 5.236249876022339 + }, + "poetry": { + "rank": 0.018950437317784258, + "z": 2.3210366465607466 + }, + "typo": { + "rank": 0.3407738095238095, + "z": 3.7059822231531143 + } + }, + "StackExchange|s0": { + "multihop": { + "rank": 0.27184466019417475, + "z": 3.6677653523324762 + }, + "multilingual": { + "rank": 0.18201595358955763, + "z": 2.905021356825296 + }, + "order-ops": { + "rank": 0.4935064935064935, + "z": 5.3095506169579245 + }, + "poetry": { + "rank": 0.01603498542274052, + "z": 2.3015884240062867 + }, + "typo": { + "rank": 0.3705357142857143, + "z": 3.8123155757784843 + } + }, + "StackExchange|s1": { + "multihop": { + "rank": 0.26907073509015256, + "z": 3.6657675024375176 + }, + "multilingual": { + "rank": 0.1794778825235678, + "z": 2.9023827644925433 + }, + "order-ops": { + "rank": 0.49480519480519486, + "z": 5.298259841312062 + }, + "poetry": { + "rank": 0.01749271137026239, + "z": 2.2904911886672585 + }, + "typo": { + "rank": 0.3586309523809524, + "z": 3.766133587807417 + } + }, + "StackExchange|s2": { + "multihop": { + "rank": 0.27045769764216365, + "z": 3.6679530126377218 + }, + "multilingual": { + "rank": 0.1787527193618564, + "z": 2.9015078599077797 + }, + "order-ops": { + "rank": 0.4857142857142857, + "z": 5.308969775113193 + }, + "poetry": { + "rank": 0.01749271137026239, + "z": 2.296619354462137 + }, + "typo": { + "rank": 0.3630952380952381, + "z": 3.7829195770124593 + } + }, + "Github|s0": { + "multihop": { + "rank": 0.17891816920943132, + "z": 3.1870120798499837 + }, + "multilingual": { + "rank": 0.11348803480783176, + "z": 2.4887163202623426 + }, + "order-ops": { + "rank": 0.6090909090909091, + "z": 5.0270470554178415 + }, + "poetry": { + "rank": 0.01749271137026239, + "z": 2.0776315218940073 + }, + "typo": { + "rank": 0.2619047619047619, + "z": 3.230017771323522 + } + }, + "Github|s1": { + "multihop": { + "rank": 0.1927877947295423, + "z": 3.2497521800902285 + }, + "multilingual": { + "rank": 0.12545322697606962, + "z": 2.557975201096934 + }, + "order-ops": { + "rank": 0.5987012987012987, + "z": 5.047873000665144 + }, + "poetry": { + "rank": 0.01749271137026239, + "z": 2.092680626073662 + }, + "typo": { + "rank": 0.28273809523809523, + "z": 3.29463261179626 + } + }, + "Github|s2": { + "multihop": { + "rank": 0.1941747572815534, + "z": 3.2394828356585457 + }, + "multilingual": { + "rank": 0.11747643219724438, + "z": 2.514337351479506 + }, + "order-ops": { + "rank": 0.6051948051948052, + "z": 5.058664525638927 + }, + "poetry": { + "rank": 0.016034985422740525, + "z": 2.088071653733448 + }, + "typo": { + "rank": 0.275297619047619, + "z": 3.2603357657790184 + } + } + }, + "by_family": { + "multihop": { + "n_pairs": 103, + "logit_rank": 0.1844660194174757, + "logit_z": 3.2861543355636225, + "per_corpus": { + "Wikipedia_en": { + "rank_mean": 0.24503005085529356, + "rank_sd": 0.0028871927870542744, + "z_mean": 3.5210921814526555, + "z_sd": 0.00664861248299945 + }, + "USPTO_Backgrounds": { + "rank_mean": 0.2552011095700416, + "rank_sd": 0.00277392510402219, + "z_mean": 3.5593321043696604, + "z_sd": 0.013587388980674152 + }, + "Pile-CC": { + "rank_mean": 0.26629680998613037, + "rank_sd": 0.004804579216557216, + "z_mean": 3.668053524972551, + "z_sd": 0.012662477015826568 + }, + "StackExchange": { + "rank_mean": 0.27045769764216365, + "rank_sd": 0.001386962552011095, + "z_mean": 3.667161955802572, + "z_sd": 0.0012112717440874363 + }, + "Github": { + "rank_mean": 0.18862690707350901, + "rank_sd": 0.008436563842299901, + "z_mean": 3.2254156985329194, + "z_sd": 0.033652537864511396 + } + }, + "rank_spread": 0.08183079056865464, + "rank_pooled_seed_sd": 0.0047373789948957955, + "rank_spread_over_sd": 17.273431291189024, + "z_spread": 0.4426378264396318, + "z_pooled_seed_sd": 0.017453481001782855, + "z_spread_over_sd": 25.36100542891225, + "order_by_z": [ + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ] + }, + "multilingual": { + "n_pairs": 394, + "logit_rank": 0.10877447425670776, + "logit_z": 2.620148496503757, + "per_corpus": { + "Wikipedia_en": { + "rank_mean": 0.14249456127628715, + "rank_sd": 0.004396068039514896, + "z_mean": 2.7098931563409856, + "z_sd": 0.01732091577706631 + }, + "USPTO_Backgrounds": { + "rank_mean": 0.15155910079767948, + "rank_sd": 0.0012560194398613903, + "z_mean": 2.762140205769773, + "z_sd": 0.013691664778865744 + }, + "Pile-CC": { + "rank_mean": 0.18491660623640319, + "rank_sd": 0.0027374309047392235, + "z_mean": 2.856937881466947, + "z_sd": 0.014918741294046299 + }, + "StackExchange": { + "rank_mean": 0.18008218515832727, + "rank_sd": 0.001713493700599198, + "z_mean": 2.9029706604085397, + "z_sd": 0.0018290383684830523 + }, + "Github": { + "rank_mean": 0.11880589799371526, + "rank_sd": 0.006092377646182529, + "z_mean": 2.5203429576129275, + "z_sd": 0.03501783291963367 + } + }, + "rank_spread": 0.06611070824268793, + "rank_pooled_seed_sd": 0.003699990622833338, + "rank_spread_over_sd": 17.867804268126065, + "z_spread": 0.3826277027956122, + "z_pooled_seed_sd": 0.019695881058294912, + "z_spread_over_sd": 19.426787847831193, + "order_by_z": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ] + }, + "order-ops": { + "n_pairs": 110, + "logit_rank": 0.5012987012987012, + "logit_z": 5.086123768849807, + "per_corpus": { + "Wikipedia_en": { + "rank_mean": 0.48225108225108226, + "rank_sd": 0.0019837990021454163, + "z_mean": 5.0899733901023865, + "z_sd": 0.00725108327899038 + }, + "USPTO_Backgrounds": { + "rank_mean": 0.512987012987013, + "rank_sd": 0.0034360406637202277, + "z_mean": 5.341631260062709, + "z_sd": 0.01332523967967692 + }, + "Pile-CC": { + "rank_mean": 0.49567099567099565, + "rank_sd": 0.003967598004290778, + "z_mean": 5.2506961826122165, + "z_sd": 0.014089778636919856 + }, + "StackExchange": { + "rank_mean": 0.49134199134199136, + "rank_sd": 0.00491680376259766, + "z_mean": 5.3055934111277265, + "z_sd": 0.006357694471509954 + }, + "Github": { + "rank_mean": 0.6043290043290044, + "rank_sd": 0.005248638810814777, + "z_mean": 5.044528193907304, + "z_sd": 0.016071929099544396 + } + }, + "rank_spread": 0.12207792207792212, + "rank_pooled_seed_sd": 0.004079383199788662, + "rank_spread_over_sd": 29.9255833784496, + "z_spread": 0.2971030661554046, + "z_pooled_seed_sd": 0.012061420042855183, + "z_spread_over_sd": 24.632511354365725, + "order_by_z": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "Github", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en" + ] + }, + "poetry": { + "n_pairs": 98, + "logit_rank": 0.020408163265306124, + "logit_z": 2.5611357868325952, + "per_corpus": { + "Wikipedia_en": { + "rank_mean": 0.03012633624878523, + "rank_sd": 0.001683236936412902, + "z_mean": 2.262391952227573, + "z_sd": 0.002959690248155813 + }, + "USPTO_Backgrounds": { + "rank_mean": 0.025267249757045675, + "rank_sd": 0.0008416184682064499, + "z_mean": 2.304783543153685, + "z_sd": 0.006984468025297743 + }, + "Pile-CC": { + "rank_mean": 0.02089407191448008, + "rank_sd": 0.0033664738728258056, + "z_mean": 2.3242364016925396, + "z_sd": 0.002771727633125726 + }, + "StackExchange": { + "rank_mean": 0.017006802721088433, + "rank_sd": 0.0008416184682064539, + "z_mean": 2.2962329890452273, + "z_sd": 0.005558697394082701 + }, + "Github": { + "rank_mean": 0.017006802721088437, + "rank_sd": 0.0008416184682064519, + "z_mean": 2.0861279339003724, + "z_sd": 0.007710539491153821 + } + }, + "rank_spread": 0.013119533527696795, + "rank_pooled_seed_sd": 0.0018050705171046362, + "rank_spread_over_sd": 7.268155677785237, + "z_spread": 0.2381084677921672, + "z_pooled_seed_sd": 0.005578110253433368, + "z_spread_over_sd": 42.686224720210525, + "order_by_z": [ + "Pile-CC", + "USPTO_Backgrounds", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "StackExchange" + ] + }, + "typo": { + "n_pairs": 96, + "logit_rank": 0.1755952380952381, + "logit_z": 3.4371277851363025, + "per_corpus": { + "Wikipedia_en": { + "rank_mean": 0.3100198412698412, + "rank_sd": 0.000859152186294115, + "z_mean": 3.519566491660145, + "z_sd": 0.006946672273509102 + }, + "USPTO_Backgrounds": { + "rank_mean": 0.36061507936507936, + "rank_sd": 0.005226018726613443, + "z_mean": 3.64455410796735, + "z_sd": 0.008274087569483566 + }, + "Pile-CC": { + "rank_mean": 0.34424603174603174, + "rank_sd": 0.0037449575571779636, + "z_mean": 3.719502582318253, + "z_sd": 0.013546591635088659 + }, + "StackExchange": { + "rank_mean": 0.36408730158730157, + "rank_sd": 0.00601406530405861, + "z_mean": 3.787122913532787, + "z_sd": 0.02337616375001026 + }, + "Github": { + "rank_mean": 0.27331349206349204, + "rank_sd": 0.010557438816365445, + "z_mean": 3.2616620496329336, + "z_sd": 0.03232783126000267 + } + }, + "rank_spread": 0.09077380952380953, + "rank_pooled_seed_sd": 0.006159587927093404, + "rank_spread_over_sd": 14.736993870082479, + "z_spread": 0.5254608638998532, + "z_pooled_seed_sd": 0.019451275697770446, + "z_spread_over_sd": 27.014210896207846, + "order_by_z": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "Github" + ], + "order_by_rank": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ] + } + }, + "provenance": { + "schema_version": 1, + "experiment": "D3", + "utc": "2026-08-30T06:31:15+00:00", + "script": { + "path": "experiments/d3_corpus_by_family.py", + "sha256": "b7c11544034c9098ebb722857d59f1d017030157aba46a24cc6d1f38fba51e6c", + "n_lines": 164 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/d3_corpus_by_family.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/cv3_margins_410m.json", + "sha256": "665094bb757d868c90e2429d48a38fbd8ca1556a5c963540c79a8322497a5d15", + "bytes": 14817, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "fc353bf005b4e4ab1bf00c84bc40013d9e5497435306c0e5ce5bdc7298085042" + } +} \ No newline at end of file diff --git a/results/da1_derangement_adjudication_410m.json b/results/da1_derangement_adjudication_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..c803a9ee366b013cc8f820e84d4501bd7f3c1589 --- /dev/null +++ b/results/da1_derangement_adjudication_410m.json @@ -0,0 +1,39896 @@ +{ + "experiment": "DA1 \u2014 adjudicating what the layer-derangement control identifies", + "prereg": "docs/experiments/preregs/DA1_derangement_adjudication.md", + "status": "PRE-REGISTERED \u2014 plan written before any DA1 number existed", + "adjudicates": [ + "results/r9_permutation_calibrated_min.json", + "results/e48_crossover_410m_rstrip.json", + "results/d1_min_union_diagnostic_410m.json" + ], + "readout": "CORRECTED \u2014 item['prompt'].rstrip(), per anchor_evals.py:32-34", + "no_refit": true, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_items": 541, + "n_pairs": 893, + "shifts": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "n_derangements": 5, + "derangement_perms": { + "shuf0": [ + 15, + 9, + 13, + 17, + 21, + 11, + 20, + 14, + 19, + 16, + 10, + 18, + 12 + ], + "shuf1": [ + 13, + 19, + 10, + 20, + 14, + 17, + 11, + 9, + 12, + 21, + 15, + 16, + 18 + ], + "shuf2": [ + 17, + 20, + 9, + 10, + 14, + 19, + 18, + 15, + 16, + 13, + 21, + 12, + 11 + ], + "shuf3": [ + 11, + 20, + 18, + 15, + 19, + 10, + 21, + 13, + 16, + 9, + 14, + 17, + 12 + ], + "shuf4": [ + 11, + 13, + 12, + 20, + 18, + 15, + 10, + 19, + 14, + 16, + 21, + 9, + 17 + ] + }, + "cyclic_perms": { + "cyc1": [ + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 9 + ], + "cyc2": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 9, + 10 + ], + "cyc3": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 9, + 10, + 11 + ], + "cyc4": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 9, + 10, + 11, + 12 + ], + "cyc5": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 9, + 10, + 11, + 12, + 13 + ], + "cyc6": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "cyc7": [ + 16, + 17, + 18, + 19, + 20, + 21, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ], + "cyc8": [ + 17, + 18, + 19, + 20, + 21, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "cyc9": [ + 18, + 19, + 20, + 21, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17 + ], + "cyc10": [ + 19, + 20, + 21, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "cyc11": [ + 20, + 21, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19 + ], + "cyc12": [ + 21, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + "note_on_the_fifteen_draws": "R9's 'fifteen derangements' are 3 seed blocks x 5 permutations, so there are five DISTINCT permutations, not fifteen. That is a property of the E48 design being adjudicated; DA1 reproduces it rather than changing it.", + "aggregation_definitions": { + "min": "admitted mean of mean_k 1[min over band layers of rank <= k] \u2014 t48_crossover", + "persist": "admitted mean of mean_k 1[#{band layers with rank<=k} >= floor(|B|/2)]", + "best1L": "max over band layers of the admitted mean of mean_k 1[rank_l <= k]", + "mean": "mean over band layers of the admitted mean of mean_k 1[rank_l <= k]", + "semantics_from": "experiments/t17_reaggregate.py:aggregate()", + "k_summary_and_set_averaging_from": "experiments/t48_crossover.py", + "declared_deviation": "t17 computes min/best1L/mean over ALL layers and persist over the band, under KS=(1,2,5,10,25,100) with a log-k trapezoid. DA1 computes all four OVER THE BAND under K=(1,2,5,10,20,50,100) with a flat mean, because the claim under adjudication is band-scoped and stated under that k-summary. Declared in the pre-registration before running.", + "best1L_caveat": "a maximum over 13 layers is itself a selection; best1L is reported as a diagnostic of where per-layer quality sits, not as a corrected recovery metric." + }, + "logit_arm": { + "min": 0.19810852520167826, + "persist": 0.08317913140635938, + "best1L": 0.14046438420191407, + "mean": 0.07690842781717387 + }, + "gate": { + "file": "results/e48_crossover_410m_rstrip.json", + "tolerance": 1e-06, + "checked": [ + { + "arm": "logit_I", + "agg": "min", + "da1": 0.19810852520167826, + "stored": 0.19810852520167826, + "abs_diff": 0.0 + }, + { + "arm": "logit_I", + "agg": "persist", + "da1": 0.08317913140635938, + "stored": 0.08317913140635938, + "abs_diff": 0.0 + }, + { + "arm": "J|Wikipedia_en|s0", + "agg": "min", + "da1": 0.24112006239593028, + "stored": 0.24112006239593028, + "abs_diff": 0.0 + }, + { + "arm": "J|Wikipedia_en|s0", + "agg": "persist", + "da1": 0.10448124296963215, + "stored": 0.10448124296963215, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d0", + "agg": "min", + "da1": 0.24828002080321313, + "stored": 0.24828002080321313, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d0", + "agg": "persist", + "da1": 0.0813209842890501, + "stored": 0.0813209842890501, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d1", + "agg": "min", + "da1": 0.25063798800110815, + "stored": 0.25063798800110815, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d1", + "agg": "persist", + "da1": 0.07815886037424206, + "stored": 0.07815886037424206, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d2", + "agg": "min", + "da1": 0.27249130234122276, + "stored": 0.27249130234122276, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d2", + "agg": "persist", + "da1": 0.07532001119107008, + "stored": 0.07532001119107008, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d3", + "agg": "min", + "da1": 0.2572912372648716, + "stored": 0.2572912372648716, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d3", + "agg": "persist", + "da1": 0.07884126789867878, + "stored": 0.07884126789867878, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d4", + "agg": "min", + "da1": 0.27122137174010275, + "stored": 0.27122137174010275, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s0|d4", + "agg": "persist", + "da1": 0.07891678363084793, + "stored": 0.07891678363084793, + "abs_diff": 0.0 + }, + { + "arm": "J|Wikipedia_en|s1", + "agg": "min", + "da1": 0.24326577521860598, + "stored": 0.24326577521860598, + "abs_diff": 0.0 + }, + { + "arm": "J|Wikipedia_en|s1", + "agg": "persist", + "da1": 0.10463058389723301, + "stored": 0.10463058389723301, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d0", + "agg": "min", + "da1": 0.24736156240105628, + "stored": 0.24736156240105628, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d0", + "agg": "persist", + "da1": 0.08280326202511787, + "stored": 0.08280326202511787, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d1", + "agg": "min", + "da1": 0.2511155292391777, + "stored": 0.2511155292391777, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d1", + "agg": "persist", + "da1": 0.07869168482720852, + "stored": 0.07869168482720852, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d2", + "agg": "min", + "da1": 0.2717252686619759, + "stored": 0.2717252686619759, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d2", + "agg": "persist", + "da1": 0.07610932774841786, + "stored": 0.07610932774841786, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d3", + "agg": "min", + "da1": 0.2539973951876163, + "stored": 0.2539973951876163, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d3", + "agg": "persist", + "da1": 0.07892063967883586, + "stored": 0.07892063967883586, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d4", + "agg": "min", + "da1": 0.26881202794611453, + "stored": 0.26881202794611453, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s1|d4", + "agg": "persist", + "da1": 0.07768305167555808, + "stored": 0.07768305167555808, + "abs_diff": 0.0 + }, + { + "arm": "J|Wikipedia_en|s2", + "agg": "min", + "da1": 0.2404395245015621, + "stored": 0.2404395245015621, + "abs_diff": 0.0 + }, + { + "arm": "J|Wikipedia_en|s2", + "agg": "persist", + "da1": 0.1035530712455511, + "stored": 0.1035530712455511, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d0", + "agg": "min", + "da1": 0.24522592797875403, + "stored": 0.24522592797875403, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d0", + "agg": "persist", + "da1": 0.08325023017823696, + "stored": 0.08325023017823696, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d1", + "agg": "min", + "da1": 0.24893060624599456, + "stored": 0.24893060624599456, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d1", + "agg": "persist", + "da1": 0.07850877046585084, + "stored": 0.07850877046585084, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d2", + "agg": "min", + "da1": 0.2693440858274698, + "stored": 0.2693440858274698, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d2", + "agg": "persist", + "da1": 0.0764667907729745, + "stored": 0.0764667907729745, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d3", + "agg": "min", + "da1": 0.2519726946949959, + "stored": 0.2519726946949959, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d3", + "agg": "persist", + "da1": 0.07845090553164483, + "stored": 0.07845090553164483, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d4", + "agg": "min", + "da1": 0.26741342917084693, + "stored": 0.26741342917084693, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Wikipedia_en|s2|d4", + "agg": "persist", + "da1": 0.07803469803184271, + "stored": 0.07803469803184271, + "abs_diff": 0.0 + }, + { + "arm": "J|USPTO_Backgrounds|s0", + "agg": "min", + "da1": 0.259354492649436, + "stored": 0.259354492649436, + "abs_diff": 0.0 + }, + { + "arm": "J|USPTO_Backgrounds|s0", + "agg": "persist", + "da1": 0.12008049613796175, + "stored": 0.12008049613796175, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d0", + "agg": "min", + "da1": 0.25584742724895476, + "stored": 0.25584742724895476, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d0", + "agg": "persist", + "da1": 0.0909794308245182, + "stored": 0.0909794308245182, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d1", + "agg": "min", + "da1": 0.26351498067379, + "stored": 0.26351498067379, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d1", + "agg": "persist", + "da1": 0.08494102142285556, + "stored": 0.08494102142285556, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d2", + "agg": "min", + "da1": 0.26983229219913485, + "stored": 0.26983229219913485, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d2", + "agg": "persist", + "da1": 0.08603019118309022, + "stored": 0.08603019118309022, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d3", + "agg": "min", + "da1": 0.2610111027956009, + "stored": 0.2610111027956009, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d3", + "agg": "persist", + "da1": 0.08238022662699222, + "stored": 0.08238022662699222, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d4", + "agg": "min", + "da1": 0.2718806996941566, + "stored": 0.2718806996941566, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s0|d4", + "agg": "persist", + "da1": 0.08888753782957792, + "stored": 0.08888753782957792, + "abs_diff": 0.0 + }, + { + "arm": "J|USPTO_Backgrounds|s1", + "agg": "min", + "da1": 0.26035019345581534, + "stored": 0.26035019345581534, + "abs_diff": 0.0 + }, + { + "arm": "J|USPTO_Backgrounds|s1", + "agg": "persist", + "da1": 0.11979121798649431, + "stored": 0.11979121798649431, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d0", + "agg": "min", + "da1": 0.25631540194153785, + "stored": 0.25631540194153785, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d0", + "agg": "persist", + "da1": 0.08966481909155846, + "stored": 0.08966481909155846, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d1", + "agg": "min", + "da1": 0.26299020983278754, + "stored": 0.26299020983278754, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d1", + "agg": "persist", + "da1": 0.08335184752941131, + "stored": 0.08335184752941131, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d2", + "agg": "min", + "da1": 0.26882662773132326, + "stored": 0.26882662773132326, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d2", + "agg": "persist", + "da1": 0.08593744933605194, + "stored": 0.08593744933605194, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d3", + "agg": "min", + "da1": 0.26178502663969994, + "stored": 0.26178502663969994, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d3", + "agg": "persist", + "da1": 0.08251192029565572, + "stored": 0.08251192029565572, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d4", + "agg": "min", + "da1": 0.2712178058922291, + "stored": 0.2712178058922291, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s1|d4", + "agg": "persist", + "da1": 0.08784533105790615, + "stored": 0.08784533105790615, + "abs_diff": 0.0 + }, + { + "arm": "J|USPTO_Backgrounds|s2", + "agg": "min", + "da1": 0.2636730432510376, + "stored": 0.2636730432510376, + "abs_diff": 0.0 + }, + { + "arm": "J|USPTO_Backgrounds|s2", + "agg": "persist", + "da1": 0.12028511771932245, + "stored": 0.12028511771932245, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d0", + "agg": "min", + "da1": 0.25852382332086565, + "stored": 0.25852382332086565, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d0", + "agg": "persist", + "da1": 0.09051615260541439, + "stored": 0.09051615260541439, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d1", + "agg": "min", + "da1": 0.2640672240406275, + "stored": 0.2640672240406275, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d1", + "agg": "persist", + "da1": 0.08496731789782644, + "stored": 0.08496731789782644, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d2", + "agg": "min", + "da1": 0.27396334931254385, + "stored": 0.27396334931254385, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d2", + "agg": "persist", + "da1": 0.08608829639852048, + "stored": 0.08608829639852048, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d3", + "agg": "min", + "da1": 0.265429225564003, + "stored": 0.265429225564003, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d3", + "agg": "persist", + "da1": 0.0844848508015275, + "stored": 0.0844848508015275, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d4", + "agg": "min", + "da1": 0.27192303650081157, + "stored": 0.27192303650081157, + "abs_diff": 0.0 + }, + { + "arm": "shuf|USPTO_Backgrounds|s2|d4", + "agg": "persist", + "da1": 0.08831145130097866, + "stored": 0.08831145130097866, + "abs_diff": 0.0 + }, + { + "arm": "J|Pile-CC|s0", + "agg": "min", + "da1": 0.26216798909008504, + "stored": 0.26216798909008504, + "abs_diff": 0.0 + }, + { + "arm": "J|Pile-CC|s0", + "agg": "persist", + "da1": 0.11054239969234914, + "stored": 0.11054239969234914, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d0", + "agg": "min", + "da1": 0.2740033358335495, + "stored": 0.2740033358335495, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d0", + "agg": "persist", + "da1": 0.08584446720778942, + "stored": 0.08584446720778942, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d1", + "agg": "min", + "da1": 0.2677078492939472, + "stored": 0.2677078492939472, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d1", + "agg": "persist", + "da1": 0.08377785235643387, + "stored": 0.08377785235643387, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d2", + "agg": "min", + "da1": 0.2843741521239281, + "stored": 0.2843741521239281, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d2", + "agg": "persist", + "da1": 0.08186221458017826, + "stored": 0.08186221458017826, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d3", + "agg": "min", + "da1": 0.26529838889837265, + "stored": 0.26529838889837265, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d3", + "agg": "persist", + "da1": 0.08023232966661453, + "stored": 0.08023232966661453, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d4", + "agg": "min", + "da1": 0.27600362077355384, + "stored": 0.27600362077355384, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s0|d4", + "agg": "persist", + "da1": 0.08126350324600935, + "stored": 0.08126350324600935, + "abs_diff": 0.0 + }, + { + "arm": "J|Pile-CC|s1", + "agg": "min", + "da1": 0.2650879543274641, + "stored": 0.2650879543274641, + "abs_diff": 0.0 + }, + { + "arm": "J|Pile-CC|s1", + "agg": "persist", + "da1": 0.1118273551343009, + "stored": 0.1118273551343009, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d0", + "agg": "min", + "da1": 0.2786212250590324, + "stored": 0.2786212250590324, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d0", + "agg": "persist", + "da1": 0.08509123313706368, + "stored": 0.08509123313706368, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d1", + "agg": "min", + "da1": 0.2705193042755127, + "stored": 0.2705193042755127, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d1", + "agg": "persist", + "da1": 0.08387383986264467, + "stored": 0.08387383986264467, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d2", + "agg": "min", + "da1": 0.2866306558251381, + "stored": 0.2866306558251381, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d2", + "agg": "persist", + "da1": 0.08191019371151924, + "stored": 0.08191019371151924, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d3", + "agg": "min", + "da1": 0.2700627721846104, + "stored": 0.2700627721846104, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d3", + "agg": "persist", + "da1": 0.08150973357260227, + "stored": 0.08150973357260227, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d4", + "agg": "min", + "da1": 0.27726985663175585, + "stored": 0.27726985663175585, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s1|d4", + "agg": "persist", + "da1": 0.08268851265311242, + "stored": 0.08268851265311242, + "abs_diff": 0.0 + }, + { + "arm": "J|Pile-CC|s2", + "agg": "min", + "da1": 0.25994115322828293, + "stored": 0.25994115322828293, + "abs_diff": 0.0 + }, + { + "arm": "J|Pile-CC|s2", + "agg": "persist", + "da1": 0.11091751407366246, + "stored": 0.11091751407366246, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d0", + "agg": "min", + "da1": 0.2740327961742878, + "stored": 0.2740327961742878, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d0", + "agg": "persist", + "da1": 0.0852287447778508, + "stored": 0.0852287447778508, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d1", + "agg": "min", + "da1": 0.2686062276363373, + "stored": 0.2686062276363373, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d1", + "agg": "persist", + "da1": 0.0828045567497611, + "stored": 0.0828045567497611, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d2", + "agg": "min", + "da1": 0.2827077329158783, + "stored": 0.2827077329158783, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d2", + "agg": "persist", + "da1": 0.08076190613210202, + "stored": 0.08076190613210202, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d3", + "agg": "min", + "da1": 0.2685593143105507, + "stored": 0.2685593143105507, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d3", + "agg": "persist", + "da1": 0.07991705778986216, + "stored": 0.07991705778986216, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d4", + "agg": "min", + "da1": 0.27765234634280206, + "stored": 0.27765234634280206, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Pile-CC|s2|d4", + "agg": "persist", + "da1": 0.0815402265638113, + "stored": 0.0815402265638113, + "abs_diff": 0.0 + }, + { + "arm": "J|StackExchange|s0", + "agg": "min", + "da1": 0.2665700364857912, + "stored": 0.2665700364857912, + "abs_diff": 0.0 + }, + { + "arm": "J|StackExchange|s0", + "agg": "persist", + "da1": 0.11692521655932069, + "stored": 0.11692521655932069, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d0", + "agg": "min", + "da1": 0.26500423550605773, + "stored": 0.26500423550605773, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d0", + "agg": "persist", + "da1": 0.08735615264158696, + "stored": 0.08735615264158696, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d1", + "agg": "min", + "da1": 0.27110425978899, + "stored": 0.27110425978899, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d1", + "agg": "persist", + "da1": 0.08621515452396125, + "stored": 0.08621515452396125, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d2", + "agg": "min", + "da1": 0.2754390150308609, + "stored": 0.2754390150308609, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d2", + "agg": "persist", + "da1": 0.08668109960854053, + "stored": 0.08668109960854053, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d3", + "agg": "min", + "da1": 0.25646587610244753, + "stored": 0.25646587610244753, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d3", + "agg": "persist", + "da1": 0.07744382806122303, + "stored": 0.07744382806122303, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d4", + "agg": "min", + "da1": 0.26446273662149905, + "stored": 0.26446273662149905, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s0|d4", + "agg": "persist", + "da1": 0.08563328310847282, + "stored": 0.08563328310847282, + "abs_diff": 0.0 + }, + { + "arm": "J|StackExchange|s1", + "agg": "min", + "da1": 0.2635776612907648, + "stored": 0.2635776612907648, + "abs_diff": 0.0 + }, + { + "arm": "J|StackExchange|s1", + "agg": "persist", + "da1": 0.11572976196184755, + "stored": 0.11572976196184755, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d0", + "agg": "min", + "da1": 0.26482513546943665, + "stored": 0.26482513546943665, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d0", + "agg": "persist", + "da1": 0.08486844457220286, + "stored": 0.08486844457220286, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d1", + "agg": "min", + "da1": 0.2690747335553169, + "stored": 0.2690747335553169, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d1", + "agg": "persist", + "da1": 0.08523496259003878, + "stored": 0.08523496259003878, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d2", + "agg": "min", + "da1": 0.2727171786129475, + "stored": 0.2727171786129475, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d2", + "agg": "persist", + "da1": 0.08461906053125859, + "stored": 0.08461906053125859, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d3", + "agg": "min", + "da1": 0.2517006419599056, + "stored": 0.2517006419599056, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d3", + "agg": "persist", + "da1": 0.0757306307554245, + "stored": 0.0757306307554245, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d4", + "agg": "min", + "da1": 0.26243913993239404, + "stored": 0.26243913993239404, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s1|d4", + "agg": "persist", + "da1": 0.08559647388756275, + "stored": 0.08559647388756275, + "abs_diff": 0.0 + }, + { + "arm": "J|StackExchange|s2", + "agg": "min", + "da1": 0.26277029849588873, + "stored": 0.26277029849588873, + "abs_diff": 0.0 + }, + { + "arm": "J|StackExchange|s2", + "agg": "persist", + "da1": 0.11578959980979561, + "stored": 0.11578959980979561, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d0", + "agg": "min", + "da1": 0.26502151116728784, + "stored": 0.26502151116728784, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d0", + "agg": "persist", + "da1": 0.08450331743806601, + "stored": 0.08450331743806601, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d1", + "agg": "min", + "da1": 0.26919378712773323, + "stored": 0.26919378712773323, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d1", + "agg": "persist", + "da1": 0.08529306538403034, + "stored": 0.08529306538403034, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d2", + "agg": "min", + "da1": 0.27218492031097413, + "stored": 0.27218492031097413, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d2", + "agg": "persist", + "da1": 0.0856543831527233, + "stored": 0.0856543831527233, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d3", + "agg": "min", + "da1": 0.25335856787860395, + "stored": 0.25335856787860395, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d3", + "agg": "persist", + "da1": 0.07592109628021718, + "stored": 0.07592109628021718, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d4", + "agg": "min", + "da1": 0.26091468334198, + "stored": 0.26091468334198, + "abs_diff": 0.0 + }, + { + "arm": "shuf|StackExchange|s2|d4", + "agg": "persist", + "da1": 0.08539159186184406, + "stored": 0.08539159186184406, + "abs_diff": 0.0 + }, + { + "arm": "J|Github|s0", + "agg": "min", + "da1": 0.23628932386636733, + "stored": 0.23628932386636733, + "abs_diff": 0.0 + }, + { + "arm": "J|Github|s0", + "agg": "persist", + "da1": 0.09901561820879579, + "stored": 0.09901561820879579, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d0", + "agg": "min", + "da1": 0.22602029740810395, + "stored": 0.22602029740810395, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d0", + "agg": "persist", + "da1": 0.08236480634659529, + "stored": 0.08236480634659529, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d1", + "agg": "min", + "da1": 0.23050989732146263, + "stored": 0.23050989732146263, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d1", + "agg": "persist", + "da1": 0.07923254787456244, + "stored": 0.07923254787456244, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d2", + "agg": "min", + "da1": 0.24628960341215134, + "stored": 0.24628960341215134, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d2", + "agg": "persist", + "da1": 0.08386189788579941, + "stored": 0.08386189788579941, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d3", + "agg": "min", + "da1": 0.20614932514727116, + "stored": 0.20614932514727116, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d3", + "agg": "persist", + "da1": 0.0770264295861125, + "stored": 0.0770264295861125, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d4", + "agg": "min", + "da1": 0.23921805135905744, + "stored": 0.23921805135905744, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s0|d4", + "agg": "persist", + "da1": 0.08045144844800234, + "stored": 0.08045144844800234, + "abs_diff": 0.0 + }, + { + "arm": "J|Github|s1", + "agg": "min", + "da1": 0.2431370198726654, + "stored": 0.2431370198726654, + "abs_diff": 0.0 + }, + { + "arm": "J|Github|s1", + "agg": "persist", + "da1": 0.10285943346098066, + "stored": 0.10285943346098066, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d0", + "agg": "min", + "da1": 0.23540843054652213, + "stored": 0.23540843054652213, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d0", + "agg": "persist", + "da1": 0.08247843962162733, + "stored": 0.08247843962162733, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d1", + "agg": "min", + "da1": 0.2322789803147316, + "stored": 0.2322789803147316, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d1", + "agg": "persist", + "da1": 0.08251860772725195, + "stored": 0.08251860772725195, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d2", + "agg": "min", + "da1": 0.25017106980085374, + "stored": 0.25017106980085374, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d2", + "agg": "persist", + "da1": 0.08669814679306001, + "stored": 0.08669814679306001, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d3", + "agg": "min", + "da1": 0.21141563914716244, + "stored": 0.21141563914716244, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d3", + "agg": "persist", + "da1": 0.07803292572498322, + "stored": 0.07803292572498322, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d4", + "agg": "min", + "da1": 0.2424768466502428, + "stored": 0.2424768466502428, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s1|d4", + "agg": "persist", + "da1": 0.08245038855820894, + "stored": 0.08245038855820894, + "abs_diff": 0.0 + }, + { + "arm": "J|Github|s2", + "agg": "min", + "da1": 0.2420657031238079, + "stored": 0.2420657031238079, + "abs_diff": 0.0 + }, + { + "arm": "J|Github|s2", + "agg": "persist", + "da1": 0.10037821242585779, + "stored": 0.10037821242585779, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d0", + "agg": "min", + "da1": 0.22873966172337531, + "stored": 0.22873966172337531, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d0", + "agg": "persist", + "da1": 0.08199466541409492, + "stored": 0.08199466541409492, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d1", + "agg": "min", + "da1": 0.23314449191093445, + "stored": 0.23314449191093445, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d1", + "agg": "persist", + "da1": 0.0824049681192264, + "stored": 0.0824049681192264, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d2", + "agg": "min", + "da1": 0.250272324681282, + "stored": 0.250272324681282, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d2", + "agg": "persist", + "da1": 0.08569017052650452, + "stored": 0.08569017052650452, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d3", + "agg": "min", + "da1": 0.2102797579020262, + "stored": 0.2102797579020262, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d3", + "agg": "persist", + "da1": 0.07784030102193355, + "stored": 0.07784030102193355, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d4", + "agg": "min", + "da1": 0.24373132549226284, + "stored": 0.24373132549226284, + "abs_diff": 0.0 + }, + { + "arm": "shuf|Github|s2|d4", + "agg": "persist", + "da1": 0.0819853462278843, + "stored": 0.0819853462278843, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_News_2024|s0", + "agg": "min", + "da1": 0.24892510399222373, + "stored": 0.24892510399222373, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_News_2024|s0", + "agg": "persist", + "da1": 0.0986472788034007, + "stored": 0.0986472788034007, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d0", + "agg": "min", + "da1": 0.2541537590324879, + "stored": 0.2541537590324879, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d0", + "agg": "persist", + "da1": 0.07370346884708852, + "stored": 0.07370346884708852, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d1", + "agg": "min", + "da1": 0.24952763319015503, + "stored": 0.24952763319015503, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d1", + "agg": "persist", + "da1": 0.072022794932127, + "stored": 0.072022794932127, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d2", + "agg": "min", + "da1": 0.27084239572286606, + "stored": 0.27084239572286606, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d2", + "agg": "persist", + "da1": 0.07044114768505097, + "stored": 0.07044114768505097, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d3", + "agg": "min", + "da1": 0.2475816510617733, + "stored": 0.2475816510617733, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d3", + "agg": "persist", + "da1": 0.07144842091947794, + "stored": 0.07144842091947794, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d4", + "agg": "min", + "da1": 0.26790230087935923, + "stored": 0.26790230087935923, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s0|d4", + "agg": "persist", + "da1": 0.06905782185494899, + "stored": 0.06905782185494899, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_News_2024|s1", + "agg": "min", + "da1": 0.2498748254030943, + "stored": 0.2498748254030943, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_News_2024|s1", + "agg": "persist", + "da1": 0.0974247383652255, + "stored": 0.0974247383652255, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d0", + "agg": "min", + "da1": 0.2539294570684433, + "stored": 0.2539294570684433, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d0", + "agg": "persist", + "da1": 0.07335275076329709, + "stored": 0.07335275076329709, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d1", + "agg": "min", + "da1": 0.2506876736879349, + "stored": 0.2506876736879349, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d1", + "agg": "persist", + "da1": 0.07231609672307968, + "stored": 0.07231609672307968, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d2", + "agg": "min", + "da1": 0.26873930990695954, + "stored": 0.26873930990695954, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d2", + "agg": "persist", + "da1": 0.07153821419924497, + "stored": 0.07153821419924497, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d3", + "agg": "min", + "da1": 0.24649405032396315, + "stored": 0.24649405032396315, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d3", + "agg": "persist", + "da1": 0.070631312020123, + "stored": 0.070631312020123, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d4", + "agg": "min", + "da1": 0.2676916144788265, + "stored": 0.2676916144788265, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s1|d4", + "agg": "persist", + "da1": 0.06975763961672783, + "stored": 0.06975763961672783, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_News_2024|s2", + "agg": "min", + "da1": 0.2513899695128202, + "stored": 0.2513899695128202, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_News_2024|s2", + "agg": "persist", + "da1": 0.0976877189008519, + "stored": 0.0976877189008519, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d0", + "agg": "min", + "da1": 0.25065327659249304, + "stored": 0.25065327659249304, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d0", + "agg": "persist", + "da1": 0.07303814720362425, + "stored": 0.07303814720362425, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d1", + "agg": "min", + "da1": 0.24929407984018326, + "stored": 0.24929407984018326, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d1", + "agg": "persist", + "da1": 0.07085146754980087, + "stored": 0.07085146754980087, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d2", + "agg": "min", + "da1": 0.2683473452925682, + "stored": 0.2683473452925682, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d2", + "agg": "persist", + "da1": 0.06865867599844933, + "stored": 0.06865867599844933, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d3", + "agg": "min", + "da1": 0.2447837095707655, + "stored": 0.2447837095707655, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d3", + "agg": "persist", + "da1": 0.07009160798043013, + "stored": 0.07009160798043013, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d4", + "agg": "min", + "da1": 0.2669975996017456, + "stored": 0.2669975996017456, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_News_2024|s2|d4", + "agg": "persist", + "da1": 0.06849358156323433, + "stored": 0.06849358156323433, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_arXiv_2023|s0", + "agg": "min", + "da1": 0.2689389446750283, + "stored": 0.2689389446750283, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_arXiv_2023|s0", + "agg": "persist", + "da1": 0.10620390176773072, + "stored": 0.10620390176773072, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d0", + "agg": "min", + "da1": 0.2580361794680357, + "stored": 0.2580361794680357, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d0", + "agg": "persist", + "da1": 0.08164319396018982, + "stored": 0.08164319396018982, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d1", + "agg": "min", + "da1": 0.2720124736428261, + "stored": 0.2720124736428261, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d1", + "agg": "persist", + "da1": 0.08086159788072109, + "stored": 0.08086159788072109, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d2", + "agg": "min", + "da1": 0.29188896268606185, + "stored": 0.29188896268606185, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d2", + "agg": "persist", + "da1": 0.07931843660771846, + "stored": 0.07931843660771846, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d3", + "agg": "min", + "da1": 0.2627900928258896, + "stored": 0.2627900928258896, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d3", + "agg": "persist", + "da1": 0.07451336067169904, + "stored": 0.07451336067169904, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d4", + "agg": "min", + "da1": 0.27306791618466375, + "stored": 0.27306791618466375, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s0|d4", + "agg": "persist", + "da1": 0.08041031993925571, + "stored": 0.08041031993925571, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_arXiv_2023|s1", + "agg": "min", + "da1": 0.26879445035010574, + "stored": 0.26879445035010574, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_arXiv_2023|s1", + "agg": "persist", + "da1": 0.10661298111081123, + "stored": 0.10661298111081123, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d0", + "agg": "min", + "da1": 0.25658972561359406, + "stored": 0.25658972561359406, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d0", + "agg": "persist", + "da1": 0.08155045211315155, + "stored": 0.08155045211315155, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d1", + "agg": "min", + "da1": 0.26868619695305823, + "stored": 0.26868619695305823, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d1", + "agg": "persist", + "da1": 0.08128509055823088, + "stored": 0.08128509055823088, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d2", + "agg": "min", + "da1": 0.29177426397800443, + "stored": 0.29177426397800443, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d2", + "agg": "persist", + "da1": 0.07937072664499283, + "stored": 0.07937072664499283, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d3", + "agg": "min", + "da1": 0.2612253025174141, + "stored": 0.2612253025174141, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d3", + "agg": "persist", + "da1": 0.0759934613481164, + "stored": 0.0759934613481164, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d4", + "agg": "min", + "da1": 0.2692541994154453, + "stored": 0.2692541994154453, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s1|d4", + "agg": "persist", + "da1": 0.07981508262455464, + "stored": 0.07981508262455464, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_arXiv_2023|s2", + "agg": "min", + "da1": 0.27008571829646827, + "stored": 0.27008571829646827, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_arXiv_2023|s2", + "agg": "persist", + "da1": 0.10486989989876747, + "stored": 0.10486989989876747, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d0", + "agg": "min", + "da1": 0.2571215949952602, + "stored": 0.2571215949952602, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d0", + "agg": "persist", + "da1": 0.08120271135121584, + "stored": 0.08120271135121584, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d1", + "agg": "min", + "da1": 0.27173766642808916, + "stored": 0.27173766642808916, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d1", + "agg": "persist", + "da1": 0.07898571649566292, + "stored": 0.07898571649566292, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d2", + "agg": "min", + "da1": 0.29140356555581093, + "stored": 0.29140356555581093, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d2", + "agg": "persist", + "da1": 0.07789169233292341, + "stored": 0.07789169233292341, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d3", + "agg": "min", + "da1": 0.25893412753939626, + "stored": 0.25893412753939626, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d3", + "agg": "persist", + "da1": 0.0748466819524765, + "stored": 0.0748466819524765, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d4", + "agg": "min", + "da1": 0.2704723007977009, + "stored": 0.2704723007977009, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_arXiv_2023|s2|d4", + "agg": "persist", + "da1": 0.07978044673800469, + "stored": 0.07978044673800469, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_CommonPile|s0", + "agg": "min", + "da1": 0.2531457645818591, + "stored": 0.2531457645818591, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_CommonPile|s0", + "agg": "persist", + "da1": 0.10742316469550132, + "stored": 0.10742316469550132, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d0", + "agg": "min", + "da1": 0.2543313570320606, + "stored": 0.2543313570320606, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d0", + "agg": "persist", + "da1": 0.07998228296637536, + "stored": 0.07998228296637536, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d1", + "agg": "min", + "da1": 0.26050583124160764, + "stored": 0.26050583124160764, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d1", + "agg": "persist", + "da1": 0.08328658873215318, + "stored": 0.08328658873215318, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d2", + "agg": "min", + "da1": 0.2809336222708225, + "stored": 0.2809336222708225, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d2", + "agg": "persist", + "da1": 0.07980556171387435, + "stored": 0.07980556171387435, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d3", + "agg": "min", + "da1": 0.24981799349188805, + "stored": 0.24981799349188805, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d3", + "agg": "persist", + "da1": 0.07883550319820642, + "stored": 0.07883550319820642, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d4", + "agg": "min", + "da1": 0.2634584128856659, + "stored": 0.2634584128856659, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s0|d4", + "agg": "persist", + "da1": 0.08045977549627423, + "stored": 0.08045977549627423, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_CommonPile|s1", + "agg": "min", + "da1": 0.25511594451963904, + "stored": 0.25511594451963904, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_CommonPile|s1", + "agg": "persist", + "da1": 0.10653829500079155, + "stored": 0.10653829500079155, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d0", + "agg": "min", + "da1": 0.2539062112569809, + "stored": 0.2539062112569809, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d0", + "agg": "persist", + "da1": 0.07729471735656261, + "stored": 0.07729471735656261, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d1", + "agg": "min", + "da1": 0.26217192634940145, + "stored": 0.26217192634940145, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d1", + "agg": "persist", + "da1": 0.07961883107200265, + "stored": 0.07961883107200265, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d2", + "agg": "min", + "da1": 0.2828520558774471, + "stored": 0.2828520558774471, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d2", + "agg": "persist", + "da1": 0.0780245989561081, + "stored": 0.0780245989561081, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d3", + "agg": "min", + "da1": 0.24786126241087914, + "stored": 0.24786126241087914, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d3", + "agg": "persist", + "da1": 0.07607217859476804, + "stored": 0.07607217859476804, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d4", + "agg": "min", + "da1": 0.26146751791238787, + "stored": 0.26146751791238787, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s1|d4", + "agg": "persist", + "da1": 0.07973866332322359, + "stored": 0.07973866332322359, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_CommonPile|s2", + "agg": "min", + "da1": 0.2503817303106189, + "stored": 0.2503817303106189, + "abs_diff": 0.0 + }, + { + "arm": "J|OOD_CommonPile|s2", + "agg": "persist", + "da1": 0.1059548944234848, + "stored": 0.1059548944234848, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d0", + "agg": "min", + "da1": 0.2520745679736137, + "stored": 0.2520745679736137, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d0", + "agg": "persist", + "da1": 0.07946023121476173, + "stored": 0.07946023121476173, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d1", + "agg": "min", + "da1": 0.2588454566895962, + "stored": 0.2588454566895962, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d1", + "agg": "persist", + "da1": 0.08081897310912609, + "stored": 0.08081897310912609, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d2", + "agg": "min", + "da1": 0.277788944542408, + "stored": 0.277788944542408, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d2", + "agg": "persist", + "da1": 0.07980298958718776, + "stored": 0.07980298958718776, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d3", + "agg": "min", + "da1": 0.24596437588334083, + "stored": 0.24596437588334083, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d3", + "agg": "persist", + "da1": 0.07772078756242991, + "stored": 0.07772078756242991, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d4", + "agg": "min", + "da1": 0.2595244117081165, + "stored": 0.2595244117081165, + "abs_diff": 0.0 + }, + { + "arm": "shuf|OOD_CommonPile|s2|d4", + "agg": "persist", + "da1": 0.07968054311349988, + "stored": 0.07968054311349988, + "abs_diff": 0.0 + } + ], + "max_abs_diff": 0.0, + "n_checked": 290, + "fires": true, + "smallest_possible_real_difference": 0.00036258158085569254, + "tolerance_justification": "one pair flipping at one k moves a set mean by at least 1/(largest set * |K|); the tolerance sits far below that and above float32 accumulation noise. Transcribed from t48_crossover.py's C0, not tuned here." + }, + "controls": { + "C2_shifts_have_no_fixed_point": { + "requirement": "every non-zero cyclic shift moves every band layer off its own Jacobian", + "fails_if": "any k in 1..12 leaves some slot receiving its own layer", + "per_shift_fixed_points": { + "cyc1": 0, + "cyc2": 0, + "cyc3": 0, + "cyc4": 0, + "cyc5": 0, + "cyc6": 0, + "cyc7": 0, + "cyc8": 0, + "cyc9": 0, + "cyc10": 0, + "cyc11": 0, + "cyc12": 0 + }, + "fires": true + }, + "C4_derangements_match_e48": { + "requirement": "the five shuf permutations equal t48_crossover.derangement(BAND, 7000+97d)", + "fails_if": "any element differs, which would mean DA1 is not scoring E48's objects", + "perms": { + "shuf0": [ + 15, + 9, + 13, + 17, + 21, + 11, + 20, + 14, + 19, + 16, + 10, + 18, + 12 + ], + "shuf1": [ + 13, + 19, + 10, + 20, + 14, + 17, + 11, + 9, + 12, + 21, + 15, + 16, + 18 + ], + "shuf2": [ + 17, + 20, + 9, + 10, + 14, + 19, + 18, + 15, + 16, + 13, + 21, + 12, + 11 + ], + "shuf3": [ + 11, + 20, + 18, + 15, + 19, + 10, + 21, + 13, + 16, + 9, + 14, + 17, + 12 + ], + "shuf4": [ + 11, + 13, + 12, + 20, + 18, + 15, + 10, + 19, + 14, + 16, + 21, + 9, + 17 + ] + }, + "fires": true + }, + "C3_gate_reproduces_e48_rstrip": { + "file": "results/e48_crossover_410m_rstrip.json", + "tolerance": 1e-06, + "n_checked": 290, + "max_abs_diff": 0.0, + "fires": true + }, + "C1_identity_permutation_is_the_real_arm": { + "requirement": "the identity permutation of the 13x13 tensor IS the real operator arm", + "note": "structural: `real` is ARM_PERMS['real'] = BAND, so its lookup is Rall[:,j,j]. C3 tests the resulting numbers against the independently-scored stored arms.", + "fires": true + } + }, + "by_corpus": { + "Wikipedia_en": { + "real": { + "min": 0.24160845403869946, + "persist": 0.10422163270413876, + "best1L": 0.14925913113790254, + "mean": 0.10188544646789058 + }, + "cyclic": { + "cyc1": { + "min": 0.24620236940681933, + "persist": 0.08967843676606814, + "best1L": 0.1534876957690964, + "mean": 0.09296688959838297 + }, + "cyc2": { + "min": 0.2468460672845443, + "persist": 0.08124806330694506, + "best1L": 0.1592970922589302, + "mean": 0.0847222715722194 + }, + "cyc3": { + "min": 0.24073332212865353, + "persist": 0.07525289316351215, + "best1L": 0.15527221911276382, + "mean": 0.07968479528715715 + }, + "cyc4": { + "min": 0.23072649637858073, + "persist": 0.07180977668613195, + "best1L": 0.14100884747070572, + "mean": 0.07503117650556258 + }, + "cyc5": { + "min": 0.21997044856349626, + "persist": 0.06891317944973707, + "best1L": 0.11027454842502872, + "mean": 0.07327572150567833 + }, + "cyc6": { + "min": 0.22075465011099973, + "persist": 0.07136421483010054, + "best1L": 0.12558587196593482, + "mean": 0.0728079155045681 + }, + "cyc7": { + "min": 0.2142627300073703, + "persist": 0.07389877264698347, + "best1L": 0.13322370536625386, + "mean": 0.07391114694496187 + }, + "cyc8": { + "min": 0.22435291185975076, + "persist": 0.07578660113116105, + "best1L": 0.14513427388543884, + "mean": 0.07626625466673898 + }, + "cyc9": { + "min": 0.2209307886660099, + "persist": 0.07675887557367485, + "best1L": 0.14100900110788644, + "mean": 0.07779738365565068 + }, + "cyc10": { + "min": 0.23384907096624374, + "persist": 0.08192808690170447, + "best1L": 0.14712283002833526, + "mean": 0.08310496067592445 + }, + "cyc11": { + "min": 0.24935587917764981, + "persist": 0.08926978521049023, + "best1L": 0.1508162988970677, + "mean": 0.0898645333157709 + }, + "cyc12": { + "min": 0.24737414146463077, + "persist": 0.09580344595791151, + "best1L": 0.14309973440443477, + "mean": 0.09492850206816235 + } + }, + "shuf": { + "shuf0": { + "min": 0.24695583706100782, + "persist": 0.08245815883080164, + "best1L": 0.14153299797326327, + "mean": 0.08469348192656748 + }, + "shuf1": { + "min": 0.25022804116209346, + "persist": 0.0784531052224338, + "best1L": 0.15527221911276382, + "mean": 0.08345885274973579 + }, + "shuf2": { + "min": 0.2711868856102228, + "persist": 0.07596537657082081, + "best1L": 0.1592970922589302, + "mean": 0.08382359682761419 + }, + "shuf3": { + "min": 0.2544204423824946, + "persist": 0.07873760436971983, + "best1L": 0.14513427388543884, + "mean": 0.08043374572925341 + }, + "shuf4": { + "min": 0.26914894295235475, + "persist": 0.07821151111274957, + "best1L": 0.1592970922589302, + "mean": 0.08392068246910826 + } + }, + "cyclic_mean": { + "min": 0.2329465730012291, + "persist": 0.07930934430203504, + "best1L": 0.1421110098909897, + "mean": 0.08119679594173149 + }, + "shuf_mean": { + "min": 0.2583880298336347, + "persist": 0.07876515122130513, + "best1L": 0.15210673509786526, + "mean": 0.08326607194045582 + }, + "real_minus_cyclic_mean": { + "min": 0.00866188103747037, + "persist": 0.024912288402103716, + "best1L": 0.007148121246912831, + "mean": 0.02068865052615909 + }, + "real_minus_shuf_mean": { + "min": -0.01677957579493522, + "persist": 0.025456481482833623, + "best1L": -0.0028476039599627245, + "mean": 0.018619374527434757 + }, + "n_cyclic_shifts_real_beats": { + "min": 8, + "persist": 12, + "best1L": 8, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 0, + "persist": 15, + "best1L": 6, + "mean": 15 + }, + "bandconst": { + "min": 0.22990795026222866, + "persist": 0.1070176790157954, + "best1L": 0.1606192346351842, + "mean": 0.10196266787914703 + }, + "real_minus_bandconst": { + "min": 0.011700503776470805, + "persist": -0.0027960463116566414, + "best1L": -0.01136010349728167, + "mean": -7.722141125644788e-05 + } + }, + "USPTO_Backgrounds": { + "real": { + "min": 0.26112590978542966, + "persist": 0.1200522772812595, + "best1L": 0.15068939132615924, + "mean": 0.11370598818862046 + }, + "cyclic": { + "cyc1": { + "min": 0.2682402078062296, + "persist": 0.10319235756372412, + "best1L": 0.1566471378939847, + "mean": 0.10463949375657532 + }, + "cyc2": { + "min": 0.2656247089927395, + "persist": 0.09123014932653556, + "best1L": 0.16099942923368266, + "mean": 0.09306690081959973 + }, + "cyc3": { + "min": 0.25357947647571566, + "persist": 0.08340693545372535, + "best1L": 0.15810412174711624, + "mean": 0.0861466884242896 + }, + "cyc4": { + "min": 0.2478711356719335, + "persist": 0.07702312618494034, + "best1L": 0.1446501533811291, + "mean": 0.08034665194125129 + }, + "cyc5": { + "min": 0.22866494059562684, + "persist": 0.07350230074177186, + "best1L": 0.11988154311353962, + "mean": 0.07774462753530735 + }, + "cyc6": { + "min": 0.2272843154768149, + "persist": 0.07261146251888324, + "best1L": 0.1238315117234985, + "mean": 0.07573309387707224 + }, + "cyc7": { + "min": 0.23269046669205032, + "persist": 0.07704993664131811, + "best1L": 0.14296874112139146, + "mean": 0.07809713682673203 + }, + "cyc8": { + "min": 0.24780923376480737, + "persist": 0.07418708155552546, + "best1L": 0.1492499282117933, + "mean": 0.08046131407400259 + }, + "cyc9": { + "min": 0.23660796135663986, + "persist": 0.08112515234388411, + "best1L": 0.14868702179131407, + "mean": 0.08229359736104902 + }, + "cyc10": { + "min": 0.2460287426908811, + "persist": 0.09164939522743225, + "best1L": 0.14877224198232095, + "mean": 0.08913614229502109 + }, + "cyc11": { + "min": 0.25315611238280933, + "persist": 0.102144414310654, + "best1L": 0.15429384193072718, + "mean": 0.09775584854128866 + }, + "cyc12": { + "min": 0.26554778416951497, + "persist": 0.1065430271672085, + "best1L": 0.1469192434102297, + "mean": 0.1045970891982926 + } + }, + "shuf": { + "shuf0": { + "min": 0.2568955508371194, + "persist": 0.09038680084049702, + "best1L": 0.14231165293604137, + "mean": 0.09168554394750689 + }, + "shuf1": { + "min": 0.26352413818240167, + "persist": 0.08442006228336443, + "best1L": 0.15810412174711624, + "mean": 0.0892647847707849 + }, + "shuf2": { + "min": 0.2708740897476673, + "persist": 0.08601864563922088, + "best1L": 0.16099942923368266, + "mean": 0.09024830246777035 + }, + "shuf3": { + "min": 0.26274178499976797, + "persist": 0.08312566590805848, + "best1L": 0.1492499282117933, + "mean": 0.08531742401015109 + }, + "shuf4": { + "min": 0.2716738473623991, + "persist": 0.08834810672948758, + "best1L": 0.16099942923368266, + "mean": 0.09036357994385374 + } + }, + "cyclic_mean": { + "min": 0.24775875717298024, + "persist": 0.08613877825296691, + "best1L": 0.14625040962839395, + "mean": 0.08750154872087346 + }, + "shuf_mean": { + "min": 0.2651418822258711, + "persist": 0.08645985628012567, + "best1L": 0.15433291227246324, + "mean": 0.08937592702801339 + }, + "real_minus_cyclic_mean": { + "min": 0.013367152612449418, + "persist": 0.03391349902829259, + "best1L": 0.0044389816977652985, + "mean": 0.026204439467747004 + }, + "real_minus_shuf_mean": { + "min": -0.00401597244044144, + "persist": 0.033592421001133826, + "best1L": -0.003643520946303991, + "mean": 0.02433006116060707 + }, + "n_cyclic_shifts_real_beats": { + "min": 9, + "persist": 12, + "best1L": 8, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 3, + "persist": 15, + "best1L": 6, + "mean": 15 + }, + "bandconst": { + "min": 0.23482114858925343, + "persist": 0.11490410533733665, + "best1L": 0.164572404542317, + "mean": 0.10892134161833196 + }, + "real_minus_bandconst": { + "min": 0.026304761196176224, + "persist": 0.00514817194392285, + "best1L": -0.013883013216157758, + "mean": 0.0047846465702885005 + } + }, + "Pile-CC": { + "real": { + "min": 0.2623990322152774, + "persist": 0.11109575630010417, + "best1L": 0.15880081144471964, + "mean": 0.11154730294872887 + }, + "cyclic": { + "cyc1": { + "min": 0.25906300209462646, + "persist": 0.09192285773654779, + "best1L": 0.15904501229524612, + "mean": 0.09876172728239535 + }, + "cyc2": { + "min": 0.25808143702646097, + "persist": 0.08455817422363908, + "best1L": 0.16192924777666728, + "mean": 0.08836059247573408 + }, + "cyc3": { + "min": 0.24634905755519867, + "persist": 0.0791391878699263, + "best1L": 0.15533130917077265, + "mean": 0.08171624454583089 + }, + "cyc4": { + "min": 0.24388572747508686, + "persist": 0.0724325987820824, + "best1L": 0.1396392092263947, + "mean": 0.0757280360814781 + }, + "cyc5": { + "min": 0.23164271985491117, + "persist": 0.07344011745881289, + "best1L": 0.12121701079886407, + "mean": 0.07515563597628035 + }, + "cyc6": { + "min": 0.23456315621733667, + "persist": 0.07477158640200893, + "best1L": 0.12888195634198685, + "mean": 0.07679661031191548 + }, + "cyc7": { + "min": 0.24082261522610982, + "persist": 0.08042583329758296, + "best1L": 0.1467723469560345, + "mean": 0.08104794094494233 + }, + "cyc8": { + "min": 0.24192518244187036, + "persist": 0.0831174769749244, + "best1L": 0.15559283926462134, + "mean": 0.08320452535453325 + }, + "cyc9": { + "min": 0.25010518928368886, + "persist": 0.08412765835722287, + "best1L": 0.15608514860893288, + "mean": 0.08601785433024932 + }, + "cyc10": { + "min": 0.2591923147439957, + "persist": 0.09157470535331716, + "best1L": 0.15980236968025566, + "mean": 0.09277939868267052 + }, + "cyc11": { + "min": 0.2692971408367157, + "persist": 0.10017426556441933, + "best1L": 0.16882663142556945, + "mean": 0.10080933632054485 + }, + "cyc12": { + "min": 0.2704550303518772, + "persist": 0.10326980154495687, + "best1L": 0.156898401863873, + "mean": 0.10551834407274445 + } + }, + "shuf": { + "shuf0": { + "min": 0.27555245235562326, + "persist": 0.08538814837423464, + "best1L": 0.1477861797437072, + "mean": 0.08993655732558228 + }, + "shuf1": { + "min": 0.2689444604019324, + "persist": 0.08348541632294655, + "best1L": 0.15573568713540833, + "mean": 0.08978998113351946 + }, + "shuf2": { + "min": 0.28457084695498147, + "persist": 0.0815114381412665, + "best1L": 0.16192924777666728, + "mean": 0.09023511599904546 + }, + "shuf3": { + "min": 0.2679734917978446, + "persist": 0.08055304034302632, + "best1L": 0.15559283926462134, + "mean": 0.08370900987229574 + }, + "shuf4": { + "min": 0.27697527458270393, + "persist": 0.08183074748764435, + "best1L": 0.16192924777666728, + "mean": 0.08808244325351926 + } + }, + "cyclic_mean": { + "min": 0.25044854775898984, + "persist": 0.08491285529712009, + "best1L": 0.15083512361743487, + "mean": 0.08715802053160991 + }, + "shuf_mean": { + "min": 0.2748033052186171, + "persist": 0.08255375813382368, + "best1L": 0.15659464033941428, + "mean": 0.08835062151679245 + }, + "real_minus_cyclic_mean": { + "min": 0.011950484456287547, + "persist": 0.02618290100298408, + "best1L": 0.007965687827284768, + "mean": 0.024389282417118963 + }, + "real_minus_shuf_mean": { + "min": -0.012404273003339739, + "persist": 0.02854199816628049, + "best1L": 0.0022061711053053545, + "mean": 0.023196681431936428 + }, + "n_cyclic_shifts_real_beats": { + "min": 10, + "persist": 12, + "best1L": 8, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 0, + "persist": 15, + "best1L": 9, + "mean": 15 + }, + "bandconst": { + "min": 0.2452594154824813, + "persist": 0.11121803348263104, + "best1L": 0.1703273767294983, + "mean": 0.10650555761232496 + }, + "real_minus_bandconst": { + "min": 0.01713961673279607, + "persist": -0.00012227718252687259, + "best1L": -0.011526565284778661, + "mean": 0.005041745336403919 + } + }, + "StackExchange": { + "real": { + "min": 0.26430599875748156, + "persist": 0.11614819277698794, + "best1L": 0.16773439900328715, + "mean": 0.1148480885847209 + }, + "cyclic": { + "cyc1": { + "min": 0.2602175726244847, + "persist": 0.10009923714678734, + "best1L": 0.16052603755767147, + "mean": 0.10274202207383963 + }, + "cyc2": { + "min": 0.2536673714717229, + "persist": 0.08799013131453345, + "best1L": 0.16434343590711553, + "mean": 0.09018820042912967 + }, + "cyc3": { + "min": 0.2457752457509438, + "persist": 0.08087204080075025, + "best1L": 0.15947164700677, + "mean": 0.08212700144951948 + }, + "cyc4": { + "min": 0.23311492390930652, + "persist": 0.07581633444254597, + "best1L": 0.14374379354218642, + "mean": 0.0750785053780792 + }, + "cyc5": { + "min": 0.21508176252245903, + "persist": 0.07449775990098716, + "best1L": 0.10756637731877466, + "mean": 0.07275755010076607 + }, + "cyc6": { + "min": 0.21412456296384336, + "persist": 0.07069029711807767, + "best1L": 0.11151212677359582, + "mean": 0.07154377383591894 + }, + "cyc7": { + "min": 0.22237966544926166, + "persist": 0.07386406383787592, + "best1L": 0.11915480793298533, + "mean": 0.07562776391412347 + }, + "cyc8": { + "min": 0.23209570720791817, + "persist": 0.07579561267048121, + "best1L": 0.13157878750935195, + "mean": 0.07828643491360335 + }, + "cyc9": { + "min": 0.2435741938650608, + "persist": 0.08074995105465253, + "best1L": 0.14232842198883494, + "mean": 0.08321409541158341 + }, + "cyc10": { + "min": 0.2561497452358405, + "persist": 0.08751093907436977, + "best1L": 0.15381717098255954, + "mean": 0.08970952608789771 + }, + "cyc11": { + "min": 0.2670311520497004, + "persist": 0.09546239662449807, + "best1L": 0.15690669262160858, + "mean": 0.09957681812274341 + }, + "cyc12": { + "min": 0.2697738895813624, + "persist": 0.09935908803405862, + "best1L": 0.15857629446933666, + "mean": 0.10632815197832549 + } + }, + "shuf": { + "shuf0": { + "min": 0.26495029404759407, + "persist": 0.08557597155061861, + "best1L": 0.14619056514153878, + "mean": 0.09105743219664034 + }, + "shuf1": { + "min": 0.2697909268240134, + "persist": 0.08558106083267679, + "best1L": 0.15947164700677, + "mean": 0.09051078407210297 + }, + "shuf2": { + "min": 0.2734470379849275, + "persist": 0.0856515144308408, + "best1L": 0.16434343590711553, + "mean": 0.0898064474414031 + }, + "shuf3": { + "min": 0.25384169531365236, + "persist": 0.07636518503228823, + "best1L": 0.15086793849865596, + "mean": 0.0802394265923888 + }, + "shuf4": { + "min": 0.262605519965291, + "persist": 0.0855404496192932, + "best1L": 0.16434343590711553, + "mean": 0.08873274703331005 + } + }, + "cyclic_mean": { + "min": 0.2427488160526587, + "persist": 0.0835589876683015, + "best1L": 0.14246046613423258, + "mean": 0.08559832030796082 + }, + "shuf_mean": { + "min": 0.26492709482709564, + "persist": 0.08374283629314352, + "best1L": 0.15704340449223916, + "mean": 0.08806936746716905 + }, + "real_minus_cyclic_mean": { + "min": 0.021557182704822875, + "persist": 0.032589205108686445, + "best1L": 0.025273932869054577, + "mean": 0.029249768276760083 + }, + "real_minus_shuf_mean": { + "min": -0.0006210960696140799, + "persist": 0.03240535648384442, + "best1L": 0.010690994511047991, + "mean": 0.02677872111755185 + }, + "n_cyclic_shifts_real_beats": { + "min": 10, + "persist": 12, + "best1L": 12, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 7, + "persist": 15, + "best1L": 15, + "mean": 15 + }, + "bandconst": { + "min": 0.2443662917862336, + "persist": 0.11185659411518524, + "best1L": 0.16442335004297395, + "mean": 0.10759148981723671 + }, + "real_minus_bandconst": { + "min": 0.019939706971247972, + "persist": 0.0042915986618027036, + "best1L": 0.003311048960313201, + "mean": 0.00725659876748419 + } + }, + "Github": { + "real": { + "min": 0.2404973489542802, + "persist": 0.10075108803187807, + "best1L": 0.1427786145048837, + "mean": 0.1017404747589563 + }, + "cyclic": { + "cyc1": { + "min": 0.250745386009415, + "persist": 0.09135591405599068, + "best1L": 0.1491064979073902, + "mean": 0.09555235214289635 + }, + "cyc2": { + "min": 0.24781977497041224, + "persist": 0.08774319112611313, + "best1L": 0.15658612690555554, + "mean": 0.0876798498733805 + }, + "cyc3": { + "min": 0.22966599961121878, + "persist": 0.08294373514751593, + "best1L": 0.14641289558882514, + "mean": 0.08176087495297767 + }, + "cyc4": { + "min": 0.21711318269371987, + "persist": 0.0777836432059606, + "best1L": 0.14069318775242817, + "mean": 0.07687635177474779 + }, + "cyc5": { + "min": 0.196179140980045, + "persist": 0.07870196259270112, + "best1L": 0.101258784532547, + "mean": 0.07411237751903872 + }, + "cyc6": { + "min": 0.18732259484628835, + "persist": 0.07552773976543298, + "best1L": 0.1110404654716452, + "mean": 0.07061376934721827 + }, + "cyc7": { + "min": 0.19087662448485693, + "persist": 0.07810676110287508, + "best1L": 0.11456959539403518, + "mean": 0.07297495943792642 + }, + "cyc8": { + "min": 0.19336260557174684, + "persist": 0.07650083076829711, + "best1L": 0.11795583966498574, + "mean": 0.07306196825393739 + }, + "cyc9": { + "min": 0.1964816431204478, + "persist": 0.07946545370699218, + "best1L": 0.12304757951448361, + "mean": 0.07296326280010339 + }, + "cyc10": { + "min": 0.20920862158139547, + "persist": 0.08091156721735994, + "best1L": 0.12490706741809844, + "mean": 0.07686499999537587 + }, + "cyc11": { + "min": 0.22258305897315345, + "persist": 0.08775251284241677, + "best1L": 0.1256153185851872, + "mean": 0.08419476814078501 + }, + "cyc12": { + "min": 0.23385596747199694, + "persist": 0.09066073295349876, + "best1L": 0.13241185080260037, + "mean": 0.09051228143059864 + } + }, + "shuf": { + "shuf0": { + "min": 0.23005612989266713, + "persist": 0.08227930379410585, + "best1L": 0.12084485841915012, + "mean": 0.08252969522026177 + }, + "shuf1": { + "min": 0.23197778984904288, + "persist": 0.08138537457368027, + "best1L": 0.14641289558882514, + "mean": 0.08080037113783571 + }, + "shuf2": { + "min": 0.2489109992980957, + "persist": 0.08541673840178798, + "best1L": 0.15658612690555554, + "mean": 0.08287016136410574 + }, + "shuf3": { + "min": 0.2092815740654866, + "persist": 0.07763321877767643, + "best1L": 0.1231656660946707, + "mean": 0.07479944115934464 + }, + "shuf4": { + "min": 0.2418087411671877, + "persist": 0.08162906107803186, + "best1L": 0.15658612690555554, + "mean": 0.08410498081610944 + } + }, + "cyclic_mean": { + "min": 0.2146012166928914, + "persist": 0.08228783704042952, + "best1L": 0.1286337674614818, + "mean": 0.07976398463908217 + }, + "shuf_mean": { + "min": 0.232407046854496, + "persist": 0.08166873932505647, + "best1L": 0.14071913478275141, + "mean": 0.08102092993953146 + }, + "real_minus_cyclic_mean": { + "min": 0.025896132261388816, + "persist": 0.01846325099144855, + "best1L": 0.014144847043401881, + "mean": 0.021976490119874126 + }, + "real_minus_shuf_mean": { + "min": 0.008090302099784202, + "persist": 0.0190823487068216, + "best1L": 0.0020594797221322736, + "mean": 0.02071954481942484 + }, + "n_cyclic_shifts_real_beats": { + "min": 10, + "persist": 12, + "best1L": 9, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 10, + "persist": 15, + "best1L": 6, + "mean": 15 + }, + "bandconst": { + "min": 0.2230183318257332, + "persist": 0.10451118210330605, + "best1L": 0.14676144449040293, + "mean": 0.1005614094185428 + }, + "real_minus_bandconst": { + "min": 0.01747901712854702, + "persist": -0.00376009407142798, + "best1L": -0.003982829985519243, + "mean": 0.001179065340413496 + } + }, + "OOD_News_2024": { + "real": { + "min": 0.25006329963604607, + "persist": 0.09791991202315936, + "best1L": 0.14915074508171527, + "mean": 0.10013916230229183 + }, + "cyclic": { + "cyc1": { + "min": 0.24838644166787466, + "persist": 0.07736670548717181, + "best1L": 0.15343523845076562, + "mean": 0.08781252768762314 + }, + "cyc2": { + "min": 0.24832093939185143, + "persist": 0.06925444814066092, + "best1L": 0.15854859087460985, + "mean": 0.07996690121407693 + }, + "cyc3": { + "min": 0.2375550676137209, + "persist": 0.06432431346426408, + "best1L": 0.1516452020344635, + "mean": 0.0728812705395588 + }, + "cyc4": { + "min": 0.22962008491158487, + "persist": 0.06180745388070742, + "best1L": 0.13410183292192718, + "mean": 0.06818618745041581 + }, + "cyc5": { + "min": 0.22249935256938141, + "persist": 0.060894018854014575, + "best1L": 0.11318647386506199, + "mean": 0.0677094748775021 + }, + "cyc6": { + "min": 0.2168354636679093, + "persist": 0.06646195489447564, + "best1L": 0.12627609216918548, + "mean": 0.06940446286760749 + }, + "cyc7": { + "min": 0.21790604069828987, + "persist": 0.07003692364475379, + "best1L": 0.13679101125647625, + "mean": 0.0709469715724341 + }, + "cyc8": { + "min": 0.22478230198224386, + "persist": 0.07076994137217602, + "best1L": 0.15115354309479395, + "mean": 0.07350301379815509 + }, + "cyc9": { + "min": 0.23548510347803434, + "persist": 0.07342660203576087, + "best1L": 0.14763435355077187, + "mean": 0.07583844905826621 + }, + "cyc10": { + "min": 0.23747988566756248, + "persist": 0.08154075775140275, + "best1L": 0.137961953257521, + "mean": 0.0818324163197898 + }, + "cyc11": { + "min": 0.24752230544884998, + "persist": 0.09057725531359513, + "best1L": 0.14421733776107432, + "mean": 0.08966705768595999 + }, + "cyc12": { + "min": 0.24759460166096686, + "persist": 0.0911966457652549, + "best1L": 0.13649143888615073, + "mean": 0.09300372272085113 + } + }, + "shuf": { + "shuf0": { + "min": 0.2529121642311414, + "persist": 0.07336478893800329, + "best1L": 0.1315648317957918, + "mean": 0.07998725675727027 + }, + "shuf1": { + "min": 0.24983646223942438, + "persist": 0.07173011973500251, + "best1L": 0.1516452020344635, + "mean": 0.07816165640281561 + }, + "shuf2": { + "min": 0.26930968364079794, + "persist": 0.07021267929424842, + "best1L": 0.15854859087460985, + "mean": 0.08115636848199825 + }, + "shuf3": { + "min": 0.246286470318834, + "persist": 0.07072378030667702, + "best1L": 0.15115354309479395, + "mean": 0.07556945861866458 + }, + "shuf4": { + "min": 0.26753050498664377, + "persist": 0.06910301434497039, + "best1L": 0.15854859087460985, + "mean": 0.07870777266753766 + } + }, + "cyclic_mean": { + "min": 0.23449896572985582, + "persist": 0.07313808505035316, + "best1L": 0.14095358901023347, + "mean": 0.07756270464935339 + }, + "shuf_mean": { + "min": 0.2571750570833683, + "persist": 0.07102687652378033, + "best1L": 0.15029215173485377, + "mean": 0.07871650258565728 + }, + "real_minus_cyclic_mean": { + "min": 0.015564333906190247, + "persist": 0.024781826972806204, + "best1L": 0.008197156071481804, + "mean": 0.022576457652938448 + }, + "real_minus_shuf_mean": { + "min": -0.007111757447322231, + "persist": 0.026893035499379034, + "best1L": -0.0011414066531384992, + "mean": 0.021422659716634554 + }, + "n_cyclic_shifts_real_beats": { + "min": 12, + "persist": 12, + "best1L": 8, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 5, + "persist": 15, + "best1L": 3, + "mean": 15 + }, + "bandconst": { + "min": 0.2279623795300722, + "persist": 0.1022291832913955, + "best1L": 0.15719435699284076, + "mean": 0.09586961104367406 + }, + "real_minus_bandconst": { + "min": 0.022100920105973865, + "persist": -0.004309271268236145, + "best1L": -0.008043611911125487, + "mean": 0.004269551258617776 + } + }, + "OOD_arXiv_2023": { + "real": { + "min": 0.26927303777386746, + "persist": 0.10589559425910314, + "best1L": 0.1499242829935004, + "mean": 0.10793894144619863 + }, + "cyclic": { + "cyc1": { + "min": 0.26060471857587497, + "persist": 0.08966357906659445, + "best1L": 0.15334509264988203, + "mean": 0.09414112445456572 + }, + "cyc2": { + "min": 0.2598521575331688, + "persist": 0.07659630191822847, + "best1L": 0.16083774268627166, + "mean": 0.08371371819491451 + }, + "cyc3": { + "min": 0.24282207923630875, + "persist": 0.07331365942955018, + "best1L": 0.1569489906852444, + "mean": 0.07719842380193326 + }, + "cyc4": { + "min": 0.24344526827335358, + "persist": 0.07008327587197224, + "best1L": 0.14271940143468478, + "mean": 0.07331205269736715 + }, + "cyc5": { + "min": 0.22919819975892702, + "persist": 0.06833742608626683, + "best1L": 0.11603862453872958, + "mean": 0.07203832941893966 + }, + "cyc6": { + "min": 0.22728197500109673, + "persist": 0.06362237200761835, + "best1L": 0.12774722687900067, + "mean": 0.07054500334466306 + }, + "cyc7": { + "min": 0.23291768978039423, + "persist": 0.06914585682873925, + "best1L": 0.13943094389202693, + "mean": 0.0751008594671312 + }, + "cyc8": { + "min": 0.24113517850637436, + "persist": 0.07280987727766236, + "best1L": 0.14222827591001988, + "mean": 0.07749475625594361 + }, + "cyc9": { + "min": 0.2415068676074346, + "persist": 0.08006956474855542, + "best1L": 0.13784233232339224, + "mean": 0.08056066902070784 + }, + "cyc10": { + "min": 0.2624349524577459, + "persist": 0.08713615220040083, + "best1L": 0.1381059776370724, + "mean": 0.08827518017690938 + }, + "cyc11": { + "min": 0.2686994945009549, + "persist": 0.09368644704421361, + "best1L": 0.1436638013459742, + "mean": 0.09648851716535715 + }, + "cyc12": { + "min": 0.2737049493938684, + "persist": 0.09519674368202687, + "best1L": 0.14379627344508966, + "mean": 0.10057515100164052 + } + }, + "shuf": { + "shuf0": { + "min": 0.25724916669229664, + "persist": 0.0814654524748524, + "best1L": 0.13681389347960551, + "mean": 0.08524249728264598 + }, + "shuf1": { + "min": 0.2708121123413245, + "persist": 0.0803774683115383, + "best1L": 0.1569489906852444, + "mean": 0.08540571896935431 + }, + "shuf2": { + "min": 0.29168893073995905, + "persist": 0.07886028519521157, + "best1L": 0.16083774268627166, + "mean": 0.08751284766745253 + }, + "shuf3": { + "min": 0.26098317429423334, + "persist": 0.07511783465743065, + "best1L": 0.14222827591001988, + "mean": 0.07852778857549986 + }, + "shuf4": { + "min": 0.2709314721326033, + "persist": 0.08000194976727168, + "best1L": 0.16083774268627166, + "mean": 0.08366112211310209 + } + }, + "cyclic_mean": { + "min": 0.2486336275521252, + "persist": 0.07830510468015241, + "best1L": 0.14189205695228238, + "mean": 0.08245364875000609 + }, + "shuf_mean": { + "min": 0.2703329712400834, + "persist": 0.07916459808126092, + "best1L": 0.1515333290894826, + "mean": 0.08406999492161095 + }, + "real_minus_cyclic_mean": { + "min": 0.02063941022174226, + "persist": 0.02759048957895073, + "best1L": 0.008032226041218032, + "mean": 0.02548529269619254 + }, + "real_minus_shuf_mean": { + "min": -0.0010599334662159121, + "persist": 0.02673099617784222, + "best1L": -0.0016090460959821973, + "mean": 0.02386894652458768 + }, + "n_cyclic_shifts_real_beats": { + "min": 11, + "persist": 12, + "best1L": 9, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 7, + "persist": 15, + "best1L": 6, + "mean": 15 + }, + "bandconst": { + "min": 0.23723225332796574, + "persist": 0.10574559147159258, + "best1L": 0.16357198410357038, + "mean": 0.10255865676997182 + }, + "real_minus_bandconst": { + "min": 0.03204078444590172, + "persist": 0.00015000278751055307, + "best1L": -0.013647701110069965, + "mean": 0.00538028467622681 + } + }, + "OOD_CommonPile": { + "real": { + "min": 0.2528811464707057, + "persist": 0.10663878470659256, + "best1L": 0.15052258966801066, + "mean": 0.1075210745289671 + }, + "cyclic": { + "cyc1": { + "min": 0.2612555713703235, + "persist": 0.09152340764800708, + "best1L": 0.1535651180582742, + "mean": 0.09599785156359968 + }, + "cyc2": { + "min": 0.25781926388541854, + "persist": 0.07844462196032206, + "best1L": 0.16019913107156752, + "mean": 0.0847824488291875 + }, + "cyc3": { + "min": 0.2484008375555277, + "persist": 0.07439009255419175, + "best1L": 0.1568204214796424, + "mean": 0.07958643469565477 + }, + "cyc4": { + "min": 0.2428180269896984, + "persist": 0.06985785427192846, + "best1L": 0.14400238114564368, + "mean": 0.07541617397293568 + }, + "cyc5": { + "min": 0.2311974103252093, + "persist": 0.06875039568791787, + "best1L": 0.12145234094932675, + "mean": 0.07367054397413411 + }, + "cyc6": { + "min": 0.21886724084615708, + "persist": 0.06804801635444165, + "best1L": 0.12648244332522154, + "mean": 0.07226377327878697 + }, + "cyc7": { + "min": 0.2232258086403211, + "persist": 0.07167736571282148, + "best1L": 0.13990804363662998, + "mean": 0.07500618123688185 + }, + "cyc8": { + "min": 0.23181134338180223, + "persist": 0.07736211278631042, + "best1L": 0.13537564662595591, + "mean": 0.07762297994445246 + }, + "cyc9": { + "min": 0.22952917367219924, + "persist": 0.0817146703756104, + "best1L": 0.13224772308021784, + "mean": 0.08039391281841418 + }, + "cyc10": { + "min": 0.24342885265747707, + "persist": 0.08733500379603355, + "best1L": 0.1366285028246542, + "mean": 0.08646240630336345 + }, + "cyc11": { + "min": 0.25070178558429085, + "persist": 0.09197888503161569, + "best1L": 0.14232506658881902, + "mean": 0.094329298073008 + }, + "cyc12": { + "min": 0.2559330403804779, + "persist": 0.09424131624400615, + "best1L": 0.1434563041354219, + "mean": 0.09878170399565417 + } + }, + "shuf": { + "shuf0": { + "min": 0.2534373787542184, + "persist": 0.07891241051256656, + "best1L": 0.13101446629346658, + "mean": 0.08479975875308857 + }, + "shuf1": { + "min": 0.2605077380935351, + "persist": 0.0812414643044273, + "best1L": 0.1568204214796424, + "mean": 0.0856663490830658 + }, + "shuf2": { + "min": 0.2805248742302259, + "persist": 0.0792110500857234, + "best1L": 0.16019913107156752, + "mean": 0.0881531578172237 + }, + "shuf3": { + "min": 0.24788121059536933, + "persist": 0.07754282311846812, + "best1L": 0.13537564662595591, + "mean": 0.07875811186953424 + }, + "shuf4": { + "min": 0.26148344750205677, + "persist": 0.07995966064433256, + "best1L": 0.16019913107156752, + "mean": 0.08329552803307963 + } + }, + "cyclic_mean": { + "min": 0.24124902960740857, + "persist": 0.07961031186860054, + "best1L": 0.14103859357678125, + "mean": 0.08285947572383941 + }, + "shuf_mean": { + "min": 0.2607669298350811, + "persist": 0.07937348173310359, + "best1L": 0.14872175930843998, + "mean": 0.08413458111119838 + }, + "real_minus_cyclic_mean": { + "min": 0.011632116863297126, + "persist": 0.02702847283799202, + "best1L": 0.009483996091229407, + "mean": 0.024661598805127694 + }, + "real_minus_shuf_mean": { + "min": -0.007885783364375398, + "persist": 0.027265302973488975, + "best1L": 0.001800830359570682, + "mean": 0.023386493417768717 + }, + "n_cyclic_shifts_real_beats": { + "min": 9, + "persist": 12, + "best1L": 9, + "mean": 12 + }, + "n_shuf_draws_real_beats": { + "min": 4, + "persist": 15, + "best1L": 6, + "mean": 15 + }, + "bandconst": { + "min": 0.22967875885466735, + "persist": 0.10634291693568229, + "best1L": 0.15963473555942376, + "mean": 0.1013671363847187 + }, + "real_minus_bandconst": { + "min": 0.02320238761603835, + "persist": 0.0002958677709102714, + "best1L": -0.009112145891413098, + "mean": 0.006153938144248403 + } + } + }, + "across_corpora": { + "min": { + "n_corpora_real_beats_cyclic_mean": 8, + "n_corpora_real_beats_shuf_mean": 1, + "median_real_minus_cyclic_mean": 0.014465743259319833, + "median_real_minus_shuf_mean": -0.0055638649438818355, + "n_corpora": 8, + "total_cyclic_cells_real_beats": 79, + "total_cyclic_cells": 96, + "total_shuf_draws_real_beats": 36, + "total_shuf_draws": 120, + "n_corpora_real_beats_bandconst": 8, + "median_real_minus_bandconst": 0.02102031353861092 + }, + "persist": { + "n_corpora_real_beats_cyclic_mean": 8, + "n_corpora_real_beats_shuf_mean": 8, + "median_real_minus_cyclic_mean": 0.02660568692048805, + "median_real_minus_shuf_mean": 0.027079169236434004, + "n_corpora": 8, + "total_cyclic_cells_real_beats": 96, + "total_cyclic_cells": 96, + "total_shuf_draws_real_beats": 120, + "total_shuf_draws": 120, + "n_corpora_real_beats_bandconst": 4, + "median_real_minus_bandconst": 1.3862802491840243e-05 + }, + "best1L": { + "n_corpora_real_beats_cyclic_mean": 8, + "n_corpora_real_beats_shuf_mean": 4, + "median_real_minus_cyclic_mean": 0.008114691056349918, + "median_real_minus_shuf_mean": 0.0003297118532160914, + "n_corpora": 8, + "total_cyclic_cells_real_beats": 71, + "total_cyclic_cells": 96, + "total_shuf_draws_real_beats": 57, + "total_shuf_draws": 120, + "n_corpora_real_beats_bandconst": 1, + "median_real_minus_bandconst": -0.010236124694347384 + }, + "mean": { + "n_corpora_real_beats_cyclic_mean": 8, + "n_corpora_real_beats_shuf_mean": 8, + "median_real_minus_cyclic_mean": 0.024525440611123328, + "median_real_minus_shuf_mean": 0.023291587424852572, + "n_corpora": 8, + "total_cyclic_cells_real_beats": 96, + "total_cyclic_cells": 96, + "total_shuf_draws_real_beats": 120, + "total_shuf_draws": 120, + "n_corpora_real_beats_bandconst": 7, + "median_real_minus_bandconst": 0.00491319595334621 + } + }, + "per_shift": { + "cyc1": { + "min": { + "median_real_minus_shift": -0.001458528699974232, + "n_corpora_real_beats": 4 + }, + "persist": { + "median_real_minus_shift": 0.016140485411354648, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": -0.003824687143787739, + "n_corpora_real_beats": 1 + }, + "mean": { + "median_real_minus_shift": 0.011814644738124346, + "n_corpora_real_beats": 8 + } + }, + "cyc2": { + "min": { + "median_real_minus_shift": -0.0013782194815575993, + "n_corpora_real_beats": 4 + }, + "persist": { + "median_real_minus_shift": 0.028176112104362494, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": -0.009857251262292269, + "n_corpora_real_beats": 1 + }, + "mean": { + "median_real_minus_shift": 0.021688856534400162, + "n_corpora_real_beats": 8 + } + }, + "cyc3": { + "min": { + "median_real_minus_shift": 0.011669790682693304, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.032415313490976885, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": -0.004823684529401365, + "n_corpora_real_beats": 2 + }, + "mean": { + "median_real_minus_shift": 0.027746969798821595, + "n_corpora_real_beats": 8 + } + }, + "cyc4": { + "min": { + "median_real_minus_shift": 0.019478259732325864, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.03644669428855802, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.007727582613006226, + "n_corpora_real_beats": 8 + }, + "mean": { + "median_real_minus_shift": 0.032732118401700296, + "n_corpora_real_beats": 8 + } + }, + "cyc5": { + "min": { + "median_real_minus_shift": 0.03160864077508452, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.03760690350706379, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.036774035931254424, + "n_corpora_real_beats": 8 + }, + "mean": { + "median_real_minus_shift": 0.03487557129104598, + "n_corpora_real_beats": 8 + } + }, + "cyc6": { + "min": { + "median_real_minus_shift": 0.03392774996658168, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.03745746912512308, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.02544901297272493, + "n_corpora_real_beats": 8 + }, + "mean": { + "median_real_minus_shift": 0.03500399694349676, + "n_corpora_real_beats": 8 + } + }, + "cyc7": { + "min": { + "median_real_minus_shift": 0.0309062983840704, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.03281567099814615, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.012194099156962082, + "n_corpora_real_beats": 8 + }, + "mean": { + "median_real_minus_shift": 0.031507127647935895, + "n_corpora_real_beats": 8 + } + }, + "cyc8": { + "min": { + "median_real_minus_shift": 0.023175400371352836, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.028855851746629926, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.005910432167972116, + "n_corpora_real_beats": 7 + }, + "mean": { + "median_real_minus_shift": 0.029288300544766777, + "n_corpora_real_beats": 8 + } + }, + "cyc9": { + "min": { + "median_real_minus_shift": 0.02204188884546361, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.026397063726714506, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.010166040350062133, + "n_corpora_real_beats": 8 + }, + "mean": { + "median_real_minus_shift": 0.027252717068021855, + "n_corpora_real_beats": 8 + } + }, + "cyc10": { + "min": { + "median_real_minus_shift": 0.008804273667434845, + "n_corpora_real_beats": 8 + }, + "persist": { + "median_real_minus_shift": 0.01968028588065257, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.011503548590311136, + "n_corpora_real_beats": 7 + }, + "mean": { + "median_real_minus_shift": 0.020361214747446447, + "n_corpora_real_beats": 8 + } + }, + "cyc11": { + "min": { + "median_real_minus_shift": 0.0013764520796636925, + "n_corpora_real_beats": 5 + }, + "persist": { + "median_real_minus_shift": 0.013829237432219088, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.005596944484083585, + "n_corpora_real_beats": 5 + }, + "mean": { + "median_real_minus_shift": 0.012606344804039389, + "n_corpora_real_beats": 8 + } + }, + "cyc12": { + "min": { + "median_real_minus_shift": -0.004426893002043131, + "n_corpora_real_beats": 2 + }, + "persist": { + "median_real_minus_shift": 0.010394602827727792, + "n_corpora_real_beats": 8 + }, + "best1L": { + "median_real_minus_shift": 0.006612841133028263, + "n_corpora_real_beats": 8 + }, + "mean": { + "median_real_minus_shift": 0.00794186352547676, + "n_corpora_real_beats": 8 + } + } + }, + "dependence": { + "what_rho_bar_measures": "mean off-diagonal Pearson correlation, over the 78 band layer pairs, of the binary recovery events 1[rank_l <= k] across admitted pairs. This is the dependence an existential union over the band consumes. It is NOT Jacobian cosine similarity, which is a different quantity and cannot answer the question.", + "what_union_gap_measures": "observed union minus the independent-layer prediction 1 - prod(1 - p_l). A sequence whose layers are redundant sits below its prediction; a decorrelated one sits near it. Same construction as D1's H3 clause.", + "rho_bar": { + "real": 0.4357606606291873, + "cyclic": 0.3371182815319607, + "shuf": 0.3282108573154325, + "bandconst": 0.44761211052536964 + }, + "union_gap": { + "real": -0.3534625401676056, + "cyclic": -0.28995150806094033, + "shuf": -0.27808403652604846, + "bandconst": -0.36034563469833564 + }, + "per_shift_rho_bar": { + "cyc1": 0.3908536880321446, + "cyc2": 0.3428606978129773, + "cyc3": 0.3350429651105688, + "cyc4": 0.31334498511361225, + "cyc5": 0.321449821770546, + "cyc6": 0.32015285960265566, + "cyc7": 0.3269367554180679, + "cyc8": 0.31870793036761735, + "cyc9": 0.320791576855949, + "cyc10": 0.33001999131270815, + "cyc11": 0.3515563111397482, + "cyc12": 0.3737017958469334 + } + }, + "dependence_per_arm": { + "Wikipedia_en|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.28782638907432556, + "union_observed": 0.04993757978081703, + "union_independent": 0.143623948097229, + "union_gap": -0.09368636831641197, + "mean_per_layer_hit_rate": 0.01181215699762106 + }, + "2": { + "rho_bar": 0.3470577895641327, + "union_observed": 0.07990012317895889, + "union_independent": 0.22066068649291992, + "union_gap": -0.14076056331396103, + "mean_per_layer_hit_rate": 0.018918659538030624 + }, + "5": { + "rho_bar": 0.37533244490623474, + "union_observed": 0.12484394758939743, + "union_independent": 0.41185641288757324, + "union_gap": -0.2870124652981758, + "mean_per_layer_hit_rate": 0.03975799307227135 + }, + "10": { + "rho_bar": 0.4545285999774933, + "union_observed": 0.17977528274059296, + "union_independent": 0.5802339315414429, + "union_gap": -0.4004586488008499, + "mean_per_layer_hit_rate": 0.06395851075649261 + }, + "20": { + "rho_bar": 0.49235549569129944, + "union_observed": 0.24469412863254547, + "union_independent": 0.7317472696304321, + "union_gap": -0.48705314099788666, + "mean_per_layer_hit_rate": 0.09497743099927902 + }, + "50": { + "rho_bar": 0.5147377252578735, + "union_observed": 0.3383270800113678, + "union_independent": 0.8635488152503967, + "union_gap": -0.5252217352390289, + "mean_per_layer_hit_rate": 0.13972918689250946 + }, + "100": { + "rho_bar": 0.5309268236160278, + "union_observed": 0.4319600462913513, + "union_independent": 0.9366166591644287, + "union_gap": -0.5046566128730774, + "mean_per_layer_hit_rate": 0.1879381388425827 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.42896646686962675, + "union_gap_mean_over_k": -0.3484070764056274 + }, + "Wikipedia_en|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.23062802851200104, + "union_observed": 0.041198503226041794, + "union_independent": 0.1138838529586792, + "union_gap": -0.0726853497326374, + "mean_per_layer_hit_rate": 0.009219245053827763 + }, + "2": { + "rho_bar": 0.25868281722068787, + "union_observed": 0.0749063640832901, + "union_independent": 0.1995745301246643, + "union_gap": -0.1246681660413742, + "mean_per_layer_hit_rate": 0.016901949420571327 + }, + "5": { + "rho_bar": 0.32218605279922485, + "union_observed": 0.13108614087104797, + "union_independent": 0.38034868240356445, + "union_gap": -0.24926254153251648, + "mean_per_layer_hit_rate": 0.03591664135456085 + }, + "10": { + "rho_bar": 0.38672423362731934, + "union_observed": 0.17478151619434357, + "union_independent": 0.5320039987564087, + "union_gap": -0.3572224825620651, + "mean_per_layer_hit_rate": 0.056179776787757874 + }, + "20": { + "rho_bar": 0.45363542437553406, + "union_observed": 0.23970037698745728, + "union_independent": 0.6962809562683105, + "union_gap": -0.45658057928085327, + "mean_per_layer_hit_rate": 0.08643042296171188 + }, + "50": { + "rho_bar": 0.5247536897659302, + "union_observed": 0.3408239781856537, + "union_independent": 0.8378467559814453, + "union_gap": -0.4970227777957916, + "mean_per_layer_hit_rate": 0.12868529558181763 + }, + "100": { + "rho_bar": 0.540533185005188, + "union_observed": 0.4344569146633148, + "union_independent": 0.9186291694641113, + "union_gap": -0.4841722548007965, + "mean_per_layer_hit_rate": 0.17266878485679626 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.3881633473294122, + "union_gap_mean_over_k": -0.32023059310657637 + }, + "Wikipedia_en|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.20733600854873657, + "union_observed": 0.04619225859642029, + "union_independent": 0.10705691576004028, + "union_gap": -0.060864657163619995, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.22113816440105438, + "union_observed": 0.06491885334253311, + "union_independent": 0.16096055507659912, + "union_gap": -0.09604170173406601, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.2653510272502899, + "union_observed": 0.13857677578926086, + "union_independent": 0.35419154167175293, + "union_gap": -0.21561476588249207, + "mean_per_layer_hit_rate": 0.032843559980392456 + }, + "10": { + "rho_bar": 0.35547131299972534, + "union_observed": 0.1860174834728241, + "union_independent": 0.5096245408058167, + "union_gap": -0.32360705733299255, + "mean_per_layer_hit_rate": 0.05281859263777733 + }, + "20": { + "rho_bar": 0.39880022406578064, + "union_observed": 0.24719101190567017, + "union_independent": 0.6582322120666504, + "union_gap": -0.4110412001609802, + "mean_per_layer_hit_rate": 0.07826754450798035 + }, + "50": { + "rho_bar": 0.4780617356300354, + "union_observed": 0.3333333432674408, + "union_independent": 0.8110557794570923, + "union_gap": -0.4777224361896515, + "mean_per_layer_hit_rate": 0.11831364035606384 + }, + "100": { + "rho_bar": 0.5106396079063416, + "union_observed": 0.4394506812095642, + "union_independent": 0.8972411155700684, + "union_gap": -0.45779043436050415, + "mean_per_layer_hit_rate": 0.15787957608699799 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3481140115431377, + "union_gap_mean_over_k": -0.29181175040347235 + }, + "Wikipedia_en|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.1363896280527115, + "union_observed": 0.04369537904858589, + "union_independent": 0.08418786525726318, + "union_gap": -0.04049248620867729, + "mean_per_layer_hit_rate": 0.0067223659716546535 + }, + "2": { + "rho_bar": 0.14667685329914093, + "union_observed": 0.07615480571985245, + "union_independent": 0.14595794677734375, + "union_gap": -0.0698031410574913, + "mean_per_layer_hit_rate": 0.012004226446151733 + }, + "5": { + "rho_bar": 0.2705223858356476, + "union_observed": 0.13233457505702972, + "union_independent": 0.32410651445388794, + "union_gap": -0.19177193939685822, + "mean_per_layer_hit_rate": 0.02948237583041191 + }, + "10": { + "rho_bar": 0.36362603306770325, + "union_observed": 0.1885143518447876, + "union_independent": 0.48921728134155273, + "union_gap": -0.30070292949676514, + "mean_per_layer_hit_rate": 0.04993757978081703 + }, + "20": { + "rho_bar": 0.4224107265472412, + "union_observed": 0.23720349371433258, + "union_independent": 0.6301975250244141, + "union_gap": -0.3929940313100815, + "mean_per_layer_hit_rate": 0.0727936178445816 + }, + "50": { + "rho_bar": 0.48592737317085266, + "union_observed": 0.33208489418029785, + "union_independent": 0.7916017174720764, + "union_gap": -0.45951682329177856, + "mean_per_layer_hit_rate": 0.11197541654109955 + }, + "100": { + "rho_bar": 0.5047319531440735, + "union_observed": 0.4219725430011749, + "union_independent": 0.8793290853500366, + "union_gap": -0.4573565423488617, + "mean_per_layer_hit_rate": 0.14779601991176605 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33289785044533865, + "union_gap_mean_over_k": -0.2732339847300734 + }, + "Wikipedia_en|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.1246553361415863, + "union_observed": 0.047440700232982635, + "union_independent": 0.08420717716217041, + "union_gap": -0.036766476929187775, + "mean_per_layer_hit_rate": 0.0067223659716546535 + }, + "2": { + "rho_bar": 0.17956475913524628, + "union_observed": 0.07865168899297714, + "union_independent": 0.15780019760131836, + "union_gap": -0.07914850860834122, + "mean_per_layer_hit_rate": 0.013060597702860832 + }, + "5": { + "rho_bar": 0.257878839969635, + "union_observed": 0.13233457505702972, + "union_independent": 0.3128083348274231, + "union_gap": -0.18047375977039337, + "mean_per_layer_hit_rate": 0.028233937919139862 + }, + "10": { + "rho_bar": 0.3311666250228882, + "union_observed": 0.17977528274059296, + "union_independent": 0.4545453190803528, + "union_gap": -0.2747700363397598, + "mean_per_layer_hit_rate": 0.04513588920235634 + }, + "20": { + "rho_bar": 0.3936232030391693, + "union_observed": 0.23220974206924438, + "union_independent": 0.6044437885284424, + "union_gap": -0.372234046459198, + "mean_per_layer_hit_rate": 0.0681840032339096 + }, + "50": { + "rho_bar": 0.46043962240219116, + "union_observed": 0.3258427083492279, + "union_independent": 0.7672813534736633, + "union_gap": -0.4414386451244354, + "mean_per_layer_hit_rate": 0.10486892610788345 + }, + "100": { + "rho_bar": 0.47940683364868164, + "union_observed": 0.41947564482688904, + "union_independent": 0.8731867074966431, + "union_gap": -0.45371106266975403, + "mean_per_layer_hit_rate": 0.1448189914226532 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.31810503133705686, + "union_gap_mean_over_k": -0.2626489337001528 + }, + "Wikipedia_en|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.1718340665102005, + "union_observed": 0.04244694113731384, + "union_independent": 0.08178675174713135, + "union_gap": -0.039339810609817505, + "mean_per_layer_hit_rate": 0.006530298851430416 + }, + "2": { + "rho_bar": 0.17076218128204346, + "union_observed": 0.06866417080163956, + "union_independent": 0.14125633239746094, + "union_gap": -0.07259216159582138, + "mean_per_layer_hit_rate": 0.01162009034305811 + }, + "5": { + "rho_bar": 0.28570273518562317, + "union_observed": 0.10486891120672226, + "union_independent": 0.295823872089386, + "union_gap": -0.19095496088266373, + "mean_per_layer_hit_rate": 0.026505330577492714 + }, + "10": { + "rho_bar": 0.3197873830795288, + "union_observed": 0.1498127281665802, + "union_independent": 0.43231362104415894, + "union_gap": -0.28250089287757874, + "mean_per_layer_hit_rate": 0.04235090687870979 + }, + "20": { + "rho_bar": 0.3628520965576172, + "union_observed": 0.19225966930389404, + "union_independent": 0.5657541751861572, + "union_gap": -0.3734945058822632, + "mean_per_layer_hit_rate": 0.061653707176446915 + }, + "50": { + "rho_bar": 0.4366202652454376, + "union_observed": 0.2971285879611969, + "union_independent": 0.7556765079498291, + "union_gap": -0.4585479199886322, + "mean_per_layer_hit_rate": 0.10169979184865952 + }, + "100": { + "rho_bar": 0.4512958526611328, + "union_observed": 0.3957552909851074, + "union_independent": 0.8715145587921143, + "union_gap": -0.47575926780700684, + "mean_per_layer_hit_rate": 0.14414674043655396 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3141220829316548, + "union_gap_mean_over_k": -0.27045564566339764 + }, + "Wikipedia_en|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.18903104960918427, + "union_observed": 0.03745318204164505, + "union_independent": 0.07137197256088257, + "union_gap": -0.03391879051923752, + "mean_per_layer_hit_rate": 0.005665994714945555 + }, + "2": { + "rho_bar": 0.19573356211185455, + "union_observed": 0.06367041170597076, + "union_independent": 0.1327836513519287, + "union_gap": -0.06911323964595795, + "mean_per_layer_hit_rate": 0.01085181999951601 + }, + "5": { + "rho_bar": 0.2549557387828827, + "union_observed": 0.10986267030239105, + "union_independent": 0.28440308570861816, + "union_gap": -0.1745404154062271, + "mean_per_layer_hit_rate": 0.025256890803575516 + }, + "10": { + "rho_bar": 0.32587930560112, + "union_observed": 0.15106117725372314, + "union_independent": 0.4345685839653015, + "union_gap": -0.28350740671157837, + "mean_per_layer_hit_rate": 0.04254297539591789 + }, + "20": { + "rho_bar": 0.3757008910179138, + "union_observed": 0.19725343585014343, + "union_independent": 0.5817817449569702, + "union_gap": -0.3845283091068268, + "mean_per_layer_hit_rate": 0.06415057927370071 + }, + "50": { + "rho_bar": 0.42300790548324585, + "union_observed": 0.3096129894256592, + "union_independent": 0.7618947625160217, + "union_gap": -0.45228177309036255, + "mean_per_layer_hit_rate": 0.10304427146911621 + }, + "100": { + "rho_bar": 0.43366140127182007, + "union_observed": 0.40199750661849976, + "union_independent": 0.8801635503768921, + "union_gap": -0.47816604375839233, + "mean_per_layer_hit_rate": 0.14760395884513855 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3139956934111459, + "union_gap_mean_over_k": -0.26800799689122606 + }, + "Wikipedia_en|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.20328477025032043, + "union_observed": 0.039950061589479446, + "union_independent": 0.1002315878868103, + "union_gap": -0.060281526297330856, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.23056362569332123, + "union_observed": 0.054931335151195526, + "union_independent": 0.15122073888778687, + "union_gap": -0.09628940373659134, + "mean_per_layer_hit_rate": 0.012484394945204258 + }, + "5": { + "rho_bar": 0.29539754986763, + "union_observed": 0.10237203538417816, + "union_independent": 0.29115647077560425, + "union_gap": -0.1887844353914261, + "mean_per_layer_hit_rate": 0.025929125025868416 + }, + "10": { + "rho_bar": 0.35439324378967285, + "union_observed": 0.14107365906238556, + "union_independent": 0.4336441159248352, + "union_gap": -0.29257045686244965, + "mean_per_layer_hit_rate": 0.04235090687870979 + }, + "20": { + "rho_bar": 0.3877582252025604, + "union_observed": 0.20474407076835632, + "union_independent": 0.5974748134613037, + "union_gap": -0.3927307426929474, + "mean_per_layer_hit_rate": 0.06664745509624481 + }, + "50": { + "rho_bar": 0.3958510756492615, + "union_observed": 0.32209739089012146, + "union_independent": 0.7753134965896606, + "union_gap": -0.4532161056995392, + "mean_per_layer_hit_rate": 0.10659752786159515 + }, + "100": { + "rho_bar": 0.4140860140323639, + "union_observed": 0.40449437499046326, + "union_independent": 0.8818570375442505, + "union_gap": -0.47736266255378723, + "mean_per_layer_hit_rate": 0.14798808097839355 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.32590492921216146, + "union_gap_mean_over_k": -0.280176476176296 + }, + "Wikipedia_en|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.17377936840057373, + "union_observed": 0.04369537904858589, + "union_independent": 0.1036335825920105, + "union_gap": -0.059938203543424606, + "mean_per_layer_hit_rate": 0.008354941383004189 + }, + "2": { + "rho_bar": 0.25113645195961, + "union_observed": 0.06491885334253311, + "union_independent": 0.1727433204650879, + "union_gap": -0.10782446712255478, + "mean_per_layer_hit_rate": 0.014405070804059505 + }, + "5": { + "rho_bar": 0.3080434203147888, + "union_observed": 0.1086142286658287, + "union_independent": 0.293795645236969, + "union_gap": -0.1851814165711403, + "mean_per_layer_hit_rate": 0.026217229664325714 + }, + "10": { + "rho_bar": 0.3479316532611847, + "union_observed": 0.15605492889881134, + "union_independent": 0.45156848430633545, + "union_gap": -0.2955135554075241, + "mean_per_layer_hit_rate": 0.04465572163462639 + }, + "20": { + "rho_bar": 0.38247182965278625, + "union_observed": 0.2122347056865692, + "union_independent": 0.6054459810256958, + "union_gap": -0.3932112753391266, + "mean_per_layer_hit_rate": 0.06799193471670151 + }, + "50": { + "rho_bar": 0.3947775363922119, + "union_observed": 0.32459425926208496, + "union_independent": 0.7824826240539551, + "union_gap": -0.4578883647918701, + "mean_per_layer_hit_rate": 0.10851819813251495 + }, + "100": { + "rho_bar": 0.4023515284061432, + "union_observed": 0.4294631779193878, + "union_independent": 0.8941240906715393, + "union_gap": -0.4646609127521515, + "mean_per_layer_hit_rate": 0.154038205742836 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.32292739834104267, + "union_gap_mean_over_k": -0.28060259936111315 + }, + "Wikipedia_en|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.205686554312706, + "union_observed": 0.039950061589479446, + "union_independent": 0.0991552472114563, + "union_gap": -0.05920518562197685, + "mean_per_layer_hit_rate": 0.007970805279910564 + }, + "2": { + "rho_bar": 0.22266560792922974, + "union_observed": 0.06117353215813637, + "union_independent": 0.1578657627105713, + "union_gap": -0.09669223055243492, + "mean_per_layer_hit_rate": 0.013060597702860832 + }, + "5": { + "rho_bar": 0.28937608003616333, + "union_observed": 0.10112359374761581, + "union_independent": 0.2929452657699585, + "union_gap": -0.19182167202234268, + "mean_per_layer_hit_rate": 0.026121195405721664 + }, + "10": { + "rho_bar": 0.3565922975540161, + "union_observed": 0.144818976521492, + "union_independent": 0.4553481340408325, + "union_gap": -0.3105291575193405, + "mean_per_layer_hit_rate": 0.04513588920235634 + }, + "20": { + "rho_bar": 0.37712758779525757, + "union_observed": 0.21847690641880035, + "union_independent": 0.6214895248413086, + "union_gap": -0.40301261842250824, + "mean_per_layer_hit_rate": 0.07077691704034805 + }, + "50": { + "rho_bar": 0.38243696093559265, + "union_observed": 0.32209739089012146, + "union_independent": 0.793201744556427, + "union_gap": -0.47110435366630554, + "mean_per_layer_hit_rate": 0.11159127205610275 + }, + "100": { + "rho_bar": 0.38963583111763, + "union_observed": 0.4394506812095642, + "union_independent": 0.9030770659446716, + "union_gap": -0.4636263847351074, + "mean_per_layer_hit_rate": 0.15951214730739594 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.3176458456686565, + "union_gap_mean_over_k": -0.2851416575057166 + }, + "Wikipedia_en|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.20613126456737518, + "union_observed": 0.041198503226041794, + "union_independent": 0.10254669189453125, + "union_gap": -0.061348188668489456, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.22030764818191528, + "union_observed": 0.06117353215813637, + "union_independent": 0.16856729984283447, + "union_gap": -0.1073937676846981, + "mean_per_layer_hit_rate": 0.014020935632288456 + }, + "5": { + "rho_bar": 0.3031577467918396, + "union_observed": 0.10486891120672226, + "union_independent": 0.3252567648887634, + "union_gap": -0.22038785368204117, + "mean_per_layer_hit_rate": 0.02957841381430626 + }, + "10": { + "rho_bar": 0.35297536849975586, + "union_observed": 0.16978776454925537, + "union_independent": 0.4890288710594177, + "union_gap": -0.31924110651016235, + "mean_per_layer_hit_rate": 0.04964947700500488 + }, + "20": { + "rho_bar": 0.3866112232208252, + "union_observed": 0.23970037698745728, + "union_independent": 0.6598300933837891, + "union_gap": -0.4201297163963318, + "mean_per_layer_hit_rate": 0.07807548344135284 + }, + "50": { + "rho_bar": 0.3913654088973999, + "union_observed": 0.3570536971092224, + "union_independent": 0.818405032157898, + "union_gap": -0.46135133504867554, + "mean_per_layer_hit_rate": 0.11994622647762299 + }, + "100": { + "rho_bar": 0.38963258266448975, + "union_observed": 0.4656679034233093, + "union_independent": 0.9152053594589233, + "union_gap": -0.449537456035614, + "mean_per_layer_hit_rate": 0.16757899522781372 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3214544632605144, + "union_gap_mean_over_k": -0.29134134628943037 + }, + "Wikipedia_en|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.2094871550798416, + "union_observed": 0.056179776787757874, + "union_independent": 0.11839979887008667, + "union_gap": -0.062220022082328796, + "mean_per_layer_hit_rate": 0.009603381156921387 + }, + "2": { + "rho_bar": 0.2677067220211029, + "union_observed": 0.0774032473564148, + "union_independent": 0.19781333208084106, + "union_gap": -0.12041008472442627, + "mean_per_layer_hit_rate": 0.016709882766008377 + }, + "5": { + "rho_bar": 0.3029088079929352, + "union_observed": 0.12234706431627274, + "union_independent": 0.3546082377433777, + "union_gap": -0.23226117342710495, + "mean_per_layer_hit_rate": 0.032843563705682755 + }, + "10": { + "rho_bar": 0.3529209792613983, + "union_observed": 0.1960049867630005, + "union_independent": 0.5230611562728882, + "union_gap": -0.3270561695098877, + "mean_per_layer_hit_rate": 0.054643236100673676 + }, + "20": { + "rho_bar": 0.3944692015647888, + "union_observed": 0.26716604828834534, + "union_independent": 0.6921995282173157, + "union_gap": -0.42503347992897034, + "mean_per_layer_hit_rate": 0.08508595079183578 + }, + "50": { + "rho_bar": 0.407718688249588, + "union_observed": 0.3770287036895752, + "union_independent": 0.8464624881744385, + "union_gap": -0.4694337844848633, + "mean_per_layer_hit_rate": 0.13118217885494232 + }, + "100": { + "rho_bar": 0.4222315847873688, + "union_observed": 0.4731585383415222, + "union_independent": 0.9268794655799866, + "union_gap": -0.45372092723846436, + "mean_per_layer_hit_rate": 0.1773744374513626 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.3367775912795748, + "union_gap_mean_over_k": -0.29859080591372084 + }, + "Wikipedia_en|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.19877882301807404, + "union_observed": 0.05243445560336113, + "union_independent": 0.1337854266166687, + "union_gap": -0.08135097101330757, + "mean_per_layer_hit_rate": 0.010947853326797485 + }, + "2": { + "rho_bar": 0.2623841166496277, + "union_observed": 0.07116104662418365, + "union_independent": 0.20278406143188477, + "union_gap": -0.1316230148077011, + "mean_per_layer_hit_rate": 0.017190052196383476 + }, + "5": { + "rho_bar": 0.3079215884208679, + "union_observed": 0.13358302414417267, + "union_independent": 0.37667614221572876, + "union_gap": -0.2430931180715561, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.3988189995288849, + "union_observed": 0.19850187003612518, + "union_independent": 0.5533203482627869, + "union_gap": -0.3548184782266617, + "mean_per_layer_hit_rate": 0.05944492295384407 + }, + "20": { + "rho_bar": 0.42787131667137146, + "union_observed": 0.27590513229370117, + "union_independent": 0.7166036367416382, + "union_gap": -0.440698504447937, + "mean_per_layer_hit_rate": 0.09094401448965073 + }, + "50": { + "rho_bar": 0.44675543904304504, + "union_observed": 0.3595505654811859, + "union_independent": 0.84466552734375, + "union_gap": -0.4851149618625641, + "mean_per_layer_hit_rate": 0.13099010288715363 + }, + "100": { + "rho_bar": 0.4728594124317169, + "union_observed": 0.4569288492202759, + "union_independent": 0.928085207939148, + "union_gap": -0.47115635871887207, + "mean_per_layer_hit_rate": 0.1794871687889099 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.359341385109084, + "union_gap_mean_over_k": -0.3154079153069428 + }, + "Wikipedia_en|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.12269321829080582, + "union_observed": 0.054931335151195526, + "union_independent": 0.10473740100860596, + "union_gap": -0.04980606585741043, + "mean_per_layer_hit_rate": 0.008450974710285664 + }, + "2": { + "rho_bar": 0.19482207298278809, + "union_observed": 0.07865168899297714, + "union_independent": 0.18915188312530518, + "union_gap": -0.11050019413232803, + "mean_per_layer_hit_rate": 0.015941612422466278 + }, + "5": { + "rho_bar": 0.3091031610965729, + "union_observed": 0.12983770668506622, + "union_independent": 0.347068727016449, + "union_gap": -0.21723102033138275, + "mean_per_layer_hit_rate": 0.03207528963685036 + }, + "10": { + "rho_bar": 0.39853227138519287, + "union_observed": 0.19725343585014343, + "union_independent": 0.5110410451889038, + "union_gap": -0.3137876093387604, + "mean_per_layer_hit_rate": 0.05310669168829918 + }, + "20": { + "rho_bar": 0.42232728004455566, + "union_observed": 0.24843944609165192, + "union_independent": 0.6601212024688721, + "union_gap": -0.41168175637722015, + "mean_per_layer_hit_rate": 0.07865168154239655 + }, + "50": { + "rho_bar": 0.4617465138435364, + "union_observed": 0.3408239781856537, + "union_independent": 0.8118493556976318, + "union_gap": -0.47102537751197815, + "mean_per_layer_hit_rate": 0.11869779229164124 + }, + "100": { + "rho_bar": 0.4461570084095001, + "union_observed": 0.45817726850509644, + "union_independent": 0.9019508361816406, + "union_gap": -0.4437735676765442, + "mean_per_layer_hit_rate": 0.16047248244285583 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3364830751504217, + "union_gap_mean_over_k": -0.2882579416036606 + }, + "Wikipedia_en|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.17603449523448944, + "union_observed": 0.04494382068514824, + "union_independent": 0.10365265607833862, + "union_gap": -0.058708835393190384, + "mean_per_layer_hit_rate": 0.008354941383004189 + }, + "2": { + "rho_bar": 0.22487086057662964, + "union_observed": 0.07615480571985245, + "union_independent": 0.17480307817459106, + "union_gap": -0.09864827245473862, + "mean_per_layer_hit_rate": 0.01459713838994503 + }, + "5": { + "rho_bar": 0.2531352937221527, + "union_observed": 0.12234706431627274, + "union_independent": 0.3209245800971985, + "union_gap": -0.19857751578092575, + "mean_per_layer_hit_rate": 0.02909824438393116 + }, + "10": { + "rho_bar": 0.34538137912750244, + "union_observed": 0.17727839946746826, + "union_independent": 0.4877496361732483, + "union_gap": -0.31047123670578003, + "mean_per_layer_hit_rate": 0.049649473279714584 + }, + "20": { + "rho_bar": 0.40551963448524475, + "union_observed": 0.24344569444656372, + "union_independent": 0.6425550580024719, + "union_gap": -0.3991093635559082, + "mean_per_layer_hit_rate": 0.0750984400510788 + }, + "50": { + "rho_bar": 0.449736624956131, + "union_observed": 0.34456929564476013, + "union_independent": 0.808234453201294, + "union_gap": -0.4636651575565338, + "mean_per_layer_hit_rate": 0.11716124415397644 + }, + "100": { + "rho_bar": 0.44770410656929016, + "union_observed": 0.4731585383415222, + "union_independent": 0.908395528793335, + "union_gap": -0.43523699045181274, + "mean_per_layer_hit_rate": 0.16440987586975098 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3289117706673486, + "union_gap_mean_over_k": -0.2806310531284128 + }, + "Wikipedia_en|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.15157507359981537, + "union_observed": 0.05742821469902992, + "union_independent": 0.10603052377700806, + "union_gap": -0.048602309077978134, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.2077951580286026, + "union_observed": 0.09238451719284058, + "union_independent": 0.17810726165771484, + "union_gap": -0.08572274446487427, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.25815054774284363, + "union_observed": 0.1548064947128296, + "union_independent": 0.3507782816886902, + "union_gap": -0.1959717869758606, + "mean_per_layer_hit_rate": 0.032363392412662506 + }, + "10": { + "rho_bar": 0.3254725933074951, + "union_observed": 0.21847690641880035, + "union_independent": 0.5155462026596069, + "union_gap": -0.2970692962408066, + "mean_per_layer_hit_rate": 0.053490832448005676 + }, + "20": { + "rho_bar": 0.37302443385124207, + "union_observed": 0.2871410846710205, + "union_independent": 0.6652213335037231, + "union_gap": -0.37808024883270264, + "mean_per_layer_hit_rate": 0.07932392507791519 + }, + "50": { + "rho_bar": 0.3910556733608246, + "union_observed": 0.4007490575313568, + "union_independent": 0.8161470890045166, + "union_gap": -0.4153980314731598, + "mean_per_layer_hit_rate": 0.11965811997652054 + }, + "100": { + "rho_bar": 0.40752488374710083, + "union_observed": 0.49687889218330383, + "union_independent": 0.903790295124054, + "union_gap": -0.4069114029407501, + "mean_per_layer_hit_rate": 0.16047246754169464 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.30208548051970346, + "union_gap_mean_over_k": -0.2611079742865903 + }, + "Wikipedia_en|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.14195336401462555, + "union_observed": 0.05118601769208908, + "union_independent": 0.10930472612380981, + "union_gap": -0.058118708431720734, + "mean_per_layer_hit_rate": 0.008835108950734138 + }, + "2": { + "rho_bar": 0.17511577904224396, + "union_observed": 0.08863919973373413, + "union_independent": 0.17800235748291016, + "union_gap": -0.08936315774917603, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2635638415813446, + "union_observed": 0.144818976521492, + "union_independent": 0.33499377965927124, + "union_gap": -0.19017480313777924, + "mean_per_layer_hit_rate": 0.03063478320837021 + }, + "10": { + "rho_bar": 0.3183702230453491, + "union_observed": 0.21348313987255096, + "union_independent": 0.4896339774131775, + "union_gap": -0.2761508375406265, + "mean_per_layer_hit_rate": 0.04974551498889923 + }, + "20": { + "rho_bar": 0.3700691759586334, + "union_observed": 0.26966291666030884, + "union_independent": 0.6492271423339844, + "union_gap": -0.37956422567367554, + "mean_per_layer_hit_rate": 0.0760587751865387 + }, + "50": { + "rho_bar": 0.40231233835220337, + "union_observed": 0.38826465606689453, + "union_independent": 0.8015168905258179, + "union_gap": -0.41325223445892334, + "mean_per_layer_hit_rate": 0.11466435343027115 + }, + "100": { + "rho_bar": 0.4171909987926483, + "union_observed": 0.49188515543937683, + "union_independent": 0.9003574252128601, + "union_gap": -0.4084722697734833, + "mean_per_layer_hit_rate": 0.15826371312141418 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2983679601124355, + "union_gap_mean_over_k": -0.25929946239505497 + }, + "Wikipedia_en|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.12847548723220825, + "union_observed": 0.047440700232982635, + "union_independent": 0.1025315523147583, + "union_gap": -0.055090852081775665, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.1586580127477646, + "union_observed": 0.08114856481552124, + "union_independent": 0.17574936151504517, + "union_gap": -0.09460079669952393, + "mean_per_layer_hit_rate": 0.014693171717226505 + }, + "5": { + "rho_bar": 0.26388201117515564, + "union_observed": 0.13857677578926086, + "union_independent": 0.3337487578392029, + "union_gap": -0.19517198204994202, + "mean_per_layer_hit_rate": 0.03053875081241131 + }, + "10": { + "rho_bar": 0.34322240948677063, + "union_observed": 0.20599250495433807, + "union_independent": 0.5025746822357178, + "union_gap": -0.2965821772813797, + "mean_per_layer_hit_rate": 0.05176221951842308 + }, + "20": { + "rho_bar": 0.3845110535621643, + "union_observed": 0.28589263558387756, + "union_independent": 0.6590526103973389, + "union_gap": -0.3731599748134613, + "mean_per_layer_hit_rate": 0.07817152142524719 + }, + "50": { + "rho_bar": 0.446697473526001, + "union_observed": 0.387016236782074, + "union_independent": 0.8166216015815735, + "union_gap": -0.4296053647994995, + "mean_per_layer_hit_rate": 0.11985017359256744 + }, + "100": { + "rho_bar": 0.467989981174469, + "union_observed": 0.4906367063522339, + "union_independent": 0.9076260328292847, + "union_gap": -0.4169893264770508, + "mean_per_layer_hit_rate": 0.16354557871818542 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3133480612720762, + "union_gap_mean_over_k": -0.2658857820289476 + }, + "Wikipedia_en|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2750057876110077, + "union_observed": 0.04619225859642029, + "union_independent": 0.1391845941543579, + "union_gap": -0.09299233555793762, + "mean_per_layer_hit_rate": 0.011428022757172585 + }, + "2": { + "rho_bar": 0.31482362747192383, + "union_observed": 0.0699126124382019, + "union_independent": 0.22868263721466064, + "union_gap": -0.15877002477645874, + "mean_per_layer_hit_rate": 0.019686929881572723 + }, + "5": { + "rho_bar": 0.39244192838668823, + "union_observed": 0.12484394758939743, + "union_independent": 0.4332089424133301, + "union_gap": -0.30836499482393265, + "mean_per_layer_hit_rate": 0.04244694486260414 + }, + "10": { + "rho_bar": 0.45434731245040894, + "union_observed": 0.17478151619434357, + "union_independent": 0.5934724807739258, + "union_gap": -0.4186909645795822, + "mean_per_layer_hit_rate": 0.06616729497909546 + }, + "20": { + "rho_bar": 0.4907458424568176, + "union_observed": 0.25218477845191956, + "union_independent": 0.7444347143173218, + "union_gap": -0.4922499358654022, + "mean_per_layer_hit_rate": 0.09805051982402802 + }, + "50": { + "rho_bar": 0.5313024520874023, + "union_observed": 0.3383270800113678, + "union_independent": 0.8711949586868286, + "union_gap": -0.5328678786754608, + "mean_per_layer_hit_rate": 0.14280226826667786 + }, + "100": { + "rho_bar": 0.5513998866081238, + "union_observed": 0.4282147288322449, + "union_independent": 0.9423686861991882, + "union_gap": -0.5141539573669434, + "mean_per_layer_hit_rate": 0.1917795091867447 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4300095481531961, + "union_gap_mean_over_k": -0.35972715594938826 + }, + "Wikipedia_en|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.29636847972869873, + "union_observed": 0.05243445560336113, + "union_independent": 0.1511741280555725, + "union_gap": -0.09873967245221138, + "mean_per_layer_hit_rate": 0.012484394945204258 + }, + "2": { + "rho_bar": 0.30460110306739807, + "union_observed": 0.07865168899297714, + "union_independent": 0.22070550918579102, + "union_gap": -0.14205382019281387, + "mean_per_layer_hit_rate": 0.018918659538030624 + }, + "5": { + "rho_bar": 0.3807051479816437, + "union_observed": 0.12983770668506622, + "union_independent": 0.41720831394195557, + "union_gap": -0.28737060725688934, + "mean_per_layer_hit_rate": 0.040430232882499695 + }, + "10": { + "rho_bar": 0.4643758237361908, + "union_observed": 0.1835206001996994, + "union_independent": 0.5847734212875366, + "union_gap": -0.4012528210878372, + "mean_per_layer_hit_rate": 0.06472678482532501 + }, + "20": { + "rho_bar": 0.4860262870788574, + "union_observed": 0.2509363293647766, + "union_independent": 0.7340924739837646, + "union_gap": -0.48315614461898804, + "mean_per_layer_hit_rate": 0.09545760601758957 + }, + "50": { + "rho_bar": 0.5188071131706238, + "union_observed": 0.34956303238868713, + "union_independent": 0.8679636120796204, + "union_gap": -0.5184005796909332, + "mean_per_layer_hit_rate": 0.14184191823005676 + }, + "100": { + "rho_bar": 0.5381633639335632, + "union_observed": 0.4344569146633148, + "union_independent": 0.9387606382369995, + "union_gap": -0.5043037235736847, + "mean_per_layer_hit_rate": 0.1900508999824524 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.42700675981385366, + "union_gap_mean_over_k": -0.34789676698190825 + }, + "Wikipedia_en|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.23270183801651, + "union_observed": 0.04369537904858589, + "union_independent": 0.11616402864456177, + "union_gap": -0.07246864959597588, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.2461475282907486, + "union_observed": 0.07615480571985245, + "union_independent": 0.19648617506027222, + "union_gap": -0.12033136934041977, + "mean_per_layer_hit_rate": 0.016613848507404327 + }, + "5": { + "rho_bar": 0.3295809328556061, + "union_observed": 0.13108614087104797, + "union_independent": 0.3804290294647217, + "union_gap": -0.2493428885936737, + "mean_per_layer_hit_rate": 0.03591664507985115 + }, + "10": { + "rho_bar": 0.3892271816730499, + "union_observed": 0.1760299652814865, + "union_independent": 0.5313341617584229, + "union_gap": -0.35530419647693634, + "mean_per_layer_hit_rate": 0.056083742529153824 + }, + "20": { + "rho_bar": 0.4496447741985321, + "union_observed": 0.24968789517879486, + "union_independent": 0.6963955760002136, + "union_gap": -0.44670768082141876, + "mean_per_layer_hit_rate": 0.08643042296171188 + }, + "50": { + "rho_bar": 0.5202914476394653, + "union_observed": 0.3458177149295807, + "union_independent": 0.8393679857254028, + "union_gap": -0.49355027079582214, + "mean_per_layer_hit_rate": 0.12926150858402252 + }, + "100": { + "rho_bar": 0.5425955653190613, + "union_observed": 0.4307115972042084, + "union_independent": 0.9190874099731445, + "union_gap": -0.48837581276893616, + "mean_per_layer_hit_rate": 0.17295686900615692 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.38716989542756763, + "union_gap_mean_over_k": -0.31801155262759756 + }, + "Wikipedia_en|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.21787400543689728, + "union_observed": 0.04244694113731384, + "union_independent": 0.10481631755828857, + "union_gap": -0.06236937642097473, + "mean_per_layer_hit_rate": 0.008450975641608238 + }, + "2": { + "rho_bar": 0.22598819434642792, + "union_observed": 0.06616728752851486, + "union_independent": 0.16636335849761963, + "union_gap": -0.10019607096910477, + "mean_per_layer_hit_rate": 0.013828867115080357 + }, + "5": { + "rho_bar": 0.28056851029396057, + "union_observed": 0.13857677578926086, + "union_independent": 0.3549714684486389, + "union_gap": -0.21639469265937805, + "mean_per_layer_hit_rate": 0.032939594238996506 + }, + "10": { + "rho_bar": 0.3497582972049713, + "union_observed": 0.1910112351179123, + "union_independent": 0.516169548034668, + "union_gap": -0.3251583129167557, + "mean_per_layer_hit_rate": 0.05377893149852753 + }, + "20": { + "rho_bar": 0.4027678072452545, + "union_observed": 0.25218477845191956, + "union_independent": 0.6591284275054932, + "union_gap": -0.4069436490535736, + "mean_per_layer_hit_rate": 0.07845962047576904 + }, + "50": { + "rho_bar": 0.47430235147476196, + "union_observed": 0.33957552909851074, + "union_independent": 0.8128020763397217, + "union_gap": -0.47322654724121094, + "mean_per_layer_hit_rate": 0.11879380792379379 + }, + "100": { + "rho_bar": 0.5089413523674011, + "union_observed": 0.4344569146633148, + "union_independent": 0.8980478048324585, + "union_gap": -0.4635908901691437, + "mean_per_layer_hit_rate": 0.158263698220253 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3514572169099535, + "union_gap_mean_over_k": -0.29255421991859165 + }, + "Wikipedia_en|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.1353609412908554, + "union_observed": 0.04494382068514824, + "union_independent": 0.08535146713256836, + "union_gap": -0.04040764644742012, + "mean_per_layer_hit_rate": 0.0068184006959199905 + }, + "2": { + "rho_bar": 0.1544460952281952, + "union_observed": 0.072409488260746, + "union_independent": 0.1492389440536499, + "union_gap": -0.0768294557929039, + "mean_per_layer_hit_rate": 0.012292327359318733 + }, + "5": { + "rho_bar": 0.276495099067688, + "union_observed": 0.13358302414417267, + "union_independent": 0.3310486078262329, + "union_gap": -0.19746558368206024, + "mean_per_layer_hit_rate": 0.03025064617395401 + }, + "10": { + "rho_bar": 0.3668694794178009, + "union_observed": 0.18976280093193054, + "union_independent": 0.49194228649139404, + "union_gap": -0.3021794855594635, + "mean_per_layer_hit_rate": 0.05032171308994293 + }, + "20": { + "rho_bar": 0.4268614947795868, + "union_observed": 0.23720349371433258, + "union_independent": 0.6321661472320557, + "union_gap": -0.3949626535177231, + "mean_per_layer_hit_rate": 0.0731777623295784 + }, + "50": { + "rho_bar": 0.4778628349304199, + "union_observed": 0.33707866072654724, + "union_independent": 0.7915534377098083, + "union_gap": -0.4544747769832611, + "mean_per_layer_hit_rate": 0.1118793860077858 + }, + "100": { + "rho_bar": 0.5058311820030212, + "union_observed": 0.4244694113731384, + "union_independent": 0.8817869424819946, + "union_gap": -0.4573175311088562, + "mean_per_layer_hit_rate": 0.14914049208164215 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3348181609596525, + "union_gap_mean_over_k": -0.274805304727384 + }, + "Wikipedia_en|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.12382839620113373, + "union_observed": 0.048689138144254684, + "union_independent": 0.08195501565933228, + "union_gap": -0.03326587751507759, + "mean_per_layer_hit_rate": 0.006530298851430416 + }, + "2": { + "rho_bar": 0.1813996136188507, + "union_observed": 0.0774032473564148, + "union_independent": 0.15565276145935059, + "union_gap": -0.07824951410293579, + "mean_per_layer_hit_rate": 0.012868530116975307 + }, + "5": { + "rho_bar": 0.2577102482318878, + "union_observed": 0.13233457505702972, + "union_independent": 0.31443536281585693, + "union_gap": -0.1821007877588272, + "mean_per_layer_hit_rate": 0.028426004573702812 + }, + "10": { + "rho_bar": 0.3287346065044403, + "union_observed": 0.18227216601371765, + "union_independent": 0.46300601959228516, + "union_gap": -0.2807338535785675, + "mean_per_layer_hit_rate": 0.04628829285502434 + }, + "20": { + "rho_bar": 0.3949849307537079, + "union_observed": 0.2297128587961197, + "union_independent": 0.6022511720657349, + "union_gap": -0.3725383132696152, + "mean_per_layer_hit_rate": 0.06779985874891281 + }, + "50": { + "rho_bar": 0.45243003964424133, + "union_observed": 0.32709112763404846, + "union_independent": 0.7674039006233215, + "union_gap": -0.44031277298927307, + "mean_per_layer_hit_rate": 0.10486892610788345 + }, + "100": { + "rho_bar": 0.47245681285858154, + "union_observed": 0.42696627974510193, + "union_independent": 0.8756312131881714, + "union_gap": -0.44866493344306946, + "mean_per_layer_hit_rate": 0.1459713876247406 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.31593494968754904, + "union_gap_mean_over_k": -0.26226657895105226 + }, + "Wikipedia_en|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.17386451363563538, + "union_observed": 0.039950061589479446, + "union_independent": 0.07831907272338867, + "union_gap": -0.038369011133909225, + "mean_per_layer_hit_rate": 0.006242197472602129 + }, + "2": { + "rho_bar": 0.18422511219978333, + "union_observed": 0.06616728752851486, + "union_independent": 0.14345097541809082, + "union_gap": -0.07728368788957596, + "mean_per_layer_hit_rate": 0.011812158860266209 + }, + "5": { + "rho_bar": 0.2765139937400818, + "union_observed": 0.10736579447984695, + "union_independent": 0.29310786724090576, + "union_gap": -0.1857420727610588, + "mean_per_layer_hit_rate": 0.026217225939035416 + }, + "10": { + "rho_bar": 0.31816455721855164, + "union_observed": 0.1498127281665802, + "union_independent": 0.4353863000869751, + "union_gap": -0.2855735719203949, + "mean_per_layer_hit_rate": 0.04273504018783569 + }, + "20": { + "rho_bar": 0.3621710538864136, + "union_observed": 0.19225966930389404, + "union_independent": 0.568220853805542, + "union_gap": -0.37596118450164795, + "mean_per_layer_hit_rate": 0.06203783303499222 + }, + "50": { + "rho_bar": 0.4324721395969391, + "union_observed": 0.2996254563331604, + "union_independent": 0.7601600289344788, + "union_gap": -0.46053457260131836, + "mean_per_layer_hit_rate": 0.10294824093580246 + }, + "100": { + "rho_bar": 0.4528769552707672, + "union_observed": 0.3982521891593933, + "union_independent": 0.8765124082565308, + "union_gap": -0.47826021909713745, + "mean_per_layer_hit_rate": 0.14664362370967865 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.31432690364973886, + "union_gap_mean_over_k": -0.27167490284357754 + }, + "Wikipedia_en|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.20025162398815155, + "union_observed": 0.0387016236782074, + "union_independent": 0.07488518953323364, + "union_gap": -0.036183565855026245, + "mean_per_layer_hit_rate": 0.005954096093773842 + }, + "2": { + "rho_bar": 0.19494980573654175, + "union_observed": 0.05992509424686432, + "union_independent": 0.1260749101638794, + "union_gap": -0.06614981591701508, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "5": { + "rho_bar": 0.2520086467266083, + "union_observed": 0.10736579447984695, + "union_independent": 0.2806701064109802, + "union_gap": -0.17330431193113327, + "mean_per_layer_hit_rate": 0.024872753769159317 + }, + "10": { + "rho_bar": 0.32776495814323425, + "union_observed": 0.15355804562568665, + "union_independent": 0.44116634130477905, + "union_gap": -0.2876082956790924, + "mean_per_layer_hit_rate": 0.04340727999806404 + }, + "20": { + "rho_bar": 0.37141671776771545, + "union_observed": 0.19975031912326813, + "union_independent": 0.5851601362228394, + "union_gap": -0.38540981709957123, + "mean_per_layer_hit_rate": 0.06472678482532501 + }, + "50": { + "rho_bar": 0.418531209230423, + "union_observed": 0.3121098577976227, + "union_independent": 0.7656160593032837, + "union_gap": -0.453506201505661, + "mean_per_layer_hit_rate": 0.10410065203905106 + }, + "100": { + "rho_bar": 0.43013089895248413, + "union_observed": 0.4132334589958191, + "union_independent": 0.8835790157318115, + "union_gap": -0.47034555673599243, + "mean_per_layer_hit_rate": 0.149428591132164 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3135791229350226, + "union_gap_mean_over_k": -0.2675010806747845 + }, + "Wikipedia_en|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.21052199602127075, + "union_observed": 0.041198503226041794, + "union_independent": 0.09568560123443604, + "union_gap": -0.05448709800839424, + "mean_per_layer_hit_rate": 0.007682703901082277 + }, + "2": { + "rho_bar": 0.2276444286108017, + "union_observed": 0.05867665261030197, + "union_independent": 0.1512444019317627, + "union_gap": -0.09256774932146072, + "mean_per_layer_hit_rate": 0.012484394945204258 + }, + "5": { + "rho_bar": 0.2968687117099762, + "union_observed": 0.10362047702074051, + "union_independent": 0.296592652797699, + "union_gap": -0.19297217577695847, + "mean_per_layer_hit_rate": 0.026505330577492714 + }, + "10": { + "rho_bar": 0.35744142532348633, + "union_observed": 0.1423220932483673, + "union_independent": 0.44112133979797363, + "union_gap": -0.2987992465496063, + "mean_per_layer_hit_rate": 0.04331124573945999 + }, + "20": { + "rho_bar": 0.38617509603500366, + "union_observed": 0.20973782241344452, + "union_independent": 0.603907585144043, + "union_gap": -0.39416976273059845, + "mean_per_layer_hit_rate": 0.06779986619949341 + }, + "50": { + "rho_bar": 0.4008461833000183, + "union_observed": 0.323345810174942, + "union_independent": 0.7760714292526245, + "union_gap": -0.4527256190776825, + "mean_per_layer_hit_rate": 0.10678958892822266 + }, + "100": { + "rho_bar": 0.41357263922691345, + "union_observed": 0.41198500990867615, + "union_independent": 0.8861485123634338, + "union_gap": -0.4741635024547577, + "mean_per_layer_hit_rate": 0.1503889411687851 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3275814971753529, + "union_gap_mean_over_k": -0.27998359341706547 + }, + "Wikipedia_en|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.20382364094257355, + "union_observed": 0.04494382068514824, + "union_independent": 0.1070716381072998, + "union_gap": -0.062127817422151566, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.26469215750694275, + "union_observed": 0.06117353215813637, + "union_independent": 0.16952890157699585, + "union_gap": -0.10835536941885948, + "mean_per_layer_hit_rate": 0.014116968959569931 + }, + "5": { + "rho_bar": 0.29567089676856995, + "union_observed": 0.10986267030239105, + "union_independent": 0.29386061429977417, + "union_gap": -0.18399794399738312, + "mean_per_layer_hit_rate": 0.026217225939035416 + }, + "10": { + "rho_bar": 0.3461548686027527, + "union_observed": 0.15355804562568665, + "union_independent": 0.45606327056884766, + "union_gap": -0.302505224943161, + "mean_per_layer_hit_rate": 0.04523191973567009 + }, + "20": { + "rho_bar": 0.3833438754081726, + "union_observed": 0.21348313987255096, + "union_independent": 0.6064327359199524, + "union_gap": -0.39294959604740143, + "mean_per_layer_hit_rate": 0.0681840032339096 + }, + "50": { + "rho_bar": 0.39369863271713257, + "union_observed": 0.3308364450931549, + "union_independent": 0.7841418385505676, + "union_gap": -0.4533053934574127, + "mean_per_layer_hit_rate": 0.10890233516693115 + }, + "100": { + "rho_bar": 0.39834633469581604, + "union_observed": 0.43320849537849426, + "union_independent": 0.8948723077774048, + "union_gap": -0.4616638123989105, + "mean_per_layer_hit_rate": 0.15451838076114655 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.32653291523456573, + "union_gap_mean_over_k": -0.2807007368121828 + }, + "Wikipedia_en|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.19253568351268768, + "union_observed": 0.039950061589479446, + "union_independent": 0.09574288129806519, + "union_gap": -0.05579281970858574, + "mean_per_layer_hit_rate": 0.007682703901082277 + }, + "2": { + "rho_bar": 0.22076332569122314, + "union_observed": 0.06117353215813637, + "union_independent": 0.15353858470916748, + "union_gap": -0.09236505255103111, + "mean_per_layer_hit_rate": 0.012676462531089783 + }, + "5": { + "rho_bar": 0.285443514585495, + "union_observed": 0.10486891120672226, + "union_independent": 0.3010510802268982, + "union_gap": -0.19618216902017593, + "mean_per_layer_hit_rate": 0.026985498145222664 + }, + "10": { + "rho_bar": 0.36384713649749756, + "union_observed": 0.14357054233551025, + "union_independent": 0.4582556486129761, + "union_gap": -0.3146851062774658, + "mean_per_layer_hit_rate": 0.04552002251148224 + }, + "20": { + "rho_bar": 0.3900846838951111, + "union_observed": 0.2122347056865692, + "union_independent": 0.6195986270904541, + "union_gap": -0.4073639214038849, + "mean_per_layer_hit_rate": 0.0704888179898262 + }, + "50": { + "rho_bar": 0.3774786591529846, + "union_observed": 0.32958802580833435, + "union_independent": 0.7979296445846558, + "union_gap": -0.4683416187763214, + "mean_per_layer_hit_rate": 0.11312782019376755 + }, + "100": { + "rho_bar": 0.3877640664577484, + "union_observed": 0.4394506812095642, + "union_independent": 0.9048154950141907, + "union_gap": -0.46536481380462646, + "mean_per_layer_hit_rate": 0.16056853532791138 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.31684529568467823, + "union_gap_mean_over_k": -0.2857279287917273 + }, + "Wikipedia_en|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.21091976761817932, + "union_observed": 0.04494382068514824, + "union_independent": 0.11165040731430054, + "union_gap": -0.0667065866291523, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.21211795508861542, + "union_observed": 0.06367041170597076, + "union_independent": 0.17289316654205322, + "union_gap": -0.10922275483608246, + "mean_per_layer_hit_rate": 0.01440507173538208 + }, + "5": { + "rho_bar": 0.3018321990966797, + "union_observed": 0.10486891120672226, + "union_independent": 0.3235897421836853, + "union_gap": -0.21872083097696304, + "mean_per_layer_hit_rate": 0.02938634343445301 + }, + "10": { + "rho_bar": 0.3577633798122406, + "union_observed": 0.17228464782238007, + "union_independent": 0.5034505128860474, + "union_gap": -0.3311658650636673, + "mean_per_layer_hit_rate": 0.05166618525981903 + }, + "20": { + "rho_bar": 0.382542222738266, + "union_observed": 0.24094881117343903, + "union_independent": 0.6618335247039795, + "union_gap": -0.42088471353054047, + "mean_per_layer_hit_rate": 0.07845962047576904 + }, + "50": { + "rho_bar": 0.3859025537967682, + "union_observed": 0.35580524802207947, + "union_independent": 0.8185790777206421, + "union_gap": -0.4627738296985626, + "mean_per_layer_hit_rate": 0.11994622647762299 + }, + "100": { + "rho_bar": 0.3934957683086395, + "union_observed": 0.46691635251045227, + "union_independent": 0.918925940990448, + "union_gap": -0.4520095884799957, + "mean_per_layer_hit_rate": 0.1702679544687271 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.32065340663705555, + "union_gap_mean_over_k": -0.29449773845928057 + }, + "Wikipedia_en|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.17962712049484253, + "union_observed": 0.054931335151195526, + "union_independent": 0.1206631064414978, + "union_gap": -0.06573177129030228, + "mean_per_layer_hit_rate": 0.009795447811484337 + }, + "2": { + "rho_bar": 0.27138176560401917, + "union_observed": 0.08239700645208359, + "union_independent": 0.20098352432250977, + "union_gap": -0.11858651787042618, + "mean_per_layer_hit_rate": 0.016997983679175377 + }, + "5": { + "rho_bar": 0.30799272656440735, + "union_observed": 0.12234706431627274, + "union_independent": 0.3621397614479065, + "union_gap": -0.23979269713163376, + "mean_per_layer_hit_rate": 0.033707864582538605 + }, + "10": { + "rho_bar": 0.36736682057380676, + "union_observed": 0.1960049867630005, + "union_independent": 0.5286262631416321, + "union_gap": -0.3326212763786316, + "mean_per_layer_hit_rate": 0.055507540702819824 + }, + "20": { + "rho_bar": 0.40045231580734253, + "union_observed": 0.2684144675731659, + "union_independent": 0.6914759278297424, + "union_gap": -0.42306146025657654, + "mean_per_layer_hit_rate": 0.08489388972520828 + }, + "50": { + "rho_bar": 0.40233075618743896, + "union_observed": 0.38826465606689453, + "union_independent": 0.8513299226760864, + "union_gap": -0.4630652666091919, + "mean_per_layer_hit_rate": 0.13319888710975647 + }, + "100": { + "rho_bar": 0.42419755458831787, + "union_observed": 0.4781523048877716, + "union_independent": 0.9291465878486633, + "union_gap": -0.4509942829608917, + "mean_per_layer_hit_rate": 0.17929509282112122 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.33619272283145357, + "union_gap_mean_over_k": -0.2991218960710934 + }, + "Wikipedia_en|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.2114536464214325, + "union_observed": 0.05243445560336113, + "union_independent": 0.12940454483032227, + "union_gap": -0.07697008922696114, + "mean_per_layer_hit_rate": 0.010563718155026436 + }, + "2": { + "rho_bar": 0.24628248810768127, + "union_observed": 0.072409488260746, + "union_independent": 0.19665861129760742, + "union_gap": -0.12424912303686142, + "mean_per_layer_hit_rate": 0.016613848507404327 + }, + "5": { + "rho_bar": 0.3401857018470764, + "union_observed": 0.13233457505702972, + "union_independent": 0.3856890797615051, + "union_gap": -0.2533545047044754, + "mean_per_layer_hit_rate": 0.03649284690618515 + }, + "10": { + "rho_bar": 0.4057559370994568, + "union_observed": 0.20724095404148102, + "union_independent": 0.5642199516296387, + "union_gap": -0.35697899758815765, + "mean_per_layer_hit_rate": 0.06117353215813637 + }, + "20": { + "rho_bar": 0.42923882603645325, + "union_observed": 0.2746566832065582, + "union_independent": 0.7217397093772888, + "union_gap": -0.4470830261707306, + "mean_per_layer_hit_rate": 0.09219244867563248 + }, + "50": { + "rho_bar": 0.4463941752910614, + "union_observed": 0.3720349669456482, + "union_independent": 0.8502131700515747, + "union_gap": -0.4781782031059265, + "mean_per_layer_hit_rate": 0.13329492509365082 + }, + "100": { + "rho_bar": 0.4673718810081482, + "union_observed": 0.4656679034233093, + "union_independent": 0.9298638701438904, + "union_gap": -0.46419596672058105, + "mean_per_layer_hit_rate": 0.1810237318277359 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.36381180797304424, + "union_gap_mean_over_k": -0.31442998722195625 + }, + "Wikipedia_en|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.13028506934642792, + "union_observed": 0.056179776787757874, + "union_independent": 0.10588473081588745, + "union_gap": -0.04970495402812958, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.20384682714939117, + "union_observed": 0.07865168899297714, + "union_independent": 0.19545906782150269, + "union_gap": -0.11680737882852554, + "mean_per_layer_hit_rate": 0.016517816111445427 + }, + "5": { + "rho_bar": 0.3143111765384674, + "union_observed": 0.13233457505702972, + "union_independent": 0.35212212800979614, + "union_gap": -0.21978755295276642, + "mean_per_layer_hit_rate": 0.03265149146318436 + }, + "10": { + "rho_bar": 0.39697393774986267, + "union_observed": 0.193508118391037, + "union_independent": 0.5110835433006287, + "union_gap": -0.3175754249095917, + "mean_per_layer_hit_rate": 0.053106699138879776 + }, + "20": { + "rho_bar": 0.42062875628471375, + "union_observed": 0.24968789517879486, + "union_independent": 0.661650538444519, + "union_gap": -0.4119626432657242, + "mean_per_layer_hit_rate": 0.07893978804349899 + }, + "50": { + "rho_bar": 0.4605593979358673, + "union_observed": 0.34456929564476013, + "union_independent": 0.8140290975570679, + "union_gap": -0.46945980191230774, + "mean_per_layer_hit_rate": 0.11937002092599869 + }, + "100": { + "rho_bar": 0.4414701759815216, + "union_observed": 0.4606741666793823, + "union_independent": 0.9051650166511536, + "union_gap": -0.44449084997177124, + "mean_per_layer_hit_rate": 0.16239316761493683 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3382964772837503, + "union_gap_mean_over_k": -0.28996980083840235 + }, + "Wikipedia_en|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.17244985699653625, + "union_observed": 0.04619225859642029, + "union_independent": 0.10366815328598022, + "union_gap": -0.057475894689559937, + "mean_per_layer_hit_rate": 0.008354941383004189 + }, + "2": { + "rho_bar": 0.23971974849700928, + "union_observed": 0.072409488260746, + "union_independent": 0.17164433002471924, + "union_gap": -0.09923484176397324, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.25775274634361267, + "union_observed": 0.12359550595283508, + "union_independent": 0.3261093497276306, + "union_gap": -0.20251384377479553, + "mean_per_layer_hit_rate": 0.02967444434762001 + }, + "10": { + "rho_bar": 0.3656885325908661, + "union_observed": 0.1760299652814865, + "union_independent": 0.49304014444351196, + "union_gap": -0.31701017916202545, + "mean_per_layer_hit_rate": 0.05041774362325668 + }, + "20": { + "rho_bar": 0.4027513861656189, + "union_observed": 0.24469412863254547, + "union_independent": 0.6454049348831177, + "union_gap": -0.4007108062505722, + "mean_per_layer_hit_rate": 0.0756746381521225 + }, + "50": { + "rho_bar": 0.4377005994319916, + "union_observed": 0.352059930562973, + "union_independent": 0.811525821685791, + "union_gap": -0.459465891122818, + "mean_per_layer_hit_rate": 0.1182175949215889 + }, + "100": { + "rho_bar": 0.4438870847225189, + "union_observed": 0.4731585383415222, + "union_independent": 0.9097602963447571, + "union_gap": -0.43660175800323486, + "mean_per_layer_hit_rate": 0.16537021100521088 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3314214221068791, + "union_gap_mean_over_k": -0.28185903068099705 + }, + "Wikipedia_en|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.16749311983585358, + "union_observed": 0.05243445560336113, + "union_independent": 0.10150963068008423, + "union_gap": -0.0490751750767231, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.2254248857498169, + "union_observed": 0.08988764137029648, + "union_independent": 0.18015152215957642, + "union_gap": -0.09026388078927994, + "mean_per_layer_hit_rate": 0.01507730782032013 + }, + "5": { + "rho_bar": 0.2676697373390198, + "union_observed": 0.15355804562568665, + "union_independent": 0.35162752866744995, + "union_gap": -0.1980694830417633, + "mean_per_layer_hit_rate": 0.03245942294597626 + }, + "10": { + "rho_bar": 0.3321952521800995, + "union_observed": 0.227215975522995, + "union_independent": 0.5257806777954102, + "union_gap": -0.29856470227241516, + "mean_per_layer_hit_rate": 0.055027369409799576 + }, + "20": { + "rho_bar": 0.3696272671222687, + "union_observed": 0.2871410846710205, + "union_independent": 0.6607370376586914, + "union_gap": -0.3735959529876709, + "mean_per_layer_hit_rate": 0.0783635824918747 + }, + "50": { + "rho_bar": 0.38550981879234314, + "union_observed": 0.40699127316474915, + "union_independent": 0.8175934553146362, + "union_gap": -0.4106021821498871, + "mean_per_layer_hit_rate": 0.12004225701093674 + }, + "100": { + "rho_bar": 0.40360724925994873, + "union_observed": 0.5018726587295532, + "union_independent": 0.9032353162765503, + "union_gap": -0.40136265754699707, + "mean_per_layer_hit_rate": 0.16018438339233398 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.3073610471827643, + "union_gap_mean_over_k": -0.26021914769496235 + }, + "Wikipedia_en|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.1465303748846054, + "union_observed": 0.05118601769208908, + "union_independent": 0.11042344570159912, + "union_gap": -0.05923742800951004, + "mean_per_layer_hit_rate": 0.008931143209338188 + }, + "2": { + "rho_bar": 0.18233205378055573, + "union_observed": 0.08489388227462769, + "union_independent": 0.1748117208480835, + "union_gap": -0.08991783857345581, + "mean_per_layer_hit_rate": 0.01459713838994503 + }, + "5": { + "rho_bar": 0.27157530188560486, + "union_observed": 0.144818976521492, + "union_independent": 0.3367077112197876, + "union_gap": -0.1918887346982956, + "mean_per_layer_hit_rate": 0.030826853588223457 + }, + "10": { + "rho_bar": 0.3173723816871643, + "union_observed": 0.21348313987255096, + "union_independent": 0.49590569734573364, + "union_gap": -0.2824225574731827, + "mean_per_layer_hit_rate": 0.05060981959104538 + }, + "20": { + "rho_bar": 0.3717440068721771, + "union_observed": 0.2659175992012024, + "union_independent": 0.6472870111465454, + "union_gap": -0.381369411945343, + "mean_per_layer_hit_rate": 0.0756746381521225 + }, + "50": { + "rho_bar": 0.4008484482765198, + "union_observed": 0.392009973526001, + "union_independent": 0.803902268409729, + "union_gap": -0.411892294883728, + "mean_per_layer_hit_rate": 0.11533660441637039 + }, + "100": { + "rho_bar": 0.4108297824859619, + "union_observed": 0.5031210780143738, + "union_independent": 0.9038932919502258, + "union_gap": -0.40077221393585205, + "mean_per_layer_hit_rate": 0.16056853532791138 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3001760499817984, + "union_gap_mean_over_k": -0.2596429256456239 + }, + "Wikipedia_en|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.12331796437501907, + "union_observed": 0.04993757978081703, + "union_independent": 0.10591530799865723, + "union_gap": -0.055977728217840195, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.16814804077148438, + "union_observed": 0.08239700645208359, + "union_independent": 0.18315130472183228, + "union_gap": -0.10075429826974869, + "mean_per_layer_hit_rate": 0.015365407802164555 + }, + "5": { + "rho_bar": 0.27529722452163696, + "union_observed": 0.13358302414417267, + "union_independent": 0.3398500084877014, + "union_gap": -0.20626698434352875, + "mean_per_layer_hit_rate": 0.031210986897349358 + }, + "10": { + "rho_bar": 0.3562842011451721, + "union_observed": 0.20599250495433807, + "union_independent": 0.5087093114852905, + "union_gap": -0.30271680653095245, + "mean_per_layer_hit_rate": 0.05262652412056923 + }, + "20": { + "rho_bar": 0.39157846570014954, + "union_observed": 0.2796504497528076, + "union_independent": 0.65608811378479, + "union_gap": -0.3764376640319824, + "mean_per_layer_hit_rate": 0.0775953158736229 + }, + "50": { + "rho_bar": 0.4427506923675537, + "union_observed": 0.3932584226131439, + "union_independent": 0.819567859172821, + "union_gap": -0.4263094365596771, + "mean_per_layer_hit_rate": 0.12081052362918854 + }, + "100": { + "rho_bar": 0.4694617688655853, + "union_observed": 0.48938825726509094, + "union_independent": 0.9080525636672974, + "union_gap": -0.4186643064022064, + "mean_per_layer_hit_rate": 0.1638336479663849 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3181197653923716, + "union_gap_mean_over_k": -0.26958960347941946 + }, + "Wikipedia_en|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2777409255504608, + "union_observed": 0.04494382068514824, + "union_independent": 0.13591116666793823, + "union_gap": -0.09096734598279, + "mean_per_layer_hit_rate": 0.01113992091268301 + }, + "2": { + "rho_bar": 0.3104162812232971, + "union_observed": 0.072409488260746, + "union_independent": 0.22577691078186035, + "union_gap": -0.15336742252111435, + "mean_per_layer_hit_rate": 0.019398828968405724 + }, + "5": { + "rho_bar": 0.39243751764297485, + "union_observed": 0.12484394758939743, + "union_independent": 0.43826931715011597, + "union_gap": -0.31342536956071854, + "mean_per_layer_hit_rate": 0.04311917722225189 + }, + "10": { + "rho_bar": 0.45111700892448425, + "union_observed": 0.17727839946746826, + "union_independent": 0.596244752407074, + "union_gap": -0.4189663529396057, + "mean_per_layer_hit_rate": 0.06664745509624481 + }, + "20": { + "rho_bar": 0.49043595790863037, + "union_observed": 0.2584269642829895, + "union_independent": 0.748569130897522, + "union_gap": -0.49014216661453247, + "mean_per_layer_hit_rate": 0.09910687804222107 + }, + "50": { + "rho_bar": 0.5364136695861816, + "union_observed": 0.32958802580833435, + "union_independent": 0.8717508912086487, + "union_gap": -0.5421628654003143, + "mean_per_layer_hit_rate": 0.1430903822183609 + }, + "100": { + "rho_bar": 0.5507604479789734, + "union_observed": 0.4307115972042084, + "union_independent": 0.9438294172286987, + "union_gap": -0.5131178200244904, + "mean_per_layer_hit_rate": 0.19331605732440948 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4299031155450003, + "union_gap_mean_over_k": -0.3603070490062237 + }, + "Wikipedia_en|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.294338583946228, + "union_observed": 0.04993757978081703, + "union_independent": 0.14468860626220703, + "union_gap": -0.09475102648139, + "mean_per_layer_hit_rate": 0.011908192187547684 + }, + "2": { + "rho_bar": 0.34207263588905334, + "union_observed": 0.07865168899297714, + "union_independent": 0.22268927097320557, + "union_gap": -0.14403758198022842, + "mean_per_layer_hit_rate": 0.019110728055238724 + }, + "5": { + "rho_bar": 0.4383620023727417, + "union_observed": 0.12484394758939743, + "union_independent": 0.4104844927787781, + "union_gap": -0.28564054518938065, + "mean_per_layer_hit_rate": 0.039565928280353546 + }, + "10": { + "rho_bar": 0.4634299576282501, + "union_observed": 0.1760299652814865, + "union_independent": 0.578712522983551, + "union_gap": -0.4026825577020645, + "mean_per_layer_hit_rate": 0.06367041170597076 + }, + "20": { + "rho_bar": 0.49380409717559814, + "union_observed": 0.24594257771968842, + "union_independent": 0.7289440631866455, + "union_gap": -0.4830014854669571, + "mean_per_layer_hit_rate": 0.09420915693044662 + }, + "50": { + "rho_bar": 0.51211017370224, + "union_observed": 0.3458177149295807, + "union_independent": 0.86455237865448, + "union_gap": -0.5187346637248993, + "mean_per_layer_hit_rate": 0.14011332392692566 + }, + "100": { + "rho_bar": 0.5284847617149353, + "union_observed": 0.4307115972042084, + "union_independent": 0.9349803924560547, + "union_gap": -0.5042687952518463, + "mean_per_layer_hit_rate": 0.18611352145671844 + } + }, + "n_degenerate_pairs": 58, + "rho_bar_mean_over_k": 0.4389431732041495, + "union_gap_mean_over_k": -0.34758809368525234 + }, + "Wikipedia_en|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.23053380846977234, + "union_observed": 0.04244694113731384, + "union_independent": 0.1150364875793457, + "union_gap": -0.07258954644203186, + "mean_per_layer_hit_rate": 0.009315279312431812 + }, + "2": { + "rho_bar": 0.22070246934890747, + "union_observed": 0.0774032473564148, + "union_independent": 0.20567065477371216, + "union_gap": -0.12826740741729736, + "mean_per_layer_hit_rate": 0.017478151246905327 + }, + "5": { + "rho_bar": 0.3287160396575928, + "union_observed": 0.13233457505702972, + "union_independent": 0.3877333402633667, + "union_gap": -0.255398765206337, + "mean_per_layer_hit_rate": 0.0367809496819973 + }, + "10": { + "rho_bar": 0.3958491384983063, + "union_observed": 0.1760299652814865, + "union_independent": 0.530693769454956, + "union_gap": -0.35466380417346954, + "mean_per_layer_hit_rate": 0.055987708270549774 + }, + "20": { + "rho_bar": 0.4322609603404999, + "union_observed": 0.24469412863254547, + "union_independent": 0.6908450722694397, + "union_gap": -0.4461509436368942, + "mean_per_layer_hit_rate": 0.08508595079183578 + }, + "50": { + "rho_bar": 0.5116322636604309, + "union_observed": 0.34207239747047424, + "union_independent": 0.834355890750885, + "union_gap": -0.49228349328041077, + "mean_per_layer_hit_rate": 0.12714876234531403 + }, + "100": { + "rho_bar": 0.5313258767127991, + "union_observed": 0.4369538128376007, + "union_independent": 0.91853266954422, + "union_gap": -0.48157885670661926, + "mean_per_layer_hit_rate": 0.17247670888900757 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.3787172223840441, + "union_gap_mean_over_k": -0.3187046881232943 + }, + "Wikipedia_en|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.20613574981689453, + "union_observed": 0.04494382068514824, + "union_independent": 0.10592377185821533, + "union_gap": -0.06097995117306709, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.1966416984796524, + "union_observed": 0.06741572916507721, + "union_independent": 0.1683497428894043, + "union_gap": -0.10093401372432709, + "mean_per_layer_hit_rate": 0.014020935632288456 + }, + "5": { + "rho_bar": 0.2657884955406189, + "union_observed": 0.13483145833015442, + "union_independent": 0.3491506576538086, + "union_gap": -0.21431919932365417, + "mean_per_layer_hit_rate": 0.032267358154058456 + }, + "10": { + "rho_bar": 0.3438336253166199, + "union_observed": 0.1885143518447876, + "union_independent": 0.513648509979248, + "union_gap": -0.32513415813446045, + "mean_per_layer_hit_rate": 0.05339479446411133 + }, + "20": { + "rho_bar": 0.3892865777015686, + "union_observed": 0.24843944609165192, + "union_independent": 0.6604523658752441, + "union_gap": -0.4120129197835922, + "mean_per_layer_hit_rate": 0.07865168154239655 + }, + "50": { + "rho_bar": 0.4562162160873413, + "union_observed": 0.3383270800113678, + "union_independent": 0.8079333305358887, + "union_gap": -0.4696062505245209, + "mean_per_layer_hit_rate": 0.11696918308734894 + }, + "100": { + "rho_bar": 0.4990934729576111, + "union_observed": 0.4394506812095642, + "union_independent": 0.8970158100128174, + "union_gap": -0.4575651288032532, + "mean_per_layer_hit_rate": 0.15759147703647614 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.336713690842901, + "union_gap_mean_over_k": -0.29150737449526787 + }, + "Wikipedia_en|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.1537432074546814, + "union_observed": 0.04244694113731384, + "union_independent": 0.08652573823928833, + "union_gap": -0.04407879710197449, + "mean_per_layer_hit_rate": 0.006914434023201466 + }, + "2": { + "rho_bar": 0.15313217043876648, + "union_observed": 0.07365792989730835, + "union_independent": 0.14812874794006348, + "union_gap": -0.07447081804275513, + "mean_per_layer_hit_rate": 0.012196293100714684 + }, + "5": { + "rho_bar": 0.2735043168067932, + "union_observed": 0.13108614087104797, + "union_independent": 0.3267873525619507, + "union_gap": -0.1957012116909027, + "mean_per_layer_hit_rate": 0.02977047488093376 + }, + "10": { + "rho_bar": 0.3522707223892212, + "union_observed": 0.1910112351179123, + "union_independent": 0.49197977781295776, + "union_gap": -0.30096854269504547, + "mean_per_layer_hit_rate": 0.05032171308994293 + }, + "20": { + "rho_bar": 0.4182857573032379, + "union_observed": 0.23970037698745728, + "union_independent": 0.630340576171875, + "union_gap": -0.3906401991844177, + "mean_per_layer_hit_rate": 0.0727936178445816 + }, + "50": { + "rho_bar": 0.46473807096481323, + "union_observed": 0.33458176255226135, + "union_independent": 0.7906262278556824, + "union_gap": -0.456044465303421, + "mean_per_layer_hit_rate": 0.1114952489733696 + }, + "100": { + "rho_bar": 0.5022397041320801, + "union_observed": 0.4244694113731384, + "union_independent": 0.8834412693977356, + "union_gap": -0.45897185802459717, + "mean_per_layer_hit_rate": 0.1500048041343689 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3311305642127991, + "union_gap_mean_over_k": -0.27441084172044483 + }, + "Wikipedia_en|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.11845012754201889, + "union_observed": 0.05118601769208908, + "union_independent": 0.08655673265457153, + "union_gap": -0.03537071496248245, + "mean_per_layer_hit_rate": 0.006914433557540178 + }, + "2": { + "rho_bar": 0.15659697353839874, + "union_observed": 0.07990012317895889, + "union_independent": 0.15669643878936768, + "union_gap": -0.07679631561040878, + "mean_per_layer_hit_rate": 0.012964563444256783 + }, + "5": { + "rho_bar": 0.2533218264579773, + "union_observed": 0.13108614087104797, + "union_independent": 0.3110050559043884, + "union_gap": -0.17991891503334045, + "mean_per_layer_hit_rate": 0.028041871264576912 + }, + "10": { + "rho_bar": 0.3334510922431946, + "union_observed": 0.17977528274059296, + "union_independent": 0.45813286304473877, + "union_gap": -0.2783575803041458, + "mean_per_layer_hit_rate": 0.04561605677008629 + }, + "20": { + "rho_bar": 0.39303937554359436, + "union_observed": 0.2297128587961197, + "union_independent": 0.601930558681488, + "union_gap": -0.37221769988536835, + "mean_per_layer_hit_rate": 0.06770383566617966 + }, + "50": { + "rho_bar": 0.4547633230686188, + "union_observed": 0.32709112763404846, + "union_independent": 0.7670189142227173, + "union_gap": -0.4399277865886688, + "mean_per_layer_hit_rate": 0.10477288067340851 + }, + "100": { + "rho_bar": 0.47824156284332275, + "union_observed": 0.42571786046028137, + "union_independent": 0.876331627368927, + "union_gap": -0.45061376690864563, + "mean_per_layer_hit_rate": 0.14645156264305115 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3125520401767322, + "union_gap_mean_over_k": -0.26188611132758005 + }, + "Wikipedia_en|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.16066572070121765, + "union_observed": 0.04244694113731384, + "union_independent": 0.07947707176208496, + "union_gap": -0.03703013062477112, + "mean_per_layer_hit_rate": 0.006338231265544891 + }, + "2": { + "rho_bar": 0.18277335166931152, + "union_observed": 0.06367041170597076, + "union_independent": 0.13902980089187622, + "union_gap": -0.07535938918590546, + "mean_per_layer_hit_rate": 0.011428022757172585 + }, + "5": { + "rho_bar": 0.27997177839279175, + "union_observed": 0.1086142286658287, + "union_independent": 0.29402923583984375, + "union_gap": -0.18541500717401505, + "mean_per_layer_hit_rate": 0.026313263922929764 + }, + "10": { + "rho_bar": 0.312178373336792, + "union_observed": 0.15106117725372314, + "union_independent": 0.4360194206237793, + "union_gap": -0.28495824337005615, + "mean_per_layer_hit_rate": 0.04283107817173004 + }, + "20": { + "rho_bar": 0.3596535325050354, + "union_observed": 0.19475655257701874, + "union_independent": 0.5652933120727539, + "union_gap": -0.37053675949573517, + "mean_per_layer_hit_rate": 0.061557669192552567 + }, + "50": { + "rho_bar": 0.4400593638420105, + "union_observed": 0.29837703704833984, + "union_independent": 0.7604930400848389, + "union_gap": -0.462116003036499, + "mean_per_layer_hit_rate": 0.10304427146911621 + }, + "100": { + "rho_bar": 0.45306670665740967, + "union_observed": 0.3982521891593933, + "union_independent": 0.8753467798233032, + "union_gap": -0.4770945906639099, + "mean_per_layer_hit_rate": 0.14606741070747375 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3126241181577955, + "union_gap_mean_over_k": -0.27035858907869886 + }, + "Wikipedia_en|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.18903104960918427, + "union_observed": 0.03745318204164505, + "union_independent": 0.07137197256088257, + "union_gap": -0.03391879051923752, + "mean_per_layer_hit_rate": 0.005665994714945555 + }, + "2": { + "rho_bar": 0.1975744664669037, + "union_observed": 0.05992509424686432, + "union_independent": 0.1260749101638794, + "union_gap": -0.06614981591701508, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "5": { + "rho_bar": 0.24636192619800568, + "union_observed": 0.10986267030239105, + "union_independent": 0.27969980239868164, + "union_gap": -0.1698371320962906, + "mean_per_layer_hit_rate": 0.024776719510555267 + }, + "10": { + "rho_bar": 0.32207298278808594, + "union_observed": 0.15106117725372314, + "union_independent": 0.43147820234298706, + "union_gap": -0.2804170250892639, + "mean_per_layer_hit_rate": 0.04215884208679199 + }, + "20": { + "rho_bar": 0.37103143334388733, + "union_observed": 0.19975031912326813, + "union_independent": 0.5816366076469421, + "union_gap": -0.381886288523674, + "mean_per_layer_hit_rate": 0.06415058672428131 + }, + "50": { + "rho_bar": 0.40871521830558777, + "union_observed": 0.31960049271583557, + "union_independent": 0.7659856081008911, + "union_gap": -0.44638511538505554, + "mean_per_layer_hit_rate": 0.10419667512178421 + }, + "100": { + "rho_bar": 0.4305548071861267, + "union_observed": 0.41198500990867615, + "union_independent": 0.8835260272026062, + "union_gap": -0.47154101729393005, + "mean_per_layer_hit_rate": 0.149428591132164 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3093345548425402, + "union_gap_mean_over_k": -0.26430502640349524 + }, + "Wikipedia_en|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.19701038300991058, + "union_observed": 0.041198503226041794, + "union_independent": 0.09686058759689331, + "union_gap": -0.05566208437085152, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.2293211668729782, + "union_observed": 0.05742821469902992, + "union_independent": 0.14797145128250122, + "union_gap": -0.0905432365834713, + "mean_per_layer_hit_rate": 0.012196293100714684 + }, + "5": { + "rho_bar": 0.294423371553421, + "union_observed": 0.09862671792507172, + "union_independent": 0.2865028977394104, + "union_gap": -0.18787617981433868, + "mean_per_layer_hit_rate": 0.025448959320783615 + }, + "10": { + "rho_bar": 0.3490460216999054, + "union_observed": 0.144818976521492, + "union_independent": 0.43452930450439453, + "union_gap": -0.2897103279829025, + "mean_per_layer_hit_rate": 0.042446937412023544 + }, + "20": { + "rho_bar": 0.3884759545326233, + "union_observed": 0.20599250495433807, + "union_independent": 0.5967812538146973, + "union_gap": -0.3907887488603592, + "mean_per_layer_hit_rate": 0.06655143201351166 + }, + "50": { + "rho_bar": 0.3914029896259308, + "union_observed": 0.32459425926208496, + "union_independent": 0.7774727940559387, + "union_gap": -0.45287853479385376, + "mean_per_layer_hit_rate": 0.10726974904537201 + }, + "100": { + "rho_bar": 0.41500943899154663, + "union_observed": 0.4157303273677826, + "union_independent": 0.8844013214111328, + "union_gap": -0.4686709940433502, + "mean_per_layer_hit_rate": 0.1494286060333252 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.32352704661233084, + "union_gap_mean_over_k": -0.27659001520701815 + }, + "Wikipedia_en|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.2048673927783966, + "union_observed": 0.04494382068514824, + "union_independent": 0.10478240251541138, + "union_gap": -0.05983858183026314, + "mean_per_layer_hit_rate": 0.008450974710285664 + }, + "2": { + "rho_bar": 0.24803473055362701, + "union_observed": 0.06491885334253311, + "union_independent": 0.17272156476974487, + "union_gap": -0.10780271142721176, + "mean_per_layer_hit_rate": 0.014405070804059505 + }, + "5": { + "rho_bar": 0.29784104228019714, + "union_observed": 0.10986267030239105, + "union_independent": 0.29391396045684814, + "union_gap": -0.1840512901544571, + "mean_per_layer_hit_rate": 0.026217229664325714 + }, + "10": { + "rho_bar": 0.3542998731136322, + "union_observed": 0.15355804562568665, + "union_independent": 0.45227694511413574, + "union_gap": -0.2987188994884491, + "mean_per_layer_hit_rate": 0.04475174844264984 + }, + "20": { + "rho_bar": 0.38665178418159485, + "union_observed": 0.20848938822746277, + "union_independent": 0.6063456535339355, + "union_gap": -0.3978562653064728, + "mean_per_layer_hit_rate": 0.0681840032339096 + }, + "50": { + "rho_bar": 0.39445650577545166, + "union_observed": 0.32709112763404846, + "union_independent": 0.7865762710571289, + "union_gap": -0.45948514342308044, + "mean_per_layer_hit_rate": 0.10967060178518295 + }, + "100": { + "rho_bar": 0.4072955548763275, + "union_observed": 0.4307115972042084, + "union_independent": 0.8944742679595947, + "union_gap": -0.46376267075538635, + "mean_per_layer_hit_rate": 0.15432633459568024 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.32763526907988955, + "union_gap_mean_over_k": -0.2816450803407601 + }, + "Wikipedia_en|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.19471803307533264, + "union_observed": 0.041198503226041794, + "union_independent": 0.10031008720397949, + "union_gap": -0.0591115839779377, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.22171686589717865, + "union_observed": 0.06367041170597076, + "union_independent": 0.16217350959777832, + "union_gap": -0.09850309789180756, + "mean_per_layer_hit_rate": 0.013444732874631882 + }, + "5": { + "rho_bar": 0.29544541239738464, + "union_observed": 0.10362047702074051, + "union_independent": 0.3011234998703003, + "union_gap": -0.19750302284955978, + "mean_per_layer_hit_rate": 0.026985498145222664 + }, + "10": { + "rho_bar": 0.3624951243400574, + "union_observed": 0.1398252248764038, + "union_independent": 0.4560762047767639, + "union_gap": -0.3162509799003601, + "mean_per_layer_hit_rate": 0.04523191973567009 + }, + "20": { + "rho_bar": 0.38789045810699463, + "union_observed": 0.20599250495433807, + "union_independent": 0.6149464845657349, + "union_gap": -0.4089539796113968, + "mean_per_layer_hit_rate": 0.06962450593709946 + }, + "50": { + "rho_bar": 0.38123819231987, + "union_observed": 0.3283395767211914, + "union_independent": 0.7947481870651245, + "union_gap": -0.4664086103439331, + "mean_per_layer_hit_rate": 0.1120714619755745 + }, + "100": { + "rho_bar": 0.3915266990661621, + "union_observed": 0.4369538128376007, + "union_independent": 0.9044398665428162, + "union_gap": -0.46748605370521545, + "mean_per_layer_hit_rate": 0.16028040647506714 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.31929011217185427, + "union_gap_mean_over_k": -0.2877453326114586 + }, + "Wikipedia_en|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.20130373537540436, + "union_observed": 0.041198503226041794, + "union_independent": 0.10257840156555176, + "union_gap": -0.061379898339509964, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.22898700833320618, + "union_observed": 0.06117353215813637, + "union_independent": 0.17175430059432983, + "union_gap": -0.11058076843619347, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.3054926097393036, + "union_observed": 0.10486891120672226, + "union_independent": 0.32619351148605347, + "union_gap": -0.2213246002793312, + "mean_per_layer_hit_rate": 0.02967444434762001 + }, + "10": { + "rho_bar": 0.3541124165058136, + "union_observed": 0.16729088127613068, + "union_independent": 0.4947029948234558, + "union_gap": -0.32741211354732513, + "mean_per_layer_hit_rate": 0.05041774362325668 + }, + "20": { + "rho_bar": 0.38268381357192993, + "union_observed": 0.24219726026058197, + "union_independent": 0.6614251136779785, + "union_gap": -0.41922785341739655, + "mean_per_layer_hit_rate": 0.0783635824918747 + }, + "50": { + "rho_bar": 0.3947831094264984, + "union_observed": 0.3483146131038666, + "union_independent": 0.814897894859314, + "union_gap": -0.4665832817554474, + "mean_per_layer_hit_rate": 0.11860174685716629 + }, + "100": { + "rho_bar": 0.39082688093185425, + "union_observed": 0.4619225859642029, + "union_independent": 0.9175917506217957, + "union_gap": -0.4556691646575928, + "mean_per_layer_hit_rate": 0.16930760443210602 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3225985105548586, + "union_gap_mean_over_k": -0.2945968114903995 + }, + "Wikipedia_en|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.19237345457077026, + "union_observed": 0.05243445560336113, + "union_independent": 0.1195218563079834, + "union_gap": -0.06708740070462227, + "mean_per_layer_hit_rate": 0.009699413552880287 + }, + "2": { + "rho_bar": 0.26657921075820923, + "union_observed": 0.07990012317895889, + "union_independent": 0.2009345293045044, + "union_gap": -0.1210344061255455, + "mean_per_layer_hit_rate": 0.016997983679175377 + }, + "5": { + "rho_bar": 0.31225869059562683, + "union_observed": 0.12109863013029099, + "union_independent": 0.3571746349334717, + "union_gap": -0.2360760048031807, + "mean_per_layer_hit_rate": 0.033131662756204605 + }, + "10": { + "rho_bar": 0.3670985996723175, + "union_observed": 0.18726591765880585, + "union_independent": 0.5211076140403748, + "union_gap": -0.3338416963815689, + "mean_per_layer_hit_rate": 0.054355137050151825 + }, + "20": { + "rho_bar": 0.3993038833141327, + "union_observed": 0.2634207308292389, + "union_independent": 0.6881812810897827, + "union_gap": -0.4247605502605438, + "mean_per_layer_hit_rate": 0.08422163873910904 + }, + "50": { + "rho_bar": 0.40083205699920654, + "union_observed": 0.3820224702358246, + "union_independent": 0.8449734449386597, + "union_gap": -0.4629509747028351, + "mean_per_layer_hit_rate": 0.13050994277000427 + }, + "100": { + "rho_bar": 0.42580100893974304, + "union_observed": 0.4756554365158081, + "union_independent": 0.9251030683517456, + "union_gap": -0.4494476318359375, + "mean_per_layer_hit_rate": 0.17593392729759216 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.33774955783571514, + "union_gap_mean_over_k": -0.29931409497346195 + }, + "Wikipedia_en|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.29340463876724243, + "union_observed": 0.04993757978081703, + "union_independent": 0.1349189281463623, + "union_gap": -0.08498134836554527, + "mean_per_layer_hit_rate": 0.011043887585401535 + }, + "2": { + "rho_bar": 0.30384501814842224, + "union_observed": 0.07116104662418365, + "union_independent": 0.19972342252731323, + "union_gap": -0.12856237590312958, + "mean_per_layer_hit_rate": 0.016901949420571327 + }, + "5": { + "rho_bar": 0.3304753303527832, + "union_observed": 0.12609238922595978, + "union_independent": 0.3750670552253723, + "union_gap": -0.24897466599941254, + "mean_per_layer_hit_rate": 0.0352444089949131 + }, + "10": { + "rho_bar": 0.40656617283821106, + "union_observed": 0.19475655257701874, + "union_independent": 0.5511003732681274, + "union_gap": -0.3563438206911087, + "mean_per_layer_hit_rate": 0.05906078964471817 + }, + "20": { + "rho_bar": 0.4283890426158905, + "union_observed": 0.27590513229370117, + "union_independent": 0.7182362079620361, + "union_gap": -0.44233107566833496, + "mean_per_layer_hit_rate": 0.09132814407348633 + }, + "50": { + "rho_bar": 0.4460451304912567, + "union_observed": 0.35830211639404297, + "union_independent": 0.8429674506187439, + "union_gap": -0.4846653342247009, + "mean_per_layer_hit_rate": 0.13022184371948242 + }, + "100": { + "rho_bar": 0.46811461448669434, + "union_observed": 0.46441948413848877, + "union_independent": 0.9293294548988342, + "union_gap": -0.46490997076034546, + "mean_per_layer_hit_rate": 0.180447518825531 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.3824057068143572, + "union_gap_mean_over_k": -0.3158240845160825 + }, + "Wikipedia_en|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.12815943360328674, + "union_observed": 0.05243445560336113, + "union_independent": 0.1013258695602417, + "union_gap": -0.04889141395688057, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.20483465492725372, + "union_observed": 0.072409488260746, + "union_independent": 0.1891525387763977, + "union_gap": -0.1167430505156517, + "mean_per_layer_hit_rate": 0.015941612422466278 + }, + "5": { + "rho_bar": 0.311403751373291, + "union_observed": 0.12858925759792328, + "union_independent": 0.34874361753463745, + "union_gap": -0.22015435993671417, + "mean_per_layer_hit_rate": 0.032267358154058456 + }, + "10": { + "rho_bar": 0.404819130897522, + "union_observed": 0.1885143518447876, + "union_independent": 0.5050623416900635, + "union_gap": -0.3165479898452759, + "mean_per_layer_hit_rate": 0.05224239081144333 + }, + "20": { + "rho_bar": 0.4249137341976166, + "union_observed": 0.24094881117343903, + "union_independent": 0.658740222454071, + "union_gap": -0.417791411280632, + "mean_per_layer_hit_rate": 0.0783635824918747 + }, + "50": { + "rho_bar": 0.45258110761642456, + "union_observed": 0.33957552909851074, + "union_independent": 0.8091150522232056, + "union_gap": -0.4695395231246948, + "mean_per_layer_hit_rate": 0.11764141172170639 + }, + "100": { + "rho_bar": 0.4443494379520416, + "union_observed": 0.4631710350513458, + "union_independent": 0.9029337167739868, + "union_gap": -0.439762681722641, + "mean_per_layer_hit_rate": 0.16104868054389954 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.338723035795348, + "union_gap_mean_over_k": -0.2899186329117843 + }, + "Wikipedia_en|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.1777355968952179, + "union_observed": 0.04619225859642029, + "union_independent": 0.10595077276229858, + "union_gap": -0.059758514165878296, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.2441806197166443, + "union_observed": 0.0749063640832901, + "union_independent": 0.17896854877471924, + "union_gap": -0.10406218469142914, + "mean_per_layer_hit_rate": 0.01498127356171608 + }, + "5": { + "rho_bar": 0.26897725462913513, + "union_observed": 0.11985018849372864, + "union_independent": 0.32719850540161133, + "union_gap": -0.2073483169078827, + "mean_per_layer_hit_rate": 0.02977048046886921 + }, + "10": { + "rho_bar": 0.3579369783401489, + "union_observed": 0.1760299652814865, + "union_independent": 0.49242347478866577, + "union_gap": -0.31639350950717926, + "mean_per_layer_hit_rate": 0.05032171681523323 + }, + "20": { + "rho_bar": 0.40541842579841614, + "union_observed": 0.24219726026058197, + "union_independent": 0.6440614461898804, + "union_gap": -0.4018641859292984, + "mean_per_layer_hit_rate": 0.07538653165102005 + }, + "50": { + "rho_bar": 0.442696750164032, + "union_observed": 0.33957552909851074, + "union_independent": 0.8076143860816956, + "union_gap": -0.4680388569831848, + "mean_per_layer_hit_rate": 0.11687313765287399 + }, + "100": { + "rho_bar": 0.4444454312324524, + "union_observed": 0.4756554365158081, + "union_independent": 0.9092774391174316, + "union_gap": -0.43362200260162354, + "mean_per_layer_hit_rate": 0.16508212685585022 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3344844366822924, + "union_gap_mean_over_k": -0.2844410815409252 + }, + "Wikipedia_en|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.16667766869068146, + "union_observed": 0.048689138144254684, + "union_independent": 0.1003577709197998, + "union_gap": -0.05166863277554512, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.22044014930725098, + "union_observed": 0.08739075809717178, + "union_independent": 0.17701983451843262, + "union_gap": -0.08962907642126083, + "mean_per_layer_hit_rate": 0.014789207838475704 + }, + "5": { + "rho_bar": 0.2696804106235504, + "union_observed": 0.14856429398059845, + "union_independent": 0.3446546792984009, + "union_gap": -0.19609038531780243, + "mean_per_layer_hit_rate": 0.03169115632772446 + }, + "10": { + "rho_bar": 0.3219653367996216, + "union_observed": 0.2172284573316574, + "union_independent": 0.5155611038208008, + "union_gap": -0.29833264648914337, + "mean_per_layer_hit_rate": 0.053490832448005676 + }, + "20": { + "rho_bar": 0.3715003728866577, + "union_observed": 0.2871410846710205, + "union_independent": 0.66206294298172, + "union_gap": -0.37492185831069946, + "mean_per_layer_hit_rate": 0.07865168154239655 + }, + "50": { + "rho_bar": 0.38978713750839233, + "union_observed": 0.39700374007225037, + "union_independent": 0.8159645199775696, + "union_gap": -0.4189607799053192, + "mean_per_layer_hit_rate": 0.11946605890989304 + }, + "100": { + "rho_bar": 0.40639087557792664, + "union_observed": 0.4981273412704468, + "union_independent": 0.9036081433296204, + "union_gap": -0.4054808020591736, + "mean_per_layer_hit_rate": 0.16037645936012268 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.30663456448486875, + "union_gap_mean_over_k": -0.26215488303984913 + }, + "Wikipedia_en|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.14522784948349, + "union_observed": 0.047440700232982635, + "union_independent": 0.10592621564865112, + "union_gap": -0.05848551541566849, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.16557928919792175, + "union_observed": 0.08614232391119003, + "union_independent": 0.1748853325843811, + "union_gap": -0.08874300867319107, + "mean_per_layer_hit_rate": 0.01459713838994503 + }, + "5": { + "rho_bar": 0.2672268748283386, + "union_observed": 0.1398252248764038, + "union_independent": 0.33233994245529175, + "union_gap": -0.19251471757888794, + "mean_per_layer_hit_rate": 0.03034668229520321 + }, + "10": { + "rho_bar": 0.31671926379203796, + "union_observed": 0.20599250495433807, + "union_independent": 0.4889453649520874, + "union_gap": -0.28295285999774933, + "mean_per_layer_hit_rate": 0.04964947700500488 + }, + "20": { + "rho_bar": 0.3713737428188324, + "union_observed": 0.260923832654953, + "union_independent": 0.6453555226325989, + "union_gap": -0.3844316899776459, + "mean_per_layer_hit_rate": 0.0752904936671257 + }, + "50": { + "rho_bar": 0.39058718085289, + "union_observed": 0.3845193386077881, + "union_independent": 0.7996411323547363, + "union_gap": -0.41512179374694824, + "mean_per_layer_hit_rate": 0.11389609426259995 + }, + "100": { + "rho_bar": 0.4108924865722656, + "union_observed": 0.49687889218330383, + "union_independent": 0.903346598148346, + "union_gap": -0.4064677059650421, + "mean_per_layer_hit_rate": 0.16008836030960083 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2953723839351109, + "union_gap_mean_over_k": -0.26124532733644756 + }, + "Wikipedia_en|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.12634405493736267, + "union_observed": 0.04494382068514824, + "union_independent": 0.09908849000930786, + "union_gap": -0.05414466932415962, + "mean_per_layer_hit_rate": 0.007970806211233139 + }, + "2": { + "rho_bar": 0.15995480120182037, + "union_observed": 0.07615480571985245, + "union_independent": 0.1715131402015686, + "union_gap": -0.09535833448171616, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.2594790756702423, + "union_observed": 0.13607990741729736, + "union_independent": 0.3390353322029114, + "union_gap": -0.202955424785614, + "mean_per_layer_hit_rate": 0.031114954501390457 + }, + "10": { + "rho_bar": 0.34587863087654114, + "union_observed": 0.20099875330924988, + "union_independent": 0.49869102239608765, + "union_gap": -0.29769226908683777, + "mean_per_layer_hit_rate": 0.05118602141737938 + }, + "20": { + "rho_bar": 0.3779154419898987, + "union_observed": 0.27715355157852173, + "union_independent": 0.6530277729034424, + "union_gap": -0.37587422132492065, + "mean_per_layer_hit_rate": 0.07692307978868484 + }, + "50": { + "rho_bar": 0.42325544357299805, + "union_observed": 0.387016236782074, + "union_independent": 0.8137510418891907, + "union_gap": -0.4267348051071167, + "mean_per_layer_hit_rate": 0.11860174685716629 + }, + "100": { + "rho_bar": 0.46973952651023865, + "union_observed": 0.4881398379802704, + "union_independent": 0.9077358841896057, + "union_gap": -0.4195960462093353, + "mean_per_layer_hit_rate": 0.16354557871818542 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.30893813925130026, + "union_gap_mean_over_k": -0.26747939575995716 + }, + "Wikipedia_en|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2824462354183197, + "union_observed": 0.04494382068514824, + "union_independent": 0.13919663429260254, + "union_gap": -0.0942528136074543, + "mean_per_layer_hit_rate": 0.011428022757172585 + }, + "2": { + "rho_bar": 0.31376197934150696, + "union_observed": 0.0699126124382019, + "union_independent": 0.22572749853134155, + "union_gap": -0.15581488609313965, + "mean_per_layer_hit_rate": 0.019398828968405724 + }, + "5": { + "rho_bar": 0.39762094616889954, + "union_observed": 0.12359550595283508, + "union_independent": 0.43761909008026123, + "union_gap": -0.31402358412742615, + "mean_per_layer_hit_rate": 0.04302314668893814 + }, + "10": { + "rho_bar": 0.4491429328918457, + "union_observed": 0.17478151619434357, + "union_independent": 0.5929425954818726, + "union_gap": -0.418161079287529, + "mean_per_layer_hit_rate": 0.06607125699520111 + }, + "20": { + "rho_bar": 0.48454809188842773, + "union_observed": 0.2534331977367401, + "union_independent": 0.7460328340530396, + "union_gap": -0.49259963631629944, + "mean_per_layer_hit_rate": 0.09843464940786362 + }, + "50": { + "rho_bar": 0.5306515693664551, + "union_observed": 0.3333333432674408, + "union_independent": 0.8701898455619812, + "union_gap": -0.5368565022945404, + "mean_per_layer_hit_rate": 0.1423220932483673 + }, + "100": { + "rho_bar": 0.5506072640419006, + "union_observed": 0.4319600462913513, + "union_independent": 0.9429354071617126, + "union_gap": -0.5109753608703613, + "mean_per_layer_hit_rate": 0.1923557072877884 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4298255741596222, + "union_gap_mean_over_k": -0.3603834089423929 + }, + "USPTO_Backgrounds|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.3306964635848999, + "union_observed": 0.05368289723992348, + "union_independent": 0.1490265130996704, + "union_gap": -0.09534361585974693, + "mean_per_layer_hit_rate": 0.012292327359318733 + }, + "2": { + "rho_bar": 0.3551974594593048, + "union_observed": 0.07990012317895889, + "union_independent": 0.25371837615966797, + "union_gap": -0.17381825298070908, + "mean_per_layer_hit_rate": 0.02218380756676197 + }, + "5": { + "rho_bar": 0.385131299495697, + "union_observed": 0.1473158597946167, + "union_independent": 0.45123451948165894, + "union_gap": -0.30391865968704224, + "mean_per_layer_hit_rate": 0.04484778642654419 + }, + "10": { + "rho_bar": 0.47957998514175415, + "union_observed": 0.20224718749523163, + "union_independent": 0.6423013210296631, + "union_gap": -0.44005413353443146, + "mean_per_layer_hit_rate": 0.0754825696349144 + }, + "20": { + "rho_bar": 0.5112429857254028, + "union_observed": 0.26966291666030884, + "union_independent": 0.7841476202011108, + "union_gap": -0.514484703540802, + "mean_per_layer_hit_rate": 0.1101507619023323 + }, + "50": { + "rho_bar": 0.5292496085166931, + "union_observed": 0.36828964948654175, + "union_independent": 0.8928669095039368, + "union_gap": -0.524577260017395, + "mean_per_layer_hit_rate": 0.15586286783218384 + }, + "100": { + "rho_bar": 0.5399994850158691, + "union_observed": 0.43820226192474365, + "union_independent": 0.9480138421058655, + "union_gap": -0.5098115801811218, + "mean_per_layer_hit_rate": 0.20032651722431183 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.44729961241994587, + "union_gap_mean_over_k": -0.3660011722573212 + }, + "USPTO_Backgrounds|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.2891143560409546, + "union_observed": 0.04993757978081703, + "union_independent": 0.13493221998214722, + "union_gap": -0.08499464020133018, + "mean_per_layer_hit_rate": 0.011043887585401535 + }, + "2": { + "rho_bar": 0.2759089469909668, + "union_observed": 0.08114856481552124, + "union_independent": 0.22365039587020874, + "union_gap": -0.1425018310546875, + "mean_per_layer_hit_rate": 0.019206760451197624 + }, + "5": { + "rho_bar": 0.31783515214920044, + "union_observed": 0.144818976521492, + "union_independent": 0.4103071093559265, + "union_gap": -0.2654881328344345, + "mean_per_layer_hit_rate": 0.039565928280353546 + }, + "10": { + "rho_bar": 0.3895147144794464, + "union_observed": 0.20474407076835632, + "union_independent": 0.5896819829940796, + "union_gap": -0.38493791222572327, + "mean_per_layer_hit_rate": 0.06568712741136551 + }, + "20": { + "rho_bar": 0.46601927280426025, + "union_observed": 0.2721598148345947, + "union_independent": 0.7451663017272949, + "union_gap": -0.4730064868927002, + "mean_per_layer_hit_rate": 0.09881877899169922 + }, + "50": { + "rho_bar": 0.5251055955886841, + "union_observed": 0.37328338623046875, + "union_independent": 0.8777307271957397, + "union_gap": -0.504447340965271, + "mean_per_layer_hit_rate": 0.14741189777851105 + }, + "100": { + "rho_bar": 0.5317472219467163, + "union_observed": 0.4594257175922394, + "union_independent": 0.9363336563110352, + "union_gap": -0.4769079387187958, + "mean_per_layer_hit_rate": 0.18822625279426575 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3993207514286041, + "union_gap_mean_over_k": -0.3331834689847061 + }, + "USPTO_Backgrounds|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.17404277622699738, + "union_observed": 0.05742821469902992, + "union_independent": 0.12607282400131226, + "union_gap": -0.06864460930228233, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "2": { + "rho_bar": 0.24652083218097687, + "union_observed": 0.08239700645208359, + "union_independent": 0.19845467805862427, + "union_gap": -0.11605767160654068, + "mean_per_layer_hit_rate": 0.016805915161967278 + }, + "5": { + "rho_bar": 0.26850444078445435, + "union_observed": 0.1523096114397049, + "union_independent": 0.3730546236038208, + "union_gap": -0.2207450121641159, + "mean_per_layer_hit_rate": 0.0350523367524147 + }, + "10": { + "rho_bar": 0.3221262991428375, + "union_observed": 0.2122347056865692, + "union_independent": 0.5452077388763428, + "union_gap": -0.33297303318977356, + "mean_per_layer_hit_rate": 0.05829251557588577 + }, + "20": { + "rho_bar": 0.387328565120697, + "union_observed": 0.2796504497528076, + "union_independent": 0.6984903216362, + "union_gap": -0.41883987188339233, + "mean_per_layer_hit_rate": 0.08710265159606934 + }, + "50": { + "rho_bar": 0.47292909026145935, + "union_observed": 0.3695380687713623, + "union_independent": 0.844571590423584, + "union_gap": -0.4750335216522217, + "mean_per_layer_hit_rate": 0.13156630098819733 + }, + "100": { + "rho_bar": 0.4930158853530884, + "union_observed": 0.4319600462913513, + "union_independent": 0.9147501587867737, + "union_gap": -0.48279011249542236, + "mean_per_layer_hit_rate": 0.16988380253314972 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33778112701007296, + "union_gap_mean_over_k": -0.3021548331848213 + }, + "USPTO_Backgrounds|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.18210411071777344, + "union_observed": 0.04619225859642029, + "union_independent": 0.10359048843383789, + "union_gap": -0.0573982298374176, + "mean_per_layer_hit_rate": 0.008354941383004189 + }, + "2": { + "rho_bar": 0.2627714276313782, + "union_observed": 0.07990012317895889, + "union_independent": 0.1870233416557312, + "union_gap": -0.10712321847677231, + "mean_per_layer_hit_rate": 0.01574954390525818 + }, + "5": { + "rho_bar": 0.2852463722229004, + "union_observed": 0.14606741070747375, + "union_independent": 0.3471643328666687, + "union_gap": -0.20109692215919495, + "mean_per_layer_hit_rate": 0.03207528963685036 + }, + "10": { + "rho_bar": 0.33720481395721436, + "union_observed": 0.20848938822746277, + "union_independent": 0.507787823677063, + "union_gap": -0.2992984354496002, + "mean_per_layer_hit_rate": 0.05262652412056923 + }, + "20": { + "rho_bar": 0.40240079164505005, + "union_observed": 0.25468164682388306, + "union_independent": 0.6607720255851746, + "union_gap": -0.4060903787612915, + "mean_per_layer_hit_rate": 0.07884375751018524 + }, + "50": { + "rho_bar": 0.46986153721809387, + "union_observed": 0.3545567989349365, + "union_independent": 0.8158677220344543, + "union_gap": -0.4613109230995178, + "mean_per_layer_hit_rate": 0.12023432552814484 + }, + "100": { + "rho_bar": 0.4928543269634247, + "union_observed": 0.4394506812095642, + "union_independent": 0.8999052047729492, + "union_gap": -0.460454523563385, + "mean_per_layer_hit_rate": 0.15970420837402344 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.347491911479405, + "union_gap_mean_over_k": -0.2846818044781685 + }, + "USPTO_Backgrounds|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.19007296860218048, + "union_observed": 0.04993757978081703, + "union_independent": 0.102486252784729, + "union_gap": -0.05254867300391197, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.23735572397708893, + "union_observed": 0.08239700645208359, + "union_independent": 0.1755639910697937, + "union_gap": -0.09316698461771011, + "mean_per_layer_hit_rate": 0.01469317078590393 + }, + "5": { + "rho_bar": 0.2693835198879242, + "union_observed": 0.14856429398059845, + "union_independent": 0.32978904247283936, + "union_gap": -0.1812247484922409, + "mean_per_layer_hit_rate": 0.03015461564064026 + }, + "10": { + "rho_bar": 0.33617299795150757, + "union_observed": 0.19725343585014343, + "union_independent": 0.48797476291656494, + "union_gap": -0.2907213270664215, + "mean_per_layer_hit_rate": 0.04984154552221298 + }, + "20": { + "rho_bar": 0.3907955288887024, + "union_observed": 0.260923832654953, + "union_independent": 0.6379675269126892, + "union_gap": -0.3770436942577362, + "mean_per_layer_hit_rate": 0.07442618906497955 + }, + "50": { + "rho_bar": 0.43881696462631226, + "union_observed": 0.34956303238868713, + "union_independent": 0.7890408039093018, + "union_gap": -0.4394777715206146, + "mean_per_layer_hit_rate": 0.11159128695726395 + }, + "100": { + "rho_bar": 0.46450087428092957, + "union_observed": 0.43570536375045776, + "union_independent": 0.880652666091919, + "union_gap": -0.4449473023414612, + "mean_per_layer_hit_rate": 0.1490444839000702 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.33244265403066364, + "union_gap_mean_over_k": -0.2684472144714424 + }, + "USPTO_Backgrounds|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.1199381873011589, + "union_observed": 0.03495630621910095, + "union_independent": 0.07010519504547119, + "union_gap": -0.03514888882637024, + "mean_per_layer_hit_rate": 0.005569960456341505 + }, + "2": { + "rho_bar": 0.2454482764005661, + "union_observed": 0.06616728752851486, + "union_independent": 0.16373902559280396, + "union_gap": -0.09757173806428909, + "mean_per_layer_hit_rate": 0.013636799529194832 + }, + "5": { + "rho_bar": 0.3043087422847748, + "union_observed": 0.11610487103462219, + "union_independent": 0.32921576499938965, + "union_gap": -0.21311089396476746, + "mean_per_layer_hit_rate": 0.03015461564064026 + }, + "10": { + "rho_bar": 0.35006996989250183, + "union_observed": 0.15355804562568665, + "union_independent": 0.4671529531478882, + "union_gap": -0.31359490752220154, + "mean_per_layer_hit_rate": 0.04705656319856644 + }, + "20": { + "rho_bar": 0.39330556988716125, + "union_observed": 0.20474407076835632, + "union_independent": 0.6040229797363281, + "union_gap": -0.3992789089679718, + "mean_per_layer_hit_rate": 0.0683760717511177 + }, + "50": { + "rho_bar": 0.4063769578933716, + "union_observed": 0.31460675597190857, + "union_independent": 0.7657197713851929, + "union_gap": -0.4511130154132843, + "mean_per_layer_hit_rate": 0.10486891865730286 + }, + "100": { + "rho_bar": 0.43522390723228455, + "union_observed": 0.4107365906238556, + "union_independent": 0.8727747201919556, + "union_gap": -0.4620381295681, + "mean_per_layer_hit_rate": 0.14529915153980255 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.322095944413117, + "union_gap_mean_over_k": -0.2816937831895692 + }, + "USPTO_Backgrounds|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.15541492402553558, + "union_observed": 0.039950061589479446, + "union_independent": 0.0771215558052063, + "union_gap": -0.03717149421572685, + "mean_per_layer_hit_rate": 0.006146163679659367 + }, + "2": { + "rho_bar": 0.19829261302947998, + "union_observed": 0.0699126124382019, + "union_independent": 0.1702277660369873, + "union_gap": -0.1003151535987854, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.29528501629829407, + "union_observed": 0.11485642939805984, + "union_independent": 0.3101070523262024, + "union_gap": -0.19525062292814255, + "mean_per_layer_hit_rate": 0.028041871264576912 + }, + "10": { + "rho_bar": 0.35394927859306335, + "union_observed": 0.15730337798595428, + "union_independent": 0.4506927728652954, + "union_gap": -0.2933893948793411, + "mean_per_layer_hit_rate": 0.04475175589323044 + }, + "20": { + "rho_bar": 0.37787240743637085, + "union_observed": 0.21098627150058746, + "union_independent": 0.6004332304000854, + "union_gap": -0.389446958899498, + "mean_per_layer_hit_rate": 0.06751176714897156 + }, + "50": { + "rho_bar": 0.42890664935112, + "union_observed": 0.3071160912513733, + "union_independent": 0.7711368799209595, + "union_gap": -0.4640207886695862, + "mean_per_layer_hit_rate": 0.1061173602938652 + }, + "100": { + "rho_bar": 0.4377650320529938, + "union_observed": 0.4007490575313568, + "union_independent": 0.867522656917572, + "union_gap": -0.4667735993862152, + "mean_per_layer_hit_rate": 0.14203399419784546 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.3210694172552654, + "union_gap_mean_over_k": -0.2780525732253279 + }, + "USPTO_Backgrounds|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.1610385626554489, + "union_observed": 0.04369537904858589, + "union_independent": 0.10136175155639648, + "union_gap": -0.05766637250781059, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.20998820662498474, + "union_observed": 0.07365792989730835, + "union_independent": 0.17886465787887573, + "union_gap": -0.10520672798156738, + "mean_per_layer_hit_rate": 0.014981274493038654 + }, + "5": { + "rho_bar": 0.2718389332294464, + "union_observed": 0.1111111119389534, + "union_independent": 0.3088235855102539, + "union_gap": -0.1977124735713005, + "mean_per_layer_hit_rate": 0.027849802747368813 + }, + "10": { + "rho_bar": 0.33890300989151, + "union_observed": 0.16604244709014893, + "union_independent": 0.4594714641571045, + "union_gap": -0.29342901706695557, + "mean_per_layer_hit_rate": 0.04580812156200409 + }, + "20": { + "rho_bar": 0.3789367079734802, + "union_observed": 0.23470661044120789, + "union_independent": 0.6201789975166321, + "union_gap": -0.3854723870754242, + "mean_per_layer_hit_rate": 0.07077690958976746 + }, + "50": { + "rho_bar": 0.4147450923919678, + "union_observed": 0.3208489418029785, + "union_independent": 0.7831819653511047, + "union_gap": -0.4623330235481262, + "mean_per_layer_hit_rate": 0.10909439623355865 + }, + "100": { + "rho_bar": 0.4294191598892212, + "union_observed": 0.4107365906238556, + "union_independent": 0.8840581774711609, + "union_gap": -0.4733215868473053, + "mean_per_layer_hit_rate": 0.14952462911605835 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31498138180800844, + "union_gap_mean_over_k": -0.28216308408549856 + }, + "USPTO_Backgrounds|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.16213999688625336, + "union_observed": 0.054931335151195526, + "union_independent": 0.11830919981002808, + "union_gap": -0.06337786465883255, + "mean_per_layer_hit_rate": 0.009603381156921387 + }, + "2": { + "rho_bar": 0.2137744426727295, + "union_observed": 0.08239700645208359, + "union_independent": 0.1873801350593567, + "union_gap": -0.1049831286072731, + "mean_per_layer_hit_rate": 0.01574954390525818 + }, + "5": { + "rho_bar": 0.2634665071964264, + "union_observed": 0.12983770668506622, + "union_independent": 0.3294792175292969, + "union_gap": -0.19964151084423065, + "mean_per_layer_hit_rate": 0.03005858138203621 + }, + "10": { + "rho_bar": 0.319343626499176, + "union_observed": 0.1785268485546112, + "union_independent": 0.47423040866851807, + "union_gap": -0.29570356011390686, + "mean_per_layer_hit_rate": 0.047728799283504486 + }, + "20": { + "rho_bar": 0.3674173951148987, + "union_observed": 0.24219726026058197, + "union_independent": 0.630870521068573, + "union_gap": -0.38867326080799103, + "mean_per_layer_hit_rate": 0.0726015567779541 + }, + "50": { + "rho_bar": 0.400134414434433, + "union_observed": 0.3383270800113678, + "union_independent": 0.8044464588165283, + "union_gap": -0.4661193788051605, + "mean_per_layer_hit_rate": 0.11543263494968414 + }, + "100": { + "rho_bar": 0.4224623143672943, + "union_observed": 0.4157303273677826, + "union_independent": 0.8949826955795288, + "union_gap": -0.4792523682117462, + "mean_per_layer_hit_rate": 0.15471045672893524 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.30696267102445873, + "union_gap_mean_over_k": -0.2853930102927344 + }, + "USPTO_Backgrounds|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.21782049536705017, + "union_observed": 0.04369537904858589, + "union_independent": 0.11045032739639282, + "union_gap": -0.06675494834780693, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.2852784991264343, + "union_observed": 0.072409488260746, + "union_independent": 0.18830156326293945, + "union_gap": -0.11589207500219345, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.26553478837013245, + "union_observed": 0.12734082341194153, + "union_independent": 0.33900028467178345, + "union_gap": -0.21165946125984192, + "mean_per_layer_hit_rate": 0.031114954501390457 + }, + "10": { + "rho_bar": 0.3445744514465332, + "union_observed": 0.17228464782238007, + "union_independent": 0.4952746629714966, + "union_gap": -0.3229900151491165, + "mean_per_layer_hit_rate": 0.05060981214046478 + }, + "20": { + "rho_bar": 0.36224958300590515, + "union_observed": 0.24968789517879486, + "union_independent": 0.648255467414856, + "union_gap": -0.3985675722360611, + "mean_per_layer_hit_rate": 0.0758667066693306 + }, + "50": { + "rho_bar": 0.4002726674079895, + "union_observed": 0.3408239781856537, + "union_independent": 0.8136163949966431, + "union_gap": -0.4727924168109894, + "mean_per_layer_hit_rate": 0.11840968579053879 + }, + "100": { + "rho_bar": 0.42946791648864746, + "union_observed": 0.4157303273677826, + "union_independent": 0.9040144085884094, + "union_gap": -0.48828408122062683, + "mean_per_layer_hit_rate": 0.16008836030960083 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.3293140573160989, + "union_gap_mean_over_k": -0.29670579571809086 + }, + "USPTO_Backgrounds|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.3024420738220215, + "union_observed": 0.048689138144254684, + "union_independent": 0.11950618028640747, + "union_gap": -0.07081704214215279, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.329086035490036, + "union_observed": 0.07865168899297714, + "union_independent": 0.21199512481689453, + "union_gap": -0.1333434358239174, + "mean_per_layer_hit_rate": 0.018054354935884476 + }, + "5": { + "rho_bar": 0.28215810656547546, + "union_observed": 0.13857677578926086, + "union_independent": 0.36597752571105957, + "union_gap": -0.2274007499217987, + "mean_per_layer_hit_rate": 0.03418803587555885 + }, + "10": { + "rho_bar": 0.3573510944843292, + "union_observed": 0.19225966930389404, + "union_independent": 0.5375221967697144, + "union_gap": -0.3452625274658203, + "mean_per_layer_hit_rate": 0.056852009147405624 + }, + "20": { + "rho_bar": 0.3716689944267273, + "union_observed": 0.2534331977367401, + "union_independent": 0.6869771480560303, + "union_gap": -0.43354395031929016, + "mean_per_layer_hit_rate": 0.08393353968858719 + }, + "50": { + "rho_bar": 0.40384116768836975, + "union_observed": 0.3545567989349365, + "union_independent": 0.8336709141731262, + "union_gap": -0.4791141152381897, + "mean_per_layer_hit_rate": 0.12580427527427673 + }, + "100": { + "rho_bar": 0.4262886941432953, + "union_observed": 0.44319599866867065, + "union_independent": 0.9186417460441589, + "union_gap": -0.4754457473754883, + "mean_per_layer_hit_rate": 0.17026792466640472 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.35326230951717924, + "union_gap_mean_over_k": -0.30927536689809393 + }, + "USPTO_Backgrounds|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.2756846845149994, + "union_observed": 0.04993757978081703, + "union_independent": 0.12724906206130981, + "union_gap": -0.07731148228049278, + "mean_per_layer_hit_rate": 0.010371650569140911 + }, + "2": { + "rho_bar": 0.27678462862968445, + "union_observed": 0.08114856481552124, + "union_independent": 0.22587859630584717, + "union_gap": -0.14473003149032593, + "mean_per_layer_hit_rate": 0.019398828968405724 + }, + "5": { + "rho_bar": 0.33159181475639343, + "union_observed": 0.1398252248764038, + "union_independent": 0.40032607316970825, + "union_gap": -0.26050084829330444, + "mean_per_layer_hit_rate": 0.0383174866437912 + }, + "10": { + "rho_bar": 0.38045796751976013, + "union_observed": 0.193508118391037, + "union_independent": 0.5750981569290161, + "union_gap": -0.3815900385379791, + "mean_per_layer_hit_rate": 0.06299817562103271 + }, + "20": { + "rho_bar": 0.4059695303440094, + "union_observed": 0.25967541337013245, + "union_independent": 0.7287158966064453, + "union_gap": -0.46904048323631287, + "mean_per_layer_hit_rate": 0.09392105787992477 + }, + "50": { + "rho_bar": 0.4341444969177246, + "union_observed": 0.3533083498477936, + "union_independent": 0.8576606512069702, + "union_gap": -0.5043523013591766, + "mean_per_layer_hit_rate": 0.13646404445171356 + }, + "100": { + "rho_bar": 0.46188637614250183, + "union_observed": 0.4394506812095642, + "union_independent": 0.9277308583259583, + "union_gap": -0.48828017711639404, + "mean_per_layer_hit_rate": 0.1785268485546112 + } + }, + "n_degenerate_pairs": 33, + "rho_bar_mean_over_k": 0.36664564268929617, + "union_gap_mean_over_k": -0.33225790890199797 + }, + "USPTO_Backgrounds|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.22402258217334747, + "union_observed": 0.054931335151195526, + "union_independent": 0.13926821947097778, + "union_gap": -0.08433688431978226, + "mean_per_layer_hit_rate": 0.011428022757172585 + }, + "2": { + "rho_bar": 0.318563312292099, + "union_observed": 0.07615480571985245, + "union_independent": 0.22746485471725464, + "union_gap": -0.1513100489974022, + "mean_per_layer_hit_rate": 0.019590895622968674 + }, + "5": { + "rho_bar": 0.3456254005432129, + "union_observed": 0.1498127281665802, + "union_independent": 0.4149492383003235, + "union_gap": -0.2651365101337433, + "mean_per_layer_hit_rate": 0.04014212638139725 + }, + "10": { + "rho_bar": 0.40792471170425415, + "union_observed": 0.20973782241344452, + "union_independent": 0.6125913858413696, + "union_gap": -0.4028535634279251, + "mean_per_layer_hit_rate": 0.06972053647041321 + }, + "20": { + "rho_bar": 0.44883856177330017, + "union_observed": 0.2734082341194153, + "union_independent": 0.7602245211601257, + "union_gap": -0.48681628704071045, + "mean_per_layer_hit_rate": 0.10266012698411942 + }, + "50": { + "rho_bar": 0.4721323251724243, + "union_observed": 0.36079901456832886, + "union_independent": 0.8707122802734375, + "union_gap": -0.5099132657051086, + "mean_per_layer_hit_rate": 0.1432824432849884 + }, + "100": { + "rho_bar": 0.4684174954891205, + "union_observed": 0.4656679034233093, + "union_independent": 0.935976505279541, + "union_gap": -0.4703086018562317, + "mean_per_layer_hit_rate": 0.18688179552555084 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3836463413068226, + "union_gap_mean_over_k": -0.33866788021155764 + }, + "USPTO_Backgrounds|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.2062552273273468, + "union_observed": 0.048689138144254684, + "union_independent": 0.11594939231872559, + "union_gap": -0.0672602541744709, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.26872238516807556, + "union_observed": 0.08114856481552124, + "union_independent": 0.20121467113494873, + "union_gap": -0.12006610631942749, + "mean_per_layer_hit_rate": 0.017094016075134277 + }, + "5": { + "rho_bar": 0.3325520157814026, + "union_observed": 0.15106117725372314, + "union_independent": 0.3790017366409302, + "union_gap": -0.22794055938720703, + "mean_per_layer_hit_rate": 0.0358206108212471 + }, + "10": { + "rho_bar": 0.38228002190589905, + "union_observed": 0.20973782241344452, + "union_independent": 0.5448381900787354, + "union_gap": -0.33510036766529083, + "mean_per_layer_hit_rate": 0.05829251557588577 + }, + "20": { + "rho_bar": 0.44197943806648254, + "union_observed": 0.255930095911026, + "union_independent": 0.6899554133415222, + "union_gap": -0.4340253174304962, + "mean_per_layer_hit_rate": 0.08527801930904388 + }, + "50": { + "rho_bar": 0.47116851806640625, + "union_observed": 0.34456929564476013, + "union_independent": 0.8339388370513916, + "union_gap": -0.48936954140663147, + "mean_per_layer_hit_rate": 0.12734082341194153 + }, + "100": { + "rho_bar": 0.4772069454193115, + "union_observed": 0.4369538128376007, + "union_independent": 0.9078424572944641, + "union_gap": -0.4708886444568634, + "mean_per_layer_hit_rate": 0.16498605906963348 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.36859493596213205, + "union_gap_mean_over_k": -0.3063786844057696 + }, + "USPTO_Backgrounds|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.21307143568992615, + "union_observed": 0.05368289723992348, + "union_independent": 0.1227186918258667, + "union_gap": -0.06903579458594322, + "mean_per_layer_hit_rate": 0.009987516328692436 + }, + "2": { + "rho_bar": 0.25844618678092957, + "union_observed": 0.08239700645208359, + "union_independent": 0.20351296663284302, + "union_gap": -0.12111596018075943, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.2767138183116913, + "union_observed": 0.1473158597946167, + "union_independent": 0.35640382766723633, + "union_gap": -0.20908796787261963, + "mean_per_layer_hit_rate": 0.033131662756204605 + }, + "10": { + "rho_bar": 0.35606661438941956, + "union_observed": 0.1910112351179123, + "union_independent": 0.5159976482391357, + "union_gap": -0.32498641312122345, + "mean_per_layer_hit_rate": 0.05377892777323723 + }, + "20": { + "rho_bar": 0.41106849908828735, + "union_observed": 0.260923832654953, + "union_independent": 0.6739075183868408, + "union_gap": -0.4129836857318878, + "mean_per_layer_hit_rate": 0.08143667131662369 + }, + "50": { + "rho_bar": 0.4451651871204376, + "union_observed": 0.37078651785850525, + "union_independent": 0.8372761011123657, + "union_gap": -0.4664895832538605, + "mean_per_layer_hit_rate": 0.12791702151298523 + }, + "100": { + "rho_bar": 0.4391462802886963, + "union_observed": 0.47191011905670166, + "union_independent": 0.9169613718986511, + "union_gap": -0.44505125284194946, + "mean_per_layer_hit_rate": 0.17017190158367157 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3428111459527697, + "union_gap_mean_over_k": -0.29267866536974907 + }, + "USPTO_Backgrounds|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.23419222235679626, + "union_observed": 0.04993757978081703, + "union_independent": 0.11168533563613892, + "union_gap": -0.061747755855321884, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.2792614698410034, + "union_observed": 0.08364544063806534, + "union_independent": 0.20169353485107422, + "union_gap": -0.11804809421300888, + "mean_per_layer_hit_rate": 0.017094017937779427 + }, + "5": { + "rho_bar": 0.28466561436653137, + "union_observed": 0.15855181217193604, + "union_independent": 0.3717600107192993, + "union_gap": -0.21320819854736328, + "mean_per_layer_hit_rate": 0.0348602719604969 + }, + "10": { + "rho_bar": 0.3297317624092102, + "union_observed": 0.20973782241344452, + "union_independent": 0.525455117225647, + "union_gap": -0.31571729481220245, + "mean_per_layer_hit_rate": 0.055123403668403625 + }, + "20": { + "rho_bar": 0.39219194650650024, + "union_observed": 0.27715355157852173, + "union_independent": 0.6939621567726135, + "union_gap": -0.4168086051940918, + "mean_per_layer_hit_rate": 0.08575819432735443 + }, + "50": { + "rho_bar": 0.4198519289493561, + "union_observed": 0.38576778769493103, + "union_independent": 0.8320824503898621, + "union_gap": -0.44631466269493103, + "mean_per_layer_hit_rate": 0.12580429017543793 + }, + "100": { + "rho_bar": 0.4312364161014557, + "union_observed": 0.47191011905670166, + "union_independent": 0.911256730556488, + "union_gap": -0.4393466114997864, + "mean_per_layer_hit_rate": 0.16623450815677643 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.33873305150440763, + "union_gap_mean_over_k": -0.28731303183095797 + }, + "USPTO_Backgrounds|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.16149355471134186, + "union_observed": 0.05243445560336113, + "union_independent": 0.10930949449539185, + "union_gap": -0.056875038892030716, + "mean_per_layer_hit_rate": 0.008835110813379288 + }, + "2": { + "rho_bar": 0.22118355333805084, + "union_observed": 0.08988764137029648, + "union_independent": 0.198461651802063, + "union_gap": -0.10857401043176651, + "mean_per_layer_hit_rate": 0.016805917024612427 + }, + "5": { + "rho_bar": 0.257946640253067, + "union_observed": 0.1548064947128296, + "union_independent": 0.35067665576934814, + "union_gap": -0.19587016105651855, + "mean_per_layer_hit_rate": 0.032459430396556854 + }, + "10": { + "rho_bar": 0.33480724692344666, + "union_observed": 0.20848938822746277, + "union_independent": 0.5140174627304077, + "union_gap": -0.30552807450294495, + "mean_per_layer_hit_rate": 0.05339479446411133 + }, + "20": { + "rho_bar": 0.37843039631843567, + "union_observed": 0.2709113657474518, + "union_independent": 0.6629014015197754, + "union_gap": -0.3919900357723236, + "mean_per_layer_hit_rate": 0.07903582602739334 + }, + "50": { + "rho_bar": 0.41304153203964233, + "union_observed": 0.36579275131225586, + "union_independent": 0.8132985234260559, + "union_gap": -0.44750577211380005, + "mean_per_layer_hit_rate": 0.11888986080884933 + }, + "100": { + "rho_bar": 0.42137739062309265, + "union_observed": 0.4656679034233093, + "union_independent": 0.9061771631240845, + "union_gap": -0.44050925970077515, + "mean_per_layer_hit_rate": 0.16248919069766998 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3126114734581539, + "union_gap_mean_over_k": -0.2781217646385942 + }, + "USPTO_Backgrounds|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.14955708384513855, + "union_observed": 0.062421973794698715, + "union_independent": 0.12274914979934692, + "union_gap": -0.06032717600464821, + "mean_per_layer_hit_rate": 0.009987516328692436 + }, + "2": { + "rho_bar": 0.23122741281986237, + "union_observed": 0.09363295882940292, + "union_independent": 0.203571617603302, + "union_gap": -0.10993865877389908, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.28063252568244934, + "union_observed": 0.15605492889881134, + "union_independent": 0.37506669759750366, + "union_gap": -0.21901176869869232, + "mean_per_layer_hit_rate": 0.035340435802936554 + }, + "10": { + "rho_bar": 0.3465149700641632, + "union_observed": 0.21598002314567566, + "union_independent": 0.5428234338760376, + "union_gap": -0.32684341073036194, + "mean_per_layer_hit_rate": 0.05790838226675987 + }, + "20": { + "rho_bar": 0.4115113615989685, + "union_observed": 0.27715355157852173, + "union_independent": 0.6937384009361267, + "union_gap": -0.416584849357605, + "mean_per_layer_hit_rate": 0.08585421741008759 + }, + "50": { + "rho_bar": 0.46521085500717163, + "union_observed": 0.36329588294029236, + "union_independent": 0.836006760597229, + "union_gap": -0.47271087765693665, + "mean_per_layer_hit_rate": 0.12753288447856903 + }, + "100": { + "rho_bar": 0.48128440976142883, + "union_observed": 0.45318353176116943, + "union_independent": 0.9111904501914978, + "union_gap": -0.45800691843032837, + "mean_per_layer_hit_rate": 0.16623450815677643 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3379912312541689, + "union_gap_mean_over_k": -0.2947748085217817 + }, + "USPTO_Backgrounds|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.3106148838996887, + "union_observed": 0.04369537904858589, + "union_independent": 0.13923239707946777, + "union_gap": -0.09553701803088188, + "mean_per_layer_hit_rate": 0.011428022757172585 + }, + "2": { + "rho_bar": 0.40563148260116577, + "union_observed": 0.07116104662418365, + "union_independent": 0.244329035282135, + "union_gap": -0.17316798865795135, + "mean_per_layer_hit_rate": 0.02122347243130207 + }, + "5": { + "rho_bar": 0.42465558648109436, + "union_observed": 0.12609238922595978, + "union_independent": 0.43072110414505005, + "union_gap": -0.30462871491909027, + "mean_per_layer_hit_rate": 0.04215884208679199 + }, + "10": { + "rho_bar": 0.5012964010238647, + "union_observed": 0.17478151619434357, + "union_independent": 0.6213725209236145, + "union_gap": -0.44659100472927094, + "mean_per_layer_hit_rate": 0.071257084608078 + }, + "20": { + "rho_bar": 0.5283972024917603, + "union_observed": 0.24719101190567017, + "union_independent": 0.7731325626373291, + "union_gap": -0.5259415507316589, + "mean_per_layer_hit_rate": 0.10621338337659836 + }, + "50": { + "rho_bar": 0.5605112910270691, + "union_observed": 0.3408239781856537, + "union_independent": 0.8914083242416382, + "union_gap": -0.5505843460559845, + "mean_per_layer_hit_rate": 0.1538461446762085 + }, + "100": { + "rho_bar": 0.5782725811004639, + "union_observed": 0.4244694113731384, + "union_independent": 0.9465247392654419, + "union_gap": -0.5220553278923035, + "mean_per_layer_hit_rate": 0.19715739786624908 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.4727684898035867, + "union_gap_mean_over_k": -0.3740722787167345 + }, + "USPTO_Backgrounds|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.31569647789001465, + "union_observed": 0.05368289723992348, + "union_independent": 0.14794522523880005, + "union_gap": -0.09426232799887657, + "mean_per_layer_hit_rate": 0.012196293100714684 + }, + "2": { + "rho_bar": 0.34977877140045166, + "union_observed": 0.07990012317895889, + "union_independent": 0.2508491277694702, + "union_gap": -0.17094900459051132, + "mean_per_layer_hit_rate": 0.02189570665359497 + }, + "5": { + "rho_bar": 0.37982892990112305, + "union_observed": 0.1498127281665802, + "union_independent": 0.45072609186172485, + "union_gap": -0.30091336369514465, + "mean_per_layer_hit_rate": 0.04475175589323044 + }, + "10": { + "rho_bar": 0.47612670063972473, + "union_observed": 0.20349563658237457, + "union_independent": 0.6406564712524414, + "union_gap": -0.43716083467006683, + "mean_per_layer_hit_rate": 0.0750984400510788 + }, + "20": { + "rho_bar": 0.5115594267845154, + "union_observed": 0.2721598148345947, + "union_independent": 0.7871055006980896, + "union_gap": -0.5149456858634949, + "mean_per_layer_hit_rate": 0.11101507395505905 + }, + "50": { + "rho_bar": 0.5286728143692017, + "union_observed": 0.3670412003993988, + "union_independent": 0.8953223824501038, + "union_gap": -0.528281182050705, + "mean_per_layer_hit_rate": 0.1573033630847931 + }, + "100": { + "rho_bar": 0.5391871333122253, + "union_observed": 0.4419475793838501, + "union_independent": 0.9477025270462036, + "union_gap": -0.5057549476623535, + "mean_per_layer_hit_rate": 0.19984634220600128 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.4429786077567509, + "union_gap_mean_over_k": -0.3646096209330218 + }, + "USPTO_Backgrounds|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.2973613739013672, + "union_observed": 0.048689138144254684, + "union_independent": 0.1338314414024353, + "union_gap": -0.08514230325818062, + "mean_per_layer_hit_rate": 0.010947853326797485 + }, + "2": { + "rho_bar": 0.2681256830692291, + "union_observed": 0.08489388227462769, + "union_independent": 0.22275179624557495, + "union_gap": -0.13785791397094727, + "mean_per_layer_hit_rate": 0.019110726192593575 + }, + "5": { + "rho_bar": 0.31662389636039734, + "union_observed": 0.144818976521492, + "union_independent": 0.40748995542526245, + "union_gap": -0.26267097890377045, + "mean_per_layer_hit_rate": 0.03918179124593735 + }, + "10": { + "rho_bar": 0.38324758410453796, + "union_observed": 0.21598002314567566, + "union_independent": 0.5936225652694702, + "union_gap": -0.37764254212379456, + "mean_per_layer_hit_rate": 0.06635935604572296 + }, + "20": { + "rho_bar": 0.4591735899448395, + "union_observed": 0.2721598148345947, + "union_independent": 0.7439330816268921, + "union_gap": -0.47177326679229736, + "mean_per_layer_hit_rate": 0.09843464940786362 + }, + "50": { + "rho_bar": 0.5238584876060486, + "union_observed": 0.38077402114868164, + "union_independent": 0.8801659345626831, + "union_gap": -0.49939191341400146, + "mean_per_layer_hit_rate": 0.14875635504722595 + }, + "100": { + "rho_bar": 0.5405027270317078, + "union_observed": 0.45068663358688354, + "union_independent": 0.93705153465271, + "union_gap": -0.4863649010658264, + "mean_per_layer_hit_rate": 0.1888984888792038 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.39841333457401823, + "union_gap_mean_over_k": -0.33154911707554546 + }, + "USPTO_Backgrounds|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.1759805679321289, + "union_observed": 0.05742821469902992, + "union_independent": 0.12498211860656738, + "union_gap": -0.06755390390753746, + "mean_per_layer_hit_rate": 0.010179583914577961 + }, + "2": { + "rho_bar": 0.25162768363952637, + "union_observed": 0.0774032473564148, + "union_independent": 0.19838851690292358, + "union_gap": -0.12098526954650879, + "mean_per_layer_hit_rate": 0.016805915161967278 + }, + "5": { + "rho_bar": 0.2658531665802002, + "union_observed": 0.1498127281665802, + "union_independent": 0.3715091347694397, + "union_gap": -0.2216964066028595, + "mean_per_layer_hit_rate": 0.034860268235206604 + }, + "10": { + "rho_bar": 0.32916396856307983, + "union_observed": 0.21098627150058746, + "union_independent": 0.5444415211677551, + "union_gap": -0.33345524966716766, + "mean_per_layer_hit_rate": 0.05819648504257202 + }, + "20": { + "rho_bar": 0.3905770480632782, + "union_observed": 0.27715355157852173, + "union_independent": 0.6986111998558044, + "union_gap": -0.4214576482772827, + "mean_per_layer_hit_rate": 0.08710265904664993 + }, + "50": { + "rho_bar": 0.4818737208843231, + "union_observed": 0.3670412003993988, + "union_independent": 0.8477150201797485, + "union_gap": -0.48067381978034973, + "mean_per_layer_hit_rate": 0.13281475007534027 + }, + "100": { + "rho_bar": 0.49967160820961, + "union_observed": 0.42696627974510193, + "union_independent": 0.9141921401023865, + "union_gap": -0.48722586035728455, + "mean_per_layer_hit_rate": 0.16940362751483917 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3421068234103067, + "union_gap_mean_over_k": -0.3047211654484272 + }, + "USPTO_Backgrounds|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.1860385537147522, + "union_observed": 0.047440700232982635, + "union_independent": 0.10584545135498047, + "union_gap": -0.05840475112199783, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.23718924820423126, + "union_observed": 0.08239700645208359, + "union_independent": 0.18810325860977173, + "union_gap": -0.10570625215768814, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.27480751276016235, + "union_observed": 0.14606741070747375, + "union_independent": 0.3455265760421753, + "union_gap": -0.19945916533470154, + "mean_per_layer_hit_rate": 0.031883224844932556 + }, + "10": { + "rho_bar": 0.34142130613327026, + "union_observed": 0.20474407076835632, + "union_independent": 0.507943868637085, + "union_gap": -0.30319979786872864, + "mean_per_layer_hit_rate": 0.05262652412056923 + }, + "20": { + "rho_bar": 0.4086417555809021, + "union_observed": 0.25717851519584656, + "union_independent": 0.6672441959381104, + "union_gap": -0.4100656807422638, + "mean_per_layer_hit_rate": 0.08018822968006134 + }, + "50": { + "rho_bar": 0.47475042939186096, + "union_observed": 0.352059930562973, + "union_independent": 0.8167185187339783, + "union_gap": -0.46465858817100525, + "mean_per_layer_hit_rate": 0.12061845511198044 + }, + "100": { + "rho_bar": 0.4916979968547821, + "union_observed": 0.4469413161277771, + "union_independent": 0.90189528465271, + "union_gap": -0.45495396852493286, + "mean_per_layer_hit_rate": 0.16095265746116638 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3449352575199945, + "union_gap_mean_over_k": -0.285206886274474 + }, + "USPTO_Backgrounds|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.15498127043247223, + "union_observed": 0.04993757978081703, + "union_independent": 0.10246878862380981, + "union_gap": -0.05253120884299278, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.2435133457183838, + "union_observed": 0.08114856481552124, + "union_independent": 0.17763805389404297, + "union_gap": -0.09648948907852173, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2591317892074585, + "union_observed": 0.15605492889881134, + "union_independent": 0.3358173370361328, + "union_gap": -0.17976240813732147, + "mean_per_layer_hit_rate": 0.03082684986293316 + }, + "10": { + "rho_bar": 0.33516591787338257, + "union_observed": 0.19725343585014343, + "union_independent": 0.489953875541687, + "union_gap": -0.2927004396915436, + "mean_per_layer_hit_rate": 0.05012964829802513 + }, + "20": { + "rho_bar": 0.38334429264068604, + "union_observed": 0.255930095911026, + "union_independent": 0.6349607706069946, + "union_gap": -0.37903067469596863, + "mean_per_layer_hit_rate": 0.07384999096393585 + }, + "50": { + "rho_bar": 0.4341104030609131, + "union_observed": 0.3570536971092224, + "union_independent": 0.7920168042182922, + "union_gap": -0.4349631071090698, + "mean_per_layer_hit_rate": 0.11255161464214325 + }, + "100": { + "rho_bar": 0.465721070766449, + "union_observed": 0.43820226192474365, + "union_independent": 0.8820124864578247, + "union_gap": -0.44381022453308105, + "mean_per_layer_hit_rate": 0.14971670508384705 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.32513829852853504, + "union_gap_mean_over_k": -0.268469650298357 + }, + "USPTO_Backgrounds|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.13065552711486816, + "union_observed": 0.0387016236782074, + "union_independent": 0.07709413766860962, + "union_gap": -0.03839251399040222, + "mean_per_layer_hit_rate": 0.006146163213998079 + }, + "2": { + "rho_bar": 0.23177284002304077, + "union_observed": 0.06616728752851486, + "union_independent": 0.16269248723983765, + "union_gap": -0.09652519971132278, + "mean_per_layer_hit_rate": 0.013540766201913357 + }, + "5": { + "rho_bar": 0.29947954416275024, + "union_observed": 0.11985018849372864, + "union_independent": 0.3317968249320984, + "union_gap": -0.21194663643836975, + "mean_per_layer_hit_rate": 0.030442718416452408 + }, + "10": { + "rho_bar": 0.3533814549446106, + "union_observed": 0.15106117725372314, + "union_independent": 0.46647536754608154, + "union_gap": -0.3154141902923584, + "mean_per_layer_hit_rate": 0.04696052893996239 + }, + "20": { + "rho_bar": 0.38562026619911194, + "union_observed": 0.20474407076835632, + "union_independent": 0.6015269756317139, + "union_gap": -0.39678290486335754, + "mean_per_layer_hit_rate": 0.06789589673280716 + }, + "50": { + "rho_bar": 0.41637200117111206, + "union_observed": 0.30586767196655273, + "union_independent": 0.7643078565597534, + "union_gap": -0.4584401845932007, + "mean_per_layer_hit_rate": 0.10448478162288666 + }, + "100": { + "rho_bar": 0.4331440031528473, + "union_observed": 0.4157303273677826, + "union_independent": 0.8740535974502563, + "union_gap": -0.45832327008247375, + "mean_per_layer_hit_rate": 0.1459713876247406 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3214893766811916, + "union_gap_mean_over_k": -0.28226069999592646 + }, + "USPTO_Backgrounds|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.1759791225194931, + "union_observed": 0.039950061589479446, + "union_independent": 0.08063632249832153, + "union_gap": -0.04068626090884209, + "mean_per_layer_hit_rate": 0.006434265058487654 + }, + "2": { + "rho_bar": 0.20527786016464233, + "union_observed": 0.07116104662418365, + "union_independent": 0.1733928918838501, + "union_gap": -0.10223184525966644, + "mean_per_layer_hit_rate": 0.01450110413134098 + }, + "5": { + "rho_bar": 0.27524957060813904, + "union_observed": 0.11860174685716629, + "union_independent": 0.3102598786354065, + "union_gap": -0.1916581317782402, + "mean_per_layer_hit_rate": 0.028041871264576912 + }, + "10": { + "rho_bar": 0.35517463088035583, + "union_observed": 0.15730337798595428, + "union_independent": 0.4520854949951172, + "union_gap": -0.2947821170091629, + "mean_per_layer_hit_rate": 0.04494382441043854 + }, + "20": { + "rho_bar": 0.38596415519714355, + "union_observed": 0.20973782241344452, + "union_independent": 0.5998085141181946, + "union_gap": -0.39007069170475006, + "mean_per_layer_hit_rate": 0.06741572916507721 + }, + "50": { + "rho_bar": 0.42687851190567017, + "union_observed": 0.30087390542030334, + "union_independent": 0.7605222463607788, + "union_gap": -0.45964834094047546, + "mean_per_layer_hit_rate": 0.10304427146911621 + }, + "100": { + "rho_bar": 0.43379005789756775, + "union_observed": 0.3982521891593933, + "union_independent": 0.8658708333969116, + "union_gap": -0.4676186442375183, + "mean_per_layer_hit_rate": 0.14126572012901306 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.3226162727390017, + "union_gap_mean_over_k": -0.27809943311980795 + }, + "USPTO_Backgrounds|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.16111719608306885, + "union_observed": 0.04244694113731384, + "union_independent": 0.09685081243515015, + "union_gap": -0.054403871297836304, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.21194511651992798, + "union_observed": 0.07116104662418365, + "union_independent": 0.17888188362121582, + "union_gap": -0.10772083699703217, + "mean_per_layer_hit_rate": 0.014981274493038654 + }, + "5": { + "rho_bar": 0.2688124179840088, + "union_observed": 0.10986267030239105, + "union_independent": 0.30717629194259644, + "union_gap": -0.19731362164020538, + "mean_per_layer_hit_rate": 0.027657734230160713 + }, + "10": { + "rho_bar": 0.3489900827407837, + "union_observed": 0.16354556381702423, + "union_independent": 0.4608919024467468, + "union_gap": -0.2973463386297226, + "mean_per_layer_hit_rate": 0.04600019007921219 + }, + "20": { + "rho_bar": 0.3812049329280853, + "union_observed": 0.22846442461013794, + "union_independent": 0.6120572090148926, + "union_gap": -0.38359278440475464, + "mean_per_layer_hit_rate": 0.0693364068865776 + }, + "50": { + "rho_bar": 0.41188859939575195, + "union_observed": 0.31960049271583557, + "union_independent": 0.7828829288482666, + "union_gap": -0.46328243613243103, + "mean_per_layer_hit_rate": 0.1089983731508255 + }, + "100": { + "rho_bar": 0.42078399658203125, + "union_observed": 0.4182272255420685, + "union_independent": 0.8852353096008301, + "union_gap": -0.4670080840587616, + "mean_per_layer_hit_rate": 0.15010084211826324 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3149631917476654, + "union_gap_mean_over_k": -0.2815239961658205 + }, + "USPTO_Backgrounds|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.15635481476783752, + "union_observed": 0.054931335151195526, + "union_independent": 0.11494654417037964, + "union_gap": -0.06001520901918411, + "mean_per_layer_hit_rate": 0.009315279312431812 + }, + "2": { + "rho_bar": 0.22362194955348969, + "union_observed": 0.08114856481552124, + "union_independent": 0.1884341835975647, + "union_gap": -0.10728561878204346, + "mean_per_layer_hit_rate": 0.01584557630121708 + }, + "5": { + "rho_bar": 0.2649903893470764, + "union_observed": 0.12234706431627274, + "union_independent": 0.31637972593307495, + "union_gap": -0.19403266161680222, + "mean_per_layer_hit_rate": 0.02861807495355606 + }, + "10": { + "rho_bar": 0.3336808383464813, + "union_observed": 0.17727839946746826, + "union_independent": 0.4742749333381653, + "union_gap": -0.296996533870697, + "mean_per_layer_hit_rate": 0.047728799283504486 + }, + "20": { + "rho_bar": 0.36983704566955566, + "union_observed": 0.24344569444656372, + "union_independent": 0.625369668006897, + "union_gap": -0.38192397356033325, + "mean_per_layer_hit_rate": 0.07154518365859985 + }, + "50": { + "rho_bar": 0.3916386365890503, + "union_observed": 0.33957552909851074, + "union_independent": 0.8019962906837463, + "union_gap": -0.4624207615852356, + "mean_per_layer_hit_rate": 0.1145683228969574 + }, + "100": { + "rho_bar": 0.41625168919563293, + "union_observed": 0.42571786046028137, + "union_independent": 0.8956857919692993, + "union_gap": -0.46996793150901794, + "mean_per_layer_hit_rate": 0.15509459376335144 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.30805362335273195, + "union_gap_mean_over_k": -0.2818060985633305 + }, + "USPTO_Backgrounds|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.2086879014968872, + "union_observed": 0.04369537904858589, + "union_independent": 0.10934567451477051, + "union_gap": -0.06565029546618462, + "mean_per_layer_hit_rate": 0.008835109882056713 + }, + "2": { + "rho_bar": 0.2531979978084564, + "union_observed": 0.0749063640832901, + "union_independent": 0.18417614698410034, + "union_gap": -0.10926978290081024, + "mean_per_layer_hit_rate": 0.015461442992091179 + }, + "5": { + "rho_bar": 0.26663368940353394, + "union_observed": 0.12609238922595978, + "union_independent": 0.33553940057754517, + "union_gap": -0.2094470113515854, + "mean_per_layer_hit_rate": 0.03073081560432911 + }, + "10": { + "rho_bar": 0.3433891236782074, + "union_observed": 0.17478151619434357, + "union_independent": 0.49608975648880005, + "union_gap": -0.3213082402944565, + "mean_per_layer_hit_rate": 0.05070584639906883 + }, + "20": { + "rho_bar": 0.3621063232421875, + "union_observed": 0.24968789517879486, + "union_independent": 0.6504897475242615, + "union_gap": -0.4008018523454666, + "mean_per_layer_hit_rate": 0.0762508362531662 + }, + "50": { + "rho_bar": 0.3927222192287445, + "union_observed": 0.33957552909851074, + "union_independent": 0.8144599199295044, + "union_gap": -0.47488439083099365, + "mean_per_layer_hit_rate": 0.11860174685716629 + }, + "100": { + "rho_bar": 0.4191134572029114, + "union_observed": 0.41697877645492554, + "union_independent": 0.9028401374816895, + "union_gap": -0.4858613610267639, + "mean_per_layer_hit_rate": 0.15932008624076843 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.32083581600870403, + "union_gap_mean_over_k": -0.29531756203089443 + }, + "USPTO_Backgrounds|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.3114275336265564, + "union_observed": 0.04619225859642029, + "union_independent": 0.1117052435874939, + "union_gap": -0.06551298499107361, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.33562010526657104, + "union_observed": 0.0749063640832901, + "union_independent": 0.20389384031295776, + "union_gap": -0.12898747622966766, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.27897176146507263, + "union_observed": 0.13857677578926086, + "union_independent": 0.36033719778060913, + "union_gap": -0.22176042199134827, + "mean_per_layer_hit_rate": 0.033515799790620804 + }, + "10": { + "rho_bar": 0.35943374037742615, + "union_observed": 0.18726591765880585, + "union_independent": 0.536345362663269, + "union_gap": -0.3490794450044632, + "mean_per_layer_hit_rate": 0.056659944355487823 + }, + "20": { + "rho_bar": 0.36249297857284546, + "union_observed": 0.25717851519584656, + "union_independent": 0.6856139898300171, + "union_gap": -0.42843547463417053, + "mean_per_layer_hit_rate": 0.08354941010475159 + }, + "50": { + "rho_bar": 0.393940269947052, + "union_observed": 0.35830211639404297, + "union_independent": 0.8348720669746399, + "union_gap": -0.4765699505805969, + "mean_per_layer_hit_rate": 0.12628445029258728 + }, + "100": { + "rho_bar": 0.4209839403629303, + "union_observed": 0.4469413161277771, + "union_independent": 0.9196046590805054, + "union_gap": -0.47266334295272827, + "mean_per_layer_hit_rate": 0.17084413766860962 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.351838618516922, + "union_gap_mean_over_k": -0.30614415662629263 + }, + "USPTO_Backgrounds|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.27978819608688354, + "union_observed": 0.048689138144254684, + "union_independent": 0.12502270936965942, + "union_gap": -0.07633357122540474, + "mean_per_layer_hit_rate": 0.010179583914577961 + }, + "2": { + "rho_bar": 0.2821376919746399, + "union_observed": 0.08114856481552124, + "union_independent": 0.22891759872436523, + "union_gap": -0.147769033908844, + "mean_per_layer_hit_rate": 0.019686929881572723 + }, + "5": { + "rho_bar": 0.325961709022522, + "union_observed": 0.1398252248764038, + "union_independent": 0.3965161442756653, + "union_gap": -0.2566909193992615, + "mean_per_layer_hit_rate": 0.03783731907606125 + }, + "10": { + "rho_bar": 0.36884427070617676, + "union_observed": 0.19850187003612518, + "union_independent": 0.5794501900672913, + "union_gap": -0.3809483200311661, + "mean_per_layer_hit_rate": 0.06367041170597076 + }, + "20": { + "rho_bar": 0.4024786949157715, + "union_observed": 0.26466915011405945, + "union_independent": 0.7272434830665588, + "union_gap": -0.4625743329524994, + "mean_per_layer_hit_rate": 0.09353692829608917 + }, + "50": { + "rho_bar": 0.4291662573814392, + "union_observed": 0.3533083498477936, + "union_independent": 0.8593580722808838, + "union_gap": -0.5060497224330902, + "mean_per_layer_hit_rate": 0.1371362805366516 + }, + "100": { + "rho_bar": 0.460906982421875, + "union_observed": 0.43820226192474365, + "union_independent": 0.927654504776001, + "union_gap": -0.4894522428512573, + "mean_per_layer_hit_rate": 0.1783347874879837 + } + }, + "n_degenerate_pairs": 33, + "rho_bar_mean_over_k": 0.3641834003584726, + "union_gap_mean_over_k": -0.331402591828789 + }, + "USPTO_Backgrounds|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.2324707806110382, + "union_observed": 0.05118601769208908, + "union_independent": 0.13821423053741455, + "union_gap": -0.08702821284532547, + "mean_per_layer_hit_rate": 0.01133198942989111 + }, + "2": { + "rho_bar": 0.3109663128852844, + "union_observed": 0.07615480571985245, + "union_independent": 0.222670316696167, + "union_gap": -0.14651551097631454, + "mean_per_layer_hit_rate": 0.019110726192593575 + }, + "5": { + "rho_bar": 0.35376450419425964, + "union_observed": 0.1498127281665802, + "union_independent": 0.41576671600341797, + "union_gap": -0.26595398783683777, + "mean_per_layer_hit_rate": 0.0402381606400013 + }, + "10": { + "rho_bar": 0.4048001766204834, + "union_observed": 0.21098627150058746, + "union_independent": 0.6117959022521973, + "union_gap": -0.4008096307516098, + "mean_per_layer_hit_rate": 0.0695284754037857 + }, + "20": { + "rho_bar": 0.4424469769001007, + "union_observed": 0.27715355157852173, + "union_independent": 0.7588802576065063, + "union_gap": -0.4817267060279846, + "mean_per_layer_hit_rate": 0.10227600485086441 + }, + "50": { + "rho_bar": 0.4694072902202606, + "union_observed": 0.3570536971092224, + "union_independent": 0.869775652885437, + "union_gap": -0.5127219557762146, + "mean_per_layer_hit_rate": 0.1427062302827835 + }, + "100": { + "rho_bar": 0.4636087715625763, + "union_observed": 0.47191011905670166, + "union_independent": 0.9367371797561646, + "union_gap": -0.4648270606994629, + "mean_per_layer_hit_rate": 0.18745797872543335 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3824949732848576, + "union_gap_mean_over_k": -0.33708329498767853 + }, + "USPTO_Backgrounds|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.19788479804992676, + "union_observed": 0.048689138144254684, + "union_independent": 0.11711764335632324, + "union_gap": -0.06842850521206856, + "mean_per_layer_hit_rate": 0.009507346898317337 + }, + "2": { + "rho_bar": 0.28325456380844116, + "union_observed": 0.07990012317895889, + "union_independent": 0.20328080654144287, + "union_gap": -0.12338068336248398, + "mean_per_layer_hit_rate": 0.017286082729697227 + }, + "5": { + "rho_bar": 0.33066582679748535, + "union_observed": 0.15355804562568665, + "union_independent": 0.3790174126625061, + "union_gap": -0.22545936703681946, + "mean_per_layer_hit_rate": 0.0358206070959568 + }, + "10": { + "rho_bar": 0.38386374711990356, + "union_observed": 0.20848938822746277, + "union_independent": 0.5461399555206299, + "union_gap": -0.3376505672931671, + "mean_per_layer_hit_rate": 0.05848459154367447 + }, + "20": { + "rho_bar": 0.4276643395423889, + "union_observed": 0.255930095911026, + "union_independent": 0.6836825609207153, + "union_gap": -0.42775246500968933, + "mean_per_layer_hit_rate": 0.08383750915527344 + }, + "50": { + "rho_bar": 0.46234849095344543, + "union_observed": 0.3458177149295807, + "union_independent": 0.83357834815979, + "union_gap": -0.48776063323020935, + "mean_per_layer_hit_rate": 0.12705272436141968 + }, + "100": { + "rho_bar": 0.4728918671607971, + "union_observed": 0.4419475793838501, + "union_independent": 0.9076675176620483, + "union_gap": -0.46571993827819824, + "mean_per_layer_hit_rate": 0.16479399800300598 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.36551051906176973, + "union_gap_mean_over_k": -0.3051645942032337 + }, + "USPTO_Backgrounds|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.2093743234872818, + "union_observed": 0.05118601769208908, + "union_independent": 0.11939007043838501, + "union_gap": -0.06820405274629593, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.26081448793411255, + "union_observed": 0.08239700645208359, + "union_independent": 0.2035536766052246, + "union_gap": -0.12115667015314102, + "mean_per_layer_hit_rate": 0.017286082729697227 + }, + "5": { + "rho_bar": 0.290111780166626, + "union_observed": 0.1423220932483673, + "union_independent": 0.34976524114608765, + "union_gap": -0.20744314789772034, + "mean_per_layer_hit_rate": 0.032363392412662506 + }, + "10": { + "rho_bar": 0.3569573760032654, + "union_observed": 0.19225966930389404, + "union_independent": 0.5194559097290039, + "union_gap": -0.32719624042510986, + "mean_per_layer_hit_rate": 0.05425909906625748 + }, + "20": { + "rho_bar": 0.40831390023231506, + "union_observed": 0.2684144675731659, + "union_independent": 0.678937554359436, + "union_gap": -0.41052308678627014, + "mean_per_layer_hit_rate": 0.08249303698539734 + }, + "50": { + "rho_bar": 0.4400183856487274, + "union_observed": 0.37578028440475464, + "union_independent": 0.8366363644599915, + "union_gap": -0.4608560800552368, + "mean_per_layer_hit_rate": 0.12753288447856903 + }, + "100": { + "rho_bar": 0.43818604946136475, + "union_observed": 0.46691635251045227, + "union_independent": 0.9169207811355591, + "union_gap": -0.4500044286251068, + "mean_per_layer_hit_rate": 0.17017190158367157 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.34339661470481325, + "union_gap_mean_over_k": -0.29219767238412586 + }, + "USPTO_Backgrounds|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.2484840452671051, + "union_observed": 0.04619225859642029, + "union_independent": 0.11052793264389038, + "union_gap": -0.06433567404747009, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.2730138897895813, + "union_observed": 0.08739075809717178, + "union_independent": 0.2037752866744995, + "union_gap": -0.11638452857732773, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.27471253275871277, + "union_observed": 0.16104869544506073, + "union_independent": 0.3751868009567261, + "union_gap": -0.21413810551166534, + "mean_per_layer_hit_rate": 0.0352444052696228 + }, + "10": { + "rho_bar": 0.3316550552845001, + "union_observed": 0.20973782241344452, + "union_independent": 0.5273847579956055, + "union_gap": -0.31764693558216095, + "mean_per_layer_hit_rate": 0.055411502718925476 + }, + "20": { + "rho_bar": 0.3969883322715759, + "union_observed": 0.27590513229370117, + "union_independent": 0.6935499906539917, + "union_gap": -0.4176448583602905, + "mean_per_layer_hit_rate": 0.08566214144229889 + }, + "50": { + "rho_bar": 0.4223960041999817, + "union_observed": 0.38327091932296753, + "union_independent": 0.8306554555892944, + "union_gap": -0.4473845362663269, + "mean_per_layer_hit_rate": 0.12522807717323303 + }, + "100": { + "rho_bar": 0.4256272315979004, + "union_observed": 0.47191011905670166, + "union_independent": 0.9125050902366638, + "union_gap": -0.44059497117996216, + "mean_per_layer_hit_rate": 0.16700278222560883 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.33898244159562246, + "union_gap_mean_over_k": -0.288304229932172 + }, + "USPTO_Backgrounds|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.17171189188957214, + "union_observed": 0.047440700232982635, + "union_independent": 0.10480797290802002, + "union_gap": -0.057367272675037384, + "mean_per_layer_hit_rate": 0.008450974710285664 + }, + "2": { + "rho_bar": 0.22365409135818481, + "union_observed": 0.08739075809717178, + "union_independent": 0.20057696104049683, + "union_gap": -0.11318620294332504, + "mean_per_layer_hit_rate": 0.016997981816530228 + }, + "5": { + "rho_bar": 0.25259050726890564, + "union_observed": 0.15605492889881134, + "union_independent": 0.3482072353363037, + "union_gap": -0.19215230643749237, + "mean_per_layer_hit_rate": 0.03217132389545441 + }, + "10": { + "rho_bar": 0.3350765109062195, + "union_observed": 0.21098627150058746, + "union_independent": 0.5166962146759033, + "union_gap": -0.30570994317531586, + "mean_per_layer_hit_rate": 0.05377893149852753 + }, + "20": { + "rho_bar": 0.37450939416885376, + "union_observed": 0.26966291666030884, + "union_independent": 0.6638685464859009, + "union_gap": -0.39420562982559204, + "mean_per_layer_hit_rate": 0.07922788709402084 + }, + "50": { + "rho_bar": 0.4031943678855896, + "union_observed": 0.37827715277671814, + "union_independent": 0.8158183097839355, + "union_gap": -0.4375411570072174, + "mean_per_layer_hit_rate": 0.11975415050983429 + }, + "100": { + "rho_bar": 0.4156484305858612, + "union_observed": 0.4706616699695587, + "union_independent": 0.9054479002952576, + "union_gap": -0.43478623032569885, + "mean_per_layer_hit_rate": 0.16191299259662628 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3109121705804552, + "union_gap_mean_over_k": -0.2764212489128113 + }, + "USPTO_Backgrounds|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.1469879299402237, + "union_observed": 0.06117353215813637, + "union_independent": 0.11941182613372803, + "union_gap": -0.05823829397559166, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.21412597596645355, + "union_observed": 0.09238451719284058, + "union_independent": 0.1995100975036621, + "union_gap": -0.10712558031082153, + "mean_per_layer_hit_rate": 0.016901949420571327 + }, + "5": { + "rho_bar": 0.27022477984428406, + "union_observed": 0.15355804562568665, + "union_independent": 0.3696329593658447, + "union_gap": -0.21607491374015808, + "mean_per_layer_hit_rate": 0.0346682034432888 + }, + "10": { + "rho_bar": 0.34039372205734253, + "union_observed": 0.21598002314567566, + "union_independent": 0.5443071126937866, + "union_gap": -0.32832708954811096, + "mean_per_layer_hit_rate": 0.05810045450925827 + }, + "20": { + "rho_bar": 0.40981948375701904, + "union_observed": 0.27715355157852173, + "union_independent": 0.6959577798843384, + "union_gap": -0.41880422830581665, + "mean_per_layer_hit_rate": 0.08633439242839813 + }, + "50": { + "rho_bar": 0.46815988421440125, + "union_observed": 0.3645443320274353, + "union_independent": 0.8369463682174683, + "union_gap": -0.47240203619003296, + "mean_per_layer_hit_rate": 0.12791702151298523 + }, + "100": { + "rho_bar": 0.4825644791126251, + "union_observed": 0.45817726850509644, + "union_independent": 0.91294264793396, + "union_gap": -0.4547653794288635, + "mean_per_layer_hit_rate": 0.16738691926002502 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33318232212747845, + "union_gap_mean_over_k": -0.2936767887856279 + }, + "USPTO_Backgrounds|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.35978907346725464, + "union_observed": 0.04244694113731384, + "union_independent": 0.13815224170684814, + "union_gap": -0.0957053005695343, + "mean_per_layer_hit_rate": 0.01133198942989111 + }, + "2": { + "rho_bar": 0.40020862221717834, + "union_observed": 0.07116104662418365, + "union_independent": 0.23845672607421875, + "union_gap": -0.1672956794500351, + "mean_per_layer_hit_rate": 0.020647268742322922 + }, + "5": { + "rho_bar": 0.4141794741153717, + "union_observed": 0.12609238922595978, + "union_independent": 0.4308229684829712, + "union_gap": -0.3047305792570114, + "mean_per_layer_hit_rate": 0.042158838361501694 + }, + "10": { + "rho_bar": 0.497457355260849, + "union_observed": 0.17478151619434357, + "union_independent": 0.6197343468666077, + "union_gap": -0.4449528306722641, + "mean_per_layer_hit_rate": 0.07096897810697556 + }, + "20": { + "rho_bar": 0.5248605012893677, + "union_observed": 0.24968789517879486, + "union_independent": 0.7737841010093689, + "union_gap": -0.524096205830574, + "mean_per_layer_hit_rate": 0.10640545189380646 + }, + "50": { + "rho_bar": 0.5607990622520447, + "union_observed": 0.3383270800113678, + "union_independent": 0.8906809091567993, + "union_gap": -0.5523538291454315, + "mean_per_layer_hit_rate": 0.1534620225429535 + }, + "100": { + "rho_bar": 0.5760120749473572, + "union_observed": 0.42696627974510193, + "union_independent": 0.9477579593658447, + "union_gap": -0.5207916796207428, + "mean_per_layer_hit_rate": 0.198501855134964 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.47618659479277475, + "union_gap_mean_over_k": -0.3728465863636562 + }, + "USPTO_Backgrounds|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.29485005140304565, + "union_observed": 0.05368289723992348, + "union_independent": 0.15114855766296387, + "union_gap": -0.09746566042304039, + "mean_per_layer_hit_rate": 0.012484394945204258 + }, + "2": { + "rho_bar": 0.3516741693019867, + "union_observed": 0.08114856481552124, + "union_independent": 0.25463950634002686, + "union_gap": -0.17349094152450562, + "mean_per_layer_hit_rate": 0.02227984182536602 + }, + "5": { + "rho_bar": 0.3884642422199249, + "union_observed": 0.1523096114397049, + "union_independent": 0.45575469732284546, + "union_gap": -0.30344508588314056, + "mean_per_layer_hit_rate": 0.04542398825287819 + }, + "10": { + "rho_bar": 0.47698602080345154, + "union_observed": 0.20224718749523163, + "union_independent": 0.6427556276321411, + "union_gap": -0.4405084401369095, + "mean_per_layer_hit_rate": 0.07557860761880875 + }, + "20": { + "rho_bar": 0.5092310905456543, + "union_observed": 0.2746566832065582, + "union_independent": 0.7883483171463013, + "union_gap": -0.513691633939743, + "mean_per_layer_hit_rate": 0.1114952489733696 + }, + "50": { + "rho_bar": 0.521935224533081, + "union_observed": 0.3720349669456482, + "union_independent": 0.8942578434944153, + "union_gap": -0.5222228765487671, + "mean_per_layer_hit_rate": 0.1567271649837494 + }, + "100": { + "rho_bar": 0.5314770936965942, + "union_observed": 0.44319599866867065, + "union_independent": 0.9483790397644043, + "union_gap": -0.5051830410957336, + "mean_per_layer_hit_rate": 0.20071063935756683 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.43923112750053406, + "union_gap_mean_over_k": -0.3651439542216914 + }, + "USPTO_Backgrounds|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.2690310478210449, + "union_observed": 0.05243445560336113, + "union_independent": 0.1349031925201416, + "union_gap": -0.08246873691678047, + "mean_per_layer_hit_rate": 0.011043887585401535 + }, + "2": { + "rho_bar": 0.265797883272171, + "union_observed": 0.08364544063806534, + "union_independent": 0.22460252046585083, + "union_gap": -0.1409570798277855, + "mean_per_layer_hit_rate": 0.019302794709801674 + }, + "5": { + "rho_bar": 0.30681970715522766, + "union_observed": 0.1473158597946167, + "union_independent": 0.4128192663192749, + "union_gap": -0.2655034065246582, + "mean_per_layer_hit_rate": 0.039854031056165695 + }, + "10": { + "rho_bar": 0.39084771275520325, + "union_observed": 0.21598002314567566, + "union_independent": 0.5939915776252747, + "union_gap": -0.378011554479599, + "mean_per_layer_hit_rate": 0.06645539402961731 + }, + "20": { + "rho_bar": 0.45532700419425964, + "union_observed": 0.27590513229370117, + "union_independent": 0.7467472553253174, + "union_gap": -0.4708421230316162, + "mean_per_layer_hit_rate": 0.09920291602611542 + }, + "50": { + "rho_bar": 0.513386070728302, + "union_observed": 0.38077402114868164, + "union_independent": 0.879133939743042, + "union_gap": -0.49835991859436035, + "mean_per_layer_hit_rate": 0.14818015694618225 + }, + "100": { + "rho_bar": 0.5270638465881348, + "union_observed": 0.4656679034233093, + "union_independent": 0.9390680193901062, + "union_gap": -0.4734001159667969, + "mean_per_layer_hit_rate": 0.1908191740512848 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3897533246449062, + "union_gap_mean_over_k": -0.3299347050487995 + }, + "USPTO_Backgrounds|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.17334724962711334, + "union_observed": 0.056179776787757874, + "union_independent": 0.1227526068687439, + "union_gap": -0.06657283008098602, + "mean_per_layer_hit_rate": 0.009987516328692436 + }, + "2": { + "rho_bar": 0.2592363953590393, + "union_observed": 0.07865168899297714, + "union_independent": 0.19936150312423706, + "union_gap": -0.12070981413125992, + "mean_per_layer_hit_rate": 0.016901949420571327 + }, + "5": { + "rho_bar": 0.2712428569793701, + "union_observed": 0.1548064947128296, + "union_independent": 0.3819958567619324, + "union_gap": -0.22718936204910278, + "mean_per_layer_hit_rate": 0.03610870987176895 + }, + "10": { + "rho_bar": 0.34034690260887146, + "union_observed": 0.21098627150058746, + "union_independent": 0.5468555688858032, + "union_gap": -0.33586929738521576, + "mean_per_layer_hit_rate": 0.05858061835169792 + }, + "20": { + "rho_bar": 0.3875184655189514, + "union_observed": 0.2821473181247711, + "union_independent": 0.7039206027984619, + "union_gap": -0.4217732846736908, + "mean_per_layer_hit_rate": 0.08835109323263168 + }, + "50": { + "rho_bar": 0.47865942120552063, + "union_observed": 0.3695380687713623, + "union_independent": 0.847387969493866, + "union_gap": -0.47784990072250366, + "mean_per_layer_hit_rate": 0.13271871209144592 + }, + "100": { + "rho_bar": 0.4936673045158386, + "union_observed": 0.4344569146633148, + "union_independent": 0.9161214828491211, + "union_gap": -0.4816645681858063, + "mean_per_layer_hit_rate": 0.17084413766860962 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.34343122797352926, + "union_gap_mean_over_k": -0.3045184367469379 + }, + "USPTO_Backgrounds|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.18636558949947357, + "union_observed": 0.04619225859642029, + "union_independent": 0.10583287477493286, + "union_gap": -0.05964061617851257, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.24094535410404205, + "union_observed": 0.08239700645208359, + "union_independent": 0.19633066654205322, + "union_gap": -0.11393366008996964, + "mean_per_layer_hit_rate": 0.016613848507404327 + }, + "5": { + "rho_bar": 0.28312256932258606, + "union_observed": 0.1498127281665802, + "union_independent": 0.3514136075973511, + "union_gap": -0.20160087943077087, + "mean_per_layer_hit_rate": 0.032555460929870605 + }, + "10": { + "rho_bar": 0.34836530685424805, + "union_observed": 0.20724095404148102, + "union_independent": 0.5136435627937317, + "union_gap": -0.30640260875225067, + "mean_per_layer_hit_rate": 0.053490832448005676 + }, + "20": { + "rho_bar": 0.39194557070732117, + "union_observed": 0.25967541337013245, + "union_independent": 0.6614189147949219, + "union_gap": -0.40174350142478943, + "mean_per_layer_hit_rate": 0.07893978804349899 + }, + "50": { + "rho_bar": 0.4682588279247284, + "union_observed": 0.3545567989349365, + "union_independent": 0.8162662982940674, + "union_gap": -0.46170949935913086, + "mean_per_layer_hit_rate": 0.12033034861087799 + }, + "100": { + "rho_bar": 0.4940272569656372, + "union_observed": 0.4444444477558136, + "union_independent": 0.9015136361122131, + "union_gap": -0.45706918835639954, + "mean_per_layer_hit_rate": 0.16076058149337769 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3447186393397195, + "union_gap_mean_over_k": -0.28601427908454624 + }, + "USPTO_Backgrounds|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.1843864917755127, + "union_observed": 0.05243445560336113, + "union_independent": 0.10251522064208984, + "union_gap": -0.050080765038728714, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.2376779466867447, + "union_observed": 0.08239700645208359, + "union_independent": 0.17969775199890137, + "union_gap": -0.09730074554681778, + "mean_per_layer_hit_rate": 0.015077306888997555 + }, + "5": { + "rho_bar": 0.26101717352867126, + "union_observed": 0.1523096114397049, + "union_independent": 0.3367706537246704, + "union_gap": -0.18446104228496552, + "mean_per_layer_hit_rate": 0.030922885984182358 + }, + "10": { + "rho_bar": 0.330681174993515, + "union_observed": 0.20349563658237457, + "union_independent": 0.49400758743286133, + "union_gap": -0.29051195085048676, + "mean_per_layer_hit_rate": 0.05070584639906883 + }, + "20": { + "rho_bar": 0.39086809754371643, + "union_observed": 0.25967541337013245, + "union_independent": 0.6338638067245483, + "union_gap": -0.3741883933544159, + "mean_per_layer_hit_rate": 0.07365792244672775 + }, + "50": { + "rho_bar": 0.43448442220687866, + "union_observed": 0.3570536971092224, + "union_independent": 0.7913185954093933, + "union_gap": -0.4342648983001709, + "mean_per_layer_hit_rate": 0.11235953867435455 + }, + "100": { + "rho_bar": 0.46296995878219604, + "union_observed": 0.43820226192474365, + "union_independent": 0.8833339214324951, + "union_gap": -0.44513165950775146, + "mean_per_layer_hit_rate": 0.15048497915267944 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.32886932364531923, + "union_gap_mean_over_k": -0.2679913506976196 + }, + "USPTO_Backgrounds|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.12344818562269211, + "union_observed": 0.04244694113731384, + "union_independent": 0.07825064659118652, + "union_gap": -0.03580370545387268, + "mean_per_layer_hit_rate": 0.006242197472602129 + }, + "2": { + "rho_bar": 0.2436220496892929, + "union_observed": 0.06616728752851486, + "union_independent": 0.16481506824493408, + "union_gap": -0.09864778071641922, + "mean_per_layer_hit_rate": 0.013732834719121456 + }, + "5": { + "rho_bar": 0.2991655468940735, + "union_observed": 0.11735330522060394, + "union_independent": 0.33181118965148926, + "union_gap": -0.21445788443088531, + "mean_per_layer_hit_rate": 0.03044271655380726 + }, + "10": { + "rho_bar": 0.36235809326171875, + "union_observed": 0.1548064947128296, + "union_independent": 0.47332704067230225, + "union_gap": -0.31852054595947266, + "mean_per_layer_hit_rate": 0.047920867800712585 + }, + "20": { + "rho_bar": 0.3885735273361206, + "union_observed": 0.20848938822746277, + "union_independent": 0.6044961214065552, + "union_gap": -0.3960067331790924, + "mean_per_layer_hit_rate": 0.06847210228443146 + }, + "50": { + "rho_bar": 0.412202388048172, + "union_observed": 0.30836454033851624, + "union_independent": 0.7624517679214478, + "union_gap": -0.4540872275829315, + "mean_per_layer_hit_rate": 0.10390857607126236 + }, + "100": { + "rho_bar": 0.43508031964302063, + "union_observed": 0.4157303273677826, + "union_independent": 0.8738605976104736, + "union_gap": -0.45813027024269104, + "mean_per_layer_hit_rate": 0.1459713876247406 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.32349287292787005, + "union_gap_mean_over_k": -0.2822363067950521 + }, + "USPTO_Backgrounds|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.15145215392112732, + "union_observed": 0.041198503226041794, + "union_independent": 0.07829129695892334, + "union_gap": -0.037092793732881546, + "mean_per_layer_hit_rate": 0.006242197472602129 + }, + "2": { + "rho_bar": 0.2006857991218567, + "union_observed": 0.072409488260746, + "union_independent": 0.17226099967956543, + "union_gap": -0.09985151141881943, + "mean_per_layer_hit_rate": 0.014405070804059505 + }, + "5": { + "rho_bar": 0.2910236418247223, + "union_observed": 0.11610487103462219, + "union_independent": 0.3119584321975708, + "union_gap": -0.1958535611629486, + "mean_per_layer_hit_rate": 0.028233937919139862 + }, + "10": { + "rho_bar": 0.355837345123291, + "union_observed": 0.1548064947128296, + "union_independent": 0.4521905779838562, + "union_gap": -0.2973840832710266, + "mean_per_layer_hit_rate": 0.04494381695985794 + }, + "20": { + "rho_bar": 0.3848330080509186, + "union_observed": 0.21098627150058746, + "union_independent": 0.6020342707633972, + "union_gap": -0.39104799926280975, + "mean_per_layer_hit_rate": 0.067799873650074 + }, + "50": { + "rho_bar": 0.42495131492614746, + "union_observed": 0.30586767196655273, + "union_independent": 0.7678912878036499, + "union_gap": -0.46202361583709717, + "mean_per_layer_hit_rate": 0.10515701025724411 + }, + "100": { + "rho_bar": 0.4388696849346161, + "union_observed": 0.40948814153671265, + "union_independent": 0.8686614036560059, + "union_gap": -0.4591732621192932, + "mean_per_layer_hit_rate": 0.14261020720005035 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.32109327827181133, + "union_gap_mean_over_k": -0.2774895466864109 + }, + "USPTO_Backgrounds|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.15004505217075348, + "union_observed": 0.04619225859642029, + "union_independent": 0.10137486457824707, + "union_gap": -0.05518260598182678, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.20898853242397308, + "union_observed": 0.07365792989730835, + "union_independent": 0.17674970626831055, + "union_gap": -0.1030917763710022, + "mean_per_layer_hit_rate": 0.01478920690715313 + }, + "5": { + "rho_bar": 0.2666364312171936, + "union_observed": 0.1111111119389534, + "union_independent": 0.3043559193611145, + "union_gap": -0.1932448074221611, + "mean_per_layer_hit_rate": 0.027369635179638863 + }, + "10": { + "rho_bar": 0.3369474709033966, + "union_observed": 0.16978776454925537, + "union_independent": 0.46443504095077515, + "union_gap": -0.2946472764015198, + "mean_per_layer_hit_rate": 0.04648036137223244 + }, + "20": { + "rho_bar": 0.3777626156806946, + "union_observed": 0.23096129298210144, + "union_independent": 0.6174401640892029, + "union_gap": -0.38647887110710144, + "mean_per_layer_hit_rate": 0.0702967494726181 + }, + "50": { + "rho_bar": 0.41991937160491943, + "union_observed": 0.3258427083492279, + "union_independent": 0.787971019744873, + "union_gap": -0.46212831139564514, + "mean_per_layer_hit_rate": 0.11063094437122345 + }, + "100": { + "rho_bar": 0.42367181181907654, + "union_observed": 0.41697877645492554, + "union_independent": 0.886644721031189, + "union_gap": -0.4696659445762634, + "mean_per_layer_hit_rate": 0.1509651392698288 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3119958979742868, + "union_gap_mean_over_k": -0.2806342276079314 + }, + "USPTO_Backgrounds|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1584574580192566, + "union_observed": 0.05742821469902992, + "union_independent": 0.11944228410720825, + "union_gap": -0.06201406940817833, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.22420533001422882, + "union_observed": 0.08114856481552124, + "union_independent": 0.18629837036132812, + "union_gap": -0.10514980554580688, + "mean_per_layer_hit_rate": 0.01565350964665413 + }, + "5": { + "rho_bar": 0.2573634386062622, + "union_observed": 0.13233457505702972, + "union_independent": 0.33040207624435425, + "union_gap": -0.19806750118732452, + "mean_per_layer_hit_rate": 0.03015461377799511 + }, + "10": { + "rho_bar": 0.31961050629615784, + "union_observed": 0.18227216601371765, + "union_independent": 0.4763861298561096, + "union_gap": -0.29411396384239197, + "mean_per_layer_hit_rate": 0.048016902059316635 + }, + "20": { + "rho_bar": 0.37327951192855835, + "union_observed": 0.24843944609165192, + "union_independent": 0.6303545236587524, + "union_gap": -0.3819150775671005, + "mean_per_layer_hit_rate": 0.07250551879405975 + }, + "50": { + "rho_bar": 0.394996702671051, + "union_observed": 0.3408239781856537, + "union_independent": 0.8071308135986328, + "union_gap": -0.4663068354129791, + "mean_per_layer_hit_rate": 0.1162969321012497 + }, + "100": { + "rho_bar": 0.4164814054965973, + "union_observed": 0.4232209622859955, + "union_independent": 0.8964682221412659, + "union_gap": -0.4732472598552704, + "mean_per_layer_hit_rate": 0.155574768781662 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3063420504331589, + "union_gap_mean_over_k": -0.2829735018312931 + }, + "USPTO_Backgrounds|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.234514519572258, + "union_observed": 0.04494382068514824, + "union_independent": 0.11607766151428223, + "union_gap": -0.07113384082913399, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.3104683756828308, + "union_observed": 0.072409488260746, + "union_independent": 0.18725800514221191, + "union_gap": -0.11484851688146591, + "mean_per_layer_hit_rate": 0.01574954390525818 + }, + "5": { + "rho_bar": 0.265453964471817, + "union_observed": 0.12484394758939743, + "union_independent": 0.33127737045288086, + "union_gap": -0.20643342286348343, + "mean_per_layer_hit_rate": 0.03025064989924431 + }, + "10": { + "rho_bar": 0.3466206192970276, + "union_observed": 0.17353308200836182, + "union_independent": 0.4966014623641968, + "union_gap": -0.32306838035583496, + "mean_per_layer_hit_rate": 0.05080188065767288 + }, + "20": { + "rho_bar": 0.36023446917533875, + "union_observed": 0.25468164682388306, + "union_independent": 0.6532542705535889, + "union_gap": -0.3985726237297058, + "mean_per_layer_hit_rate": 0.0768270418047905 + }, + "50": { + "rho_bar": 0.3886599540710449, + "union_observed": 0.3533083498477936, + "union_independent": 0.8175585269927979, + "union_gap": -0.4642501771450043, + "mean_per_layer_hit_rate": 0.11975415796041489 + }, + "100": { + "rho_bar": 0.42057737708091736, + "union_observed": 0.41947564482688904, + "union_independent": 0.9041401147842407, + "union_gap": -0.4846644699573517, + "mean_per_layer_hit_rate": 0.16018438339233398 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.3323613256216049, + "union_gap_mean_over_k": -0.29471020453742575 + }, + "USPTO_Backgrounds|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.31283095479011536, + "union_observed": 0.048689138144254684, + "union_independent": 0.11951321363449097, + "union_gap": -0.07082407549023628, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.3346294164657593, + "union_observed": 0.0774032473564148, + "union_independent": 0.20993733406066895, + "union_gap": -0.13253408670425415, + "mean_per_layer_hit_rate": 0.017862288281321526 + }, + "5": { + "rho_bar": 0.3029831349849701, + "union_observed": 0.13108614087104797, + "union_independent": 0.36341893672943115, + "union_gap": -0.23233279585838318, + "mean_per_layer_hit_rate": 0.033899933099746704 + }, + "10": { + "rho_bar": 0.356869101524353, + "union_observed": 0.19225966930389404, + "union_independent": 0.5432752370834351, + "union_gap": -0.351015567779541, + "mean_per_layer_hit_rate": 0.05771631747484207 + }, + "20": { + "rho_bar": 0.36595845222473145, + "union_observed": 0.25218477845191956, + "union_independent": 0.6851529479026794, + "union_gap": -0.4329681694507599, + "mean_per_layer_hit_rate": 0.08345337957143784 + }, + "50": { + "rho_bar": 0.3983525037765503, + "union_observed": 0.35830211639404297, + "union_independent": 0.8359726667404175, + "union_gap": -0.4776705503463745, + "mean_per_layer_hit_rate": 0.12666858732700348 + }, + "100": { + "rho_bar": 0.42142534255981445, + "union_observed": 0.44818976521492004, + "union_independent": 0.920255720615387, + "union_gap": -0.4720659554004669, + "mean_per_layer_hit_rate": 0.17161241173744202 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.35614984376089914, + "union_gap_mean_over_k": -0.3099158858614309 + }, + "USPTO_Backgrounds|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.26213303208351135, + "union_observed": 0.054931335151195526, + "union_independent": 0.1316489577293396, + "union_gap": -0.07671762257814407, + "mean_per_layer_hit_rate": 0.010755786672234535 + }, + "2": { + "rho_bar": 0.2901610732078552, + "union_observed": 0.07990012317895889, + "union_independent": 0.23276352882385254, + "union_gap": -0.15286340564489365, + "mean_per_layer_hit_rate": 0.020071063190698624 + }, + "5": { + "rho_bar": 0.33489540219306946, + "union_observed": 0.1398252248764038, + "union_independent": 0.4058275818824768, + "union_gap": -0.266002357006073, + "mean_per_layer_hit_rate": 0.038989726454019547 + }, + "10": { + "rho_bar": 0.38876470923423767, + "union_observed": 0.19725343585014343, + "union_independent": 0.5836936235427856, + "union_gap": -0.3864401876926422, + "mean_per_layer_hit_rate": 0.06443868577480316 + }, + "20": { + "rho_bar": 0.3995060324668884, + "union_observed": 0.25967541337013245, + "union_independent": 0.7275329828262329, + "union_gap": -0.46785756945610046, + "mean_per_layer_hit_rate": 0.09363295137882233 + }, + "50": { + "rho_bar": 0.42643508315086365, + "union_observed": 0.35830211639404297, + "union_independent": 0.8590030670166016, + "union_gap": -0.5007009506225586, + "mean_per_layer_hit_rate": 0.13704022765159607 + }, + "100": { + "rho_bar": 0.4501410126686096, + "union_observed": 0.449438214302063, + "union_independent": 0.9287087917327881, + "union_gap": -0.4792705774307251, + "mean_per_layer_hit_rate": 0.1792951226234436 + } + }, + "n_degenerate_pairs": 33, + "rho_bar_mean_over_k": 0.3645766207150051, + "union_gap_mean_over_k": -0.33283609577587675 + }, + "USPTO_Backgrounds|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.19342394173145294, + "union_observed": 0.054931335151195526, + "union_independent": 0.14361929893493652, + "union_gap": -0.088687963783741, + "mean_per_layer_hit_rate": 0.01181215699762106 + }, + "2": { + "rho_bar": 0.3246362507343292, + "union_observed": 0.0774032473564148, + "union_independent": 0.23051303625106812, + "union_gap": -0.15310978889465332, + "mean_per_layer_hit_rate": 0.019878996536135674 + }, + "5": { + "rho_bar": 0.3524963855743408, + "union_observed": 0.144818976521492, + "union_independent": 0.4112010598182678, + "union_gap": -0.2663820832967758, + "mean_per_layer_hit_rate": 0.039661962538957596 + }, + "10": { + "rho_bar": 0.4068441689014435, + "union_observed": 0.2122347056865692, + "union_independent": 0.6169178485870361, + "union_gap": -0.4046831429004669, + "mean_per_layer_hit_rate": 0.0704888105392456 + }, + "20": { + "rho_bar": 0.4440668523311615, + "union_observed": 0.2808988690376282, + "union_independent": 0.7604138255119324, + "union_gap": -0.4795149564743042, + "mean_per_layer_hit_rate": 0.10275617241859436 + }, + "50": { + "rho_bar": 0.46134117245674133, + "union_observed": 0.37078651785850525, + "union_independent": 0.8723171949386597, + "union_gap": -0.5015306770801544, + "mean_per_layer_hit_rate": 0.1440507024526596 + }, + "100": { + "rho_bar": 0.45922327041625977, + "union_observed": 0.4706616699695587, + "union_independent": 0.936189591884613, + "union_gap": -0.4655279219150543, + "mean_per_layer_hit_rate": 0.18707385659217834 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3774331488779613, + "union_gap_mean_over_k": -0.3370623620493071 + }, + "USPTO_Backgrounds|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.21406413614749908, + "union_observed": 0.05243445560336113, + "union_independent": 0.11930519342422485, + "union_gap": -0.06687073782086372, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.2877957224845886, + "union_observed": 0.07990012317895889, + "union_independent": 0.2062707543373108, + "union_gap": -0.1263706311583519, + "mean_per_layer_hit_rate": 0.017574187368154526 + }, + "5": { + "rho_bar": 0.34082314372062683, + "union_observed": 0.14606741070747375, + "union_independent": 0.38214111328125, + "union_gap": -0.23607370257377625, + "mean_per_layer_hit_rate": 0.0362047478556633 + }, + "10": { + "rho_bar": 0.38969430327415466, + "union_observed": 0.20848938822746277, + "union_independent": 0.5501813888549805, + "union_gap": -0.3416920006275177, + "mean_per_layer_hit_rate": 0.05915682017803192 + }, + "20": { + "rho_bar": 0.43882444500923157, + "union_observed": 0.25717851519584656, + "union_independent": 0.6864243745803833, + "union_gap": -0.42924585938453674, + "mean_per_layer_hit_rate": 0.08450973778963089 + }, + "50": { + "rho_bar": 0.47199690341949463, + "union_observed": 0.34706616401672363, + "union_independent": 0.8357213735580444, + "union_gap": -0.4886552095413208, + "mean_per_layer_hit_rate": 0.12801307439804077 + }, + "100": { + "rho_bar": 0.4715226888656616, + "union_observed": 0.44818976521492004, + "union_independent": 0.9113223552703857, + "union_gap": -0.4631325900554657, + "mean_per_layer_hit_rate": 0.16738691926002502 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3735316204173224, + "union_gap_mean_over_k": -0.3074343901659761 + }, + "USPTO_Backgrounds|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.24512718617916107, + "union_observed": 0.05118601769208908, + "union_independent": 0.12272846698760986, + "union_gap": -0.07154244929552078, + "mean_per_layer_hit_rate": 0.009987516328692436 + }, + "2": { + "rho_bar": 0.2977573871612549, + "union_observed": 0.08489388227462769, + "union_independent": 0.20961660146713257, + "union_gap": -0.12472271919250488, + "mean_per_layer_hit_rate": 0.017862286418676376 + }, + "5": { + "rho_bar": 0.2980281114578247, + "union_observed": 0.14107365906238556, + "union_independent": 0.358107328414917, + "union_gap": -0.21703366935253143, + "mean_per_layer_hit_rate": 0.033323731273412704 + }, + "10": { + "rho_bar": 0.35459238290786743, + "union_observed": 0.19725343585014343, + "union_independent": 0.521226704120636, + "union_gap": -0.32397326827049255, + "mean_per_layer_hit_rate": 0.05454719811677933 + }, + "20": { + "rho_bar": 0.4078642725944519, + "union_observed": 0.2659175992012024, + "union_independent": 0.6789309978485107, + "union_gap": -0.41301339864730835, + "mean_per_layer_hit_rate": 0.08249303698539734 + }, + "50": { + "rho_bar": 0.4416004717350006, + "union_observed": 0.3745318353176117, + "union_independent": 0.8388907313346863, + "union_gap": -0.4643588960170746, + "mean_per_layer_hit_rate": 0.12849321961402893 + }, + "100": { + "rho_bar": 0.4358294606208801, + "union_observed": 0.4731585383415222, + "union_independent": 0.9176616072654724, + "union_gap": -0.4445030689239502, + "mean_per_layer_hit_rate": 0.17074809968471527 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.35439989609377726, + "union_gap_mean_over_k": -0.294163924242769 + }, + "USPTO_Backgrounds|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.23054447770118713, + "union_observed": 0.05243445560336113, + "union_independent": 0.1150064468383789, + "union_gap": -0.06257199123501778, + "mean_per_layer_hit_rate": 0.009315278381109238 + }, + "2": { + "rho_bar": 0.26949188113212585, + "union_observed": 0.09113608300685883, + "union_independent": 0.20673757791519165, + "union_gap": -0.11560149490833282, + "mean_per_layer_hit_rate": 0.017574187368154526 + }, + "5": { + "rho_bar": 0.2784726023674011, + "union_observed": 0.15855181217193604, + "union_independent": 0.37589502334594727, + "union_gap": -0.21734321117401123, + "mean_per_layer_hit_rate": 0.03534043952822685 + }, + "10": { + "rho_bar": 0.3404409885406494, + "union_observed": 0.2147315889596939, + "union_independent": 0.5373814105987549, + "union_gap": -0.322649821639061, + "mean_per_layer_hit_rate": 0.05694804713129997 + }, + "20": { + "rho_bar": 0.3943055272102356, + "union_observed": 0.27715355157852173, + "union_independent": 0.6922730803489685, + "union_gap": -0.4151195287704468, + "mean_per_layer_hit_rate": 0.08537404239177704 + }, + "50": { + "rho_bar": 0.41639474034309387, + "union_observed": 0.387016236782074, + "union_independent": 0.8339123129844666, + "union_gap": -0.4468960762023926, + "mean_per_layer_hit_rate": 0.12647651135921478 + }, + "100": { + "rho_bar": 0.4224514961242676, + "union_observed": 0.47440698742866516, + "union_independent": 0.9130493402481079, + "union_gap": -0.43864235281944275, + "mean_per_layer_hit_rate": 0.16748295724391937 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.33601453048842295, + "union_gap_mean_over_k": -0.28840349667838644 + }, + "USPTO_Backgrounds|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.17661572992801666, + "union_observed": 0.04993757978081703, + "union_independent": 0.10816478729248047, + "union_gap": -0.05822720751166344, + "mean_per_layer_hit_rate": 0.008739075623452663 + }, + "2": { + "rho_bar": 0.2101220041513443, + "union_observed": 0.09488140046596527, + "union_independent": 0.20148944854736328, + "union_gap": -0.10660804808139801, + "mean_per_layer_hit_rate": 0.017094017937779427 + }, + "5": { + "rho_bar": 0.25076189637184143, + "union_observed": 0.15605492889881134, + "union_independent": 0.3482464551925659, + "union_gap": -0.19219152629375458, + "mean_per_layer_hit_rate": 0.03217132389545441 + }, + "10": { + "rho_bar": 0.3334372937679291, + "union_observed": 0.21348313987255096, + "union_independent": 0.5199554562568665, + "union_gap": -0.3064723163843155, + "mean_per_layer_hit_rate": 0.05425909906625748 + }, + "20": { + "rho_bar": 0.37313780188560486, + "union_observed": 0.2746566832065582, + "union_independent": 0.6626735925674438, + "union_gap": -0.3880169093608856, + "mean_per_layer_hit_rate": 0.07903582602739334 + }, + "50": { + "rho_bar": 0.40735140442848206, + "union_observed": 0.37078651785850525, + "union_independent": 0.8139959573745728, + "union_gap": -0.4432094395160675, + "mean_per_layer_hit_rate": 0.11908192187547684 + }, + "100": { + "rho_bar": 0.41218605637550354, + "union_observed": 0.47440698742866516, + "union_independent": 0.9066915512084961, + "union_gap": -0.43228456377983093, + "mean_per_layer_hit_rate": 0.16287334263324738 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.30908745527267456, + "union_gap_mean_over_k": -0.2752871444182737 + }, + "USPTO_Backgrounds|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.14826418459415436, + "union_observed": 0.05992509424686432, + "union_independent": 0.1216738224029541, + "union_gap": -0.06174872815608978, + "mean_per_layer_hit_rate": 0.009891482070088387 + }, + "2": { + "rho_bar": 0.23093055188655853, + "union_observed": 0.09238451719284058, + "union_independent": 0.19940465688705444, + "union_gap": -0.10702013969421387, + "mean_per_layer_hit_rate": 0.016901951283216476 + }, + "5": { + "rho_bar": 0.28429386019706726, + "union_observed": 0.1523096114397049, + "union_independent": 0.3752697706222534, + "union_gap": -0.22296015918254852, + "mean_per_layer_hit_rate": 0.03534043952822685 + }, + "10": { + "rho_bar": 0.3594384789466858, + "union_observed": 0.2122347056865692, + "union_independent": 0.548338770866394, + "union_gap": -0.33610406517982483, + "mean_per_layer_hit_rate": 0.05877268686890602 + }, + "20": { + "rho_bar": 0.4076225459575653, + "union_observed": 0.2846441864967346, + "union_independent": 0.6972907185554504, + "union_gap": -0.4126465320587158, + "mean_per_layer_hit_rate": 0.08662248402833939 + }, + "50": { + "rho_bar": 0.46467703580856323, + "union_observed": 0.36828964948654175, + "union_independent": 0.8375394344329834, + "union_gap": -0.46924978494644165, + "mean_per_layer_hit_rate": 0.12810909748077393 + }, + "100": { + "rho_bar": 0.48000916838645935, + "union_observed": 0.4569288492202759, + "union_independent": 0.913393497467041, + "union_gap": -0.45646464824676514, + "mean_per_layer_hit_rate": 0.16777104139328003 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33931940368243624, + "union_gap_mean_over_k": -0.29517057963779997 + }, + "USPTO_Backgrounds|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.3580121695995331, + "union_observed": 0.04244694113731384, + "union_independent": 0.13923484086990356, + "union_gap": -0.09678789973258972, + "mean_per_layer_hit_rate": 0.01142802368849516 + }, + "2": { + "rho_bar": 0.35386356711387634, + "union_observed": 0.0699126124382019, + "union_independent": 0.24336767196655273, + "union_gap": -0.17345505952835083, + "mean_per_layer_hit_rate": 0.02112743631005287 + }, + "5": { + "rho_bar": 0.42460691928863525, + "union_observed": 0.12734082341194153, + "union_independent": 0.4359341859817505, + "union_gap": -0.30859336256980896, + "mean_per_layer_hit_rate": 0.04283107817173004 + }, + "10": { + "rho_bar": 0.5081053376197815, + "union_observed": 0.17478151619434357, + "union_independent": 0.6216033697128296, + "union_gap": -0.446821853518486, + "mean_per_layer_hit_rate": 0.07135311514139175 + }, + "20": { + "rho_bar": 0.5331935286521912, + "union_observed": 0.24968789517879486, + "union_independent": 0.7735448479652405, + "union_gap": -0.5238569527864456, + "mean_per_layer_hit_rate": 0.10640545189380646 + }, + "50": { + "rho_bar": 0.5626528859138489, + "union_observed": 0.34207239747047424, + "union_independent": 0.8929175734519958, + "union_gap": -0.5508451759815216, + "mean_per_layer_hit_rate": 0.1548064798116684 + }, + "100": { + "rho_bar": 0.5763981938362122, + "union_observed": 0.4282147288322449, + "union_independent": 0.9473599195480347, + "union_gap": -0.5191451907157898, + "mean_per_layer_hit_rate": 0.19802170991897583 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.4738332288605826, + "union_gap_mean_over_k": -0.37421507069042753 + }, + "Pile-CC|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.29167187213897705, + "union_observed": 0.06616728752851486, + "union_independent": 0.16515594720840454, + "union_gap": -0.09898865967988968, + "mean_per_layer_hit_rate": 0.013732833787798882 + }, + "2": { + "rho_bar": 0.35792630910873413, + "union_observed": 0.09737827628850937, + "union_independent": 0.2643497586250305, + "union_gap": -0.16697148233652115, + "mean_per_layer_hit_rate": 0.02324018068611622 + }, + "5": { + "rho_bar": 0.40170538425445557, + "union_observed": 0.15730337798595428, + "union_independent": 0.4630669355392456, + "union_gap": -0.3057635575532913, + "mean_per_layer_hit_rate": 0.046384330838918686 + }, + "10": { + "rho_bar": 0.4656842052936554, + "union_observed": 0.2197253406047821, + "union_independent": 0.6330821514129639, + "union_gap": -0.41335681080818176, + "mean_per_layer_hit_rate": 0.0733698233962059 + }, + "20": { + "rho_bar": 0.49066925048828125, + "union_observed": 0.29088640213012695, + "union_independent": 0.782598614692688, + "union_gap": -0.49171221256256104, + "mean_per_layer_hit_rate": 0.1089983731508255 + }, + "50": { + "rho_bar": 0.5267922878265381, + "union_observed": 0.36579275131225586, + "union_independent": 0.8924798369407654, + "union_gap": -0.5266870856285095, + "mean_per_layer_hit_rate": 0.1548064798116684 + }, + "100": { + "rho_bar": 0.5319085717201233, + "union_observed": 0.4619225859642029, + "union_independent": 0.9513444900512695, + "union_gap": -0.48942190408706665, + "mean_per_layer_hit_rate": 0.20349563658237457 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4380511258329664, + "union_gap_mean_over_k": -0.3561288160937173 + }, + "Pile-CC|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.257121205329895, + "union_observed": 0.054931335151195526, + "union_independent": 0.14369726181030273, + "union_gap": -0.08876592665910721, + "mean_per_layer_hit_rate": 0.01181215699762106 + }, + "2": { + "rho_bar": 0.2862302362918854, + "union_observed": 0.07990012317895889, + "union_independent": 0.21267229318618774, + "union_gap": -0.13277217000722885, + "mean_per_layer_hit_rate": 0.018150389194488525 + }, + "5": { + "rho_bar": 0.32507362961769104, + "union_observed": 0.14856429398059845, + "union_independent": 0.41536736488342285, + "union_gap": -0.2668030709028244, + "mean_per_layer_hit_rate": 0.04014212638139725 + }, + "10": { + "rho_bar": 0.37568727135658264, + "union_observed": 0.2172284573316574, + "union_independent": 0.5681241750717163, + "union_gap": -0.3508957177400589, + "mean_per_layer_hit_rate": 0.061845771968364716 + }, + "20": { + "rho_bar": 0.4559273421764374, + "union_observed": 0.2808988690376282, + "union_independent": 0.740209698677063, + "union_gap": -0.4593108296394348, + "mean_per_layer_hit_rate": 0.09689811617136002 + }, + "50": { + "rho_bar": 0.5130223631858826, + "union_observed": 0.3745318353176117, + "union_independent": 0.8659021258354187, + "union_gap": -0.491370290517807, + "mean_per_layer_hit_rate": 0.14049746096134186 + }, + "100": { + "rho_bar": 0.5325757265090942, + "union_observed": 0.45443195104599, + "union_independent": 0.9302593469619751, + "union_gap": -0.4758273959159851, + "mean_per_layer_hit_rate": 0.18131183087825775 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3922339677810669, + "union_gap_mean_over_k": -0.3236779144832066 + }, + "Pile-CC|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.24341022968292236, + "union_observed": 0.05368289723992348, + "union_independent": 0.12050777673721313, + "union_gap": -0.06682487949728966, + "mean_per_layer_hit_rate": 0.009795447811484337 + }, + "2": { + "rho_bar": 0.28154635429382324, + "union_observed": 0.0774032473564148, + "union_independent": 0.1841256022453308, + "union_gap": -0.10672235488891602, + "mean_per_layer_hit_rate": 0.015461442992091179 + }, + "5": { + "rho_bar": 0.27660173177719116, + "union_observed": 0.1548064947128296, + "union_independent": 0.36270689964294434, + "union_gap": -0.20790040493011475, + "mean_per_layer_hit_rate": 0.033803898841142654 + }, + "10": { + "rho_bar": 0.3430929481983185, + "union_observed": 0.21098627150058746, + "union_independent": 0.5338906645774841, + "union_gap": -0.32290439307689667, + "mean_per_layer_hit_rate": 0.05637184530496597 + }, + "20": { + "rho_bar": 0.36519521474838257, + "union_observed": 0.2746566832065582, + "union_independent": 0.6829653978347778, + "union_gap": -0.4083087146282196, + "mean_per_layer_hit_rate": 0.08316528052091599 + }, + "50": { + "rho_bar": 0.4480898976325989, + "union_observed": 0.3770287036895752, + "union_independent": 0.8388225436210632, + "union_gap": -0.46179383993148804, + "mean_per_layer_hit_rate": 0.12830115854740143 + }, + "100": { + "rho_bar": 0.48429518938064575, + "union_observed": 0.44569289684295654, + "union_independent": 0.9113821983337402, + "union_gap": -0.4656893014907837, + "mean_per_layer_hit_rate": 0.16642658412456512 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.34889022367341177, + "union_gap_mean_over_k": -0.29144912692052977 + }, + "Pile-CC|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.22200432419776917, + "union_observed": 0.047440700232982635, + "union_independent": 0.09803098440170288, + "union_gap": -0.050590284168720245, + "mean_per_layer_hit_rate": 0.007874771021306515 + }, + "2": { + "rho_bar": 0.2109798789024353, + "union_observed": 0.0774032473564148, + "union_independent": 0.1620948314666748, + "union_gap": -0.08469158411026001, + "mean_per_layer_hit_rate": 0.013444731943309307 + }, + "5": { + "rho_bar": 0.3001527488231659, + "union_observed": 0.14107365906238556, + "union_independent": 0.3432600498199463, + "union_gap": -0.20218639075756073, + "mean_per_layer_hit_rate": 0.03159511834383011 + }, + "10": { + "rho_bar": 0.35160186886787415, + "union_observed": 0.20099875330924988, + "union_independent": 0.49129557609558105, + "union_gap": -0.2902968227863312, + "mean_per_layer_hit_rate": 0.05012964829802513 + }, + "20": { + "rho_bar": 0.37180039286613464, + "union_observed": 0.26466915011405945, + "union_independent": 0.6514811515808105, + "union_gap": -0.3868120014667511, + "mean_per_layer_hit_rate": 0.07673101127147675 + }, + "50": { + "rho_bar": 0.42230653762817383, + "union_observed": 0.36329588294029236, + "union_independent": 0.8098018765449524, + "union_gap": -0.44650599360466003, + "mean_per_layer_hit_rate": 0.11773744225502014 + }, + "100": { + "rho_bar": 0.4529702961444855, + "union_observed": 0.4294631779193878, + "union_independent": 0.8908523917198181, + "union_gap": -0.4613892138004303, + "mean_per_layer_hit_rate": 0.1534620225429535 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3331165782042912, + "union_gap_mean_over_k": -0.27463889867067337 + }, + "Pile-CC|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.1650272160768509, + "union_observed": 0.05867665261030197, + "union_independent": 0.0991026759147644, + "union_gap": -0.04042602330446243, + "mean_per_layer_hit_rate": 0.007970805279910564 + }, + "2": { + "rho_bar": 0.19496075809001923, + "union_observed": 0.08489388227462769, + "union_independent": 0.17764371633529663, + "union_gap": -0.09274983406066895, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.26684221625328064, + "union_observed": 0.144818976521492, + "union_independent": 0.33068543672561646, + "union_gap": -0.18586646020412445, + "mean_per_layer_hit_rate": 0.03025064803659916 + }, + "10": { + "rho_bar": 0.32877135276794434, + "union_observed": 0.1910112351179123, + "union_independent": 0.4767102599143982, + "union_gap": -0.2856990247964859, + "mean_per_layer_hit_rate": 0.048208970576524734 + }, + "20": { + "rho_bar": 0.35167396068573, + "union_observed": 0.25967541337013245, + "union_independent": 0.6224384307861328, + "union_gap": -0.36276301741600037, + "mean_per_layer_hit_rate": 0.07135311514139175 + }, + "50": { + "rho_bar": 0.3843185007572174, + "union_observed": 0.3595505654811859, + "union_independent": 0.7777944207191467, + "union_gap": -0.4182438552379608, + "mean_per_layer_hit_rate": 0.10755784809589386 + }, + "100": { + "rho_bar": 0.40995413064956665, + "union_observed": 0.4394506812095642, + "union_independent": 0.8684036731719971, + "union_gap": -0.42895299196243286, + "mean_per_layer_hit_rate": 0.1419379711151123 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.30022116218294415, + "union_gap_mean_over_k": -0.2592430295688765 + }, + "Pile-CC|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.15167450904846191, + "union_observed": 0.05368289723992348, + "union_independent": 0.08978122472763062, + "union_gap": -0.03609832748770714, + "mean_per_layer_hit_rate": 0.00720253586769104 + }, + "2": { + "rho_bar": 0.21334770321846008, + "union_observed": 0.0749063640832901, + "union_independent": 0.16485166549682617, + "union_gap": -0.08994530141353607, + "mean_per_layer_hit_rate": 0.013732834719121456 + }, + "5": { + "rho_bar": 0.30144086480140686, + "union_observed": 0.11610487103462219, + "union_independent": 0.3154556155204773, + "union_gap": -0.1993507444858551, + "mean_per_layer_hit_rate": 0.02861807495355606 + }, + "10": { + "rho_bar": 0.3403030335903168, + "union_observed": 0.15730337798595428, + "union_independent": 0.4520038366317749, + "union_gap": -0.2947004586458206, + "mean_per_layer_hit_rate": 0.04494382441043854 + }, + "20": { + "rho_bar": 0.37659332156181335, + "union_observed": 0.2147315889596939, + "union_independent": 0.5827027559280396, + "union_gap": -0.36797116696834564, + "mean_per_layer_hit_rate": 0.06453471630811691 + }, + "50": { + "rho_bar": 0.3896596431732178, + "union_observed": 0.30337077379226685, + "union_independent": 0.755546510219574, + "union_gap": -0.45217573642730713, + "mean_per_layer_hit_rate": 0.10160376131534576 + }, + "100": { + "rho_bar": 0.4160177707672119, + "union_observed": 0.4032459557056427, + "union_independent": 0.8741741180419922, + "union_gap": -0.4709281623363495, + "mean_per_layer_hit_rate": 0.145203098654747 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3127195494515555, + "union_gap_mean_over_k": -0.27302427110927446 + }, + "Pile-CC|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.16831909120082855, + "union_observed": 0.05118601769208908, + "union_independent": 0.09214907884597778, + "union_gap": -0.0409630611538887, + "mean_per_layer_hit_rate": 0.007394603453576565 + }, + "2": { + "rho_bar": 0.20051300525665283, + "union_observed": 0.07116104662418365, + "union_independent": 0.1477639079093933, + "union_gap": -0.07660286128520966, + "mean_per_layer_hit_rate": 0.012196293100714684 + }, + "5": { + "rho_bar": 0.2986719310283661, + "union_observed": 0.10986267030239105, + "union_independent": 0.29402077198028564, + "union_gap": -0.1841581016778946, + "mean_per_layer_hit_rate": 0.026313263922929764 + }, + "10": { + "rho_bar": 0.3567734658718109, + "union_observed": 0.16229712963104248, + "union_independent": 0.4602205157279968, + "union_gap": -0.29792338609695435, + "mean_per_layer_hit_rate": 0.04600019007921219 + }, + "20": { + "rho_bar": 0.38892263174057007, + "union_observed": 0.20724095404148102, + "union_independent": 0.6072618961334229, + "union_gap": -0.40002094209194183, + "mean_per_layer_hit_rate": 0.06866416335105896 + }, + "50": { + "rho_bar": 0.4023667871952057, + "union_observed": 0.3158551752567291, + "union_independent": 0.7715941071510315, + "union_gap": -0.45573893189430237, + "mean_per_layer_hit_rate": 0.105733223259449 + }, + "100": { + "rho_bar": 0.4080384373664856, + "union_observed": 0.4294631779193878, + "union_independent": 0.887729823589325, + "union_gap": -0.45826664566993713, + "mean_per_layer_hit_rate": 0.151349276304245 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31765790709427427, + "union_gap_mean_over_k": -0.27338198998144697 + }, + "Pile-CC|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.20070792734622955, + "union_observed": 0.03745318204164505, + "union_independent": 0.08984041213989258, + "union_gap": -0.05238723009824753, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.25513845682144165, + "union_observed": 0.06117353215813637, + "union_independent": 0.16078317165374756, + "union_gap": -0.09960963949561119, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.32459595799446106, + "union_observed": 0.1061173528432846, + "union_independent": 0.31861412525177, + "union_gap": -0.2124967724084854, + "mean_per_layer_hit_rate": 0.02890617586672306 + }, + "10": { + "rho_bar": 0.35513919591903687, + "union_observed": 0.16479401290416718, + "union_independent": 0.47935914993286133, + "union_gap": -0.31456513702869415, + "mean_per_layer_hit_rate": 0.048497073352336884 + }, + "20": { + "rho_bar": 0.38246047496795654, + "union_observed": 0.2297128587961197, + "union_independent": 0.641477108001709, + "union_gap": -0.4117642492055893, + "mean_per_layer_hit_rate": 0.07481033354997635 + }, + "50": { + "rho_bar": 0.41036903858184814, + "union_observed": 0.34456929564476013, + "union_independent": 0.805466890335083, + "union_gap": -0.4608975946903229, + "mean_per_layer_hit_rate": 0.1159127950668335 + }, + "100": { + "rho_bar": 0.40895235538482666, + "union_observed": 0.4606741666793823, + "union_independent": 0.9038498401641846, + "union_gap": -0.44317567348480225, + "mean_per_layer_hit_rate": 0.16028040647506714 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33390905814511435, + "union_gap_mean_over_k": -0.284985185201679 + }, + "Pile-CC|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1829574555158615, + "union_observed": 0.04619225859642029, + "union_independent": 0.10247927904129028, + "union_gap": -0.056287020444869995, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.2609376609325409, + "union_observed": 0.06491885334253311, + "union_independent": 0.17048370838165283, + "union_gap": -0.10556485503911972, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.2708655595779419, + "union_observed": 0.12234706431627274, + "union_independent": 0.3259716033935547, + "union_gap": -0.20362453907728195, + "mean_per_layer_hit_rate": 0.02967444807291031 + }, + "10": { + "rho_bar": 0.35153117775917053, + "union_observed": 0.16604244709014893, + "union_independent": 0.48889774084091187, + "union_gap": -0.32285529375076294, + "mean_per_layer_hit_rate": 0.049745507538318634 + }, + "20": { + "rho_bar": 0.37957334518432617, + "union_observed": 0.2297128587961197, + "union_independent": 0.6481313705444336, + "union_gap": -0.4184185117483139, + "mean_per_layer_hit_rate": 0.07596273720264435 + }, + "50": { + "rho_bar": 0.3966296315193176, + "union_observed": 0.34456929564476013, + "union_independent": 0.8195773959159851, + "union_gap": -0.475008100271225, + "mean_per_layer_hit_rate": 0.12033036351203918 + }, + "100": { + "rho_bar": 0.39395835995674133, + "union_observed": 0.4594257175922394, + "union_independent": 0.9126892685890198, + "union_gap": -0.4532635509967804, + "mean_per_layer_hit_rate": 0.16537021100521088 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3194933129208429, + "union_gap_mean_over_k": -0.2907174101897648 + }, + "Pile-CC|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.2063000649213791, + "union_observed": 0.0387016236782074, + "union_independent": 0.0979577898979187, + "union_gap": -0.059256166219711304, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.25501927733421326, + "union_observed": 0.05742821469902992, + "union_independent": 0.16195863485336304, + "union_gap": -0.10453042015433311, + "mean_per_layer_hit_rate": 0.013444732874631882 + }, + "5": { + "rho_bar": 0.2647910416126251, + "union_observed": 0.12484394758939743, + "union_independent": 0.33136802911758423, + "union_gap": -0.2065240815281868, + "mean_per_layer_hit_rate": 0.03025064803659916 + }, + "10": { + "rho_bar": 0.35053572058677673, + "union_observed": 0.18227216601371765, + "union_independent": 0.514639675617218, + "union_gap": -0.33236750960350037, + "mean_per_layer_hit_rate": 0.05339479446411133 + }, + "20": { + "rho_bar": 0.3816164433956146, + "union_observed": 0.24968789517879486, + "union_independent": 0.6765122413635254, + "union_gap": -0.42682434618473053, + "mean_per_layer_hit_rate": 0.08153270184993744 + }, + "50": { + "rho_bar": 0.3756611943244934, + "union_observed": 0.36828964948654175, + "union_independent": 0.8302963972091675, + "union_gap": -0.46200674772262573, + "mean_per_layer_hit_rate": 0.12407566606998444 + }, + "100": { + "rho_bar": 0.3793331980705261, + "union_observed": 0.4906367063522339, + "union_independent": 0.9239421486854553, + "union_gap": -0.43330544233322144, + "mean_per_layer_hit_rate": 0.17343704402446747 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.31617956289223265, + "union_gap_mean_over_k": -0.2892592448209013 + }, + "Pile-CC|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.21429342031478882, + "union_observed": 0.047440700232982635, + "union_independent": 0.10704648494720459, + "union_gap": -0.059605784714221954, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.24421413242816925, + "union_observed": 0.07116104662418365, + "union_independent": 0.17895597219467163, + "union_gap": -0.10779492557048798, + "mean_per_layer_hit_rate": 0.014981274493038654 + }, + "5": { + "rho_bar": 0.29410967230796814, + "union_observed": 0.13607990741729736, + "union_independent": 0.3635883927345276, + "union_gap": -0.22750848531723022, + "mean_per_layer_hit_rate": 0.033899929374456406 + }, + "10": { + "rho_bar": 0.3708396852016449, + "union_observed": 0.19475655257701874, + "union_independent": 0.5457143783569336, + "union_gap": -0.35095782577991486, + "mean_per_layer_hit_rate": 0.05810045450925827 + }, + "20": { + "rho_bar": 0.38123366236686707, + "union_observed": 0.26217228174209595, + "union_independent": 0.702361524105072, + "union_gap": -0.4401892423629761, + "mean_per_layer_hit_rate": 0.08719869703054428 + }, + "50": { + "rho_bar": 0.3854500353336334, + "union_observed": 0.387016236782074, + "union_independent": 0.8563579320907593, + "union_gap": -0.4693416953086853, + "mean_per_layer_hit_rate": 0.13454335927963257 + }, + "100": { + "rho_bar": 0.3919110894203186, + "union_observed": 0.4906367063522339, + "union_independent": 0.9364417791366577, + "union_gap": -0.44580507278442383, + "mean_per_layer_hit_rate": 0.18419283628463745 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.32600738533905577, + "union_gap_mean_over_k": -0.3001718616911343 + }, + "Pile-CC|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.228917196393013, + "union_observed": 0.056179776787757874, + "union_independent": 0.13162201642990112, + "union_gap": -0.07544223964214325, + "mean_per_layer_hit_rate": 0.010755786672234535 + }, + "2": { + "rho_bar": 0.283945232629776, + "union_observed": 0.08863919973373413, + "union_independent": 0.22394859790802002, + "union_gap": -0.1353093981742859, + "mean_per_layer_hit_rate": 0.019206760451197624 + }, + "5": { + "rho_bar": 0.3251785933971405, + "union_observed": 0.1498127281665802, + "union_independent": 0.3879413604736328, + "union_gap": -0.2381286323070526, + "mean_per_layer_hit_rate": 0.036780945956707 + }, + "10": { + "rho_bar": 0.3879443407058716, + "union_observed": 0.20474407076835632, + "union_independent": 0.5802707672119141, + "union_gap": -0.37552669644355774, + "mean_per_layer_hit_rate": 0.06376644223928452 + }, + "20": { + "rho_bar": 0.41098982095718384, + "union_observed": 0.28589263558387756, + "union_independent": 0.7401514053344727, + "union_gap": -0.4542587697505951, + "mean_per_layer_hit_rate": 0.09661000221967697 + }, + "50": { + "rho_bar": 0.41603341698646545, + "union_observed": 0.4032459557056427, + "union_independent": 0.8733623027801514, + "union_gap": -0.47011634707450867, + "mean_per_layer_hit_rate": 0.1432824432849884 + }, + "100": { + "rho_bar": 0.4287523627281189, + "union_observed": 0.4981273412704468, + "union_independent": 0.9459344744682312, + "union_gap": -0.4478071331977844, + "mean_per_layer_hit_rate": 0.19494861364364624 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3545372805425099, + "union_gap_mean_over_k": -0.3137984595128468 + }, + "Pile-CC|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.24585509300231934, + "union_observed": 0.05118601769208908, + "union_independent": 0.13709932565689087, + "union_gap": -0.08591330796480179, + "mean_per_layer_hit_rate": 0.011235956102609634 + }, + "2": { + "rho_bar": 0.3033137619495392, + "union_observed": 0.08614232391119003, + "union_independent": 0.23657673597335815, + "union_gap": -0.15043441206216812, + "mean_per_layer_hit_rate": 0.020455200225114822 + }, + "5": { + "rho_bar": 0.3564813435077667, + "union_observed": 0.1523096114397049, + "union_independent": 0.42360204458236694, + "union_gap": -0.27129243314266205, + "mean_per_layer_hit_rate": 0.041198499500751495 + }, + "10": { + "rho_bar": 0.42283278703689575, + "union_observed": 0.22846442461013794, + "union_independent": 0.6073230504989624, + "union_gap": -0.37885862588882446, + "mean_per_layer_hit_rate": 0.06856813281774521 + }, + "20": { + "rho_bar": 0.4299064576625824, + "union_observed": 0.30836454033851624, + "union_independent": 0.7634614109992981, + "union_gap": -0.45509687066078186, + "mean_per_layer_hit_rate": 0.10314030945301056 + }, + "50": { + "rho_bar": 0.44820165634155273, + "union_observed": 0.3907615542411804, + "union_independent": 0.8813815116882324, + "union_gap": -0.490619957447052, + "mean_per_layer_hit_rate": 0.1480841189622879 + }, + "100": { + "rho_bar": 0.4531407356262207, + "union_observed": 0.4931335747241974, + "union_independent": 0.9447490572929382, + "union_gap": -0.45161548256874084, + "mean_per_layer_hit_rate": 0.19514068961143494 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.379961690732411, + "union_gap_mean_over_k": -0.3262615842478616 + }, + "Pile-CC|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.21153509616851807, + "union_observed": 0.056179776787757874, + "union_independent": 0.11265790462493896, + "union_gap": -0.05647812783718109, + "mean_per_layer_hit_rate": 0.009123211726546288 + }, + "2": { + "rho_bar": 0.23668205738067627, + "union_observed": 0.09612983465194702, + "union_independent": 0.2034313678741455, + "union_gap": -0.10730153322219849, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.31699657440185547, + "union_observed": 0.15730337798595428, + "union_independent": 0.3695070743560791, + "union_gap": -0.21220369637012482, + "mean_per_layer_hit_rate": 0.0346682034432888 + }, + "10": { + "rho_bar": 0.3736793100833893, + "union_observed": 0.2197253406047821, + "union_independent": 0.5407162308692932, + "union_gap": -0.3209908902645111, + "mean_per_layer_hit_rate": 0.05752424895763397 + }, + "20": { + "rho_bar": 0.39480581879615784, + "union_observed": 0.28339576721191406, + "union_independent": 0.6903986930847168, + "union_gap": -0.40700292587280273, + "mean_per_layer_hit_rate": 0.08498991280794144 + }, + "50": { + "rho_bar": 0.4180309474468231, + "union_observed": 0.392009973526001, + "union_independent": 0.8378489017486572, + "union_gap": -0.44583892822265625, + "mean_per_layer_hit_rate": 0.12791703641414642 + }, + "100": { + "rho_bar": 0.4304451644420624, + "union_observed": 0.4906367063522339, + "union_independent": 0.9161590337753296, + "union_gap": -0.4255223274230957, + "mean_per_layer_hit_rate": 0.16959570348262787 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3403107098170689, + "union_gap_mean_over_k": -0.2821912041732243 + }, + "Pile-CC|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.21298609673976898, + "union_observed": 0.047440700232982635, + "union_independent": 0.11370867490768433, + "union_gap": -0.06626797467470169, + "mean_per_layer_hit_rate": 0.009219245053827763 + }, + "2": { + "rho_bar": 0.2639032304286957, + "union_observed": 0.07116104662418365, + "union_independent": 0.18804174661636353, + "union_gap": -0.11688069999217987, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.29906782507896423, + "union_observed": 0.13607990741729736, + "union_independent": 0.3583982586860657, + "union_gap": -0.2223183512687683, + "mean_per_layer_hit_rate": 0.033323731273412704 + }, + "10": { + "rho_bar": 0.36092737317085266, + "union_observed": 0.20224718749523163, + "union_independent": 0.5266132354736328, + "union_gap": -0.3243660479784012, + "mean_per_layer_hit_rate": 0.055315472185611725 + }, + "20": { + "rho_bar": 0.38720929622650146, + "union_observed": 0.27590513229370117, + "union_independent": 0.6899858713150024, + "union_gap": -0.41408073902130127, + "mean_per_layer_hit_rate": 0.08489387482404709 + }, + "50": { + "rho_bar": 0.41966620087623596, + "union_observed": 0.3770287036895752, + "union_independent": 0.84001624584198, + "union_gap": -0.4629875421524048, + "mean_per_layer_hit_rate": 0.12868529558181763 + }, + "100": { + "rho_bar": 0.41218939423561096, + "union_observed": 0.5031210780143738, + "union_independent": 0.9218518733978271, + "union_gap": -0.41873079538345337, + "mean_per_layer_hit_rate": 0.17324498295783997 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3365642023938043, + "union_gap_mean_over_k": -0.2893760214958872 + }, + "Pile-CC|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.15053877234458923, + "union_observed": 0.05867665261030197, + "union_independent": 0.1150292158126831, + "union_gap": -0.056352563202381134, + "mean_per_layer_hit_rate": 0.009315278381109238 + }, + "2": { + "rho_bar": 0.18813657760620117, + "union_observed": 0.09987515956163406, + "union_independent": 0.19462871551513672, + "union_gap": -0.09475355595350266, + "mean_per_layer_hit_rate": 0.01642177812755108 + }, + "5": { + "rho_bar": 0.30159467458724976, + "union_observed": 0.16104869544506073, + "union_independent": 0.3785973787307739, + "union_gap": -0.2175486832857132, + "mean_per_layer_hit_rate": 0.035628542304039 + }, + "10": { + "rho_bar": 0.35219117999076843, + "union_observed": 0.23595505952835083, + "union_independent": 0.5514011383056641, + "union_gap": -0.31544607877731323, + "mean_per_layer_hit_rate": 0.05906078964471817 + }, + "20": { + "rho_bar": 0.38257700204849243, + "union_observed": 0.30337077379226685, + "union_independent": 0.6933009624481201, + "union_gap": -0.38993018865585327, + "mean_per_layer_hit_rate": 0.08537404984235764 + }, + "50": { + "rho_bar": 0.3996654450893402, + "union_observed": 0.40948814153671265, + "union_independent": 0.8364207744598389, + "union_gap": -0.4269326329231262, + "mean_per_layer_hit_rate": 0.12714876234531403 + }, + "100": { + "rho_bar": 0.40707337856292725, + "union_observed": 0.5081148743629456, + "union_independent": 0.9171575307846069, + "union_gap": -0.4090426564216614, + "mean_per_layer_hit_rate": 0.16988380253314972 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.31168243288993835, + "union_gap_mean_over_k": -0.27285805131707874 + }, + "Pile-CC|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.17371885478496552, + "union_observed": 0.056179776787757874, + "union_independent": 0.11606466770172119, + "union_gap": -0.05988489091396332, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.20827807486057281, + "union_observed": 0.08489388227462769, + "union_independent": 0.1779612898826599, + "union_gap": -0.09306740760803223, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2539805769920349, + "union_observed": 0.15855181217193604, + "union_independent": 0.3393303155899048, + "union_gap": -0.18077850341796875, + "mean_per_layer_hit_rate": 0.031114954501390457 + }, + "10": { + "rho_bar": 0.30612993240356445, + "union_observed": 0.227215975522995, + "union_independent": 0.5135254859924316, + "union_gap": -0.28630951046943665, + "mean_per_layer_hit_rate": 0.053106699138879776 + }, + "20": { + "rho_bar": 0.325767457485199, + "union_observed": 0.28838950395584106, + "union_independent": 0.6649376749992371, + "union_gap": -0.376548171043396, + "mean_per_layer_hit_rate": 0.07893978804349899 + }, + "50": { + "rho_bar": 0.3904561996459961, + "union_observed": 0.39450687170028687, + "union_independent": 0.8236943483352661, + "union_gap": -0.42918747663497925, + "mean_per_layer_hit_rate": 0.12186690419912338 + }, + "100": { + "rho_bar": 0.4018428921699524, + "union_observed": 0.5068663954734802, + "union_independent": 0.9101611375808716, + "union_gap": -0.40329474210739136, + "mean_per_layer_hit_rate": 0.16392971575260162 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.2943105697631836, + "union_gap_mean_over_k": -0.2612958145993097 + }, + "Pile-CC|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.1792897880077362, + "union_observed": 0.048689138144254684, + "union_independent": 0.10581958293914795, + "union_gap": -0.057130444794893265, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.2510053813457489, + "union_observed": 0.0774032473564148, + "union_independent": 0.17875796556472778, + "union_gap": -0.10135471820831299, + "mean_per_layer_hit_rate": 0.01498127356171608 + }, + "5": { + "rho_bar": 0.28528183698654175, + "union_observed": 0.15355804562568665, + "union_independent": 0.36484473943710327, + "union_gap": -0.21128669381141663, + "mean_per_layer_hit_rate": 0.0340920016169548 + }, + "10": { + "rho_bar": 0.34484872221946716, + "union_observed": 0.20724095404148102, + "union_independent": 0.5215753316879272, + "union_gap": -0.31433437764644623, + "mean_per_layer_hit_rate": 0.05454719811677933 + }, + "20": { + "rho_bar": 0.38594117760658264, + "union_observed": 0.2871410846710205, + "union_independent": 0.6871428489685059, + "union_gap": -0.40000176429748535, + "mean_per_layer_hit_rate": 0.08412560820579529 + }, + "50": { + "rho_bar": 0.42871278524398804, + "union_observed": 0.40199750661849976, + "union_independent": 0.8423885703086853, + "union_gap": -0.44039106369018555, + "mean_per_layer_hit_rate": 0.12916545569896698 + }, + "100": { + "rho_bar": 0.4498971700668335, + "union_observed": 0.5043695569038391, + "union_independent": 0.9215906262397766, + "union_gap": -0.4172210693359375, + "mean_per_layer_hit_rate": 0.17218860983848572 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3321395516395569, + "union_gap_mean_over_k": -0.27738859025495394 + }, + "Pile-CC|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.3259626030921936, + "union_observed": 0.04244694113731384, + "union_independent": 0.1358737349510193, + "union_gap": -0.09342679381370544, + "mean_per_layer_hit_rate": 0.01113992091268301 + }, + "2": { + "rho_bar": 0.3641229271888733, + "union_observed": 0.08239700645208359, + "union_independent": 0.239423930644989, + "union_gap": -0.15702692419290543, + "mean_per_layer_hit_rate": 0.020743301138281822 + }, + "5": { + "rho_bar": 0.397307425737381, + "union_observed": 0.13857677578926086, + "union_independent": 0.436809778213501, + "union_gap": -0.2982330024242401, + "mean_per_layer_hit_rate": 0.04292711243033409 + }, + "10": { + "rho_bar": 0.4424513876438141, + "union_observed": 0.1960049867630005, + "union_independent": 0.6234996914863586, + "union_gap": -0.42749470472335815, + "mean_per_layer_hit_rate": 0.0714491531252861 + }, + "20": { + "rho_bar": 0.4816131591796875, + "union_observed": 0.26716604828834534, + "union_independent": 0.7740073204040527, + "union_gap": -0.5068412721157074, + "mean_per_layer_hit_rate": 0.10602132976055145 + }, + "50": { + "rho_bar": 0.5206268429756165, + "union_observed": 0.35580524802207947, + "union_independent": 0.8894457817077637, + "union_gap": -0.5336405336856842, + "mean_per_layer_hit_rate": 0.15182945132255554 + }, + "100": { + "rho_bar": 0.5166968107223511, + "union_observed": 0.45318353176116943, + "union_independent": 0.9495806694030762, + "union_gap": -0.49639713764190674, + "mean_per_layer_hit_rate": 0.19869393110275269 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4355401652199881, + "union_gap_mean_over_k": -0.3590086240853582 + }, + "Pile-CC|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.28430747985839844, + "union_observed": 0.06741572916507721, + "union_independent": 0.1672348976135254, + "union_gap": -0.09981916844844818, + "mean_per_layer_hit_rate": 0.013924901373684406 + }, + "2": { + "rho_bar": 0.358557790517807, + "union_observed": 0.09987515956163406, + "union_independent": 0.26898765563964844, + "union_gap": -0.16911249607801437, + "mean_per_layer_hit_rate": 0.023720350116491318 + }, + "5": { + "rho_bar": 0.40958037972450256, + "union_observed": 0.15980024635791779, + "union_independent": 0.4699798822402954, + "union_gap": -0.3101796358823776, + "mean_per_layer_hit_rate": 0.047344665974378586 + }, + "10": { + "rho_bar": 0.47370705008506775, + "union_observed": 0.2222222238779068, + "union_independent": 0.63929283618927, + "union_gap": -0.4170706123113632, + "mean_per_layer_hit_rate": 0.07461826503276825 + }, + "20": { + "rho_bar": 0.4951421916484833, + "union_observed": 0.29338327050209045, + "union_independent": 0.7858394384384155, + "union_gap": -0.4924561679363251, + "mean_per_layer_hit_rate": 0.11005474627017975 + }, + "50": { + "rho_bar": 0.5201461911201477, + "union_observed": 0.37328338623046875, + "union_independent": 0.8915126323699951, + "union_gap": -0.5182292461395264, + "mean_per_layer_hit_rate": 0.15413425862789154 + }, + "100": { + "rho_bar": 0.5294596552848816, + "union_observed": 0.4631710350513458, + "union_independent": 0.9522001147270203, + "union_gap": -0.48902907967567444, + "mean_per_layer_hit_rate": 0.20455199480056763 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.43870010546275545, + "union_gap_mean_over_k": -0.35655662949596134 + }, + "Pile-CC|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.2575928270816803, + "union_observed": 0.054931335151195526, + "union_independent": 0.14371097087860107, + "union_gap": -0.08877963572740555, + "mean_per_layer_hit_rate": 0.011812158860266209 + }, + "2": { + "rho_bar": 0.3188400864601135, + "union_observed": 0.08489388227462769, + "union_independent": 0.22457784414291382, + "union_gap": -0.13968396186828613, + "mean_per_layer_hit_rate": 0.019302794709801674 + }, + "5": { + "rho_bar": 0.3373371660709381, + "union_observed": 0.15106117725372314, + "union_independent": 0.4236278533935547, + "union_gap": -0.27256667613983154, + "mean_per_layer_hit_rate": 0.041198499500751495 + }, + "10": { + "rho_bar": 0.3941393196582794, + "union_observed": 0.2172284573316574, + "union_independent": 0.5776676535606384, + "union_gap": -0.360439196228981, + "mean_per_layer_hit_rate": 0.06347834318876266 + }, + "20": { + "rho_bar": 0.4686194062232971, + "union_observed": 0.2821473181247711, + "union_independent": 0.7444337606430054, + "union_gap": -0.46228644251823425, + "mean_per_layer_hit_rate": 0.09814655035734177 + }, + "50": { + "rho_bar": 0.5127934813499451, + "union_observed": 0.3770287036895752, + "union_independent": 0.8673961162567139, + "union_gap": -0.49036741256713867, + "mean_per_layer_hit_rate": 0.14126572012901306 + }, + "100": { + "rho_bar": 0.5279992818832397, + "union_observed": 0.45443195104599, + "union_independent": 0.9298588633537292, + "union_gap": -0.47542691230773926, + "mean_per_layer_hit_rate": 0.1810237020254135 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4024745098182133, + "union_gap_mean_over_k": -0.32707860533680233 + }, + "Pile-CC|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.220384418964386, + "union_observed": 0.05368289723992348, + "union_independent": 0.11830079555511475, + "union_gap": -0.06461789831519127, + "mean_per_layer_hit_rate": 0.009603380225598812 + }, + "2": { + "rho_bar": 0.27127504348754883, + "union_observed": 0.07990012317895889, + "union_independent": 0.1882041096687317, + "union_gap": -0.1083039864897728, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.28855621814727783, + "union_observed": 0.15355804562568665, + "union_independent": 0.36585837602615356, + "union_gap": -0.21230033040046692, + "mean_per_layer_hit_rate": 0.03418803587555885 + }, + "10": { + "rho_bar": 0.352666437625885, + "union_observed": 0.20973782241344452, + "union_independent": 0.5332314968109131, + "union_gap": -0.32349367439746857, + "mean_per_layer_hit_rate": 0.056275807321071625 + }, + "20": { + "rho_bar": 0.37390661239624023, + "union_observed": 0.2709113657474518, + "union_independent": 0.689673662185669, + "union_gap": -0.41876229643821716, + "mean_per_layer_hit_rate": 0.08470181375741959 + }, + "50": { + "rho_bar": 0.4619735777378082, + "union_observed": 0.3770287036895752, + "union_independent": 0.8430877327919006, + "union_gap": -0.46605902910232544, + "mean_per_layer_hit_rate": 0.13022182881832123 + }, + "100": { + "rho_bar": 0.4942154884338379, + "union_observed": 0.44319599866867065, + "union_independent": 0.9132559895515442, + "union_gap": -0.47005999088287354, + "mean_per_layer_hit_rate": 0.16786709427833557 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3518539709704263, + "union_gap_mean_over_k": -0.29479960086090223 + }, + "Pile-CC|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.18807490170001984, + "union_observed": 0.047440700232982635, + "union_independent": 0.09913092851638794, + "union_gap": -0.051690228283405304, + "mean_per_layer_hit_rate": 0.007970805279910564 + }, + "2": { + "rho_bar": 0.20433592796325684, + "union_observed": 0.0774032473564148, + "union_independent": 0.1694570779800415, + "union_gap": -0.09205383062362671, + "mean_per_layer_hit_rate": 0.014116968959569931 + }, + "5": { + "rho_bar": 0.3035358488559723, + "union_observed": 0.1423220932483673, + "union_independent": 0.3491254448890686, + "union_gap": -0.2068033516407013, + "mean_per_layer_hit_rate": 0.032267358154058456 + }, + "10": { + "rho_bar": 0.3547956645488739, + "union_observed": 0.19975031912326813, + "union_independent": 0.4957836866378784, + "union_gap": -0.2960333675146103, + "mean_per_layer_hit_rate": 0.05080188065767288 + }, + "20": { + "rho_bar": 0.39118948578834534, + "union_observed": 0.260923832654953, + "union_independent": 0.6511762142181396, + "union_gap": -0.39025238156318665, + "mean_per_layer_hit_rate": 0.07673101127147675 + }, + "50": { + "rho_bar": 0.429818719625473, + "union_observed": 0.36828964948654175, + "union_independent": 0.8139002323150635, + "union_gap": -0.44561058282852173, + "mean_per_layer_hit_rate": 0.11927399039268494 + }, + "100": { + "rho_bar": 0.46139007806777954, + "union_observed": 0.4319600462913513, + "union_independent": 0.8917943835258484, + "union_gap": -0.45983433723449707, + "mean_per_layer_hit_rate": 0.15413425862789154 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33330580379281727, + "union_gap_mean_over_k": -0.2774682970983641 + }, + "Pile-CC|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.13762366771697998, + "union_observed": 0.05992509424686432, + "union_independent": 0.09909844398498535, + "union_gap": -0.03917334973812103, + "mean_per_layer_hit_rate": 0.007970805279910564 + }, + "2": { + "rho_bar": 0.1924789398908615, + "union_observed": 0.08489388227462769, + "union_independent": 0.1755271553993225, + "union_gap": -0.09063327312469482, + "mean_per_layer_hit_rate": 0.01469317264854908 + }, + "5": { + "rho_bar": 0.28008368611335754, + "union_observed": 0.14357054233551025, + "union_independent": 0.33410847187042236, + "union_gap": -0.1905379295349121, + "mean_per_layer_hit_rate": 0.03063478320837021 + }, + "10": { + "rho_bar": 0.3318847119808197, + "union_observed": 0.18976280093193054, + "union_independent": 0.4787803888320923, + "union_gap": -0.28901758790016174, + "mean_per_layer_hit_rate": 0.048497073352336884 + }, + "20": { + "rho_bar": 0.3584735691547394, + "union_observed": 0.255930095911026, + "union_independent": 0.6222753524780273, + "union_gap": -0.36634525656700134, + "mean_per_layer_hit_rate": 0.07135312259197235 + }, + "50": { + "rho_bar": 0.3783515393733978, + "union_observed": 0.36079901456832886, + "union_independent": 0.7752165794372559, + "union_gap": -0.414417564868927, + "mean_per_layer_hit_rate": 0.10678959637880325 + }, + "100": { + "rho_bar": 0.4149847626686096, + "union_observed": 0.4469413161277771, + "union_independent": 0.8717884421348572, + "union_gap": -0.4248471260070801, + "mean_per_layer_hit_rate": 0.1436665803194046 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.29912583955696653, + "union_gap_mean_over_k": -0.2592817268201283 + }, + "Pile-CC|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.15190325677394867, + "union_observed": 0.05742821469902992, + "union_independent": 0.09664374589920044, + "union_gap": -0.03921553120017052, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.2103666216135025, + "union_observed": 0.07865168899297714, + "union_independent": 0.16697275638580322, + "union_gap": -0.08832106739282608, + "mean_per_layer_hit_rate": 0.013924902305006981 + }, + "5": { + "rho_bar": 0.3100215494632721, + "union_observed": 0.12109863013029099, + "union_independent": 0.32596808671951294, + "union_gap": -0.20486945658922195, + "mean_per_layer_hit_rate": 0.02977048046886921 + }, + "10": { + "rho_bar": 0.3428821265697479, + "union_observed": 0.16229712963104248, + "union_independent": 0.45474129915237427, + "union_gap": -0.2924441695213318, + "mean_per_layer_hit_rate": 0.04532795771956444 + }, + "20": { + "rho_bar": 0.3746367394924164, + "union_observed": 0.2147315889596939, + "union_independent": 0.5904195308685303, + "union_gap": -0.37568794190883636, + "mean_per_layer_hit_rate": 0.06587918847799301 + }, + "50": { + "rho_bar": 0.39084210991859436, + "union_observed": 0.2971285879611969, + "union_independent": 0.7540217041969299, + "union_gap": -0.45689311623573303, + "mean_per_layer_hit_rate": 0.10121963918209076 + }, + "100": { + "rho_bar": 0.42062142491340637, + "union_observed": 0.39950063824653625, + "union_independent": 0.8746034502983093, + "union_gap": -0.47510281205177307, + "mean_per_layer_hit_rate": 0.1453951895236969 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31446768982069834, + "union_gap_mean_over_k": -0.27607629927141325 + }, + "Pile-CC|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.17607422173023224, + "union_observed": 0.048689138144254684, + "union_independent": 0.0932997465133667, + "union_gap": -0.044610608369112015, + "mean_per_layer_hit_rate": 0.00749063678085804 + }, + "2": { + "rho_bar": 0.21043385565280914, + "union_observed": 0.0749063640832901, + "union_independent": 0.1596059799194336, + "union_gap": -0.0846996158361435, + "mean_per_layer_hit_rate": 0.013252666220068932 + }, + "5": { + "rho_bar": 0.28984183073043823, + "union_observed": 0.10736579447984695, + "union_independent": 0.2903803586959839, + "union_gap": -0.18301456421613693, + "mean_per_layer_hit_rate": 0.025929125025868416 + }, + "10": { + "rho_bar": 0.3527635931968689, + "union_observed": 0.16229712963104248, + "union_independent": 0.46033012866973877, + "union_gap": -0.2980329990386963, + "mean_per_layer_hit_rate": 0.04600019007921219 + }, + "20": { + "rho_bar": 0.3814917504787445, + "union_observed": 0.21348313987255096, + "union_independent": 0.6093263030052185, + "union_gap": -0.39584316313266754, + "mean_per_layer_hit_rate": 0.06904830038547516 + }, + "50": { + "rho_bar": 0.40430936217308044, + "union_observed": 0.31086140871047974, + "union_independent": 0.7727093696594238, + "union_gap": -0.4618479609489441, + "mean_per_layer_hit_rate": 0.10611733794212341 + }, + "100": { + "rho_bar": 0.40859466791152954, + "union_observed": 0.4319600462913513, + "union_independent": 0.8859995007514954, + "union_gap": -0.45403945446014404, + "mean_per_layer_hit_rate": 0.15048497915267944 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31764418312481474, + "union_gap_mean_over_k": -0.27458405228597776 + }, + "Pile-CC|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.20031026005744934, + "union_observed": 0.041198503226041794, + "union_independent": 0.09672123193740845, + "union_gap": -0.05552272871136665, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.26048141717910767, + "union_observed": 0.06367041170597076, + "union_independent": 0.16716808080673218, + "union_gap": -0.10349766910076141, + "mean_per_layer_hit_rate": 0.013924901373684406 + }, + "5": { + "rho_bar": 0.3192865252494812, + "union_observed": 0.1061173528432846, + "union_independent": 0.31941354274749756, + "union_gap": -0.21329618990421295, + "mean_per_layer_hit_rate": 0.02900220826268196 + }, + "10": { + "rho_bar": 0.36186471581459045, + "union_observed": 0.16479401290416718, + "union_independent": 0.47666800022125244, + "union_gap": -0.31187398731708527, + "mean_per_layer_hit_rate": 0.048112932592630386 + }, + "20": { + "rho_bar": 0.3804289996623993, + "union_observed": 0.23345817625522614, + "union_independent": 0.6442016363143921, + "union_gap": -0.41074346005916595, + "mean_per_layer_hit_rate": 0.0752904936671257 + }, + "50": { + "rho_bar": 0.4139719307422638, + "union_observed": 0.34207239747047424, + "union_independent": 0.8033465147018433, + "union_gap": -0.461274117231369, + "mean_per_layer_hit_rate": 0.11524057388305664 + }, + "100": { + "rho_bar": 0.40579065680503845, + "union_observed": 0.45817726850509644, + "union_independent": 0.9046265482902527, + "union_gap": -0.44644927978515625, + "mean_per_layer_hit_rate": 0.16066455841064453 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3345906436443329, + "union_gap_mean_over_k": -0.28609391887273106 + }, + "Pile-CC|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.19023901224136353, + "union_observed": 0.04494382068514824, + "union_independent": 0.10588932037353516, + "union_gap": -0.06094549968838692, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.2522578239440918, + "union_observed": 0.06741572916507721, + "union_independent": 0.17160600423812866, + "union_gap": -0.10419027507305145, + "mean_per_layer_hit_rate": 0.014309035614132881 + }, + "5": { + "rho_bar": 0.27302101254463196, + "union_observed": 0.12234706431627274, + "union_independent": 0.3354334831237793, + "union_gap": -0.21308641880750656, + "mean_per_layer_hit_rate": 0.030730821192264557 + }, + "10": { + "rho_bar": 0.35060200095176697, + "union_observed": 0.16853933036327362, + "union_independent": 0.49288153648376465, + "union_gap": -0.32434220612049103, + "mean_per_layer_hit_rate": 0.05032171681523323 + }, + "20": { + "rho_bar": 0.38462021946907043, + "union_observed": 0.2222222238779068, + "union_independent": 0.6486652493476868, + "union_gap": -0.42644302546977997, + "mean_per_layer_hit_rate": 0.0760587677359581 + }, + "50": { + "rho_bar": 0.3916630446910858, + "union_observed": 0.3570536971092224, + "union_independent": 0.8247835636138916, + "union_gap": -0.4677298665046692, + "mean_per_layer_hit_rate": 0.12205897271633148 + }, + "100": { + "rho_bar": 0.3971385955810547, + "union_observed": 0.4594257175922394, + "union_independent": 0.9137952327728271, + "union_gap": -0.45436951518058777, + "mean_per_layer_hit_rate": 0.16613848507404327 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.31993452991758076, + "union_gap_mean_over_k": -0.293015258120639 + }, + "Pile-CC|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.2168833464384079, + "union_observed": 0.041198503226041794, + "union_independent": 0.10249084234237671, + "union_gap": -0.061292339116334915, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.23538503050804138, + "union_observed": 0.06491885334253311, + "union_independent": 0.1683121919631958, + "union_gap": -0.10339333862066269, + "mean_per_layer_hit_rate": 0.014020935632288456 + }, + "5": { + "rho_bar": 0.26760217547416687, + "union_observed": 0.12484394758939743, + "union_independent": 0.33656030893325806, + "union_gap": -0.21171636134386063, + "mean_per_layer_hit_rate": 0.03082684986293316 + }, + "10": { + "rho_bar": 0.3500959873199463, + "union_observed": 0.18476903438568115, + "union_independent": 0.517291784286499, + "union_gap": -0.33252274990081787, + "mean_per_layer_hit_rate": 0.05377892777323723 + }, + "20": { + "rho_bar": 0.3847280740737915, + "union_observed": 0.24968789517879486, + "union_independent": 0.6796926259994507, + "union_gap": -0.4300047308206558, + "mean_per_layer_hit_rate": 0.08220493793487549 + }, + "50": { + "rho_bar": 0.37070006132125854, + "union_observed": 0.3820224702358246, + "union_independent": 0.8365095853805542, + "union_gap": -0.4544871151447296, + "mean_per_layer_hit_rate": 0.12628445029258728 + }, + "100": { + "rho_bar": 0.38164860010147095, + "union_observed": 0.48439452052116394, + "union_independent": 0.9243683815002441, + "union_gap": -0.4399738609790802, + "mean_per_layer_hit_rate": 0.17372514307498932 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3152918964624405, + "union_gap_mean_over_k": -0.2904843565608774 + }, + "Pile-CC|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.19703684747219086, + "union_observed": 0.048689138144254684, + "union_independent": 0.1115494966506958, + "union_gap": -0.06286035850644112, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.24615631997585297, + "union_observed": 0.072409488260746, + "union_independent": 0.18107908964157104, + "union_gap": -0.10866960138082504, + "mean_per_layer_hit_rate": 0.015173341147601604 + }, + "5": { + "rho_bar": 0.3120429813861847, + "union_observed": 0.13358302414417267, + "union_independent": 0.3719085454940796, + "union_gap": -0.23832552134990692, + "mean_per_layer_hit_rate": 0.0348602719604969 + }, + "10": { + "rho_bar": 0.37153658270835876, + "union_observed": 0.1960049867630005, + "union_independent": 0.5565428733825684, + "union_gap": -0.36053788661956787, + "mean_per_layer_hit_rate": 0.05982905998826027 + }, + "20": { + "rho_bar": 0.3735813498497009, + "union_observed": 0.2684144675731659, + "union_independent": 0.705733060836792, + "union_gap": -0.4373185932636261, + "mean_per_layer_hit_rate": 0.08796696364879608 + }, + "50": { + "rho_bar": 0.3821154832839966, + "union_observed": 0.3957552909851074, + "union_independent": 0.8618922233581543, + "union_gap": -0.4661369323730469, + "mean_per_layer_hit_rate": 0.1371362805366516 + }, + "100": { + "rho_bar": 0.3918182849884033, + "union_observed": 0.49687889218330383, + "union_independent": 0.939219057559967, + "union_gap": -0.4423401653766632, + "mean_per_layer_hit_rate": 0.18688178062438965 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3248982642378126, + "union_gap_mean_over_k": -0.3023127226957253 + }, + "Pile-CC|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.20894168317317963, + "union_observed": 0.05742821469902992, + "union_independent": 0.1327589750289917, + "union_gap": -0.07533076032996178, + "mean_per_layer_hit_rate": 0.01085181999951601 + }, + "2": { + "rho_bar": 0.27518194913864136, + "union_observed": 0.08988764137029648, + "union_independent": 0.22680097818374634, + "union_gap": -0.13691333681344986, + "mean_per_layer_hit_rate": 0.019494863227009773 + }, + "5": { + "rho_bar": 0.32219383120536804, + "union_observed": 0.15605492889881134, + "union_independent": 0.3982505202293396, + "union_gap": -0.24219559133052826, + "mean_per_layer_hit_rate": 0.03802938759326935 + }, + "10": { + "rho_bar": 0.38514137268066406, + "union_observed": 0.21348313987255096, + "union_independent": 0.5866057872772217, + "union_gap": -0.3731226474046707, + "mean_per_layer_hit_rate": 0.06482282280921936 + }, + "20": { + "rho_bar": 0.40551745891571045, + "union_observed": 0.29088640213012695, + "union_independent": 0.745087206363678, + "union_gap": -0.454200804233551, + "mean_per_layer_hit_rate": 0.09785844385623932 + }, + "50": { + "rho_bar": 0.4139869511127472, + "union_observed": 0.40199750661849976, + "union_independent": 0.8763495087623596, + "union_gap": -0.47435200214385986, + "mean_per_layer_hit_rate": 0.14472293853759766 + }, + "100": { + "rho_bar": 0.4237392246723175, + "union_observed": 0.5006242394447327, + "union_independent": 0.9462591409683228, + "union_gap": -0.4456349015235901, + "mean_per_layer_hit_rate": 0.19514068961143494 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.34781463869980406, + "union_gap_mean_over_k": -0.3145357205399445 + }, + "Pile-CC|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.24668434262275696, + "union_observed": 0.05243445560336113, + "union_independent": 0.1381610631942749, + "union_gap": -0.08572660759091377, + "mean_per_layer_hit_rate": 0.01133198942989111 + }, + "2": { + "rho_bar": 0.29938772320747375, + "union_observed": 0.09113608300685883, + "union_independent": 0.25113242864608765, + "union_gap": -0.15999634563922882, + "mean_per_layer_hit_rate": 0.02189570665359497 + }, + "5": { + "rho_bar": 0.3642997145652771, + "union_observed": 0.1548064947128296, + "union_independent": 0.43479979038238525, + "union_gap": -0.27999329566955566, + "mean_per_layer_hit_rate": 0.04263900965452194 + }, + "10": { + "rho_bar": 0.4293459951877594, + "union_observed": 0.2197253406047821, + "union_independent": 0.6088154315948486, + "union_gap": -0.38909009099006653, + "mean_per_layer_hit_rate": 0.06885623931884766 + }, + "20": { + "rho_bar": 0.4333435595035553, + "union_observed": 0.30836454033851624, + "union_independent": 0.768714964389801, + "union_gap": -0.4603504240512848, + "mean_per_layer_hit_rate": 0.10467685014009476 + }, + "50": { + "rho_bar": 0.4427677094936371, + "union_observed": 0.3957552909851074, + "union_independent": 0.8836444020271301, + "union_gap": -0.4878891110420227, + "mean_per_layer_hit_rate": 0.14933256804943085 + }, + "100": { + "rho_bar": 0.45054078102111816, + "union_observed": 0.4931335747241974, + "union_independent": 0.9449697136878967, + "union_gap": -0.45183613896369934, + "mean_per_layer_hit_rate": 0.19533275067806244 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.38090997508593966, + "union_gap_mean_over_k": -0.3306974305638245 + }, + "Pile-CC|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.1524844467639923, + "union_observed": 0.06117353215813637, + "union_independent": 0.11597687005996704, + "union_gap": -0.05480333790183067, + "mean_per_layer_hit_rate": 0.009411313571035862 + }, + "2": { + "rho_bar": 0.21739375591278076, + "union_observed": 0.09862671792507172, + "union_independent": 0.20541948080062866, + "union_gap": -0.10679276287555695, + "mean_per_layer_hit_rate": 0.017478153109550476 + }, + "5": { + "rho_bar": 0.3115396797657013, + "union_observed": 0.16229712963104248, + "union_independent": 0.36876749992370605, + "union_gap": -0.20647037029266357, + "mean_per_layer_hit_rate": 0.034572165459394455 + }, + "10": { + "rho_bar": 0.3802165389060974, + "union_observed": 0.22097378969192505, + "union_independent": 0.5443302392959595, + "union_gap": -0.3233564496040344, + "mean_per_layer_hit_rate": 0.05810045450925827 + }, + "20": { + "rho_bar": 0.3971102237701416, + "union_observed": 0.28339576721191406, + "union_independent": 0.693260908126831, + "union_gap": -0.409865140914917, + "mean_per_layer_hit_rate": 0.08566215634346008 + }, + "50": { + "rho_bar": 0.42370665073394775, + "union_observed": 0.3932584226131439, + "union_independent": 0.8409640192985535, + "union_gap": -0.44770559668540955, + "mean_per_layer_hit_rate": 0.12926150858402252 + }, + "100": { + "rho_bar": 0.4309995472431183, + "union_observed": 0.49438202381134033, + "union_independent": 0.9172250032424927, + "union_gap": -0.42284297943115234, + "mean_per_layer_hit_rate": 0.17046000063419342 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33049297758511137, + "union_gap_mean_over_k": -0.2816909482436521 + }, + "Pile-CC|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.2181253731250763, + "union_observed": 0.04619225859642029, + "union_independent": 0.11260658502578735, + "union_gap": -0.06641432642936707, + "mean_per_layer_hit_rate": 0.009123211726546288 + }, + "2": { + "rho_bar": 0.24203965067863464, + "union_observed": 0.0749063640832901, + "union_independent": 0.1870453953742981, + "union_gap": -0.112139031291008, + "mean_per_layer_hit_rate": 0.01574954204261303 + }, + "5": { + "rho_bar": 0.2959766685962677, + "union_observed": 0.13857677578926086, + "union_independent": 0.3633538484573364, + "union_gap": -0.22477707266807556, + "mean_per_layer_hit_rate": 0.033899929374456406 + }, + "10": { + "rho_bar": 0.36579737067222595, + "union_observed": 0.19850187003612518, + "union_independent": 0.5291704535484314, + "union_gap": -0.3306685835123062, + "mean_per_layer_hit_rate": 0.055699605494737625 + }, + "20": { + "rho_bar": 0.3922770917415619, + "union_observed": 0.2734082341194153, + "union_independent": 0.688774824142456, + "union_gap": -0.41536659002304077, + "mean_per_layer_hit_rate": 0.08460578322410583 + }, + "50": { + "rho_bar": 0.4134473502635956, + "union_observed": 0.38826465606689453, + "union_independent": 0.8442178964614868, + "union_gap": -0.4559532403945923, + "mean_per_layer_hit_rate": 0.13031786680221558 + }, + "100": { + "rho_bar": 0.41127073764801025, + "union_observed": 0.5068663954734802, + "union_independent": 0.9233648777008057, + "union_gap": -0.41649848222732544, + "mean_per_layer_hit_rate": 0.17449341714382172 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33413346324648174, + "union_gap_mean_over_k": -0.2888310466493879 + }, + "Pile-CC|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.14695113897323608, + "union_observed": 0.05992509424686432, + "union_independent": 0.1161351203918457, + "union_gap": -0.056210026144981384, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.18571917712688446, + "union_observed": 0.1061173528432846, + "union_independent": 0.20895999670028687, + "union_gap": -0.10284264385700226, + "mean_per_layer_hit_rate": 0.017766254022717476 + }, + "5": { + "rho_bar": 0.30917564034461975, + "union_observed": 0.16104869544506073, + "union_independent": 0.38894158601760864, + "union_gap": -0.2278928905725479, + "mean_per_layer_hit_rate": 0.03687698021531105 + }, + "10": { + "rho_bar": 0.3540707230567932, + "union_observed": 0.23470661044120789, + "union_independent": 0.5502634644508362, + "union_gap": -0.3155568540096283, + "mean_per_layer_hit_rate": 0.05886872485280037 + }, + "20": { + "rho_bar": 0.3834260404109955, + "union_observed": 0.3071160912513733, + "union_independent": 0.6967876553535461, + "union_gap": -0.38967156410217285, + "mean_per_layer_hit_rate": 0.08614231646060944 + }, + "50": { + "rho_bar": 0.39646151661872864, + "union_observed": 0.41198500990867615, + "union_independent": 0.8399779200553894, + "union_gap": -0.42799291014671326, + "mean_per_layer_hit_rate": 0.12858925759792328 + }, + "100": { + "rho_bar": 0.4044834077358246, + "union_observed": 0.5106117129325867, + "union_independent": 0.9169317483901978, + "union_gap": -0.4063200354576111, + "mean_per_layer_hit_rate": 0.16959568858146667 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31146966346672605, + "union_gap_mean_over_k": -0.2752124177558081 + }, + "Pile-CC|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.13747912645339966, + "union_observed": 0.056179776787757874, + "union_independent": 0.11492836475372314, + "union_gap": -0.05874858796596527, + "mean_per_layer_hit_rate": 0.009315279312431812 + }, + "2": { + "rho_bar": 0.18814246356487274, + "union_observed": 0.09238451719284058, + "union_independent": 0.1884709596633911, + "union_gap": -0.09608644247055054, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.2673051059246063, + "union_observed": 0.1548064947128296, + "union_independent": 0.3478144407272339, + "union_gap": -0.1930079460144043, + "mean_per_layer_hit_rate": 0.03207528963685036 + }, + "10": { + "rho_bar": 0.3112749457359314, + "union_observed": 0.22846442461013794, + "union_independent": 0.5198647975921631, + "union_gap": -0.29140037298202515, + "mean_per_layer_hit_rate": 0.054067034274339676 + }, + "20": { + "rho_bar": 0.3352947533130646, + "union_observed": 0.2871410846710205, + "union_independent": 0.6712675094604492, + "union_gap": -0.3841264247894287, + "mean_per_layer_hit_rate": 0.0802842527627945 + }, + "50": { + "rho_bar": 0.38688182830810547, + "union_observed": 0.4082396924495697, + "union_independent": 0.8293254971504211, + "union_gap": -0.42108580470085144, + "mean_per_layer_hit_rate": 0.12388360500335693 + }, + "100": { + "rho_bar": 0.40184012055397034, + "union_observed": 0.5143570303916931, + "union_independent": 0.9144349098205566, + "union_gap": -0.4000778794288635, + "mean_per_layer_hit_rate": 0.16700278222560883 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2897454776934215, + "union_gap_mean_over_k": -0.26350477976458414 + }, + "Pile-CC|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.15662187337875366, + "union_observed": 0.04993757978081703, + "union_independent": 0.10693436861038208, + "union_gap": -0.05699678882956505, + "mean_per_layer_hit_rate": 0.008643041364848614 + }, + "2": { + "rho_bar": 0.24148373305797577, + "union_observed": 0.08239700645208359, + "union_independent": 0.18498116731643677, + "union_gap": -0.10258416086435318, + "mean_per_layer_hit_rate": 0.015557477250695229 + }, + "5": { + "rho_bar": 0.3036516010761261, + "union_observed": 0.1473158597946167, + "union_independent": 0.37456637620925903, + "union_gap": -0.22725051641464233, + "mean_per_layer_hit_rate": 0.0352444052696228 + }, + "10": { + "rho_bar": 0.36560407280921936, + "union_observed": 0.20724095404148102, + "union_independent": 0.5297939777374268, + "union_gap": -0.32255302369594574, + "mean_per_layer_hit_rate": 0.05579564347863197 + }, + "20": { + "rho_bar": 0.3898853659629822, + "union_observed": 0.289637953042984, + "union_independent": 0.6914364099502563, + "union_gap": -0.40179845690727234, + "mean_per_layer_hit_rate": 0.08508595079183578 + }, + "50": { + "rho_bar": 0.4308274984359741, + "union_observed": 0.4032459557056427, + "union_independent": 0.8440024852752686, + "union_gap": -0.44075652956962585, + "mean_per_layer_hit_rate": 0.12993374466896057 + }, + "100": { + "rho_bar": 0.45333024859428406, + "union_observed": 0.5056179761886597, + "union_independent": 0.9219406843185425, + "union_gap": -0.4163227081298828, + "mean_per_layer_hit_rate": 0.17257274687290192 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3344863419021879, + "union_gap_mean_over_k": -0.2811803120587553 + }, + "Pile-CC|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.32239851355552673, + "union_observed": 0.04619225859642029, + "union_independent": 0.13699966669082642, + "union_gap": -0.09080740809440613, + "mean_per_layer_hit_rate": 0.011235954239964485 + }, + "2": { + "rho_bar": 0.3499799966812134, + "union_observed": 0.08614232391119003, + "union_independent": 0.2462138533592224, + "union_gap": -0.16007152944803238, + "mean_per_layer_hit_rate": 0.02141553908586502 + }, + "5": { + "rho_bar": 0.4041699469089508, + "union_observed": 0.13857677578926086, + "union_independent": 0.450700044631958, + "union_gap": -0.31212326884269714, + "mean_per_layer_hit_rate": 0.04475175589323044 + }, + "10": { + "rho_bar": 0.4566480219364166, + "union_observed": 0.19225966930389404, + "union_independent": 0.6253179311752319, + "union_gap": -0.4330582618713379, + "mean_per_layer_hit_rate": 0.0718332827091217 + }, + "20": { + "rho_bar": 0.4883818030357361, + "union_observed": 0.2709113657474518, + "union_independent": 0.7777509093284607, + "union_gap": -0.5068395435810089, + "mean_per_layer_hit_rate": 0.10717371851205826 + }, + "50": { + "rho_bar": 0.5265969634056091, + "union_observed": 0.3533083498477936, + "union_independent": 0.8896646499633789, + "union_gap": -0.5363563001155853, + "mean_per_layer_hit_rate": 0.15202151238918304 + }, + "100": { + "rho_bar": 0.5186823606491089, + "union_observed": 0.45443195104599, + "union_independent": 0.9503283500671387, + "union_gap": -0.4958963990211487, + "mean_per_layer_hit_rate": 0.19965428113937378 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.43812251516750883, + "union_gap_mean_over_k": -0.36216467299631666 + }, + "Pile-CC|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.28988489508628845, + "union_observed": 0.06491885334253311, + "union_independent": 0.16299599409103394, + "union_gap": -0.09807714074850082, + "mean_per_layer_hit_rate": 0.013540765270590782 + }, + "2": { + "rho_bar": 0.352774441242218, + "union_observed": 0.09737827628850937, + "union_independent": 0.2614227533340454, + "union_gap": -0.16404447704553604, + "mean_per_layer_hit_rate": 0.02295207977294922 + }, + "5": { + "rho_bar": 0.39142903685569763, + "union_observed": 0.15605492889881134, + "union_independent": 0.46294742822647095, + "union_gap": -0.3068924993276596, + "mean_per_layer_hit_rate": 0.04638432711362839 + }, + "10": { + "rho_bar": 0.46550843119621277, + "union_observed": 0.21348313987255096, + "union_independent": 0.6304022073745728, + "union_gap": -0.4169190675020218, + "mean_per_layer_hit_rate": 0.07288965582847595 + }, + "20": { + "rho_bar": 0.49973687529563904, + "union_observed": 0.2871410846710205, + "union_independent": 0.7818342447280884, + "union_gap": -0.49469316005706787, + "mean_per_layer_hit_rate": 0.1088063046336174 + }, + "50": { + "rho_bar": 0.5278193354606628, + "union_observed": 0.3670412003993988, + "union_independent": 0.8889011144638062, + "union_gap": -0.5218599140644073, + "mean_per_layer_hit_rate": 0.15278978645801544 + }, + "100": { + "rho_bar": 0.5343029499053955, + "union_observed": 0.45443195104599, + "union_independent": 0.949806809425354, + "union_gap": -0.495374858379364, + "mean_per_layer_hit_rate": 0.20167097449302673 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4373508521488735, + "union_gap_mean_over_k": -0.35683730244636536 + }, + "Pile-CC|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.29057276248931885, + "union_observed": 0.05243445560336113, + "union_independent": 0.14154070615768433, + "union_gap": -0.0891062505543232, + "mean_per_layer_hit_rate": 0.01162009034305811 + }, + "2": { + "rho_bar": 0.26801368594169617, + "union_observed": 0.07990012317895889, + "union_independent": 0.21467715501785278, + "union_gap": -0.1347770318388939, + "mean_per_layer_hit_rate": 0.018342455849051476 + }, + "5": { + "rho_bar": 0.327493816614151, + "union_observed": 0.144818976521492, + "union_independent": 0.410511314868927, + "union_gap": -0.265692338347435, + "mean_per_layer_hit_rate": 0.03956592455506325 + }, + "10": { + "rho_bar": 0.38966384530067444, + "union_observed": 0.20973782241344452, + "union_independent": 0.5718977451324463, + "union_gap": -0.36215992271900177, + "mean_per_layer_hit_rate": 0.06251800805330276 + }, + "20": { + "rho_bar": 0.4581371247768402, + "union_observed": 0.2746566832065582, + "union_independent": 0.7346939444541931, + "union_gap": -0.4600372612476349, + "mean_per_layer_hit_rate": 0.09555363655090332 + }, + "50": { + "rho_bar": 0.5198225378990173, + "union_observed": 0.3695380687713623, + "union_independent": 0.8608776330947876, + "union_gap": -0.4913395643234253, + "mean_per_layer_hit_rate": 0.13809660077095032 + }, + "100": { + "rho_bar": 0.5272283554077148, + "union_observed": 0.45068663358688354, + "union_independent": 0.928945779800415, + "union_gap": -0.4782591462135315, + "mean_per_layer_hit_rate": 0.1802554428577423 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.397276018347059, + "union_gap_mean_over_k": -0.32591021646346363 + }, + "Pile-CC|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.22030429542064667, + "union_observed": 0.05118601769208908, + "union_independent": 0.11608529090881348, + "union_gap": -0.0648992732167244, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.26345929503440857, + "union_observed": 0.07615480571985245, + "union_independent": 0.1830424666404724, + "union_gap": -0.10688766092061996, + "mean_per_layer_hit_rate": 0.015365407802164555 + }, + "5": { + "rho_bar": 0.28081801533699036, + "union_observed": 0.15730337798595428, + "union_independent": 0.3716278076171875, + "union_gap": -0.21432442963123322, + "mean_per_layer_hit_rate": 0.034860268235206604 + }, + "10": { + "rho_bar": 0.3353893458843231, + "union_observed": 0.21098627150058746, + "union_independent": 0.5300723314285278, + "union_gap": -0.31908605992794037, + "mean_per_layer_hit_rate": 0.05579564347863197 + }, + "20": { + "rho_bar": 0.3706821799278259, + "union_observed": 0.2746566832065582, + "union_independent": 0.6835662722587585, + "union_gap": -0.4089095890522003, + "mean_per_layer_hit_rate": 0.08335734158754349 + }, + "50": { + "rho_bar": 0.45716363191604614, + "union_observed": 0.3695380687713623, + "union_independent": 0.8371764421463013, + "union_gap": -0.46763837337493896, + "mean_per_layer_hit_rate": 0.12762893736362457 + }, + "100": { + "rho_bar": 0.4881969392299652, + "union_observed": 0.44319599866867065, + "union_independent": 0.908641517162323, + "union_gap": -0.46544551849365234, + "mean_per_layer_hit_rate": 0.16440989077091217 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.34514481467860086, + "union_gap_mean_over_k": -0.2924558435167585 + }, + "Pile-CC|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.21953274309635162, + "union_observed": 0.04369537904858589, + "union_independent": 0.09571218490600586, + "union_gap": -0.05201680585741997, + "mean_per_layer_hit_rate": 0.007682703901082277 + }, + "2": { + "rho_bar": 0.19824324548244476, + "union_observed": 0.07865168899297714, + "union_independent": 0.16319423913955688, + "union_gap": -0.08454255014657974, + "mean_per_layer_hit_rate": 0.013540765270590782 + }, + "5": { + "rho_bar": 0.2894802689552307, + "union_observed": 0.1473158597946167, + "union_independent": 0.34576815366744995, + "union_gap": -0.19845229387283325, + "mean_per_layer_hit_rate": 0.031883224844932556 + }, + "10": { + "rho_bar": 0.34902092814445496, + "union_observed": 0.19850187003612518, + "union_independent": 0.49236732721328735, + "union_gap": -0.29386545717716217, + "mean_per_layer_hit_rate": 0.05032171681523323 + }, + "20": { + "rho_bar": 0.3824569880962372, + "union_observed": 0.2634207308292389, + "union_independent": 0.6482762098312378, + "union_gap": -0.3848554790019989, + "mean_per_layer_hit_rate": 0.07615481317043304 + }, + "50": { + "rho_bar": 0.42322516441345215, + "union_observed": 0.3670412003993988, + "union_independent": 0.8091813325881958, + "union_gap": -0.442140132188797, + "mean_per_layer_hit_rate": 0.11754537373781204 + }, + "100": { + "rho_bar": 0.4572872817516327, + "union_observed": 0.4294631779193878, + "union_independent": 0.8899432420730591, + "union_gap": -0.46048006415367126, + "mean_per_layer_hit_rate": 0.1530779004096985 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3313209457056863, + "union_gap_mean_over_k": -0.2737646831997803 + }, + "Pile-CC|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.16034796833992004, + "union_observed": 0.05742821469902992, + "union_independent": 0.09455406665802002, + "union_gap": -0.0371258519589901, + "mean_per_layer_hit_rate": 0.007586670573800802 + }, + "2": { + "rho_bar": 0.17834946513175964, + "union_observed": 0.08614232391119003, + "union_independent": 0.1734272837638855, + "union_gap": -0.08728495985269547, + "mean_per_layer_hit_rate": 0.01450110413134098 + }, + "5": { + "rho_bar": 0.26100000739097595, + "union_observed": 0.14606741070747375, + "union_independent": 0.33238357305526733, + "union_gap": -0.18631616234779358, + "mean_per_layer_hit_rate": 0.03044271469116211 + }, + "10": { + "rho_bar": 0.33070996403694153, + "union_observed": 0.18726591765880585, + "union_independent": 0.4747682213783264, + "union_gap": -0.28750230371952057, + "mean_per_layer_hit_rate": 0.047920867800712585 + }, + "20": { + "rho_bar": 0.35167473554611206, + "union_observed": 0.25468164682388306, + "union_independent": 0.615850567817688, + "union_gap": -0.36116892099380493, + "mean_per_layer_hit_rate": 0.07010468095541 + }, + "50": { + "rho_bar": 0.3800397515296936, + "union_observed": 0.3595505654811859, + "union_independent": 0.7733020782470703, + "union_gap": -0.4137515127658844, + "mean_per_layer_hit_rate": 0.10621338337659836 + }, + "100": { + "rho_bar": 0.4075906276702881, + "union_observed": 0.44069913029670715, + "union_independent": 0.8689523339271545, + "union_gap": -0.4282532036304474, + "mean_per_layer_hit_rate": 0.14222607016563416 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29567321709224154, + "union_gap_mean_over_k": -0.25734327360987663 + }, + "Pile-CC|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.15851882100105286, + "union_observed": 0.05368289723992348, + "union_independent": 0.09324055910110474, + "union_gap": -0.03955766186118126, + "mean_per_layer_hit_rate": 0.00749063678085804 + }, + "2": { + "rho_bar": 0.21018759906291962, + "union_observed": 0.07365792989730835, + "union_independent": 0.16274935007095337, + "union_gap": -0.08909142017364502, + "mean_per_layer_hit_rate": 0.013540765270590782 + }, + "5": { + "rho_bar": 0.2983151972293854, + "union_observed": 0.11860174685716629, + "union_independent": 0.3164072632789612, + "union_gap": -0.1978055164217949, + "mean_per_layer_hit_rate": 0.02871410921216011 + }, + "10": { + "rho_bar": 0.33344629406929016, + "union_observed": 0.16479401290416718, + "union_independent": 0.4490954875946045, + "union_gap": -0.2843014746904373, + "mean_per_layer_hit_rate": 0.04455967992544174 + }, + "20": { + "rho_bar": 0.37587273120880127, + "union_observed": 0.20848938822746277, + "union_independent": 0.5850400924682617, + "union_gap": -0.37655070424079895, + "mean_per_layer_hit_rate": 0.06491884589195251 + }, + "50": { + "rho_bar": 0.3895547688007355, + "union_observed": 0.29837703704833984, + "union_independent": 0.7487764358520508, + "union_gap": -0.45039939880371094, + "mean_per_layer_hit_rate": 0.09977912902832031 + }, + "100": { + "rho_bar": 0.4127654433250427, + "union_observed": 0.40449437499046326, + "union_independent": 0.872637152671814, + "union_gap": -0.4681427776813507, + "mean_per_layer_hit_rate": 0.1444348394870758 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3112372649567468, + "union_gap_mean_over_k": -0.27226413626755985 + }, + "Pile-CC|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.15313869714736938, + "union_observed": 0.048689138144254684, + "union_independent": 0.0863943099975586, + "union_gap": -0.03770517185330391, + "mean_per_layer_hit_rate": 0.006914434023201466 + }, + "2": { + "rho_bar": 0.20412202179431915, + "union_observed": 0.07116104662418365, + "union_independent": 0.15206533670425415, + "union_gap": -0.0809042900800705, + "mean_per_layer_hit_rate": 0.012580428272485733 + }, + "5": { + "rho_bar": 0.2808833122253418, + "union_observed": 0.1086142286658287, + "union_independent": 0.2876773476600647, + "union_gap": -0.179063118994236, + "mean_per_layer_hit_rate": 0.025641024112701416 + }, + "10": { + "rho_bar": 0.34649375081062317, + "union_observed": 0.15980024635791779, + "union_independent": 0.4582160711288452, + "union_gap": -0.29841582477092743, + "mean_per_layer_hit_rate": 0.04571209102869034 + }, + "20": { + "rho_bar": 0.3815614879131317, + "union_observed": 0.20973782241344452, + "union_independent": 0.6057459115982056, + "union_gap": -0.39600808918476105, + "mean_per_layer_hit_rate": 0.0683760717511177 + }, + "50": { + "rho_bar": 0.4025799334049225, + "union_observed": 0.3133583068847656, + "union_independent": 0.7725933194160461, + "union_gap": -0.4592350125312805, + "mean_per_layer_hit_rate": 0.10602132230997086 + }, + "100": { + "rho_bar": 0.4033855199813843, + "union_observed": 0.4369538128376007, + "union_independent": 0.8884646892547607, + "union_gap": -0.45151087641716003, + "mean_per_layer_hit_rate": 0.15163737535476685 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3103092461824417, + "union_gap_mean_over_k": -0.27183462626167704 + }, + "Pile-CC|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.18599417805671692, + "union_observed": 0.039950061589479446, + "union_independent": 0.08986055850982666, + "union_gap": -0.049910496920347214, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.2511794865131378, + "union_observed": 0.05992509424686432, + "union_independent": 0.15758931636810303, + "union_gap": -0.09766422212123871, + "mean_per_layer_hit_rate": 0.013060597702860832 + }, + "5": { + "rho_bar": 0.31965503096580505, + "union_observed": 0.10486891120672226, + "union_independent": 0.3124334216117859, + "union_gap": -0.20756451040506363, + "mean_per_layer_hit_rate": 0.028233937919139862 + }, + "10": { + "rho_bar": 0.3609791696071625, + "union_observed": 0.15980024635791779, + "union_independent": 0.46953797340393066, + "union_gap": -0.3097377270460129, + "mean_per_layer_hit_rate": 0.047152601182460785 + }, + "20": { + "rho_bar": 0.3835943043231964, + "union_observed": 0.2247191071510315, + "union_independent": 0.6382035613059998, + "union_gap": -0.41348445415496826, + "mean_per_layer_hit_rate": 0.0741380974650383 + }, + "50": { + "rho_bar": 0.4045182466506958, + "union_observed": 0.34706616401672363, + "union_independent": 0.8045430183410645, + "union_gap": -0.4574768543243408, + "mean_per_layer_hit_rate": 0.11562470346689224 + }, + "100": { + "rho_bar": 0.40786635875701904, + "union_observed": 0.45817726850509644, + "union_independent": 0.9031944274902344, + "union_gap": -0.44501715898513794, + "mean_per_layer_hit_rate": 0.15980026125907898 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3305409678391048, + "union_gap_mean_over_k": -0.28297934627958704 + }, + "Pile-CC|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.19252260029315948, + "union_observed": 0.04494382068514824, + "union_independent": 0.1024966835975647, + "union_gap": -0.05755286291241646, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.2509611248970032, + "union_observed": 0.06367041170597076, + "union_independent": 0.164139986038208, + "union_gap": -0.10046957433223724, + "mean_per_layer_hit_rate": 0.013636799529194832 + }, + "5": { + "rho_bar": 0.2682034373283386, + "union_observed": 0.12234706431627274, + "union_independent": 0.3224904537200928, + "union_gap": -0.20014338940382004, + "mean_per_layer_hit_rate": 0.02929030917584896 + }, + "10": { + "rho_bar": 0.3463946580886841, + "union_observed": 0.16604244709014893, + "union_independent": 0.48398876190185547, + "union_gap": -0.31794631481170654, + "mean_per_layer_hit_rate": 0.04907327517867088 + }, + "20": { + "rho_bar": 0.3833483159542084, + "union_observed": 0.22596754133701324, + "union_independent": 0.6451258659362793, + "union_gap": -0.41915832459926605, + "mean_per_layer_hit_rate": 0.07538653165102005 + }, + "50": { + "rho_bar": 0.3938848376274109, + "union_observed": 0.3508114814758301, + "union_independent": 0.8195250630378723, + "union_gap": -0.46871358156204224, + "mean_per_layer_hit_rate": 0.12033036351203918 + }, + "100": { + "rho_bar": 0.39706915616989136, + "union_observed": 0.45568040013313293, + "union_independent": 0.9121964573860168, + "union_gap": -0.4565160572528839, + "mean_per_layer_hit_rate": 0.16489003598690033 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3189120186226709, + "union_gap_mean_over_k": -0.28864287212491035 + }, + "Pile-CC|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.194503054022789, + "union_observed": 0.03745318204164505, + "union_independent": 0.09793639183044434, + "union_gap": -0.060483209788799286, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.23165087401866913, + "union_observed": 0.06491885334253311, + "union_independent": 0.16400617361068726, + "union_gap": -0.09908732026815414, + "mean_per_layer_hit_rate": 0.013636799529194832 + }, + "5": { + "rho_bar": 0.2607000768184662, + "union_observed": 0.12359550595283508, + "union_independent": 0.327045738697052, + "union_gap": -0.20345023274421692, + "mean_per_layer_hit_rate": 0.02977048046886921 + }, + "10": { + "rho_bar": 0.3499527871608734, + "union_observed": 0.1760299652814865, + "union_independent": 0.5048253536224365, + "union_gap": -0.32879538834095, + "mean_per_layer_hit_rate": 0.05195428803563118 + }, + "20": { + "rho_bar": 0.37918102741241455, + "union_observed": 0.24344569444656372, + "union_independent": 0.6690309047698975, + "union_gap": -0.42558521032333374, + "mean_per_layer_hit_rate": 0.07999616116285324 + }, + "50": { + "rho_bar": 0.3715417683124542, + "union_observed": 0.3745318353176117, + "union_independent": 0.8323535919189453, + "union_gap": -0.4578217566013336, + "mean_per_layer_hit_rate": 0.12474791705608368 + }, + "100": { + "rho_bar": 0.3774290680885315, + "union_observed": 0.4906367063522339, + "union_independent": 0.9228872060775757, + "union_gap": -0.4322504997253418, + "mean_per_layer_hit_rate": 0.17247670888900757 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.309279807976314, + "union_gap_mean_over_k": -0.28678194539887564 + }, + "Pile-CC|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.18752117455005646, + "union_observed": 0.04494382068514824, + "union_independent": 0.10253506898880005, + "union_gap": -0.05759124830365181, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.23120693862438202, + "union_observed": 0.07365792989730835, + "union_independent": 0.17585158348083496, + "union_gap": -0.10219365358352661, + "mean_per_layer_hit_rate": 0.014693171717226505 + }, + "5": { + "rho_bar": 0.29517287015914917, + "union_observed": 0.1398252248764038, + "union_independent": 0.36115896701812744, + "union_gap": -0.22133374214172363, + "mean_per_layer_hit_rate": 0.033611830323934555 + }, + "10": { + "rho_bar": 0.3672419488430023, + "union_observed": 0.19475655257701874, + "union_independent": 0.5481232404708862, + "union_gap": -0.3533666878938675, + "mean_per_layer_hit_rate": 0.05848458781838417 + }, + "20": { + "rho_bar": 0.3759349286556244, + "union_observed": 0.2746566832065582, + "union_independent": 0.7058535218238831, + "union_gap": -0.43119683861732483, + "mean_per_layer_hit_rate": 0.08796696364879608 + }, + "50": { + "rho_bar": 0.3854769170284271, + "union_observed": 0.392009973526001, + "union_independent": 0.8547799587249756, + "union_gap": -0.4627699851989746, + "mean_per_layer_hit_rate": 0.13387110829353333 + }, + "100": { + "rho_bar": 0.38587427139282227, + "union_observed": 0.4993757903575897, + "union_independent": 0.9369748830795288, + "union_gap": -0.4375990927219391, + "mean_per_layer_hit_rate": 0.18457697331905365 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3183470070362091, + "union_gap_mean_over_k": -0.2951501783515726 + }, + "Pile-CC|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.22805656492710114, + "union_observed": 0.054931335151195526, + "union_independent": 0.1305965781211853, + "union_gap": -0.07566524296998978, + "mean_per_layer_hit_rate": 0.010659752413630486 + }, + "2": { + "rho_bar": 0.2538450062274933, + "union_observed": 0.09238451719284058, + "union_independent": 0.22488701343536377, + "union_gap": -0.1325024962425232, + "mean_per_layer_hit_rate": 0.019302794709801674 + }, + "5": { + "rho_bar": 0.31170374155044556, + "union_observed": 0.1498127281665802, + "union_independent": 0.3887057304382324, + "union_gap": -0.23889300227165222, + "mean_per_layer_hit_rate": 0.03687698021531105 + }, + "10": { + "rho_bar": 0.3842947781085968, + "union_observed": 0.2147315889596939, + "union_independent": 0.5854109525680542, + "union_gap": -0.3706793636083603, + "mean_per_layer_hit_rate": 0.06463074684143066 + }, + "20": { + "rho_bar": 0.40450531244277954, + "union_observed": 0.289637953042984, + "union_independent": 0.7376137971878052, + "union_gap": -0.44797584414482117, + "mean_per_layer_hit_rate": 0.09584173560142517 + }, + "50": { + "rho_bar": 0.41740232706069946, + "union_observed": 0.39700374007225037, + "union_independent": 0.8729583621025085, + "union_gap": -0.4759546220302582, + "mean_per_layer_hit_rate": 0.14299432933330536 + }, + "100": { + "rho_bar": 0.4221929907798767, + "union_observed": 0.49687889218330383, + "union_independent": 0.9446371793746948, + "union_gap": -0.447758287191391, + "mean_per_layer_hit_rate": 0.19341208040714264 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3460001030138561, + "union_gap_mean_over_k": -0.312775551208428 + }, + "Pile-CC|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.22733883559703827, + "union_observed": 0.05368289723992348, + "union_independent": 0.13495677709579468, + "union_gap": -0.0812738798558712, + "mean_per_layer_hit_rate": 0.011043887585401535 + }, + "2": { + "rho_bar": 0.2875542640686035, + "union_observed": 0.09238451719284058, + "union_independent": 0.2434287667274475, + "union_gap": -0.15104424953460693, + "mean_per_layer_hit_rate": 0.02112743631005287 + }, + "5": { + "rho_bar": 0.3545258939266205, + "union_observed": 0.1523096114397049, + "union_independent": 0.42511558532714844, + "union_gap": -0.27280597388744354, + "mean_per_layer_hit_rate": 0.04139057174324989 + }, + "10": { + "rho_bar": 0.4195328652858734, + "union_observed": 0.2247191071510315, + "union_independent": 0.605675458908081, + "union_gap": -0.38095635175704956, + "mean_per_layer_hit_rate": 0.06828003376722336 + }, + "20": { + "rho_bar": 0.4333149790763855, + "union_observed": 0.3071160912513733, + "union_independent": 0.764742374420166, + "union_gap": -0.4576262831687927, + "mean_per_layer_hit_rate": 0.10352443903684616 + }, + "50": { + "rho_bar": 0.4500916302204132, + "union_observed": 0.38576778769493103, + "union_independent": 0.8795814514160156, + "union_gap": -0.4938136637210846, + "mean_per_layer_hit_rate": 0.147123783826828 + }, + "100": { + "rho_bar": 0.45271673798561096, + "union_observed": 0.48439452052116394, + "union_independent": 0.9427291750907898, + "union_gap": -0.45833465456962585, + "mean_per_layer_hit_rate": 0.1929319053888321 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.37501074373722076, + "union_gap_mean_over_k": -0.3279792937849249 + }, + "Pile-CC|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.1862296462059021, + "union_observed": 0.056179776787757874, + "union_independent": 0.1104269027709961, + "union_gap": -0.05424712598323822, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.20840640366077423, + "union_observed": 0.10112359374761581, + "union_independent": 0.2023981213569641, + "union_gap": -0.1012745276093483, + "mean_per_layer_hit_rate": 0.017190050333738327 + }, + "5": { + "rho_bar": 0.30875512957572937, + "union_observed": 0.1548064947128296, + "union_independent": 0.3645416498184204, + "union_gap": -0.20973515510559082, + "mean_per_layer_hit_rate": 0.0340920016169548 + }, + "10": { + "rho_bar": 0.3673650920391083, + "union_observed": 0.2197253406047821, + "union_independent": 0.5382961630821228, + "union_gap": -0.3185708224773407, + "mean_per_layer_hit_rate": 0.05714011192321777 + }, + "20": { + "rho_bar": 0.4008077085018158, + "union_observed": 0.2808988690376282, + "union_independent": 0.6859821081161499, + "union_gap": -0.40508323907852173, + "mean_per_layer_hit_rate": 0.08402958512306213 + }, + "50": { + "rho_bar": 0.4256214201450348, + "union_observed": 0.387016236782074, + "union_independent": 0.8361051678657532, + "union_gap": -0.4490889310836792, + "mean_per_layer_hit_rate": 0.12724478542804718 + }, + "100": { + "rho_bar": 0.4305490255355835, + "union_observed": 0.4906367063522339, + "union_independent": 0.9155258536338806, + "union_gap": -0.42488914728164673, + "mean_per_layer_hit_rate": 0.16901949048042297 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.332533489380564, + "union_gap_mean_over_k": -0.28041270694562365 + }, + "Pile-CC|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.2146778553724289, + "union_observed": 0.04494382068514824, + "union_independent": 0.11370867490768433, + "union_gap": -0.06876485422253609, + "mean_per_layer_hit_rate": 0.009219245053827763 + }, + "2": { + "rho_bar": 0.2263321578502655, + "union_observed": 0.07615480571985245, + "union_independent": 0.1818823218345642, + "union_gap": -0.10572751611471176, + "mean_per_layer_hit_rate": 0.015269373543560505 + }, + "5": { + "rho_bar": 0.28789690136909485, + "union_observed": 0.14107365906238556, + "union_independent": 0.356770396232605, + "union_gap": -0.21569673717021942, + "mean_per_layer_hit_rate": 0.033131662756204605 + }, + "10": { + "rho_bar": 0.35907071828842163, + "union_observed": 0.1960049867630005, + "union_independent": 0.5215352177619934, + "union_gap": -0.3255302309989929, + "mean_per_layer_hit_rate": 0.05454719811677933 + }, + "20": { + "rho_bar": 0.3918284475803375, + "union_observed": 0.2709113657474518, + "union_independent": 0.6815754175186157, + "union_gap": -0.41066405177116394, + "mean_per_layer_hit_rate": 0.08306924253702164 + }, + "50": { + "rho_bar": 0.4143056571483612, + "union_observed": 0.3820224702358246, + "union_independent": 0.8386147618293762, + "union_gap": -0.45659229159355164, + "mean_per_layer_hit_rate": 0.12810908257961273 + }, + "100": { + "rho_bar": 0.4137963652610779, + "union_observed": 0.4981273412704468, + "union_independent": 0.920988917350769, + "union_gap": -0.42286157608032227, + "mean_per_layer_hit_rate": 0.17257274687290192 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.32970115755285534, + "union_gap_mean_over_k": -0.28654817970735685 + }, + "Pile-CC|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.14510655403137207, + "union_observed": 0.05992509424686432, + "union_independent": 0.11621212959289551, + "union_gap": -0.05628703534603119, + "mean_per_layer_hit_rate": 0.009411313571035862 + }, + "2": { + "rho_bar": 0.17036782205104828, + "union_observed": 0.10486891120672226, + "union_independent": 0.20191919803619385, + "union_gap": -0.09705028682947159, + "mean_per_layer_hit_rate": 0.017094017937779427 + }, + "5": { + "rho_bar": 0.28865447640419006, + "union_observed": 0.16229712963104248, + "union_independent": 0.3785902261734009, + "union_gap": -0.2162930965423584, + "mean_per_layer_hit_rate": 0.035628542304039 + }, + "10": { + "rho_bar": 0.34629136323928833, + "union_observed": 0.24219726026058197, + "union_independent": 0.5504589676856995, + "union_gap": -0.3082617074251175, + "mean_per_layer_hit_rate": 0.05886872112751007 + }, + "20": { + "rho_bar": 0.38487082719802856, + "union_observed": 0.2996254563331604, + "union_independent": 0.6905257701873779, + "union_gap": -0.39090031385421753, + "mean_per_layer_hit_rate": 0.08470181375741959 + }, + "50": { + "rho_bar": 0.3992820084095001, + "union_observed": 0.40449437499046326, + "union_independent": 0.8353172540664673, + "union_gap": -0.43082287907600403, + "mean_per_layer_hit_rate": 0.12666858732700348 + }, + "100": { + "rho_bar": 0.40418127179145813, + "union_observed": 0.4993757903575897, + "union_independent": 0.914941668510437, + "union_gap": -0.4155658781528473, + "mean_per_layer_hit_rate": 0.16815519332885742 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.30553633187498364, + "union_gap_mean_over_k": -0.27359731388943537 + }, + "Pile-CC|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.14203433692455292, + "union_observed": 0.05742821469902992, + "union_independent": 0.11277508735656738, + "union_gap": -0.05534687265753746, + "mean_per_layer_hit_rate": 0.009123211726546288 + }, + "2": { + "rho_bar": 0.18144848942756653, + "union_observed": 0.09238451719284058, + "union_independent": 0.18440121412277222, + "union_gap": -0.09201669692993164, + "mean_per_layer_hit_rate": 0.015461442992091179 + }, + "5": { + "rho_bar": 0.2536172568798065, + "union_observed": 0.15355804562568665, + "union_independent": 0.3332434892654419, + "union_gap": -0.17968544363975525, + "mean_per_layer_hit_rate": 0.03044271655380726 + }, + "10": { + "rho_bar": 0.3097146153450012, + "union_observed": 0.22596754133701324, + "union_independent": 0.5142726898193359, + "union_gap": -0.2883051484823227, + "mean_per_layer_hit_rate": 0.05320272594690323 + }, + "20": { + "rho_bar": 0.32895299792289734, + "union_observed": 0.289637953042984, + "union_independent": 0.6662812232971191, + "union_gap": -0.37664327025413513, + "mean_per_layer_hit_rate": 0.07922788709402084 + }, + "50": { + "rho_bar": 0.38524869084358215, + "union_observed": 0.4057428240776062, + "union_independent": 0.8224387168884277, + "union_gap": -0.41669589281082153, + "mean_per_layer_hit_rate": 0.12138672918081284 + }, + "100": { + "rho_bar": 0.39595696330070496, + "union_observed": 0.5131086111068726, + "union_independent": 0.9115252494812012, + "union_gap": -0.3984166383743286, + "mean_per_layer_hit_rate": 0.16489003598690033 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.28528190723487307, + "union_gap_mean_over_k": -0.2581585661641189 + }, + "Pile-CC|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.16141512989997864, + "union_observed": 0.047440700232982635, + "union_independent": 0.10470747947692871, + "union_gap": -0.057266779243946075, + "mean_per_layer_hit_rate": 0.008450975641608238 + }, + "2": { + "rho_bar": 0.21845188736915588, + "union_observed": 0.0774032473564148, + "union_independent": 0.18189328908920288, + "union_gap": -0.10449004173278809, + "mean_per_layer_hit_rate": 0.01526937447488308 + }, + "5": { + "rho_bar": 0.28243276476860046, + "union_observed": 0.1548064947128296, + "union_independent": 0.36149853467941284, + "union_gap": -0.20669203996658325, + "mean_per_layer_hit_rate": 0.033707864582538605 + }, + "10": { + "rho_bar": 0.3486783802509308, + "union_observed": 0.20973782241344452, + "union_independent": 0.5290400981903076, + "union_gap": -0.3193022757768631, + "mean_per_layer_hit_rate": 0.055699605494737625 + }, + "20": { + "rho_bar": 0.384367436170578, + "union_observed": 0.2946317195892334, + "union_independent": 0.6872836351394653, + "union_gap": -0.39265191555023193, + "mean_per_layer_hit_rate": 0.08412560820579529 + }, + "50": { + "rho_bar": 0.42828261852264404, + "union_observed": 0.4032459557056427, + "union_independent": 0.8397143483161926, + "union_gap": -0.4364683926105499, + "mean_per_layer_hit_rate": 0.12801305949687958 + }, + "100": { + "rho_bar": 0.4494682848453522, + "union_observed": 0.5043695569038391, + "union_independent": 0.9211302399635315, + "union_gap": -0.4167606830596924, + "mean_per_layer_hit_rate": 0.17190051078796387 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3247280716896057, + "union_gap_mean_over_k": -0.27623316113437923 + }, + "Pile-CC|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.29043588042259216, + "union_observed": 0.04494382068514824, + "union_independent": 0.13374316692352295, + "union_gap": -0.08879934623837471, + "mean_per_layer_hit_rate": 0.010947853326797485 + }, + "2": { + "rho_bar": 0.3114595413208008, + "union_observed": 0.08364544063806534, + "union_independent": 0.23760277032852173, + "union_gap": -0.1539573296904564, + "mean_per_layer_hit_rate": 0.020551234483718872 + }, + "5": { + "rho_bar": 0.40354201197624207, + "union_observed": 0.13857677578926086, + "union_independent": 0.4411887526512146, + "union_gap": -0.30261197686195374, + "mean_per_layer_hit_rate": 0.04350331053137779 + }, + "10": { + "rho_bar": 0.44620466232299805, + "union_observed": 0.19475655257701874, + "union_independent": 0.619770884513855, + "union_gap": -0.42501433193683624, + "mean_per_layer_hit_rate": 0.07077691704034805 + }, + "20": { + "rho_bar": 0.48947015404701233, + "union_observed": 0.2709113657474518, + "union_independent": 0.7742990255355835, + "union_gap": -0.5033876597881317, + "mean_per_layer_hit_rate": 0.1061173602938652 + }, + "50": { + "rho_bar": 0.5200310945510864, + "union_observed": 0.3595505654811859, + "union_independent": 0.8874223232269287, + "union_gap": -0.5278717577457428, + "mean_per_layer_hit_rate": 0.15067704021930695 + }, + "100": { + "rho_bar": 0.5181586146354675, + "union_observed": 0.45443195104599, + "union_independent": 0.948816180229187, + "union_gap": -0.494384229183197, + "mean_per_layer_hit_rate": 0.19792567193508148 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4256145656108856, + "union_gap_mean_over_k": -0.3565752330635275 + }, + "StackExchange|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.3484920561313629, + "union_observed": 0.06616728752851486, + "union_independent": 0.18387174606323242, + "union_gap": -0.11770445853471756, + "mean_per_layer_hit_rate": 0.015461442992091179 + }, + "2": { + "rho_bar": 0.4017656743526459, + "union_observed": 0.09862671792507172, + "union_independent": 0.28945034742355347, + "union_gap": -0.19082362949848175, + "mean_per_layer_hit_rate": 0.025833096355199814 + }, + "5": { + "rho_bar": 0.41504737734794617, + "union_observed": 0.16354556381702423, + "union_independent": 0.4861414432525635, + "union_gap": -0.32259587943553925, + "mean_per_layer_hit_rate": 0.049553439021110535 + }, + "10": { + "rho_bar": 0.47187477350234985, + "union_observed": 0.22596754133701324, + "union_independent": 0.6508774757385254, + "union_gap": -0.42490993440151215, + "mean_per_layer_hit_rate": 0.07692307978868484 + }, + "20": { + "rho_bar": 0.5232518315315247, + "union_observed": 0.29088640213012695, + "union_independent": 0.7946739196777344, + "union_gap": -0.5037875175476074, + "mean_per_layer_hit_rate": 0.1130317896604538 + }, + "50": { + "rho_bar": 0.5543729066848755, + "union_observed": 0.37078651785850525, + "union_independent": 0.8992158770561218, + "union_gap": -0.5284293591976166, + "mean_per_layer_hit_rate": 0.15912801027297974 + }, + "100": { + "rho_bar": 0.5666717290878296, + "union_observed": 0.45068663358688354, + "union_independent": 0.9528072476387024, + "union_gap": -0.5021206140518188, + "mean_per_layer_hit_rate": 0.20522426068782806 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.46878233551979065, + "union_gap_mean_over_k": -0.3700530560953276 + }, + "StackExchange|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.3202815055847168, + "union_observed": 0.06367041170597076, + "union_independent": 0.16502374410629272, + "union_gap": -0.10135333240032196, + "mean_per_layer_hit_rate": 0.013732834719121456 + }, + "2": { + "rho_bar": 0.31324487924575806, + "union_observed": 0.09987515956163406, + "union_independent": 0.26430439949035645, + "union_gap": -0.16442923992872238, + "mean_per_layer_hit_rate": 0.02324018068611622 + }, + "5": { + "rho_bar": 0.3707295060157776, + "union_observed": 0.1548064947128296, + "union_independent": 0.4344770908355713, + "union_gap": -0.2796705961227417, + "mean_per_layer_hit_rate": 0.04254297539591789 + }, + "10": { + "rho_bar": 0.41140028834342957, + "union_observed": 0.227215975522995, + "union_independent": 0.6058653593063354, + "union_gap": -0.37864938378334045, + "mean_per_layer_hit_rate": 0.06828003376722336 + }, + "20": { + "rho_bar": 0.4476088583469391, + "union_observed": 0.2971285879611969, + "union_independent": 0.7532042860984802, + "union_gap": -0.4560756981372833, + "mean_per_layer_hit_rate": 0.10035533457994461 + }, + "50": { + "rho_bar": 0.5147044062614441, + "union_observed": 0.37328338623046875, + "union_independent": 0.877133309841156, + "union_gap": -0.5038499236106873, + "mean_per_layer_hit_rate": 0.1461634486913681 + }, + "100": { + "rho_bar": 0.5421274900436401, + "union_observed": 0.4519350826740265, + "union_independent": 0.9350795745849609, + "union_gap": -0.48314449191093445, + "mean_per_layer_hit_rate": 0.18553730845451355 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.4171567048345293, + "union_gap_mean_over_k": -0.3381675236991474 + }, + "StackExchange|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.29662659764289856, + "union_observed": 0.05368289723992348, + "union_independent": 0.1249585747718811, + "union_gap": -0.07127567753195763, + "mean_per_layer_hit_rate": 0.010179583914577961 + }, + "2": { + "rho_bar": 0.25953197479248047, + "union_observed": 0.08863919973373413, + "union_independent": 0.20674270391464233, + "union_gap": -0.1181035041809082, + "mean_per_layer_hit_rate": 0.017574185505509377 + }, + "5": { + "rho_bar": 0.3069082498550415, + "union_observed": 0.1523096114397049, + "union_independent": 0.3825836181640625, + "union_gap": -0.2302740067243576, + "mean_per_layer_hit_rate": 0.03610870987176895 + }, + "10": { + "rho_bar": 0.3549667298793793, + "union_observed": 0.20848938822746277, + "union_independent": 0.5390464067459106, + "union_gap": -0.3305570185184479, + "mean_per_layer_hit_rate": 0.05704408511519432 + }, + "20": { + "rho_bar": 0.38544484972953796, + "union_observed": 0.2784020006656647, + "union_independent": 0.6996620893478394, + "union_gap": -0.4212600886821747, + "mean_per_layer_hit_rate": 0.08671852201223373 + }, + "50": { + "rho_bar": 0.45875078439712524, + "union_observed": 0.3595505654811859, + "union_independent": 0.8399758338928223, + "union_gap": -0.48042526841163635, + "mean_per_layer_hit_rate": 0.12878133356571198 + }, + "100": { + "rho_bar": 0.4849640429019928, + "union_observed": 0.4594257175922394, + "union_independent": 0.9180101752281189, + "union_gap": -0.4585844576358795, + "mean_per_layer_hit_rate": 0.17094016075134277 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.36388474702835083, + "union_gap_mean_over_k": -0.30149714595505167 + }, + "StackExchange|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.2659773826599121, + "union_observed": 0.05368289723992348, + "union_independent": 0.1149473786354065, + "union_gap": -0.06126448139548302, + "mean_per_layer_hit_rate": 0.009315280243754387 + }, + "2": { + "rho_bar": 0.23425467312335968, + "union_observed": 0.07990012317895889, + "union_independent": 0.17887061834335327, + "union_gap": -0.09897049516439438, + "mean_per_layer_hit_rate": 0.01498127169907093 + }, + "5": { + "rho_bar": 0.30300822854042053, + "union_observed": 0.14606741070747375, + "union_independent": 0.34783226251602173, + "union_gap": -0.20176485180854797, + "mean_per_layer_hit_rate": 0.03207528963685036 + }, + "10": { + "rho_bar": 0.35347995162010193, + "union_observed": 0.19725343585014343, + "union_independent": 0.4894252419471741, + "union_gap": -0.29217180609703064, + "mean_per_layer_hit_rate": 0.049745507538318634 + }, + "20": { + "rho_bar": 0.38151639699935913, + "union_observed": 0.2721598148345947, + "union_independent": 0.6552923917770386, + "union_gap": -0.38313257694244385, + "mean_per_layer_hit_rate": 0.0774032399058342 + }, + "50": { + "rho_bar": 0.43553727865219116, + "union_observed": 0.3595505654811859, + "union_independent": 0.810845136642456, + "union_gap": -0.45129457116127014, + "mean_per_layer_hit_rate": 0.11783348768949509 + }, + "100": { + "rho_bar": 0.4934070408344269, + "union_observed": 0.43820226192474365, + "union_independent": 0.8972224593162537, + "union_gap": -0.45902019739151, + "mean_per_layer_hit_rate": 0.15739941596984863 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.35245442177568165, + "union_gap_mean_over_k": -0.2782312828515257 + }, + "StackExchange|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.17618873715400696, + "union_observed": 0.05742821469902992, + "union_independent": 0.10353225469589233, + "union_gap": -0.04610403999686241, + "mean_per_layer_hit_rate": 0.008354941383004189 + }, + "2": { + "rho_bar": 0.21581879258155823, + "union_observed": 0.08114856481552124, + "union_independent": 0.17873680591583252, + "union_gap": -0.09758824110031128, + "mean_per_layer_hit_rate": 0.01498127169907093 + }, + "5": { + "rho_bar": 0.2873459458351135, + "union_observed": 0.13233457505702972, + "union_independent": 0.3206290602684021, + "union_gap": -0.18829448521137238, + "mean_per_layer_hit_rate": 0.02909824252128601 + }, + "10": { + "rho_bar": 0.3403073251247406, + "union_observed": 0.18227216601371765, + "union_independent": 0.4662020802497864, + "union_gap": -0.2839299142360687, + "mean_per_layer_hit_rate": 0.04667242616415024 + }, + "20": { + "rho_bar": 0.3611651360988617, + "union_observed": 0.25468164682388306, + "union_independent": 0.6225012540817261, + "union_gap": -0.367819607257843, + "mean_per_layer_hit_rate": 0.071257084608078 + }, + "50": { + "rho_bar": 0.3974519968032837, + "union_observed": 0.3483146131038666, + "union_independent": 0.7749878168106079, + "union_gap": -0.42667320370674133, + "mean_per_layer_hit_rate": 0.1066935583949089 + }, + "100": { + "rho_bar": 0.4521878659725189, + "union_observed": 0.43320849537849426, + "union_independent": 0.8753243088722229, + "union_gap": -0.44211581349372864, + "mean_per_layer_hit_rate": 0.1455872505903244 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3186379713671548, + "union_gap_mean_over_k": -0.2646464721432754 + }, + "StackExchange|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.19956953823566437, + "union_observed": 0.047440700232982635, + "union_independent": 0.10227376222610474, + "union_gap": -0.0548330619931221, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.2412947118282318, + "union_observed": 0.06741572916507721, + "union_independent": 0.16052037477493286, + "union_gap": -0.09310464560985565, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.31058233976364136, + "union_observed": 0.1086142286658287, + "union_independent": 0.3083353638648987, + "union_gap": -0.19972113519906998, + "mean_per_layer_hit_rate": 0.027849802747368813 + }, + "10": { + "rho_bar": 0.35362300276756287, + "union_observed": 0.15605492889881134, + "union_independent": 0.4527021050453186, + "union_gap": -0.29664717614650726, + "mean_per_layer_hit_rate": 0.04503985494375229 + }, + "20": { + "rho_bar": 0.38591402769088745, + "union_observed": 0.20099875330924988, + "union_independent": 0.5849112272262573, + "union_gap": -0.38391247391700745, + "mean_per_layer_hit_rate": 0.06482282280921936 + }, + "50": { + "rho_bar": 0.42306312918663025, + "union_observed": 0.29338327050209045, + "union_independent": 0.7520300149917603, + "union_gap": -0.4586467444896698, + "mean_per_layer_hit_rate": 0.10073946416378021 + }, + "100": { + "rho_bar": 0.4552607834339142, + "union_observed": 0.3820224702358246, + "union_independent": 0.8559554815292358, + "union_gap": -0.47393301129341125, + "mean_per_layer_hit_rate": 0.13704022765159607 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3384725047009332, + "union_gap_mean_over_k": -0.28011403552123476 + }, + "StackExchange|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.18123866617679596, + "union_observed": 0.0387016236782074, + "union_independent": 0.07938838005065918, + "union_gap": -0.04068675637245178, + "mean_per_layer_hit_rate": 0.006338230799883604 + }, + "2": { + "rho_bar": 0.24351435899734497, + "union_observed": 0.062421973794698715, + "union_independent": 0.14874297380447388, + "union_gap": -0.08632100000977516, + "mean_per_layer_hit_rate": 0.012292326427996159 + }, + "5": { + "rho_bar": 0.3265973627567291, + "union_observed": 0.10736579447984695, + "union_independent": 0.2964916229248047, + "union_gap": -0.18912582844495773, + "mean_per_layer_hit_rate": 0.026601364836096764 + }, + "10": { + "rho_bar": 0.35532623529434204, + "union_observed": 0.15855181217193604, + "union_independent": 0.44149863719940186, + "union_gap": -0.2829468250274658, + "mean_per_layer_hit_rate": 0.04359934851527214 + }, + "20": { + "rho_bar": 0.39614275097846985, + "union_observed": 0.20848938822746277, + "union_independent": 0.589707612991333, + "union_gap": -0.38121822476387024, + "mean_per_layer_hit_rate": 0.06568711996078491 + }, + "50": { + "rho_bar": 0.43346497416496277, + "union_observed": 0.30087390542030334, + "union_independent": 0.7531899809837341, + "union_gap": -0.4523160755634308, + "mean_per_layer_hit_rate": 0.10102756321430206 + }, + "100": { + "rho_bar": 0.45062097907066345, + "union_observed": 0.39450687170028687, + "union_independent": 0.8641587495803833, + "union_gap": -0.46965187788009644, + "mean_per_layer_hit_rate": 0.1405934989452362 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3409864753484726, + "union_gap_mean_over_k": -0.2717523697231497 + }, + "StackExchange|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.18357983231544495, + "union_observed": 0.0387016236782074, + "union_independent": 0.08977383375167847, + "union_gap": -0.05107221007347107, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.24933257699012756, + "union_observed": 0.05742821469902992, + "union_independent": 0.1530798077583313, + "union_gap": -0.09565159305930138, + "mean_per_layer_hit_rate": 0.012676461599767208 + }, + "5": { + "rho_bar": 0.33765241503715515, + "union_observed": 0.10986267030239105, + "union_independent": 0.31198590993881226, + "union_gap": -0.2021232396364212, + "mean_per_layer_hit_rate": 0.028233937919139862 + }, + "10": { + "rho_bar": 0.36486974358558655, + "union_observed": 0.15980024635791779, + "union_independent": 0.4681674838066101, + "union_gap": -0.3083672374486923, + "mean_per_layer_hit_rate": 0.04705656319856644 + }, + "20": { + "rho_bar": 0.42297691106796265, + "union_observed": 0.21098627150058746, + "union_independent": 0.6219416260719299, + "union_gap": -0.41095535457134247, + "mean_per_layer_hit_rate": 0.07135311514139175 + }, + "50": { + "rho_bar": 0.43141311407089233, + "union_observed": 0.31960049271583557, + "union_independent": 0.7752640843391418, + "union_gap": -0.4556635916233063, + "mean_per_layer_hit_rate": 0.10688561946153641 + }, + "100": { + "rho_bar": 0.4382096230983734, + "union_observed": 0.4082396924495697, + "union_independent": 0.8793322443962097, + "union_gap": -0.47109255194664, + "mean_per_layer_hit_rate": 0.14721980690956116 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3468620308807918, + "union_gap_mean_over_k": -0.28498939690845354 + }, + "StackExchange|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.2201061248779297, + "union_observed": 0.04494382068514824, + "union_independent": 0.0967179536819458, + "union_gap": -0.05177413299679756, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.2603791356086731, + "union_observed": 0.06741572916507721, + "union_independent": 0.1564875841140747, + "union_gap": -0.0890718549489975, + "mean_per_layer_hit_rate": 0.012964563444256783 + }, + "5": { + "rho_bar": 0.28373953700065613, + "union_observed": 0.1136079877614975, + "union_independent": 0.3167654275894165, + "union_gap": -0.203157439827919, + "mean_per_layer_hit_rate": 0.02871410734951496 + }, + "10": { + "rho_bar": 0.37746769189834595, + "union_observed": 0.16729088127613068, + "union_independent": 0.4771775007247925, + "union_gap": -0.3098866194486618, + "mean_per_layer_hit_rate": 0.048208970576524734 + }, + "20": { + "rho_bar": 0.4089444875717163, + "union_observed": 0.23470661044120789, + "union_independent": 0.6434555053710938, + "union_gap": -0.40874889492988586, + "mean_per_layer_hit_rate": 0.0752905011177063 + }, + "50": { + "rho_bar": 0.4314749240875244, + "union_observed": 0.33208489418029785, + "union_independent": 0.7983753085136414, + "union_gap": -0.4662904143333435, + "mean_per_layer_hit_rate": 0.1136079952120781 + }, + "100": { + "rho_bar": 0.4111614227294922, + "union_observed": 0.4294631779193878, + "union_independent": 0.891662061214447, + "union_gap": -0.4621988832950592, + "mean_per_layer_hit_rate": 0.15317392349243164 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3418961891106197, + "union_gap_mean_over_k": -0.2844468913972378 + }, + "StackExchange|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.23221614956855774, + "union_observed": 0.048689138144254684, + "union_independent": 0.11488497257232666, + "union_gap": -0.06619583442807198, + "mean_per_layer_hit_rate": 0.009315278381109238 + }, + "2": { + "rho_bar": 0.25881946086883545, + "union_observed": 0.06616728752851486, + "union_independent": 0.17781537771224976, + "union_gap": -0.1116480901837349, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.3183125853538513, + "union_observed": 0.12609238922595978, + "union_independent": 0.348341703414917, + "union_gap": -0.22224931418895721, + "mean_per_layer_hit_rate": 0.03217132389545441 + }, + "10": { + "rho_bar": 0.3687274158000946, + "union_observed": 0.1810237169265747, + "union_independent": 0.4993305802345276, + "union_gap": -0.3183068633079529, + "mean_per_layer_hit_rate": 0.05128205195069313 + }, + "20": { + "rho_bar": 0.39373651146888733, + "union_observed": 0.26217228174209595, + "union_independent": 0.6707106828689575, + "union_gap": -0.4085384011268616, + "mean_per_layer_hit_rate": 0.0804763212800026 + }, + "50": { + "rho_bar": 0.40354055166244507, + "union_observed": 0.3620474338531494, + "union_independent": 0.8220016956329346, + "union_gap": -0.45995426177978516, + "mean_per_layer_hit_rate": 0.12138672918081284 + }, + "100": { + "rho_bar": 0.4022861421108246, + "union_observed": 0.44569289684295654, + "union_independent": 0.9037646055221558, + "union_gap": -0.4580717086791992, + "mean_per_layer_hit_rate": 0.16018438339233398 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.33966268811907085, + "union_gap_mean_over_k": -0.29213778195636614 + }, + "StackExchange|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.27642470598220825, + "union_observed": 0.05243445560336113, + "union_independent": 0.1259954571723938, + "union_gap": -0.07356100156903267, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "2": { + "rho_bar": 0.28969770669937134, + "union_observed": 0.08114856481552124, + "union_independent": 0.2025536298751831, + "union_gap": -0.12140506505966187, + "mean_per_layer_hit_rate": 0.017190052196383476 + }, + "5": { + "rho_bar": 0.336749404668808, + "union_observed": 0.13857677578926086, + "union_independent": 0.3768998384475708, + "union_gap": -0.23832306265830994, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.3498110771179199, + "union_observed": 0.20099875330924988, + "union_independent": 0.5373572707176208, + "union_gap": -0.33635851740837097, + "mean_per_layer_hit_rate": 0.05685201659798622 + }, + "20": { + "rho_bar": 0.38335859775543213, + "union_observed": 0.26966291666030884, + "union_independent": 0.7011514902114868, + "union_gap": -0.431488573551178, + "mean_per_layer_hit_rate": 0.08710265904664993 + }, + "50": { + "rho_bar": 0.4002555310726166, + "union_observed": 0.36329588294029236, + "union_independent": 0.8390078544616699, + "union_gap": -0.47571197152137756, + "mean_per_layer_hit_rate": 0.12782098352909088 + }, + "100": { + "rho_bar": 0.40665116906166077, + "union_observed": 0.46941322088241577, + "union_independent": 0.9217129349708557, + "union_gap": -0.45229971408843994, + "mean_per_layer_hit_rate": 0.17238067090511322 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.34899259890828815, + "union_gap_mean_over_k": -0.30416398655091015 + }, + "StackExchange|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.2825756371021271, + "union_observed": 0.05368289723992348, + "union_independent": 0.14025896787643433, + "union_gap": -0.08657607063651085, + "mean_per_layer_hit_rate": 0.01152405608445406 + }, + "2": { + "rho_bar": 0.33545395731925964, + "union_observed": 0.08239700645208359, + "union_independent": 0.22860771417617798, + "union_gap": -0.1462107077240944, + "mean_per_layer_hit_rate": 0.019686929881572723 + }, + "5": { + "rho_bar": 0.38229337334632874, + "union_observed": 0.13857677578926086, + "union_independent": 0.4082942605018616, + "union_gap": -0.2697174847126007, + "mean_per_layer_hit_rate": 0.0392778255045414 + }, + "10": { + "rho_bar": 0.39671850204467773, + "union_observed": 0.20724095404148102, + "union_independent": 0.5759178400039673, + "union_gap": -0.36867688596248627, + "mean_per_layer_hit_rate": 0.06309421360492706 + }, + "20": { + "rho_bar": 0.4194552004337311, + "union_observed": 0.29088640213012695, + "union_independent": 0.7441107630729675, + "union_gap": -0.4532243609428406, + "mean_per_layer_hit_rate": 0.09785844385623932 + }, + "50": { + "rho_bar": 0.4344128370285034, + "union_observed": 0.3982521891593933, + "union_independent": 0.8700565099716187, + "union_gap": -0.47180432081222534, + "mean_per_layer_hit_rate": 0.14203399419784546 + }, + "100": { + "rho_bar": 0.44280654191970825, + "union_observed": 0.48439452052116394, + "union_independent": 0.9380895495414734, + "union_gap": -0.45369502902030945, + "mean_per_layer_hit_rate": 0.187361940741539 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3848165784563337, + "union_gap_mean_over_k": -0.32141497997300966 + }, + "StackExchange|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.32062745094299316, + "union_observed": 0.05992509424686432, + "union_independent": 0.15969735383987427, + "union_gap": -0.09977225959300995, + "mean_per_layer_hit_rate": 0.013252665288746357 + }, + "2": { + "rho_bar": 0.3317965865135193, + "union_observed": 0.09737827628850937, + "union_independent": 0.26067662239074707, + "union_gap": -0.1632983461022377, + "mean_per_layer_hit_rate": 0.02285604551434517 + }, + "5": { + "rho_bar": 0.3799980580806732, + "union_observed": 0.15980024635791779, + "union_independent": 0.45045119524002075, + "union_gap": -0.29065094888210297, + "mean_per_layer_hit_rate": 0.04465571790933609 + }, + "10": { + "rho_bar": 0.4212495982646942, + "union_observed": 0.227215975522995, + "union_independent": 0.6133577227592468, + "union_gap": -0.38614174723625183, + "mean_per_layer_hit_rate": 0.06962451338768005 + }, + "20": { + "rho_bar": 0.4649556875228882, + "union_observed": 0.28838950395584106, + "union_independent": 0.763929009437561, + "union_gap": -0.47553950548171997, + "mean_per_layer_hit_rate": 0.10333237051963806 + }, + "50": { + "rho_bar": 0.48784738779067993, + "union_observed": 0.3932584226131439, + "union_independent": 0.8831901550292969, + "union_gap": -0.48993173241615295, + "mean_per_layer_hit_rate": 0.14933255314826965 + }, + "100": { + "rho_bar": 0.4881054759025574, + "union_observed": 0.4806492030620575, + "union_independent": 0.9450806379318237, + "union_gap": -0.46443143486976624, + "mean_per_layer_hit_rate": 0.19533275067806244 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.41351146357400076, + "union_gap_mean_over_k": -0.3385379963687488 + }, + "StackExchange|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.23646099865436554, + "union_observed": 0.05867665261030197, + "union_independent": 0.12924480438232422, + "union_gap": -0.07056815177202225, + "mean_per_layer_hit_rate": 0.010563717223703861 + }, + "2": { + "rho_bar": 0.2825610935688019, + "union_observed": 0.09113608300685883, + "union_independent": 0.21240073442459106, + "union_gap": -0.12126465141773224, + "mean_per_layer_hit_rate": 0.018150389194488525 + }, + "5": { + "rho_bar": 0.3561849594116211, + "union_observed": 0.15355804562568665, + "union_independent": 0.3793625831604004, + "union_gap": -0.22580453753471375, + "mean_per_layer_hit_rate": 0.0358206108212471 + }, + "10": { + "rho_bar": 0.3914402723312378, + "union_observed": 0.21598002314567566, + "union_independent": 0.5414608716964722, + "union_gap": -0.3254808485507965, + "mean_per_layer_hit_rate": 0.05762027949094772 + }, + "20": { + "rho_bar": 0.4272715747356415, + "union_observed": 0.2921348214149475, + "union_independent": 0.7020958662033081, + "union_gap": -0.4099610447883606, + "mean_per_layer_hit_rate": 0.08758282661437988 + }, + "50": { + "rho_bar": 0.46072572469711304, + "union_observed": 0.3907615542411804, + "union_independent": 0.8469241857528687, + "union_gap": -0.45616263151168823, + "mean_per_layer_hit_rate": 0.13233457505702972 + }, + "100": { + "rho_bar": 0.479422390460968, + "union_observed": 0.4656679034233093, + "union_independent": 0.9172819256782532, + "union_gap": -0.45161402225494385, + "mean_per_layer_hit_rate": 0.17094017565250397 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.37629528769424986, + "union_gap_mean_over_k": -0.2944079839757511 + }, + "StackExchange|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.23657730221748352, + "union_observed": 0.05867665261030197, + "union_independent": 0.1281348466873169, + "union_gap": -0.06945819407701492, + "mean_per_layer_hit_rate": 0.01046768482774496 + }, + "2": { + "rho_bar": 0.2468835562467575, + "union_observed": 0.08489388227462769, + "union_independent": 0.20345282554626465, + "union_gap": -0.11855894327163696, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.3249559700489044, + "union_observed": 0.1523096114397049, + "union_independent": 0.37963151931762695, + "union_gap": -0.22732190787792206, + "mean_per_layer_hit_rate": 0.0358206070959568 + }, + "10": { + "rho_bar": 0.3958669602870941, + "union_observed": 0.20099875330924988, + "union_independent": 0.5345955491065979, + "union_gap": -0.333596795797348, + "mean_per_layer_hit_rate": 0.056563910096883774 + }, + "20": { + "rho_bar": 0.4237036108970642, + "union_observed": 0.2746566832065582, + "union_independent": 0.7002097368240356, + "union_gap": -0.4255530536174774, + "mean_per_layer_hit_rate": 0.08719869703054428 + }, + "50": { + "rho_bar": 0.43147003650665283, + "union_observed": 0.38327091932296753, + "union_independent": 0.8431057929992676, + "union_gap": -0.45983487367630005, + "mean_per_layer_hit_rate": 0.13002976775169373 + }, + "100": { + "rho_bar": 0.43820473551750183, + "union_observed": 0.48189762234687805, + "union_independent": 0.9187330603599548, + "union_gap": -0.4368354380130768, + "mean_per_layer_hit_rate": 0.17142033576965332 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.35680888167449404, + "union_gap_mean_over_k": -0.2958798866186823 + }, + "StackExchange|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.21911273896694183, + "union_observed": 0.05243445560336113, + "union_independent": 0.11610943078994751, + "union_gap": -0.06367497518658638, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.2449522763490677, + "union_observed": 0.09612983465194702, + "union_independent": 0.20496469736099243, + "union_gap": -0.10883486270904541, + "mean_per_layer_hit_rate": 0.017382118850946426 + }, + "5": { + "rho_bar": 0.32126423716545105, + "union_observed": 0.16354556381702423, + "union_independent": 0.3849034905433655, + "union_gap": -0.22135792672634125, + "mean_per_layer_hit_rate": 0.0363968126475811 + }, + "10": { + "rho_bar": 0.3712623119354248, + "union_observed": 0.2197253406047821, + "union_independent": 0.536919116973877, + "union_gap": -0.31719377636909485, + "mean_per_layer_hit_rate": 0.056852009147405624 + }, + "20": { + "rho_bar": 0.4173961579799652, + "union_observed": 0.2971285879611969, + "union_independent": 0.7045851945877075, + "union_gap": -0.4074566066265106, + "mean_per_layer_hit_rate": 0.08825507760047913 + }, + "50": { + "rho_bar": 0.443539559841156, + "union_observed": 0.4007490575313568, + "union_independent": 0.8381091952323914, + "union_gap": -0.43736013770103455, + "mean_per_layer_hit_rate": 0.12830115854740143 + }, + "100": { + "rho_bar": 0.4358159601688385, + "union_observed": 0.4956304728984833, + "union_independent": 0.913419246673584, + "union_gap": -0.4177887737751007, + "mean_per_layer_hit_rate": 0.16767501831054688 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3504776060581207, + "union_gap_mean_over_k": -0.2819524370133877 + }, + "StackExchange|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.1926777958869934, + "union_observed": 0.05368289723992348, + "union_independent": 0.1138758659362793, + "union_gap": -0.06019296869635582, + "mean_per_layer_hit_rate": 0.009219245053827763 + }, + "2": { + "rho_bar": 0.24573898315429688, + "union_observed": 0.08863919973373413, + "union_independent": 0.1905885934829712, + "union_gap": -0.10194939374923706, + "mean_per_layer_hit_rate": 0.01603764481842518 + }, + "5": { + "rho_bar": 0.2783297598361969, + "union_observed": 0.15605492889881134, + "union_independent": 0.3431382179260254, + "union_gap": -0.18708328902721405, + "mean_per_layer_hit_rate": 0.03149908781051636 + }, + "10": { + "rho_bar": 0.3455347418785095, + "union_observed": 0.2122347056865692, + "union_independent": 0.5032780170440674, + "union_gap": -0.29104331135749817, + "mean_per_layer_hit_rate": 0.05166618525981903 + }, + "20": { + "rho_bar": 0.3590930998325348, + "union_observed": 0.28589263558387756, + "union_independent": 0.6603193283081055, + "union_gap": -0.3744266927242279, + "mean_per_layer_hit_rate": 0.07817152142524719 + }, + "50": { + "rho_bar": 0.38937896490097046, + "union_observed": 0.38327091932296753, + "union_independent": 0.8059161305427551, + "union_gap": -0.4226452112197876, + "mean_per_layer_hit_rate": 0.11572074145078659 + }, + "100": { + "rho_bar": 0.42135778069496155, + "union_observed": 0.4756554365158081, + "union_independent": 0.8983193635940552, + "union_gap": -0.42266392707824707, + "mean_per_layer_hit_rate": 0.15701524913311005 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31887301802635193, + "union_gap_mean_over_k": -0.2657149705503668 + }, + "StackExchange|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.222370445728302, + "union_observed": 0.05742821469902992, + "union_independent": 0.12924253940582275, + "union_gap": -0.07181432470679283, + "mean_per_layer_hit_rate": 0.010563718155026436 + }, + "2": { + "rho_bar": 0.27366483211517334, + "union_observed": 0.09238451719284058, + "union_independent": 0.21258842945098877, + "union_gap": -0.1202039122581482, + "mean_per_layer_hit_rate": 0.018150389194488525 + }, + "5": { + "rho_bar": 0.3284347355365753, + "union_observed": 0.13857677578926086, + "union_independent": 0.37230998277664185, + "union_gap": -0.23373320698738098, + "mean_per_layer_hit_rate": 0.034956302493810654 + }, + "10": { + "rho_bar": 0.36070579290390015, + "union_observed": 0.19725343585014343, + "union_independent": 0.5263868570327759, + "union_gap": -0.32913342118263245, + "mean_per_layer_hit_rate": 0.055315468460321426 + }, + "20": { + "rho_bar": 0.397307425737381, + "union_observed": 0.2734082341194153, + "union_independent": 0.6895735263824463, + "union_gap": -0.416165292263031, + "mean_per_layer_hit_rate": 0.08479785174131393 + }, + "50": { + "rho_bar": 0.4461449980735779, + "union_observed": 0.3695380687713623, + "union_independent": 0.8347922563552856, + "union_gap": -0.46525418758392334, + "mean_per_layer_hit_rate": 0.12676462531089783 + }, + "100": { + "rho_bar": 0.4694725573062897, + "union_observed": 0.4756554365158081, + "union_independent": 0.914713978767395, + "union_gap": -0.4390585422515869, + "mean_per_layer_hit_rate": 0.16834726929664612 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.35687154105731417, + "union_gap_mean_over_k": -0.29648041246192797 + }, + "StackExchange|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.3735601305961609, + "union_observed": 0.048689138144254684, + "union_independent": 0.1487826704978943, + "union_gap": -0.1000935323536396, + "mean_per_layer_hit_rate": 0.012292327359318733 + }, + "2": { + "rho_bar": 0.4047161638736725, + "union_observed": 0.07865168899297714, + "union_independent": 0.23522216081619263, + "union_gap": -0.15657047182321548, + "mean_per_layer_hit_rate": 0.020359167829155922 + }, + "5": { + "rho_bar": 0.4051291048526764, + "union_observed": 0.1373283416032791, + "union_independent": 0.4491105079650879, + "union_gap": -0.3117821663618088, + "mean_per_layer_hit_rate": 0.04455968365073204 + }, + "10": { + "rho_bar": 0.48046714067459106, + "union_observed": 0.20224718749523163, + "union_independent": 0.6430600881576538, + "union_gap": -0.4408129006624222, + "mean_per_layer_hit_rate": 0.07538653165102005 + }, + "20": { + "rho_bar": 0.5307708382606506, + "union_observed": 0.2746566832065582, + "union_independent": 0.7855684757232666, + "union_gap": -0.5109117925167084, + "mean_per_layer_hit_rate": 0.1097666472196579 + }, + "50": { + "rho_bar": 0.5605862736701965, + "union_observed": 0.34956303238868713, + "union_independent": 0.8880476951599121, + "union_gap": -0.538484662771225, + "mean_per_layer_hit_rate": 0.1517334133386612 + }, + "100": { + "rho_bar": 0.5619422793388367, + "union_observed": 0.44569289684295654, + "union_independent": 0.947818398475647, + "union_gap": -0.5021255016326904, + "mean_per_layer_hit_rate": 0.19792567193508148 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.47388170446668354, + "union_gap_mean_over_k": -0.36582586116024424 + }, + "StackExchange|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.37612318992614746, + "union_observed": 0.06117353215813637, + "union_independent": 0.1808156967163086, + "union_gap": -0.11964216455817223, + "mean_per_layer_hit_rate": 0.015173341147601604 + }, + "2": { + "rho_bar": 0.3960147798061371, + "union_observed": 0.09363295882940292, + "union_independent": 0.2793164849281311, + "union_gap": -0.18568352609872818, + "mean_per_layer_hit_rate": 0.024776723235845566 + }, + "5": { + "rho_bar": 0.4176141321659088, + "union_observed": 0.16104869544506073, + "union_independent": 0.484885573387146, + "union_gap": -0.32383687794208527, + "mean_per_layer_hit_rate": 0.049361374229192734 + }, + "10": { + "rho_bar": 0.4772386848926544, + "union_observed": 0.22347065806388855, + "union_independent": 0.6422665119171143, + "union_gap": -0.4187958538532257, + "mean_per_layer_hit_rate": 0.07519447058439255 + }, + "20": { + "rho_bar": 0.5237294435501099, + "union_observed": 0.28589263558387756, + "union_independent": 0.792663037776947, + "union_gap": -0.5067704021930695, + "mean_per_layer_hit_rate": 0.11235956102609634 + }, + "50": { + "rho_bar": 0.550518810749054, + "union_observed": 0.37328338623046875, + "union_independent": 0.9006006121635437, + "union_gap": -0.527317225933075, + "mean_per_layer_hit_rate": 0.15999232232570648 + }, + "100": { + "rho_bar": 0.562576174736023, + "union_observed": 0.4519350826740265, + "union_independent": 0.9524850249290466, + "union_gap": -0.5005499422550201, + "mean_per_layer_hit_rate": 0.20493613183498383 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4719736022608621, + "union_gap_mean_over_k": -0.3689422846904823 + }, + "StackExchange|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.30654311180114746, + "union_observed": 0.06491885334253311, + "union_independent": 0.1650540828704834, + "union_gap": -0.10013522952795029, + "mean_per_layer_hit_rate": 0.013732834719121456 + }, + "2": { + "rho_bar": 0.3526042103767395, + "union_observed": 0.09987515956163406, + "union_independent": 0.2549595236778259, + "union_gap": -0.15508436411619186, + "mean_per_layer_hit_rate": 0.02227984182536602 + }, + "5": { + "rho_bar": 0.3807676434516907, + "union_observed": 0.15355804562568665, + "union_independent": 0.43625426292419434, + "union_gap": -0.2826962172985077, + "mean_per_layer_hit_rate": 0.04273504391312599 + }, + "10": { + "rho_bar": 0.41076165437698364, + "union_observed": 0.22596754133701324, + "union_independent": 0.6020609140396118, + "union_gap": -0.37609337270259857, + "mean_per_layer_hit_rate": 0.0676078051328659 + }, + "20": { + "rho_bar": 0.440838485956192, + "union_observed": 0.29588013887405396, + "union_independent": 0.7517614364624023, + "union_gap": -0.4558812975883484, + "mean_per_layer_hit_rate": 0.09997118264436722 + }, + "50": { + "rho_bar": 0.5003620982170105, + "union_observed": 0.37578028440475464, + "union_independent": 0.8759251832962036, + "union_gap": -0.500144898891449, + "mean_per_layer_hit_rate": 0.1453951746225357 + }, + "100": { + "rho_bar": 0.5327473282814026, + "union_observed": 0.4569288492202759, + "union_independent": 0.9348780512809753, + "union_gap": -0.47794920206069946, + "mean_per_layer_hit_rate": 0.1854412704706192 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.4178035046373095, + "union_gap_mean_over_k": -0.3354263688836779 + }, + "StackExchange|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.29782989621162415, + "union_observed": 0.056179776787757874, + "union_independent": 0.12826895713806152, + "union_gap": -0.07208918035030365, + "mean_per_layer_hit_rate": 0.01046768482774496 + }, + "2": { + "rho_bar": 0.2685385048389435, + "union_observed": 0.08489388227462769, + "union_independent": 0.20270293951034546, + "union_gap": -0.11780905723571777, + "mean_per_layer_hit_rate": 0.017190050333738327 + }, + "5": { + "rho_bar": 0.3037830889225006, + "union_observed": 0.14856429398059845, + "union_independent": 0.37691599130630493, + "union_gap": -0.22835169732570648, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.3454990088939667, + "union_observed": 0.20848938822746277, + "union_independent": 0.5285321474075317, + "union_gap": -0.32004275918006897, + "mean_per_layer_hit_rate": 0.055411502718925476 + }, + "20": { + "rho_bar": 0.38536161184310913, + "union_observed": 0.2808988690376282, + "union_independent": 0.7013018727302551, + "union_gap": -0.42040300369262695, + "mean_per_layer_hit_rate": 0.08710265904664993 + }, + "50": { + "rho_bar": 0.4563156068325043, + "union_observed": 0.36329588294029236, + "union_independent": 0.8410478830337524, + "union_gap": -0.4777520000934601, + "mean_per_layer_hit_rate": 0.12926150858402252 + }, + "100": { + "rho_bar": 0.48732253909111023, + "union_observed": 0.4569288492202759, + "union_independent": 0.9183422327041626, + "union_gap": -0.4614133834838867, + "mean_per_layer_hit_rate": 0.17132428288459778 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.36352146523339407, + "union_gap_mean_over_k": -0.29969444019453867 + }, + "StackExchange|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.25541847944259644, + "union_observed": 0.05243445560336113, + "union_independent": 0.11270356178283691, + "union_gap": -0.060269106179475784, + "mean_per_layer_hit_rate": 0.009123211726546288 + }, + "2": { + "rho_bar": 0.2335069179534912, + "union_observed": 0.08114856481552124, + "union_independent": 0.1799594759941101, + "union_gap": -0.09881091117858887, + "mean_per_layer_hit_rate": 0.015077306888997555 + }, + "5": { + "rho_bar": 0.297957181930542, + "union_observed": 0.14107365906238556, + "union_independent": 0.3420741558074951, + "union_gap": -0.20100049674510956, + "mean_per_layer_hit_rate": 0.03140305355191231 + }, + "10": { + "rho_bar": 0.3541674017906189, + "union_observed": 0.19850187003612518, + "union_independent": 0.4876592755317688, + "union_gap": -0.2891574054956436, + "mean_per_layer_hit_rate": 0.04955344647169113 + }, + "20": { + "rho_bar": 0.3774660527706146, + "union_observed": 0.2721598148345947, + "union_independent": 0.6505962610244751, + "union_gap": -0.37843644618988037, + "mean_per_layer_hit_rate": 0.0764429047703743 + }, + "50": { + "rho_bar": 0.4387550950050354, + "union_observed": 0.3545567989349365, + "union_independent": 0.8081173896789551, + "union_gap": -0.45356059074401855, + "mean_per_layer_hit_rate": 0.11696917563676834 + }, + "100": { + "rho_bar": 0.4968695044517517, + "union_observed": 0.4319600462913513, + "union_independent": 0.899472713470459, + "union_gap": -0.46751266717910767, + "mean_per_layer_hit_rate": 0.15893594920635223 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.35059151904923574, + "union_gap_mean_over_k": -0.2783925176731178 + }, + "StackExchange|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.1884945183992386, + "union_observed": 0.05243445560336113, + "union_independent": 0.10130703449249268, + "union_gap": -0.048872578889131546, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.21470627188682556, + "union_observed": 0.07865168899297714, + "union_independent": 0.16623669862747192, + "union_gap": -0.08758500963449478, + "mean_per_layer_hit_rate": 0.013828867115080357 + }, + "5": { + "rho_bar": 0.27265700697898865, + "union_observed": 0.13483145833015442, + "union_independent": 0.31715208292007446, + "union_gap": -0.18232062458992004, + "mean_per_layer_hit_rate": 0.02871410734951496 + }, + "10": { + "rho_bar": 0.3170187473297119, + "union_observed": 0.17977528274059296, + "union_independent": 0.45783495903015137, + "union_gap": -0.2780596762895584, + "mean_per_layer_hit_rate": 0.04552002251148224 + }, + "20": { + "rho_bar": 0.35859817266464233, + "union_observed": 0.2534331977367401, + "union_independent": 0.6154338121414185, + "union_gap": -0.36200061440467834, + "mean_per_layer_hit_rate": 0.06991260498762131 + }, + "50": { + "rho_bar": 0.401917427778244, + "union_observed": 0.34456929564476013, + "union_independent": 0.7723046541213989, + "union_gap": -0.4277353584766388, + "mean_per_layer_hit_rate": 0.10592528432607651 + }, + "100": { + "rho_bar": 0.4587768018245697, + "union_observed": 0.4294631779193878, + "union_independent": 0.8760038614273071, + "union_gap": -0.4465406835079193, + "mean_per_layer_hit_rate": 0.14606741070747375 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3160241352660315, + "union_gap_mean_over_k": -0.26187350654176306 + }, + "StackExchange|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.18535754084587097, + "union_observed": 0.04244694113731384, + "union_independent": 0.09091305732727051, + "union_gap": -0.048466116189956665, + "mean_per_layer_hit_rate": 0.007298569194972515 + }, + "2": { + "rho_bar": 0.24803340435028076, + "union_observed": 0.06367041170597076, + "union_independent": 0.158405601978302, + "union_gap": -0.09473519027233124, + "mean_per_layer_hit_rate": 0.013156631961464882 + }, + "5": { + "rho_bar": 0.30449384450912476, + "union_observed": 0.10362047702074051, + "union_independent": 0.2975490689277649, + "union_gap": -0.19392859190702438, + "mean_per_layer_hit_rate": 0.026697397232055664 + }, + "10": { + "rho_bar": 0.3421874940395355, + "union_observed": 0.15605492889881134, + "union_independent": 0.44862043857574463, + "union_gap": -0.2925655096769333, + "mean_per_layer_hit_rate": 0.04446365311741829 + }, + "20": { + "rho_bar": 0.38064178824424744, + "union_observed": 0.19850187003612518, + "union_independent": 0.5870723724365234, + "union_gap": -0.38857050240039825, + "mean_per_layer_hit_rate": 0.06520695239305496 + }, + "50": { + "rho_bar": 0.41929230093955994, + "union_observed": 0.2871410846710205, + "union_independent": 0.7484197020530701, + "union_gap": -0.46127861738204956, + "mean_per_layer_hit_rate": 0.09977912157773972 + }, + "100": { + "rho_bar": 0.4498252868652344, + "union_observed": 0.38327091932296753, + "union_independent": 0.8549713492393494, + "union_gap": -0.47170042991638184, + "mean_per_layer_hit_rate": 0.13665610551834106 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.33283309425626484, + "union_gap_mean_over_k": -0.2787492796778679 + }, + "StackExchange|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.16582836210727692, + "union_observed": 0.0387016236782074, + "union_independent": 0.07359838485717773, + "union_gap": -0.03489676117897034, + "mean_per_layer_hit_rate": 0.005858062766492367 + }, + "2": { + "rho_bar": 0.2401646375656128, + "union_observed": 0.062421973794698715, + "union_independent": 0.14337033033370972, + "union_gap": -0.080948356539011, + "mean_per_layer_hit_rate": 0.01181215699762106 + }, + "5": { + "rho_bar": 0.30915412306785583, + "union_observed": 0.1086142286658287, + "union_independent": 0.29109835624694824, + "union_gap": -0.18248412758111954, + "mean_per_layer_hit_rate": 0.026025163009762764 + }, + "10": { + "rho_bar": 0.357394278049469, + "union_observed": 0.15106117725372314, + "union_independent": 0.43217629194259644, + "union_gap": -0.2811151146888733, + "mean_per_layer_hit_rate": 0.04235090687870979 + }, + "20": { + "rho_bar": 0.3845120072364807, + "union_observed": 0.20848938822746277, + "union_independent": 0.5847740173339844, + "union_gap": -0.3762846291065216, + "mean_per_layer_hit_rate": 0.06482282280921936 + }, + "50": { + "rho_bar": 0.4313085973262787, + "union_observed": 0.29588013887405396, + "union_independent": 0.7488369941711426, + "union_gap": -0.4529568552970886, + "mean_per_layer_hit_rate": 0.09987514466047287 + }, + "100": { + "rho_bar": 0.4565848410129547, + "union_observed": 0.3845193386077881, + "union_independent": 0.8601459860801697, + "union_gap": -0.4756266474723816, + "mean_per_layer_hit_rate": 0.13876885175704956 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3349924066237041, + "union_gap_mean_over_k": -0.2691874988377094 + }, + "StackExchange|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.18176347017288208, + "union_observed": 0.03495630621910095, + "union_independent": 0.08172869682312012, + "union_gap": -0.046772390604019165, + "mean_per_layer_hit_rate": 0.006530298851430416 + }, + "2": { + "rho_bar": 0.2448241412639618, + "union_observed": 0.05742821469902992, + "union_independent": 0.15418469905853271, + "union_gap": -0.09675648435950279, + "mean_per_layer_hit_rate": 0.012772495858371258 + }, + "5": { + "rho_bar": 0.34194129705429077, + "union_observed": 0.11235955357551575, + "union_independent": 0.3148878216743469, + "union_gap": -0.20252826809883118, + "mean_per_layer_hit_rate": 0.028522036969661713 + }, + "10": { + "rho_bar": 0.35630860924720764, + "union_observed": 0.16229712963104248, + "union_independent": 0.460421085357666, + "union_gap": -0.29812395572662354, + "mean_per_layer_hit_rate": 0.04600019007921219 + }, + "20": { + "rho_bar": 0.41688859462738037, + "union_observed": 0.2122347056865692, + "union_independent": 0.6216890811920166, + "union_gap": -0.4094543755054474, + "mean_per_layer_hit_rate": 0.071257084608078 + }, + "50": { + "rho_bar": 0.43724432587623596, + "union_observed": 0.31086140871047974, + "union_independent": 0.7708134055137634, + "union_gap": -0.4599519968032837, + "mean_per_layer_hit_rate": 0.10563718527555466 + }, + "100": { + "rho_bar": 0.4428037405014038, + "union_observed": 0.3932584226131439, + "union_independent": 0.8732554912567139, + "union_gap": -0.47999706864356995, + "mean_per_layer_hit_rate": 0.14414674043655396 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.34596773982048035, + "union_gap_mean_over_k": -0.2847977913916111 + }, + "StackExchange|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.22377921640872955, + "union_observed": 0.04244694113731384, + "union_independent": 0.0932847261428833, + "union_gap": -0.05083778500556946, + "mean_per_layer_hit_rate": 0.00749063678085804 + }, + "2": { + "rho_bar": 0.2734733819961548, + "union_observed": 0.06616728752851486, + "union_independent": 0.15861886739730835, + "union_gap": -0.09245157986879349, + "mean_per_layer_hit_rate": 0.013156631030142307 + }, + "5": { + "rho_bar": 0.2828952670097351, + "union_observed": 0.11735330522060394, + "union_independent": 0.31780409812927246, + "union_gap": -0.20045079290866852, + "mean_per_layer_hit_rate": 0.02881014160811901 + }, + "10": { + "rho_bar": 0.36481744050979614, + "union_observed": 0.16853933036327362, + "union_independent": 0.4660123586654663, + "union_gap": -0.2974730283021927, + "mean_per_layer_hit_rate": 0.04667242616415024 + }, + "20": { + "rho_bar": 0.40603402256965637, + "union_observed": 0.2297128587961197, + "union_independent": 0.638626217842102, + "union_gap": -0.40891335904598236, + "mean_per_layer_hit_rate": 0.0743301659822464 + }, + "50": { + "rho_bar": 0.42761465907096863, + "union_observed": 0.32709112763404846, + "union_independent": 0.7927272319793701, + "union_gap": -0.46563610434532166, + "mean_per_layer_hit_rate": 0.11178334802389145 + }, + "100": { + "rho_bar": 0.42536017298698425, + "union_observed": 0.41697877645492554, + "union_independent": 0.8896641731262207, + "union_gap": -0.47268539667129517, + "mean_per_layer_hit_rate": 0.15221358835697174 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3434248800788607, + "union_gap_mean_over_k": -0.2840640065925462 + }, + "StackExchange|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.2194085568189621, + "union_observed": 0.048689138144254684, + "union_independent": 0.11041617393493652, + "union_gap": -0.06172703579068184, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.2556696832180023, + "union_observed": 0.0699126124382019, + "union_independent": 0.17996352910995483, + "union_gap": -0.11005091667175293, + "mean_per_layer_hit_rate": 0.015077306888997555 + }, + "5": { + "rho_bar": 0.3148193955421448, + "union_observed": 0.12734082341194153, + "union_independent": 0.3450354337692261, + "union_gap": -0.21769461035728455, + "mean_per_layer_hit_rate": 0.031787190586328506 + }, + "10": { + "rho_bar": 0.3650988042354584, + "union_observed": 0.17727839946746826, + "union_independent": 0.49339455366134644, + "union_gap": -0.3161161541938782, + "mean_per_layer_hit_rate": 0.05041774362325668 + }, + "20": { + "rho_bar": 0.39089712500572205, + "union_observed": 0.2534331977367401, + "union_independent": 0.6631826162338257, + "union_gap": -0.40974941849708557, + "mean_per_layer_hit_rate": 0.07893978804349899 + }, + "50": { + "rho_bar": 0.4054720103740692, + "union_observed": 0.35830211639404297, + "union_independent": 0.8167192935943604, + "union_gap": -0.4584171772003174, + "mean_per_layer_hit_rate": 0.11956209689378738 + }, + "100": { + "rho_bar": 0.4018173813819885, + "union_observed": 0.44319599866867065, + "union_independent": 0.9032193422317505, + "union_gap": -0.46002334356307983, + "mean_per_layer_hit_rate": 0.15980026125907898 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3361689937966211, + "union_gap_mean_over_k": -0.2905398080391543 + }, + "StackExchange|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.27834048867225647, + "union_observed": 0.05118601769208908, + "union_independent": 0.1260545253753662, + "union_gap": -0.07486850768327713, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "2": { + "rho_bar": 0.28561368584632874, + "union_observed": 0.0774032473564148, + "union_independent": 0.19330036640167236, + "union_gap": -0.11589711904525757, + "mean_per_layer_hit_rate": 0.016325747594237328 + }, + "5": { + "rho_bar": 0.3245534598827362, + "union_observed": 0.1423220932483673, + "union_independent": 0.3769108057022095, + "union_gap": -0.23458871245384216, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.3477729856967926, + "union_observed": 0.19850187003612518, + "union_independent": 0.5266287326812744, + "union_gap": -0.32812686264514923, + "mean_per_layer_hit_rate": 0.055219437927007675 + }, + "20": { + "rho_bar": 0.3787272274494171, + "union_observed": 0.2659175992012024, + "union_independent": 0.6978511214256287, + "union_gap": -0.43193352222442627, + "mean_per_layer_hit_rate": 0.08633439242839813 + }, + "50": { + "rho_bar": 0.3991745412349701, + "union_observed": 0.36079901456832886, + "union_independent": 0.8354583978652954, + "union_gap": -0.47465938329696655, + "mean_per_layer_hit_rate": 0.12647652626037598 + }, + "100": { + "rho_bar": 0.4087613523006439, + "union_observed": 0.4606741666793823, + "union_independent": 0.9202454090118408, + "union_gap": -0.4595712423324585, + "mean_per_layer_hit_rate": 0.17122825980186462 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.346134820154735, + "union_gap_mean_over_k": -0.3028064785259111 + }, + "StackExchange|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.29493317008018494, + "union_observed": 0.04993757978081703, + "union_independent": 0.1337304711341858, + "union_gap": -0.08379289135336876, + "mean_per_layer_hit_rate": 0.010947853326797485 + }, + "2": { + "rho_bar": 0.3186749219894409, + "union_observed": 0.08489388227462769, + "union_independent": 0.2178058624267578, + "union_gap": -0.13291198015213013, + "mean_per_layer_hit_rate": 0.018630558624863625 + }, + "5": { + "rho_bar": 0.36301854252815247, + "union_observed": 0.1423220932483673, + "union_independent": 0.40230149030685425, + "union_gap": -0.25997939705848694, + "mean_per_layer_hit_rate": 0.038509551435709 + }, + "10": { + "rho_bar": 0.3831363916397095, + "union_observed": 0.20474407076835632, + "union_independent": 0.5718682408332825, + "union_gap": -0.36712417006492615, + "mean_per_layer_hit_rate": 0.062421973794698715 + }, + "20": { + "rho_bar": 0.41977882385253906, + "union_observed": 0.28339576721191406, + "union_independent": 0.742961049079895, + "union_gap": -0.45956528186798096, + "mean_per_layer_hit_rate": 0.09757034480571747 + }, + "50": { + "rho_bar": 0.4344683885574341, + "union_observed": 0.38826465606689453, + "union_independent": 0.8678579330444336, + "union_gap": -0.47959327697753906, + "mean_per_layer_hit_rate": 0.1409776210784912 + }, + "100": { + "rho_bar": 0.4466657340526581, + "union_observed": 0.4781523048877716, + "union_independent": 0.9364191293716431, + "union_gap": -0.45826682448387146, + "mean_per_layer_hit_rate": 0.18572939932346344 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.38009656752858845, + "union_gap_mean_over_k": -0.32017626027975765 + }, + "StackExchange|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.2971181273460388, + "union_observed": 0.06117353215813637, + "union_independent": 0.155403733253479, + "union_gap": -0.09423020109534264, + "mean_per_layer_hit_rate": 0.012868528254330158 + }, + "2": { + "rho_bar": 0.31459876894950867, + "union_observed": 0.10112359374761581, + "union_independent": 0.2541337013244629, + "union_gap": -0.15301010757684708, + "mean_per_layer_hit_rate": 0.02218380756676197 + }, + "5": { + "rho_bar": 0.37010008096694946, + "union_observed": 0.16104869544506073, + "union_independent": 0.45042186975479126, + "union_gap": -0.28937317430973053, + "mean_per_layer_hit_rate": 0.04465571790933609 + }, + "10": { + "rho_bar": 0.4255429804325104, + "union_observed": 0.21348313987255096, + "union_independent": 0.6057800054550171, + "union_gap": -0.3922968655824661, + "mean_per_layer_hit_rate": 0.06828003376722336 + }, + "20": { + "rho_bar": 0.46345391869544983, + "union_observed": 0.28589263558387756, + "union_independent": 0.762767493724823, + "union_gap": -0.47687485814094543, + "mean_per_layer_hit_rate": 0.10304427146911621 + }, + "50": { + "rho_bar": 0.4941646456718445, + "union_observed": 0.37827715277671814, + "union_independent": 0.8798648715019226, + "union_gap": -0.5015877187252045, + "mean_per_layer_hit_rate": 0.1475079208612442 + }, + "100": { + "rho_bar": 0.4854891002178192, + "union_observed": 0.47940075397491455, + "union_independent": 0.944151759147644, + "union_gap": -0.4647510051727295, + "mean_per_layer_hit_rate": 0.1944684386253357 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4072096603257315, + "union_gap_mean_over_k": -0.338874847229038 + }, + "StackExchange|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.24344806373119354, + "union_observed": 0.056179776787757874, + "union_independent": 0.12040978670120239, + "union_gap": -0.06423000991344452, + "mean_per_layer_hit_rate": 0.009795447811484337 + }, + "2": { + "rho_bar": 0.27297747135162354, + "union_observed": 0.09488140046596527, + "union_independent": 0.20642518997192383, + "union_gap": -0.11154378950595856, + "mean_per_layer_hit_rate": 0.017574187368154526 + }, + "5": { + "rho_bar": 0.3565309941768646, + "union_observed": 0.15855181217193604, + "union_independent": 0.3809974193572998, + "union_gap": -0.22244560718536377, + "mean_per_layer_hit_rate": 0.0360126756131649 + }, + "10": { + "rho_bar": 0.3900875151157379, + "union_observed": 0.2222222238779068, + "union_independent": 0.5389639735221863, + "union_gap": -0.3167417496442795, + "mean_per_layer_hit_rate": 0.05723614990711212 + }, + "20": { + "rho_bar": 0.4246011972427368, + "union_observed": 0.29338327050209045, + "union_independent": 0.7030916213989258, + "union_gap": -0.4097083508968353, + "mean_per_layer_hit_rate": 0.08777490258216858 + }, + "50": { + "rho_bar": 0.46578583121299744, + "union_observed": 0.38077402114868164, + "union_independent": 0.8441362977027893, + "union_gap": -0.46336227655410767, + "mean_per_layer_hit_rate": 0.13108614087104797 + }, + "100": { + "rho_bar": 0.4853188097476959, + "union_observed": 0.4619225859642029, + "union_independent": 0.9186679124832153, + "union_gap": -0.45674532651901245, + "mean_per_layer_hit_rate": 0.17209258675575256 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3769642689398357, + "union_gap_mean_over_k": -0.2921110157455717 + }, + "StackExchange|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.23862922191619873, + "union_observed": 0.05742821469902992, + "union_independent": 0.12369340658187866, + "union_gap": -0.06626519188284874, + "mean_per_layer_hit_rate": 0.010083549655973911 + }, + "2": { + "rho_bar": 0.2396833747625351, + "union_observed": 0.08988764137029648, + "union_independent": 0.2004479169845581, + "union_gap": -0.11056027561426163, + "mean_per_layer_hit_rate": 0.016997983679175377 + }, + "5": { + "rho_bar": 0.3179890215396881, + "union_observed": 0.14856429398059845, + "union_independent": 0.3756237030029297, + "union_gap": -0.22705940902233124, + "mean_per_layer_hit_rate": 0.035340435802936554 + }, + "10": { + "rho_bar": 0.3765377998352051, + "union_observed": 0.20099875330924988, + "union_independent": 0.5213824510574341, + "union_gap": -0.3203836977481842, + "mean_per_layer_hit_rate": 0.05454719811677933 + }, + "20": { + "rho_bar": 0.4202304780483246, + "union_observed": 0.2721598148345947, + "union_independent": 0.6959587931632996, + "union_gap": -0.42379897832870483, + "mean_per_layer_hit_rate": 0.08623835444450378 + }, + "50": { + "rho_bar": 0.4289073348045349, + "union_observed": 0.3745318353176117, + "union_independent": 0.837252676486969, + "union_gap": -0.4627208411693573, + "mean_per_layer_hit_rate": 0.12772496044635773 + }, + "100": { + "rho_bar": 0.4426068067550659, + "union_observed": 0.47940075397491455, + "union_independent": 0.9197444319725037, + "union_gap": -0.4403436779975891, + "mean_per_layer_hit_rate": 0.17228463292121887 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.35208343395165037, + "union_gap_mean_over_k": -0.29301886739475386 + }, + "StackExchange|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.23021945357322693, + "union_observed": 0.048689138144254684, + "union_independent": 0.1104886531829834, + "union_gap": -0.061799515038728714, + "mean_per_layer_hit_rate": 0.008931143209338188 + }, + "2": { + "rho_bar": 0.2388092428445816, + "union_observed": 0.09862671792507172, + "union_independent": 0.20411592721939087, + "union_gap": -0.10548920929431915, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.3152945339679718, + "union_observed": 0.16354556381702423, + "union_independent": 0.3769974708557129, + "union_gap": -0.21345190703868866, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.3625980019569397, + "union_observed": 0.21348313987255096, + "union_independent": 0.5268195271492004, + "union_gap": -0.3133363872766495, + "mean_per_layer_hit_rate": 0.055315468460321426 + }, + "20": { + "rho_bar": 0.4187172055244446, + "union_observed": 0.2946317195892334, + "union_independent": 0.7019962072372437, + "union_gap": -0.40736448764801025, + "mean_per_layer_hit_rate": 0.08767886459827423 + }, + "50": { + "rho_bar": 0.44422003626823425, + "union_observed": 0.392009973526001, + "union_independent": 0.8340284824371338, + "union_gap": -0.4420185089111328, + "mean_per_layer_hit_rate": 0.12666858732700348 + }, + "100": { + "rho_bar": 0.44113510847091675, + "union_observed": 0.49438202381134033, + "union_independent": 0.9125799536705017, + "union_gap": -0.4181979298591614, + "mean_per_layer_hit_rate": 0.16719484329223633 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3501419403723308, + "union_gap_mean_over_k": -0.2802368492952415 + }, + "StackExchange|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.19407476484775543, + "union_observed": 0.05118601769208908, + "union_independent": 0.10931980609893799, + "union_gap": -0.05813378840684891, + "mean_per_layer_hit_rate": 0.008835108950734138 + }, + "2": { + "rho_bar": 0.24418681859970093, + "union_observed": 0.09238451719284058, + "union_independent": 0.18759572505950928, + "union_gap": -0.0952112078666687, + "mean_per_layer_hit_rate": 0.01574954390525818 + }, + "5": { + "rho_bar": 0.26468873023986816, + "union_observed": 0.15355804562568665, + "union_independent": 0.3397763967514038, + "union_gap": -0.18621835112571716, + "mean_per_layer_hit_rate": 0.03111494891345501 + }, + "10": { + "rho_bar": 0.338264524936676, + "union_observed": 0.20474407076835632, + "union_independent": 0.489867627620697, + "union_gap": -0.2851235568523407, + "mean_per_layer_hit_rate": 0.049745507538318634 + }, + "20": { + "rho_bar": 0.36299291253089905, + "union_observed": 0.2808988690376282, + "union_independent": 0.6535897254943848, + "union_gap": -0.3726908564567566, + "mean_per_layer_hit_rate": 0.0768270418047905 + }, + "50": { + "rho_bar": 0.3978811800479889, + "union_observed": 0.37078651785850525, + "union_independent": 0.8033444881439209, + "union_gap": -0.43255797028541565, + "mean_per_layer_hit_rate": 0.11495247483253479 + }, + "100": { + "rho_bar": 0.4212612211704254, + "union_observed": 0.46941322088241577, + "union_independent": 0.8966394066810608, + "union_gap": -0.427226185798645, + "mean_per_layer_hit_rate": 0.15605494379997253 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3176214503390448, + "union_gap_mean_over_k": -0.2653088452560561 + }, + "StackExchange|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.21685971319675446, + "union_observed": 0.05742821469902992, + "union_independent": 0.12926644086837769, + "union_gap": -0.07183822616934776, + "mean_per_layer_hit_rate": 0.010563718155026436 + }, + "2": { + "rho_bar": 0.27335578203201294, + "union_observed": 0.08489388227462769, + "union_independent": 0.20149719715118408, + "union_gap": -0.1166033148765564, + "mean_per_layer_hit_rate": 0.017094017937779427 + }, + "5": { + "rho_bar": 0.32966744899749756, + "union_observed": 0.1373283416032791, + "union_independent": 0.37067341804504395, + "union_gap": -0.23334507644176483, + "mean_per_layer_hit_rate": 0.03476423770189285 + }, + "10": { + "rho_bar": 0.34995710849761963, + "union_observed": 0.19725343585014343, + "union_independent": 0.5181773900985718, + "union_gap": -0.32092395424842834, + "mean_per_layer_hit_rate": 0.05406703054904938 + }, + "20": { + "rho_bar": 0.3973495364189148, + "union_observed": 0.2746566832065582, + "union_independent": 0.6895633339881897, + "union_gap": -0.41490665078163147, + "mean_per_layer_hit_rate": 0.08479785174131393 + }, + "50": { + "rho_bar": 0.445075660943985, + "union_observed": 0.36828964948654175, + "union_independent": 0.8348382711410522, + "union_gap": -0.4665486216545105, + "mean_per_layer_hit_rate": 0.12686064839363098 + }, + "100": { + "rho_bar": 0.4746202230453491, + "union_observed": 0.4706616699695587, + "union_independent": 0.9164173007011414, + "union_gap": -0.44575563073158264, + "mean_per_layer_hit_rate": 0.16959570348262787 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3552693533045905, + "union_gap_mean_over_k": -0.2957030678434031 + }, + "StackExchange|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.3948681354522705, + "union_observed": 0.047440700232982635, + "union_independent": 0.14446401596069336, + "union_gap": -0.09702331572771072, + "mean_per_layer_hit_rate": 0.011908192187547684 + }, + "2": { + "rho_bar": 0.40802034735679626, + "union_observed": 0.07865168899297714, + "union_independent": 0.23620277643203735, + "union_gap": -0.1575510874390602, + "mean_per_layer_hit_rate": 0.020455200225114822 + }, + "5": { + "rho_bar": 0.3972659111022949, + "union_observed": 0.14107365906238556, + "union_independent": 0.44629228115081787, + "union_gap": -0.3052186220884323, + "mean_per_layer_hit_rate": 0.04417555034160614 + }, + "10": { + "rho_bar": 0.47428855299949646, + "union_observed": 0.20099875330924988, + "union_independent": 0.6361825466156006, + "union_gap": -0.4351837933063507, + "mean_per_layer_hit_rate": 0.07404205948114395 + }, + "20": { + "rho_bar": 0.5247583985328674, + "union_observed": 0.27590513229370117, + "union_independent": 0.7785323858261108, + "union_gap": -0.5026272535324097, + "mean_per_layer_hit_rate": 0.1076539009809494 + }, + "50": { + "rho_bar": 0.5543363690376282, + "union_observed": 0.34456929564476013, + "union_independent": 0.8869134187698364, + "union_gap": -0.5423441231250763, + "mean_per_layer_hit_rate": 0.15106116235256195 + }, + "100": { + "rho_bar": 0.5590511560440063, + "union_observed": 0.4469413161277771, + "union_independent": 0.9480438232421875, + "union_gap": -0.5011025071144104, + "mean_per_layer_hit_rate": 0.19821375608444214 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.47322698150362286, + "union_gap_mean_over_k": -0.3630072431904929 + }, + "StackExchange|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.352664053440094, + "union_observed": 0.06616728752851486, + "union_independent": 0.18284821510314941, + "union_gap": -0.11668092757463455, + "mean_per_layer_hit_rate": 0.01536540873348713 + }, + "2": { + "rho_bar": 0.3987720310688019, + "union_observed": 0.09363295882940292, + "union_independent": 0.282950758934021, + "union_gap": -0.18931780010461807, + "mean_per_layer_hit_rate": 0.025160858407616615 + }, + "5": { + "rho_bar": 0.42338743805885315, + "union_observed": 0.16104869544506073, + "union_independent": 0.48556017875671387, + "union_gap": -0.32451148331165314, + "mean_per_layer_hit_rate": 0.04945741221308708 + }, + "10": { + "rho_bar": 0.48616674542427063, + "union_observed": 0.2147315889596939, + "union_independent": 0.6388757824897766, + "union_gap": -0.4241441935300827, + "mean_per_layer_hit_rate": 0.0745222419500351 + }, + "20": { + "rho_bar": 0.5125806927680969, + "union_observed": 0.28838950395584106, + "union_independent": 0.7898733615875244, + "union_gap": -0.5014838576316833, + "mean_per_layer_hit_rate": 0.11139921844005585 + }, + "50": { + "rho_bar": 0.5533336997032166, + "union_observed": 0.36828964948654175, + "union_independent": 0.8986788988113403, + "union_gap": -0.5303892493247986, + "mean_per_layer_hit_rate": 0.15874387323856354 + }, + "100": { + "rho_bar": 0.5642926692962646, + "union_observed": 0.45068663358688354, + "union_independent": 0.9522813558578491, + "union_gap": -0.5015947222709656, + "mean_per_layer_hit_rate": 0.20464803278446198 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.470171047108514, + "union_gap_mean_over_k": -0.369731747678348 + }, + "StackExchange|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.3172583281993866, + "union_observed": 0.05867665261030197, + "union_independent": 0.15968328714370728, + "union_gap": -0.1010066345334053, + "mean_per_layer_hit_rate": 0.013252664357423782 + }, + "2": { + "rho_bar": 0.3550403416156769, + "union_observed": 0.09987515956163406, + "union_independent": 0.25868862867355347, + "union_gap": -0.1588134691119194, + "mean_per_layer_hit_rate": 0.02266397885978222 + }, + "5": { + "rho_bar": 0.37814030051231384, + "union_observed": 0.15106117725372314, + "union_independent": 0.43226730823516846, + "union_gap": -0.2812061309814453, + "mean_per_layer_hit_rate": 0.042254868894815445 + }, + "10": { + "rho_bar": 0.40634483098983765, + "union_observed": 0.2247191071510315, + "union_independent": 0.6011097431182861, + "union_gap": -0.37639063596725464, + "mean_per_layer_hit_rate": 0.06741572916507721 + }, + "20": { + "rho_bar": 0.4443076252937317, + "union_observed": 0.2946317195892334, + "union_independent": 0.7512320280075073, + "union_gap": -0.4566003084182739, + "mean_per_layer_hit_rate": 0.09977912902832031 + }, + "50": { + "rho_bar": 0.5020819306373596, + "union_observed": 0.37078651785850525, + "union_independent": 0.873676598072052, + "union_gap": -0.5028900802135468, + "mean_per_layer_hit_rate": 0.14433881640434265 + }, + "100": { + "rho_bar": 0.5368505120277405, + "union_observed": 0.44818976521492004, + "union_independent": 0.9343822002410889, + "union_gap": -0.4861924350261688, + "mean_per_layer_hit_rate": 0.1848650723695755 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.4200034098965781, + "union_gap_mean_over_k": -0.3375856706074306 + }, + "StackExchange|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.2957693934440613, + "union_observed": 0.054931335151195526, + "union_independent": 0.12603658437728882, + "union_gap": -0.07110524922609329, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "2": { + "rho_bar": 0.2945016324520111, + "union_observed": 0.08614232391119003, + "union_independent": 0.199692964553833, + "union_gap": -0.11355064064264297, + "mean_per_layer_hit_rate": 0.016901951283216476 + }, + "5": { + "rho_bar": 0.30243292450904846, + "union_observed": 0.1473158597946167, + "union_independent": 0.3769475817680359, + "union_gap": -0.2296317219734192, + "mean_per_layer_hit_rate": 0.035436470061540604 + }, + "10": { + "rho_bar": 0.33928442001342773, + "union_observed": 0.20474407076835632, + "union_independent": 0.5291379690170288, + "union_gap": -0.3243938982486725, + "mean_per_layer_hit_rate": 0.055507540702819824 + }, + "20": { + "rho_bar": 0.37475213408470154, + "union_observed": 0.2784020006656647, + "union_independent": 0.6972740888595581, + "union_gap": -0.41887208819389343, + "mean_per_layer_hit_rate": 0.08614231646060944 + }, + "50": { + "rho_bar": 0.45949339866638184, + "union_observed": 0.3595505654811859, + "union_independent": 0.8403233289718628, + "union_gap": -0.4807727634906769, + "mean_per_layer_hit_rate": 0.12897339463233948 + }, + "100": { + "rho_bar": 0.48622140288352966, + "union_observed": 0.45568040013313293, + "union_independent": 0.9183626770973206, + "union_gap": -0.4626822769641876, + "mean_per_layer_hit_rate": 0.17122825980186462 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3646364722933088, + "union_gap_mean_over_k": -0.30014409124851227 + }, + "StackExchange|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.26617345213890076, + "union_observed": 0.05368289723992348, + "union_independent": 0.11381405591964722, + "union_gap": -0.06013115867972374, + "mean_per_layer_hit_rate": 0.009219245053827763 + }, + "2": { + "rho_bar": 0.2297319769859314, + "union_observed": 0.08239700645208359, + "union_independent": 0.17889654636383057, + "union_gap": -0.09649953991174698, + "mean_per_layer_hit_rate": 0.01498127169907093 + }, + "5": { + "rho_bar": 0.3006631135940552, + "union_observed": 0.1423220932483673, + "union_independent": 0.3428664803504944, + "union_gap": -0.20054438710212708, + "mean_per_layer_hit_rate": 0.03149908781051636 + }, + "10": { + "rho_bar": 0.3482665419578552, + "union_observed": 0.19475655257701874, + "union_independent": 0.48510611057281494, + "union_gap": -0.2903495579957962, + "mean_per_layer_hit_rate": 0.049169305711984634 + }, + "20": { + "rho_bar": 0.3769039511680603, + "union_observed": 0.2721598148345947, + "union_independent": 0.6510589122772217, + "union_gap": -0.37889909744262695, + "mean_per_layer_hit_rate": 0.07653893530368805 + }, + "50": { + "rho_bar": 0.43755683302879333, + "union_observed": 0.3545567989349365, + "union_independent": 0.808556079864502, + "union_gap": -0.45399928092956543, + "mean_per_layer_hit_rate": 0.1170651987195015 + }, + "100": { + "rho_bar": 0.4933265149593353, + "union_observed": 0.43320849537849426, + "union_independent": 0.898047149181366, + "union_gap": -0.4648386538028717, + "mean_per_layer_hit_rate": 0.15797559916973114 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.35037462626184734, + "union_gap_mean_over_k": -0.277894525123494 + }, + "StackExchange|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.18422456085681915, + "union_observed": 0.05368289723992348, + "union_independent": 0.1001550555229187, + "union_gap": -0.046472158282995224, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.21530292928218842, + "union_observed": 0.07865168899297714, + "union_independent": 0.1703857183456421, + "union_gap": -0.09173402935266495, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.27692362666130066, + "union_observed": 0.13483145833015442, + "union_independent": 0.31630247831344604, + "union_gap": -0.18147101998329163, + "mean_per_layer_hit_rate": 0.02861807495355606 + }, + "10": { + "rho_bar": 0.3249002993106842, + "union_observed": 0.18227216601371765, + "union_independent": 0.4643040895462036, + "union_gap": -0.28203192353248596, + "mean_per_layer_hit_rate": 0.04638432711362839 + }, + "20": { + "rho_bar": 0.36046919226646423, + "union_observed": 0.25468164682388306, + "union_independent": 0.6166542768478394, + "union_gap": -0.3619726300239563, + "mean_per_layer_hit_rate": 0.0701046735048294 + }, + "50": { + "rho_bar": 0.40135324001312256, + "union_observed": 0.3433208465576172, + "union_independent": 0.7705005407333374, + "union_gap": -0.4271796941757202, + "mean_per_layer_hit_rate": 0.10534907877445221 + }, + "100": { + "rho_bar": 0.45482420921325684, + "union_observed": 0.4319600462913513, + "union_independent": 0.8741205930709839, + "union_gap": -0.44216054677963257, + "mean_per_layer_hit_rate": 0.1450110375881195 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3168568653719766, + "union_gap_mean_over_k": -0.2618602860186781 + }, + "StackExchange|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.1907837688922882, + "union_observed": 0.04619225859642029, + "union_independent": 0.09889912605285645, + "union_gap": -0.05270686745643616, + "mean_per_layer_hit_rate": 0.007970806211233139 + }, + "2": { + "rho_bar": 0.237838476896286, + "union_observed": 0.062421973794698715, + "union_independent": 0.15411722660064697, + "union_gap": -0.09169525280594826, + "mean_per_layer_hit_rate": 0.012772495858371258 + }, + "5": { + "rho_bar": 0.29374468326568604, + "union_observed": 0.1086142286658287, + "union_independent": 0.3029922842979431, + "union_gap": -0.1943780556321144, + "mean_per_layer_hit_rate": 0.027273599058389664 + }, + "10": { + "rho_bar": 0.34467774629592896, + "union_observed": 0.15355804562568665, + "union_independent": 0.44707560539245605, + "union_gap": -0.2935175597667694, + "mean_per_layer_hit_rate": 0.04427158460021019 + }, + "20": { + "rho_bar": 0.38220420479774475, + "union_observed": 0.20099875330924988, + "union_independent": 0.5882887840270996, + "union_gap": -0.38729003071784973, + "mean_per_layer_hit_rate": 0.06539902091026306 + }, + "50": { + "rho_bar": 0.4202321469783783, + "union_observed": 0.2846441864967346, + "union_independent": 0.746029794216156, + "union_gap": -0.4613856077194214, + "mean_per_layer_hit_rate": 0.09910687804222107 + }, + "100": { + "rho_bar": 0.4495912194252014, + "union_observed": 0.3745318353176117, + "union_independent": 0.8535999059677124, + "union_gap": -0.4790680706501007, + "mean_per_layer_hit_rate": 0.13598385453224182 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3312960352216448, + "union_gap_mean_over_k": -0.28000592067837715 + }, + "StackExchange|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.17130455374717712, + "union_observed": 0.0387016236782074, + "union_independent": 0.07708698511123657, + "union_gap": -0.038385361433029175, + "mean_per_layer_hit_rate": 0.006146163213998079 + }, + "2": { + "rho_bar": 0.2392464131116867, + "union_observed": 0.06117353215813637, + "union_independent": 0.13682270050048828, + "union_gap": -0.07564916834235191, + "mean_per_layer_hit_rate": 0.011235954239964485 + }, + "5": { + "rho_bar": 0.3122366964817047, + "union_observed": 0.1086142286658287, + "union_independent": 0.29112404584884644, + "union_gap": -0.18250981718301773, + "mean_per_layer_hit_rate": 0.026025159284472466 + }, + "10": { + "rho_bar": 0.3462171256542206, + "union_observed": 0.15855181217193604, + "union_independent": 0.4350557327270508, + "union_gap": -0.27650392055511475, + "mean_per_layer_hit_rate": 0.04273504391312599 + }, + "20": { + "rho_bar": 0.39076635241508484, + "union_observed": 0.20474407076835632, + "union_independent": 0.5864604711532593, + "union_gap": -0.38171640038490295, + "mean_per_layer_hit_rate": 0.06511091440916061 + }, + "50": { + "rho_bar": 0.43343695998191833, + "union_observed": 0.2971285879611969, + "union_independent": 0.7474977970123291, + "union_gap": -0.4503692090511322, + "mean_per_layer_hit_rate": 0.09949101507663727 + }, + "100": { + "rho_bar": 0.45106232166290283, + "union_observed": 0.3907615542411804, + "union_independent": 0.86121666431427, + "union_gap": -0.4704551100730896, + "mean_per_layer_hit_rate": 0.1392490267753601 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3348957747220993, + "union_gap_mean_over_k": -0.2679412838603769 + }, + "StackExchange|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.17863307893276215, + "union_observed": 0.03745318204164505, + "union_independent": 0.08747673034667969, + "union_gap": -0.05002354830503464, + "mean_per_layer_hit_rate": 0.007010467816144228 + }, + "2": { + "rho_bar": 0.2470502108335495, + "union_observed": 0.06117353215813637, + "union_independent": 0.157392680644989, + "union_gap": -0.09621914848685265, + "mean_per_layer_hit_rate": 0.013060597702860832 + }, + "5": { + "rho_bar": 0.3244347870349884, + "union_observed": 0.10986267030239105, + "union_independent": 0.3085510730743408, + "union_gap": -0.19868840277194977, + "mean_per_layer_hit_rate": 0.027849802747368813 + }, + "10": { + "rho_bar": 0.36776384711265564, + "union_observed": 0.15980024635791779, + "union_independent": 0.45757871866226196, + "union_gap": -0.2977784723043442, + "mean_per_layer_hit_rate": 0.04561605677008629 + }, + "20": { + "rho_bar": 0.4168292284011841, + "union_observed": 0.20973782241344452, + "union_independent": 0.6191911697387695, + "union_gap": -0.409453347325325, + "mean_per_layer_hit_rate": 0.07077690958976746 + }, + "50": { + "rho_bar": 0.43693244457244873, + "union_observed": 0.3096129894256592, + "union_independent": 0.7694293856620789, + "union_gap": -0.4598163962364197, + "mean_per_layer_hit_rate": 0.10525304824113846 + }, + "100": { + "rho_bar": 0.4404471814632416, + "union_observed": 0.4007490575313568, + "union_independent": 0.875225841999054, + "union_gap": -0.47447678446769714, + "mean_per_layer_hit_rate": 0.145203098654747 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.34458439690726145, + "union_gap_mean_over_k": -0.2837794428425176 + }, + "StackExchange|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.20903366804122925, + "union_observed": 0.04244694113731384, + "union_independent": 0.09214866161346436, + "union_gap": -0.04970172047615051, + "mean_per_layer_hit_rate": 0.007394603453576565 + }, + "2": { + "rho_bar": 0.2719302177429199, + "union_observed": 0.06491885334253311, + "union_independent": 0.15755373239517212, + "union_gap": -0.09263487905263901, + "mean_per_layer_hit_rate": 0.013060597702860832 + }, + "5": { + "rho_bar": 0.27837175130844116, + "union_observed": 0.11610487103462219, + "union_independent": 0.3160296082496643, + "union_gap": -0.19992473721504211, + "mean_per_layer_hit_rate": 0.02861807495355606 + }, + "10": { + "rho_bar": 0.36121341586112976, + "union_observed": 0.16853933036327362, + "union_independent": 0.46881216764450073, + "union_gap": -0.3002728372812271, + "mean_per_layer_hit_rate": 0.04705656319856644 + }, + "20": { + "rho_bar": 0.40913519263267517, + "union_observed": 0.23096129298210144, + "union_independent": 0.6383624076843262, + "union_gap": -0.40740111470222473, + "mean_per_layer_hit_rate": 0.0743301659822464 + }, + "50": { + "rho_bar": 0.4377559721469879, + "union_observed": 0.323345810174942, + "union_independent": 0.7926860451698303, + "union_gap": -0.4693402349948883, + "mean_per_layer_hit_rate": 0.11178334802389145 + }, + "100": { + "rho_bar": 0.4171309769153595, + "union_observed": 0.4294631779193878, + "union_independent": 0.8899627923965454, + "union_gap": -0.4604996144771576, + "mean_per_layer_hit_rate": 0.1523096263408661 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3406530278069632, + "union_gap_mean_over_k": -0.28282501974276136 + }, + "StackExchange|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.22476191818714142, + "union_observed": 0.047440700232982635, + "union_independent": 0.10924601554870605, + "union_gap": -0.06180531531572342, + "mean_per_layer_hit_rate": 0.008835110813379288 + }, + "2": { + "rho_bar": 0.2685675323009491, + "union_observed": 0.06491885334253311, + "union_independent": 0.17890828847885132, + "union_gap": -0.1139894351363182, + "mean_per_layer_hit_rate": 0.014981274493038654 + }, + "5": { + "rho_bar": 0.3017650246620178, + "union_observed": 0.12609238922595978, + "union_independent": 0.34239280223846436, + "union_gap": -0.21630041301250458, + "mean_per_layer_hit_rate": 0.03149908781051636 + }, + "10": { + "rho_bar": 0.36441460251808167, + "union_observed": 0.1760299652814865, + "union_independent": 0.4926631450653076, + "union_gap": -0.3166331797838211, + "mean_per_layer_hit_rate": 0.05032171308994293 + }, + "20": { + "rho_bar": 0.3916185200214386, + "union_observed": 0.25218477845191956, + "union_independent": 0.6618623733520508, + "union_gap": -0.4096775949001312, + "mean_per_layer_hit_rate": 0.07865168154239655 + }, + "50": { + "rho_bar": 0.40136274695396423, + "union_observed": 0.35830211639404297, + "union_independent": 0.8168940544128418, + "union_gap": -0.45859193801879883, + "mean_per_layer_hit_rate": 0.11956209689378738 + }, + "100": { + "rho_bar": 0.40414339303970337, + "union_observed": 0.44319599866867065, + "union_independent": 0.9025807976722717, + "union_gap": -0.4593847990036011, + "mean_per_layer_hit_rate": 0.15941612422466278 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3366619625261852, + "union_gap_mean_over_k": -0.29091181073869976 + }, + "StackExchange|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.27487891912460327, + "union_observed": 0.04993757978081703, + "union_independent": 0.12163335084915161, + "union_gap": -0.07169577106833458, + "mean_per_layer_hit_rate": 0.009891481138765812 + }, + "2": { + "rho_bar": 0.28408992290496826, + "union_observed": 0.07990012317895889, + "union_independent": 0.19744926691055298, + "union_gap": -0.11754914373159409, + "mean_per_layer_hit_rate": 0.016709880903363228 + }, + "5": { + "rho_bar": 0.324739009141922, + "union_observed": 0.14107365906238556, + "union_independent": 0.3720462918281555, + "union_gap": -0.23097263276576996, + "mean_per_layer_hit_rate": 0.0348602719604969 + }, + "10": { + "rho_bar": 0.34188714623451233, + "union_observed": 0.19475655257701874, + "union_independent": 0.5263760089874268, + "union_gap": -0.331619456410408, + "mean_per_layer_hit_rate": 0.055219437927007675 + }, + "20": { + "rho_bar": 0.3781401813030243, + "union_observed": 0.2709113657474518, + "union_independent": 0.6961506605148315, + "union_gap": -0.42523929476737976, + "mean_per_layer_hit_rate": 0.08595025539398193 + }, + "50": { + "rho_bar": 0.39775949716567993, + "union_observed": 0.36329588294029236, + "union_independent": 0.836713969707489, + "union_gap": -0.47341808676719666, + "mean_per_layer_hit_rate": 0.12695668637752533 + }, + "100": { + "rho_bar": 0.40577855706214905, + "union_observed": 0.4619225859642029, + "union_independent": 0.9187095165252686, + "union_gap": -0.4567869305610657, + "mean_per_layer_hit_rate": 0.17007586359977722 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.343896176133837, + "union_gap_mean_over_k": -0.30104018801024984 + }, + "StackExchange|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.29589852690696716, + "union_observed": 0.05118601769208908, + "union_independent": 0.13370829820632935, + "union_gap": -0.08252228051424026, + "mean_per_layer_hit_rate": 0.010947853326797485 + }, + "2": { + "rho_bar": 0.30318984389305115, + "union_observed": 0.08489388227462769, + "union_independent": 0.2137662172317505, + "union_gap": -0.1288723349571228, + "mean_per_layer_hit_rate": 0.018246423453092575 + }, + "5": { + "rho_bar": 0.35535916686058044, + "union_observed": 0.14107365906238556, + "union_independent": 0.3935837149620056, + "union_gap": -0.25251005589962006, + "mean_per_layer_hit_rate": 0.03745318204164505 + }, + "10": { + "rho_bar": 0.3919140100479126, + "union_observed": 0.20474407076835632, + "union_independent": 0.5707873106002808, + "union_gap": -0.36604323983192444, + "mean_per_layer_hit_rate": 0.062229905277490616 + }, + "20": { + "rho_bar": 0.4107598066329956, + "union_observed": 0.2846441864967346, + "union_independent": 0.7410097122192383, + "union_gap": -0.45636552572250366, + "mean_per_layer_hit_rate": 0.09699413180351257 + }, + "50": { + "rho_bar": 0.432545930147171, + "union_observed": 0.3982521891593933, + "union_independent": 0.8668558597564697, + "union_gap": -0.4686036705970764, + "mean_per_layer_hit_rate": 0.14030538499355316 + }, + "100": { + "rho_bar": 0.44394752383232117, + "union_observed": 0.4806492030620575, + "union_independent": 0.9358465671539307, + "union_gap": -0.45519736409187317, + "mean_per_layer_hit_rate": 0.1852492243051529 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3762306869029999, + "union_gap_mean_over_k": -0.31573063880205154 + }, + "StackExchange|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.28974905610084534, + "union_observed": 0.056179776787757874, + "union_independent": 0.15113401412963867, + "union_gap": -0.0949542373418808, + "mean_per_layer_hit_rate": 0.012484394945204258 + }, + "2": { + "rho_bar": 0.32911738753318787, + "union_observed": 0.09488140046596527, + "union_independent": 0.24827229976654053, + "union_gap": -0.15339089930057526, + "mean_per_layer_hit_rate": 0.02160760574042797 + }, + "5": { + "rho_bar": 0.3571892976760864, + "union_observed": 0.16104869544506073, + "union_independent": 0.446927547454834, + "union_gap": -0.28587885200977325, + "mean_per_layer_hit_rate": 0.04417555034160614 + }, + "10": { + "rho_bar": 0.416979044675827, + "union_observed": 0.22596754133701324, + "union_independent": 0.6134454011917114, + "union_gap": -0.3874778598546982, + "mean_per_layer_hit_rate": 0.06962450593709946 + }, + "20": { + "rho_bar": 0.46539217233657837, + "union_observed": 0.2846441864967346, + "union_independent": 0.7598610520362854, + "union_gap": -0.4752168655395508, + "mean_per_layer_hit_rate": 0.10217996686697006 + }, + "50": { + "rho_bar": 0.48503440618515015, + "union_observed": 0.38576778769493103, + "union_independent": 0.8780269622802734, + "union_gap": -0.4922591745853424, + "mean_per_layer_hit_rate": 0.14645154774188995 + }, + "100": { + "rho_bar": 0.4857919216156006, + "union_observed": 0.4731585383415222, + "union_independent": 0.9428827166557312, + "union_gap": -0.469724178314209, + "mean_per_layer_hit_rate": 0.1929319053888321 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4041790408747537, + "union_gap_mean_over_k": -0.33698600956371855 + }, + "StackExchange|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.2366645485162735, + "union_observed": 0.05742821469902992, + "union_independent": 0.12372636795043945, + "union_gap": -0.06629815325140953, + "mean_per_layer_hit_rate": 0.010083549655973911 + }, + "2": { + "rho_bar": 0.26594728231430054, + "union_observed": 0.09488140046596527, + "union_independent": 0.20634448528289795, + "union_gap": -0.11146308481693268, + "mean_per_layer_hit_rate": 0.017574187368154526 + }, + "5": { + "rho_bar": 0.33729153871536255, + "union_observed": 0.15605492889881134, + "union_independent": 0.3745243549346924, + "union_gap": -0.21846942603588104, + "mean_per_layer_hit_rate": 0.0352444052696228 + }, + "10": { + "rho_bar": 0.38636550307273865, + "union_observed": 0.22097378969192505, + "union_independent": 0.5389389991760254, + "union_gap": -0.31796520948410034, + "mean_per_layer_hit_rate": 0.05723614990711212 + }, + "20": { + "rho_bar": 0.4235544800758362, + "union_observed": 0.2921348214149475, + "union_independent": 0.7015844583511353, + "union_gap": -0.40944963693618774, + "mean_per_layer_hit_rate": 0.08739076554775238 + }, + "50": { + "rho_bar": 0.4607379734516144, + "union_observed": 0.38077402114868164, + "union_independent": 0.8399821519851685, + "union_gap": -0.4592081308364868, + "mean_per_layer_hit_rate": 0.12935754656791687 + }, + "100": { + "rho_bar": 0.4797922670841217, + "union_observed": 0.4606741666793823, + "union_independent": 0.9151566624641418, + "union_gap": -0.4544824957847595, + "mean_per_layer_hit_rate": 0.16930760443210602 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3700505133186068, + "union_gap_mean_over_k": -0.2910480195922511 + }, + "StackExchange|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.22894138097763062, + "union_observed": 0.056179776787757874, + "union_independent": 0.12149602174758911, + "union_gap": -0.06531624495983124, + "mean_per_layer_hit_rate": 0.009891481138765812 + }, + "2": { + "rho_bar": 0.23666617274284363, + "union_observed": 0.09113608300685883, + "union_independent": 0.2035241723060608, + "union_gap": -0.11238808929920197, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.3148215115070343, + "union_observed": 0.1498127281665802, + "union_independent": 0.374769389629364, + "union_gap": -0.2249566614627838, + "mean_per_layer_hit_rate": 0.0352444052696228 + }, + "10": { + "rho_bar": 0.3824106752872467, + "union_observed": 0.20224718749523163, + "union_independent": 0.5263065695762634, + "union_gap": -0.3240593820810318, + "mean_per_layer_hit_rate": 0.055315472185611725 + }, + "20": { + "rho_bar": 0.42507049441337585, + "union_observed": 0.26966291666030884, + "union_independent": 0.6956567764282227, + "union_gap": -0.4259938597679138, + "mean_per_layer_hit_rate": 0.08614231646060944 + }, + "50": { + "rho_bar": 0.42544838786125183, + "union_observed": 0.3770287036895752, + "union_independent": 0.8355162143707275, + "union_gap": -0.45848751068115234, + "mean_per_layer_hit_rate": 0.12686064839363098 + }, + "100": { + "rho_bar": 0.4389433264732361, + "union_observed": 0.4781523048877716, + "union_independent": 0.9189306497573853, + "union_gap": -0.44077834486961365, + "mean_per_layer_hit_rate": 0.17161241173744202 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.35032884989465984, + "union_gap_mean_over_k": -0.29314001330307554 + }, + "StackExchange|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.2276536077260971, + "union_observed": 0.047440700232982635, + "union_independent": 0.10707759857177734, + "union_gap": -0.05963689833879471, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.24189934134483337, + "union_observed": 0.09612983465194702, + "union_independent": 0.20095562934875488, + "union_gap": -0.10482579469680786, + "mean_per_layer_hit_rate": 0.016997983679175377 + }, + "5": { + "rho_bar": 0.30584055185317993, + "union_observed": 0.16354556381702423, + "union_independent": 0.377034068107605, + "union_gap": -0.21348850429058075, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.361429899930954, + "union_observed": 0.2197253406047821, + "union_independent": 0.5308027267456055, + "union_gap": -0.31107738614082336, + "mean_per_layer_hit_rate": 0.055891674011945724 + }, + "20": { + "rho_bar": 0.40871185064315796, + "union_observed": 0.2921348214149475, + "union_independent": 0.6992663145065308, + "union_gap": -0.40713149309158325, + "mean_per_layer_hit_rate": 0.08700662106275558 + }, + "50": { + "rho_bar": 0.4375845193862915, + "union_observed": 0.392009973526001, + "union_independent": 0.8317450881004333, + "union_gap": -0.4397351145744324, + "mean_per_layer_hit_rate": 0.12570825219154358 + }, + "100": { + "rho_bar": 0.4399273097515106, + "union_observed": 0.48938825726509094, + "union_independent": 0.9121547341346741, + "union_gap": -0.42276647686958313, + "mean_per_layer_hit_rate": 0.16681072115898132 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3461495829480035, + "union_gap_mean_over_k": -0.27980880971465794 + }, + "StackExchange|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.1960163712501526, + "union_observed": 0.05118601769208908, + "union_independent": 0.11044734716415405, + "union_gap": -0.05926132947206497, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.24837715923786163, + "union_observed": 0.08988764137029648, + "union_independent": 0.18335318565368652, + "union_gap": -0.09346554428339005, + "mean_per_layer_hit_rate": 0.015365407802164555 + }, + "5": { + "rho_bar": 0.2502641975879669, + "union_observed": 0.15855181217193604, + "union_independent": 0.34153085947036743, + "union_gap": -0.1829790472984314, + "mean_per_layer_hit_rate": 0.03130701929330826 + }, + "10": { + "rho_bar": 0.3371860682964325, + "union_observed": 0.20973782241344452, + "union_independent": 0.49674785137176514, + "union_gap": -0.2870100289583206, + "mean_per_layer_hit_rate": 0.05070584639906883 + }, + "20": { + "rho_bar": 0.35825783014297485, + "union_observed": 0.2784020006656647, + "union_independent": 0.6541430950164795, + "union_gap": -0.3757410943508148, + "mean_per_layer_hit_rate": 0.07692307978868484 + }, + "50": { + "rho_bar": 0.3976080119609833, + "union_observed": 0.3745318353176117, + "union_independent": 0.8032761812210083, + "union_gap": -0.4287443459033966, + "mean_per_layer_hit_rate": 0.114952452480793 + }, + "100": { + "rho_bar": 0.42111581563949585, + "union_observed": 0.4681648015975952, + "union_independent": 0.8954002261161804, + "union_gap": -0.4272354245185852, + "mean_per_layer_hit_rate": 0.15528663992881775 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3155464934451239, + "union_gap_mean_over_k": -0.2649195449692862 + }, + "StackExchange|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.19730877876281738, + "union_observed": 0.05742821469902992, + "union_independent": 0.12595397233963013, + "union_gap": -0.0685257576406002, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "2": { + "rho_bar": 0.27439284324645996, + "union_observed": 0.08739075809717178, + "union_independent": 0.20252710580825806, + "union_gap": -0.11513634771108627, + "mean_per_layer_hit_rate": 0.017190050333738327 + }, + "5": { + "rho_bar": 0.33092355728149414, + "union_observed": 0.13483145833015442, + "union_independent": 0.3681543469429016, + "union_gap": -0.2333228886127472, + "mean_per_layer_hit_rate": 0.034476134926080704 + }, + "10": { + "rho_bar": 0.35620179772377014, + "union_observed": 0.19475655257701874, + "union_independent": 0.5173876881599426, + "union_gap": -0.3226311355829239, + "mean_per_layer_hit_rate": 0.05397099629044533 + }, + "20": { + "rho_bar": 0.38854268193244934, + "union_observed": 0.2746566832065582, + "union_independent": 0.6888563632965088, + "union_gap": -0.41419968008995056, + "mean_per_layer_hit_rate": 0.08460578322410583 + }, + "50": { + "rho_bar": 0.4426553249359131, + "union_observed": 0.3670412003993988, + "union_independent": 0.8334391713142395, + "union_gap": -0.4663979709148407, + "mean_per_layer_hit_rate": 0.12628445029258728 + }, + "100": { + "rho_bar": 0.47169092297554016, + "union_observed": 0.4731585383415222, + "union_independent": 0.9148144721984863, + "union_gap": -0.4416559338569641, + "mean_per_layer_hit_rate": 0.16853931546211243 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.35167370097977774, + "union_gap_mean_over_k": -0.294552816344159 + }, + "StackExchange|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.38984543085098267, + "union_observed": 0.047440700232982635, + "union_independent": 0.1433921456336975, + "union_gap": -0.09595144540071487, + "mean_per_layer_hit_rate": 0.01181215699762106 + }, + "2": { + "rho_bar": 0.40811386704444885, + "union_observed": 0.0774032473564148, + "union_independent": 0.23327624797821045, + "union_gap": -0.15587300062179565, + "mean_per_layer_hit_rate": 0.020167099311947823 + }, + "5": { + "rho_bar": 0.4103420078754425, + "union_observed": 0.13483145833015442, + "union_independent": 0.4417893886566162, + "union_gap": -0.3069579303264618, + "mean_per_layer_hit_rate": 0.04359934851527214 + }, + "10": { + "rho_bar": 0.469199001789093, + "union_observed": 0.20099875330924988, + "union_independent": 0.6376586556434631, + "union_gap": -0.43665990233421326, + "mean_per_layer_hit_rate": 0.0743301659822464 + }, + "20": { + "rho_bar": 0.5235543847084045, + "union_observed": 0.2734082341194153, + "union_independent": 0.7806446552276611, + "union_gap": -0.5072364211082458, + "mean_per_layer_hit_rate": 0.1082300990819931 + }, + "50": { + "rho_bar": 0.5541795492172241, + "union_observed": 0.34706616401672363, + "union_independent": 0.886867880821228, + "union_gap": -0.5398017168045044, + "mean_per_layer_hit_rate": 0.1509651392698288 + }, + "100": { + "rho_bar": 0.5603076219558716, + "union_observed": 0.44569289684295654, + "union_independent": 0.9480534791946411, + "union_gap": -0.5023605823516846, + "mean_per_layer_hit_rate": 0.19821375608444214 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.47364883763449533, + "union_gap_mean_over_k": -0.3635487141353743 + }, + "Github|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.2796321213245392, + "union_observed": 0.04993757978081703, + "union_independent": 0.13239920139312744, + "union_gap": -0.08246162161231041, + "mean_per_layer_hit_rate": 0.01085181999951601 + }, + "2": { + "rho_bar": 0.39370712637901306, + "union_observed": 0.0774032473564148, + "union_independent": 0.23583632707595825, + "union_gap": -0.15843307971954346, + "mean_per_layer_hit_rate": 0.020455200225114822 + }, + "5": { + "rho_bar": 0.399393230676651, + "union_observed": 0.1373283416032791, + "union_independent": 0.4340370297431946, + "union_gap": -0.29670868813991547, + "mean_per_layer_hit_rate": 0.04273504391312599 + }, + "10": { + "rho_bar": 0.44654974341392517, + "union_observed": 0.1910112351179123, + "union_independent": 0.592507004737854, + "union_gap": -0.4014957696199417, + "mean_per_layer_hit_rate": 0.06645539402961731 + }, + "20": { + "rho_bar": 0.5138513445854187, + "union_observed": 0.23470661044120789, + "union_independent": 0.7394143342971802, + "union_gap": -0.5047077238559723, + "mean_per_layer_hit_rate": 0.09766637533903122 + }, + "50": { + "rho_bar": 0.5401190519332886, + "union_observed": 0.3096129894256592, + "union_independent": 0.847254753112793, + "union_gap": -0.5376417636871338, + "mean_per_layer_hit_rate": 0.13348700106143951 + }, + "100": { + "rho_bar": 0.5542593598365784, + "union_observed": 0.37328338623046875, + "union_independent": 0.9173719882965088, + "union_gap": -0.54408860206604, + "mean_per_layer_hit_rate": 0.17276480793952942 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4467874254499163, + "union_gap_mean_over_k": -0.3607910355286939 + }, + "Github|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.3003944754600525, + "union_observed": 0.054931335151195526, + "union_independent": 0.1444597840309143, + "union_gap": -0.08952844887971878, + "mean_per_layer_hit_rate": 0.011908192187547684 + }, + "2": { + "rho_bar": 0.35033097863197327, + "union_observed": 0.08114856481552124, + "union_independent": 0.2192482352256775, + "union_gap": -0.13809967041015625, + "mean_per_layer_hit_rate": 0.018822625279426575 + }, + "5": { + "rho_bar": 0.3648640811443329, + "union_observed": 0.1523096114397049, + "union_independent": 0.42625194787979126, + "union_gap": -0.27394233644008636, + "mean_per_layer_hit_rate": 0.041678670793771744 + }, + "10": { + "rho_bar": 0.42988601326942444, + "union_observed": 0.20599250495433807, + "union_independent": 0.5770877599716187, + "union_gap": -0.3710952550172806, + "mean_per_layer_hit_rate": 0.06367041170597076 + }, + "20": { + "rho_bar": 0.46392351388931274, + "union_observed": 0.25468164682388306, + "union_independent": 0.7062498331069946, + "union_gap": -0.4515681862831116, + "mean_per_layer_hit_rate": 0.08911936730146408 + }, + "50": { + "rho_bar": 0.5278127193450928, + "union_observed": 0.3208489418029785, + "union_independent": 0.8314030170440674, + "union_gap": -0.5105540752410889, + "mean_per_layer_hit_rate": 0.12666860222816467 + }, + "100": { + "rho_bar": 0.5513759255409241, + "union_observed": 0.39700374007225037, + "union_independent": 0.9073653221130371, + "union_gap": -0.5103615820407867, + "mean_per_layer_hit_rate": 0.16517814993858337 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.426941101040159, + "union_gap_mean_over_k": -0.335021364901747 + }, + "Github|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.23279476165771484, + "union_observed": 0.054931335151195526, + "union_independent": 0.1313897967338562, + "union_gap": -0.07645846158266068, + "mean_per_layer_hit_rate": 0.01075578574091196 + }, + "2": { + "rho_bar": 0.3315160274505615, + "union_observed": 0.08239700645208359, + "union_independent": 0.22216182947158813, + "union_gap": -0.13976482301950455, + "mean_per_layer_hit_rate": 0.019110726192593575 + }, + "5": { + "rho_bar": 0.35839638113975525, + "union_observed": 0.14856429398059845, + "union_independent": 0.4024978280067444, + "union_gap": -0.25393353402614594, + "mean_per_layer_hit_rate": 0.0387016199529171 + }, + "10": { + "rho_bar": 0.408954381942749, + "union_observed": 0.19725343585014343, + "union_independent": 0.5376461744308472, + "union_gap": -0.34039273858070374, + "mean_per_layer_hit_rate": 0.05723614990711212 + }, + "20": { + "rho_bar": 0.4357724189758301, + "union_observed": 0.25468164682388306, + "union_independent": 0.676672101020813, + "union_gap": -0.42199045419692993, + "mean_per_layer_hit_rate": 0.08239699900150299 + }, + "50": { + "rho_bar": 0.4804283678531647, + "union_observed": 0.3283395767211914, + "union_independent": 0.8033404350280762, + "union_gap": -0.47500085830688477, + "mean_per_layer_hit_rate": 0.11591280251741409 + }, + "100": { + "rho_bar": 0.5141588449478149, + "union_observed": 0.38826465606689453, + "union_independent": 0.8849314451217651, + "union_gap": -0.4966667890548706, + "mean_per_layer_hit_rate": 0.15086911618709564 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3945744548525129, + "union_gap_mean_over_k": -0.31488680839538574 + }, + "Github|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.20622891187667847, + "union_observed": 0.047440700232982635, + "union_independent": 0.11484098434448242, + "union_gap": -0.06740028411149979, + "mean_per_layer_hit_rate": 0.009315278381109238 + }, + "2": { + "rho_bar": 0.2829740345478058, + "union_observed": 0.06616728752851486, + "union_independent": 0.17855632305145264, + "union_gap": -0.11238903552293777, + "mean_per_layer_hit_rate": 0.014981274493038654 + }, + "5": { + "rho_bar": 0.37225088477134705, + "union_observed": 0.1373283416032791, + "union_independent": 0.368807852268219, + "union_gap": -0.23147951066493988, + "mean_per_layer_hit_rate": 0.0346682034432888 + }, + "10": { + "rho_bar": 0.4055638909339905, + "union_observed": 0.18476903438568115, + "union_independent": 0.5083242654800415, + "union_gap": -0.32355523109436035, + "mean_per_layer_hit_rate": 0.05281858891248703 + }, + "20": { + "rho_bar": 0.4580589830875397, + "union_observed": 0.23220974206924438, + "union_independent": 0.6431164145469666, + "union_gap": -0.41090667247772217, + "mean_per_layer_hit_rate": 0.0754825696349144 + }, + "50": { + "rho_bar": 0.5029197335243225, + "union_observed": 0.31460675597190857, + "union_independent": 0.7831693291664124, + "union_gap": -0.4685625731945038, + "mean_per_layer_hit_rate": 0.10947852581739426 + }, + "100": { + "rho_bar": 0.525895357131958, + "union_observed": 0.39700374007225037, + "union_independent": 0.8775596022605896, + "union_gap": -0.48055586218833923, + "mean_per_layer_hit_rate": 0.14721980690956116 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.39341311369623455, + "union_gap_mean_over_k": -0.299264167036329 + }, + "Github|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.20947225391864777, + "union_observed": 0.05118601769208908, + "union_independent": 0.10692775249481201, + "union_gap": -0.05574173480272293, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.269273579120636, + "union_observed": 0.07615480571985245, + "union_independent": 0.17953252792358398, + "union_gap": -0.10337772220373154, + "mean_per_layer_hit_rate": 0.01507730782032013 + }, + "5": { + "rho_bar": 0.35537680983543396, + "union_observed": 0.13108614087104797, + "union_independent": 0.3448767066001892, + "union_gap": -0.21379056572914124, + "mean_per_layer_hit_rate": 0.031883224844932556 + }, + "10": { + "rho_bar": 0.4142688810825348, + "union_observed": 0.17727839946746826, + "union_independent": 0.4775083065032959, + "union_gap": -0.30022990703582764, + "mean_per_layer_hit_rate": 0.048401035368442535 + }, + "20": { + "rho_bar": 0.4734277129173279, + "union_observed": 0.22347065806388855, + "union_independent": 0.621817946434021, + "union_gap": -0.39834728837013245, + "mean_per_layer_hit_rate": 0.0714491531252861 + }, + "50": { + "rho_bar": 0.5053053498268127, + "union_observed": 0.2996254563331604, + "union_independent": 0.762528657913208, + "union_gap": -0.4629032015800476, + "mean_per_layer_hit_rate": 0.1036204844713211 + }, + "100": { + "rho_bar": 0.5307185053825378, + "union_observed": 0.36828964948654175, + "union_independent": 0.8547587394714355, + "union_gap": -0.4864690899848938, + "mean_per_layer_hit_rate": 0.13646402955055237 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3939775845834187, + "union_gap_mean_over_k": -0.28869421567235676 + }, + "Github|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.1833609789609909, + "union_observed": 0.048689138144254684, + "union_independent": 0.10801494121551514, + "union_gap": -0.05932580307126045, + "mean_per_layer_hit_rate": 0.008739076554775238 + }, + "2": { + "rho_bar": 0.2960841655731201, + "union_observed": 0.062421973794698715, + "union_independent": 0.17423194646835327, + "union_gap": -0.11180997267365456, + "mean_per_layer_hit_rate": 0.01459713838994503 + }, + "5": { + "rho_bar": 0.41443732380867004, + "union_observed": 0.10112359374761581, + "union_independent": 0.323789119720459, + "union_gap": -0.22266552597284317, + "mean_per_layer_hit_rate": 0.02957841008901596 + }, + "10": { + "rho_bar": 0.45113813877105713, + "union_observed": 0.13857677578926086, + "union_independent": 0.4448266625404358, + "union_gap": -0.3062498867511749, + "mean_per_layer_hit_rate": 0.04407951608300209 + }, + "20": { + "rho_bar": 0.47360503673553467, + "union_observed": 0.1810237169265747, + "union_independent": 0.5851945281028748, + "union_gap": -0.40417081117630005, + "mean_per_layer_hit_rate": 0.06511092185974121 + }, + "50": { + "rho_bar": 0.525072455406189, + "union_observed": 0.24344569444656372, + "union_independent": 0.7369164824485779, + "union_gap": -0.49347078800201416, + "mean_per_layer_hit_rate": 0.09709016978740692 + }, + "100": { + "rho_bar": 0.5298778414726257, + "union_observed": 0.31710362434387207, + "union_independent": 0.83451247215271, + "union_gap": -0.5174088478088379, + "mean_per_layer_hit_rate": 0.12858925759792328 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.41051084867545534, + "union_gap_mean_over_k": -0.3021573764937265 + }, + "Github|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.17307165265083313, + "union_observed": 0.041198503226041794, + "union_independent": 0.08764678239822388, + "union_gap": -0.04644827917218208, + "mean_per_layer_hit_rate": 0.007010467816144228 + }, + "2": { + "rho_bar": 0.2593044638633728, + "union_observed": 0.05867665261030197, + "union_independent": 0.16069656610488892, + "union_gap": -0.10201991349458694, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.35087916254997253, + "union_observed": 0.09488140046596527, + "union_independent": 0.2901637554168701, + "union_gap": -0.19528235495090485, + "mean_per_layer_hit_rate": 0.025929125025868416 + }, + "10": { + "rho_bar": 0.4100501537322998, + "union_observed": 0.13108614087104797, + "union_independent": 0.4167240262031555, + "union_gap": -0.28563788533210754, + "mean_per_layer_hit_rate": 0.040430229157209396 + }, + "20": { + "rho_bar": 0.49185922741889954, + "union_observed": 0.16354556381702423, + "union_independent": 0.5645560026168823, + "union_gap": -0.4010104387998581, + "mean_per_layer_hit_rate": 0.061557669192552567 + }, + "50": { + "rho_bar": 0.523942232131958, + "union_observed": 0.23220974206924438, + "union_independent": 0.7222447991371155, + "union_gap": -0.4900350570678711, + "mean_per_layer_hit_rate": 0.09324882924556732 + }, + "100": { + "rho_bar": 0.5417482852935791, + "union_observed": 0.3046192228794098, + "union_independent": 0.8225507140159607, + "union_gap": -0.5179314911365509, + "mean_per_layer_hit_rate": 0.12369154393672943 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.39297931109155926, + "union_gap_mean_over_k": -0.29119505999343737 + }, + "Github|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.15142594277858734, + "union_observed": 0.04494382068514824, + "union_independent": 0.10017669200897217, + "union_gap": -0.05523287132382393, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.2978711426258087, + "union_observed": 0.06117353215813637, + "union_independent": 0.172438383102417, + "union_gap": -0.11126485094428062, + "mean_per_layer_hit_rate": 0.014405070804059505 + }, + "5": { + "rho_bar": 0.336356520652771, + "union_observed": 0.09737827628850937, + "union_independent": 0.2995089888572693, + "union_gap": -0.20213071256875992, + "mean_per_layer_hit_rate": 0.026889465749263763 + }, + "10": { + "rho_bar": 0.4044387936592102, + "union_observed": 0.1373283416032791, + "union_independent": 0.44277966022491455, + "union_gap": -0.30545131862163544, + "mean_per_layer_hit_rate": 0.04369538277387619 + }, + "20": { + "rho_bar": 0.46614205837249756, + "union_observed": 0.16853933036327362, + "union_independent": 0.582679033279419, + "union_gap": -0.4141397029161453, + "mean_per_layer_hit_rate": 0.06443867832422256 + }, + "50": { + "rho_bar": 0.5224929451942444, + "union_observed": 0.24094881117343903, + "union_independent": 0.7412147521972656, + "union_gap": -0.5002659410238266, + "mean_per_layer_hit_rate": 0.09776241332292557 + }, + "100": { + "rho_bar": 0.517081618309021, + "union_observed": 0.3133583068847656, + "union_independent": 0.8362642526626587, + "union_gap": -0.5229059457778931, + "mean_per_layer_hit_rate": 0.12849323451519012 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.38511557451316286, + "union_gap_mean_over_k": -0.3016273347394807 + }, + "Github|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1560104638338089, + "union_observed": 0.048689138144254684, + "union_independent": 0.10712122917175293, + "union_gap": -0.058432091027498245, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.26546669006347656, + "union_observed": 0.072409488260746, + "union_independent": 0.17262017726898193, + "union_gap": -0.10021068900823593, + "mean_per_layer_hit_rate": 0.01440507173538208 + }, + "5": { + "rho_bar": 0.30794498324394226, + "union_observed": 0.09737827628850937, + "union_independent": 0.2962891459465027, + "union_gap": -0.19891086965799332, + "mean_per_layer_hit_rate": 0.026505330577492714 + }, + "10": { + "rho_bar": 0.40472543239593506, + "union_observed": 0.13233457505702972, + "union_independent": 0.43715524673461914, + "union_gap": -0.3048206716775894, + "mean_per_layer_hit_rate": 0.04292710870504379 + }, + "20": { + "rho_bar": 0.47656533122062683, + "union_observed": 0.17228464782238007, + "union_independent": 0.6042546033859253, + "union_gap": -0.4319699555635452, + "mean_per_layer_hit_rate": 0.06808796525001526 + }, + "50": { + "rho_bar": 0.5036436319351196, + "union_observed": 0.23845192790031433, + "union_independent": 0.7489554286003113, + "union_gap": -0.510503500699997, + "mean_per_layer_hit_rate": 0.09958705306053162 + }, + "100": { + "rho_bar": 0.504168689250946, + "union_observed": 0.318352073431015, + "union_independent": 0.8345221877098083, + "union_gap": -0.5161701142787933, + "mean_per_layer_hit_rate": 0.12753288447856903 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.37407503170626505, + "union_gap_mean_over_k": -0.3030025559876646 + }, + "Github|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.16152016818523407, + "union_observed": 0.04244694113731384, + "union_independent": 0.08767002820968628, + "union_gap": -0.045223087072372437, + "mean_per_layer_hit_rate": 0.007010467816144228 + }, + "2": { + "rho_bar": 0.25411102175712585, + "union_observed": 0.054931335151195526, + "union_independent": 0.14802861213684082, + "union_gap": -0.0930972769856453, + "mean_per_layer_hit_rate": 0.012196293100714684 + }, + "5": { + "rho_bar": 0.30464696884155273, + "union_observed": 0.10112359374761581, + "union_independent": 0.30159837007522583, + "union_gap": -0.20047477632761002, + "mean_per_layer_hit_rate": 0.027081532403826714 + }, + "10": { + "rho_bar": 0.3756808638572693, + "union_observed": 0.13483145833015442, + "union_independent": 0.4384966492652893, + "union_gap": -0.3036651909351349, + "mean_per_layer_hit_rate": 0.04311917722225189 + }, + "20": { + "rho_bar": 0.45074042677879333, + "union_observed": 0.1810237169265747, + "union_independent": 0.6000429391860962, + "union_gap": -0.4190192222595215, + "mean_per_layer_hit_rate": 0.06731969863176346 + }, + "50": { + "rho_bar": 0.47878822684288025, + "union_observed": 0.23845192790031433, + "union_independent": 0.745445966720581, + "union_gap": -0.5069940388202667, + "mean_per_layer_hit_rate": 0.09853067994117737 + }, + "100": { + "rho_bar": 0.4857349395751953, + "union_observed": 0.33458176255226135, + "union_independent": 0.8474715948104858, + "union_gap": -0.5128898322582245, + "mean_per_layer_hit_rate": 0.13262268900871277 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3587460879768644, + "union_gap_mean_over_k": -0.29733763209411074 + }, + "Github|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.1806667149066925, + "union_observed": 0.04369537904858589, + "union_independent": 0.09793323278427124, + "union_gap": -0.05423785373568535, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.23712566494941711, + "union_observed": 0.06741572916507721, + "union_independent": 0.178974449634552, + "union_gap": -0.11155872046947479, + "mean_per_layer_hit_rate": 0.01498127356171608 + }, + "5": { + "rho_bar": 0.27231764793395996, + "union_observed": 0.1111111119389534, + "union_independent": 0.30780166387557983, + "union_gap": -0.19669055193662643, + "mean_per_layer_hit_rate": 0.027753770351409912 + }, + "10": { + "rho_bar": 0.34936201572418213, + "union_observed": 0.15106117725372314, + "union_independent": 0.4716547131538391, + "union_gap": -0.32059353590011597, + "mean_per_layer_hit_rate": 0.04753673076629639 + }, + "20": { + "rho_bar": 0.41029998660087585, + "union_observed": 0.193508118391037, + "union_independent": 0.6217260360717773, + "union_gap": -0.42821791768074036, + "mean_per_layer_hit_rate": 0.071257084608078 + }, + "50": { + "rho_bar": 0.4562440514564514, + "union_observed": 0.26966291666030884, + "union_independent": 0.7665845155715942, + "union_gap": -0.4969215989112854, + "mean_per_layer_hit_rate": 0.10448478162288666 + }, + "100": { + "rho_bar": 0.4630105793476105, + "union_observed": 0.352059930562973, + "union_independent": 0.8589268922805786, + "union_gap": -0.5068669617176056, + "mean_per_layer_hit_rate": 0.1377124786376953 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.33843238013131277, + "union_gap_mean_over_k": -0.30215530576450483 + }, + "Github|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.21197199821472168, + "union_observed": 0.04494382068514824, + "union_independent": 0.10590040683746338, + "union_gap": -0.06095658615231514, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.27399322390556335, + "union_observed": 0.07615480571985245, + "union_independent": 0.19131523370742798, + "union_gap": -0.11516042798757553, + "mean_per_layer_hit_rate": 0.016133679077029228 + }, + "5": { + "rho_bar": 0.3126082420349121, + "union_observed": 0.11860174685716629, + "union_independent": 0.34408390522003174, + "union_gap": -0.22548215836286545, + "mean_per_layer_hit_rate": 0.031787190586328506 + }, + "10": { + "rho_bar": 0.37304171919822693, + "union_observed": 0.16229712963104248, + "union_independent": 0.5070556998252869, + "union_gap": -0.3447585701942444, + "mean_per_layer_hit_rate": 0.05262652784585953 + }, + "20": { + "rho_bar": 0.4308183491230011, + "union_observed": 0.21598002314567566, + "union_independent": 0.665733814239502, + "union_gap": -0.4497537910938263, + "mean_per_layer_hit_rate": 0.08009219169616699 + }, + "50": { + "rho_bar": 0.4647662341594696, + "union_observed": 0.2946317195892334, + "union_independent": 0.7961189150810242, + "union_gap": -0.5014871954917908, + "mean_per_layer_hit_rate": 0.11389609426259995 + }, + "100": { + "rho_bar": 0.48401808738708496, + "union_observed": 0.37078651785850525, + "union_independent": 0.8782322406768799, + "union_gap": -0.5074457228183746, + "mean_per_layer_hit_rate": 0.14760395884513855 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.36445969343185425, + "union_gap_mean_over_k": -0.31500635030014174 + }, + "Github|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.2288331538438797, + "union_observed": 0.04494382068514824, + "union_independent": 0.10803347826004028, + "union_gap": -0.06308965757489204, + "mean_per_layer_hit_rate": 0.008739076554775238 + }, + "2": { + "rho_bar": 0.3136356472969055, + "union_observed": 0.0699126124382019, + "union_independent": 0.1941247582435608, + "union_gap": -0.12421214580535889, + "mean_per_layer_hit_rate": 0.016421779990196228 + }, + "5": { + "rho_bar": 0.33145472407341003, + "union_observed": 0.12484394758939743, + "union_independent": 0.362246572971344, + "union_gap": -0.23740262538194656, + "mean_per_layer_hit_rate": 0.033899929374456406 + }, + "10": { + "rho_bar": 0.3880610466003418, + "union_observed": 0.17478151619434357, + "union_independent": 0.5376927256584167, + "union_gap": -0.3629112094640732, + "mean_per_layer_hit_rate": 0.05733218044042587 + }, + "20": { + "rho_bar": 0.42609739303588867, + "union_observed": 0.23845192790031433, + "union_independent": 0.6941009759902954, + "union_gap": -0.4556490480899811, + "mean_per_layer_hit_rate": 0.08643042296171188 + }, + "50": { + "rho_bar": 0.47668784856796265, + "union_observed": 0.323345810174942, + "union_independent": 0.8260869979858398, + "union_gap": -0.5027411878108978, + "mean_per_layer_hit_rate": 0.12445981055498123 + }, + "100": { + "rho_bar": 0.49960267543792725, + "union_observed": 0.37578028440475464, + "union_independent": 0.8958889245986938, + "union_gap": -0.5201086401939392, + "mean_per_layer_hit_rate": 0.1576875001192093 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.38062464126518797, + "union_gap_mean_over_k": -0.3237306449030127 + }, + "Github|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.18143261969089508, + "union_observed": 0.04494382068514824, + "union_independent": 0.10572820901870728, + "union_gap": -0.060784388333559036, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.2719173729419708, + "union_observed": 0.0749063640832901, + "union_independent": 0.19086623191833496, + "union_gap": -0.11595986783504486, + "mean_per_layer_hit_rate": 0.016133679077029228 + }, + "5": { + "rho_bar": 0.37535560131073, + "union_observed": 0.12983770668506622, + "union_independent": 0.34519439935684204, + "union_gap": -0.21535669267177582, + "mean_per_layer_hit_rate": 0.03197925537824631 + }, + "10": { + "rho_bar": 0.42129549384117126, + "union_observed": 0.18227216601371765, + "union_independent": 0.5008981227874756, + "union_gap": -0.31862595677375793, + "mean_per_layer_hit_rate": 0.05185825005173683 + }, + "20": { + "rho_bar": 0.4748269021511078, + "union_observed": 0.23470661044120789, + "union_independent": 0.6576175093650818, + "union_gap": -0.4229108989238739, + "mean_per_layer_hit_rate": 0.07865168154239655 + }, + "50": { + "rho_bar": 0.5118793845176697, + "union_observed": 0.2971285879611969, + "union_independent": 0.7848027944564819, + "union_gap": -0.48767420649528503, + "mean_per_layer_hit_rate": 0.1105349063873291 + }, + "100": { + "rho_bar": 0.5178583264350891, + "union_observed": 0.3745318353176117, + "union_independent": 0.8703071475028992, + "union_gap": -0.4957753121852875, + "mean_per_layer_hit_rate": 0.14395466446876526 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3935093858412334, + "union_gap_mean_over_k": -0.30244104617408346 + }, + "Github|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.15856687724590302, + "union_observed": 0.054931335151195526, + "union_independent": 0.11379837989807129, + "union_gap": -0.05886704474687576, + "mean_per_layer_hit_rate": 0.009219245053827763 + }, + "2": { + "rho_bar": 0.24701836705207825, + "union_observed": 0.0749063640832901, + "union_independent": 0.18183815479278564, + "union_gap": -0.10693179070949554, + "mean_per_layer_hit_rate": 0.01526937447488308 + }, + "5": { + "rho_bar": 0.31897154450416565, + "union_observed": 0.12484394758939743, + "union_independent": 0.3451896905899048, + "union_gap": -0.22034574300050735, + "mean_per_layer_hit_rate": 0.03188322111964226 + }, + "10": { + "rho_bar": 0.37364599108695984, + "union_observed": 0.16229712963104248, + "union_independent": 0.46930772066116333, + "union_gap": -0.30701059103012085, + "mean_per_layer_hit_rate": 0.047248635441064835 + }, + "20": { + "rho_bar": 0.45480504631996155, + "union_observed": 0.2197253406047821, + "union_independent": 0.6428943872451782, + "union_gap": -0.4231690466403961, + "mean_per_layer_hit_rate": 0.0754825696349144 + }, + "50": { + "rho_bar": 0.5054558515548706, + "union_observed": 0.30337077379226685, + "union_independent": 0.783534824848175, + "union_gap": -0.4801640510559082, + "mean_per_layer_hit_rate": 0.10967060923576355 + }, + "100": { + "rho_bar": 0.5038316249847412, + "union_observed": 0.4007490575313568, + "union_independent": 0.8756410479545593, + "union_gap": -0.4748919904232025, + "mean_per_layer_hit_rate": 0.14606742560863495 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3660421861069543, + "union_gap_mean_over_k": -0.2959114653723581 + }, + "Github|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.15005512535572052, + "union_observed": 0.062421973794698715, + "union_independent": 0.11826503276824951, + "union_gap": -0.055843058973550797, + "mean_per_layer_hit_rate": 0.009603380225598812 + }, + "2": { + "rho_bar": 0.2415090948343277, + "union_observed": 0.08239700645208359, + "union_independent": 0.18800956010818481, + "union_gap": -0.10561255365610123, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.2933080196380615, + "union_observed": 0.13607990741729736, + "union_independent": 0.3509390950202942, + "union_gap": -0.21485918760299683, + "mean_per_layer_hit_rate": 0.03255545720458031 + }, + "10": { + "rho_bar": 0.37806180119514465, + "union_observed": 0.1785268485546112, + "union_independent": 0.4996682405471802, + "union_gap": -0.32114139199256897, + "mean_per_layer_hit_rate": 0.05147411301732063 + }, + "20": { + "rho_bar": 0.43861672282218933, + "union_observed": 0.24344569444656372, + "union_independent": 0.6534217000007629, + "union_gap": -0.4099760055541992, + "mean_per_layer_hit_rate": 0.07749927788972855 + }, + "50": { + "rho_bar": 0.48876091837882996, + "union_observed": 0.3308364450931549, + "union_independent": 0.7965235710144043, + "union_gap": -0.4656871259212494, + "mean_per_layer_hit_rate": 0.1136079952120781 + }, + "100": { + "rho_bar": 0.4956091344356537, + "union_observed": 0.38826465606689453, + "union_independent": 0.8718194961547852, + "union_gap": -0.4835548400878906, + "mean_per_layer_hit_rate": 0.14395466446876526 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3551315452371325, + "union_gap_mean_over_k": -0.2938105948269367 + }, + "Github|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.2002982646226883, + "union_observed": 0.04619225859642029, + "union_independent": 0.10580182075500488, + "union_gap": -0.059609562158584595, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.2751811146736145, + "union_observed": 0.06616728752851486, + "union_independent": 0.17343354225158691, + "union_gap": -0.10726625472307205, + "mean_per_layer_hit_rate": 0.01450110413134098 + }, + "5": { + "rho_bar": 0.3068335950374603, + "union_observed": 0.1086142286658287, + "union_independent": 0.30497437715530396, + "union_gap": -0.19636014848947525, + "mean_per_layer_hit_rate": 0.027465669438242912 + }, + "10": { + "rho_bar": 0.38913223147392273, + "union_observed": 0.14856429398059845, + "union_independent": 0.4644639492034912, + "union_gap": -0.31589965522289276, + "mean_per_layer_hit_rate": 0.04657638818025589 + }, + "20": { + "rho_bar": 0.43553221225738525, + "union_observed": 0.19725343585014343, + "union_independent": 0.6060912609100342, + "union_gap": -0.40883782505989075, + "mean_per_layer_hit_rate": 0.06847210228443146 + }, + "50": { + "rho_bar": 0.466135174036026, + "union_observed": 0.2871410846710205, + "union_independent": 0.754595935344696, + "union_gap": -0.46745485067367554, + "mean_per_layer_hit_rate": 0.10112358629703522 + }, + "100": { + "rho_bar": 0.48288488388061523, + "union_observed": 0.3595505654811859, + "union_independent": 0.8415615558624268, + "union_gap": -0.48201099038124084, + "mean_per_layer_hit_rate": 0.13050994277000427 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.36514249656881603, + "union_gap_mean_over_k": -0.29106275524411884 + }, + "Github|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.14541883766651154, + "union_observed": 0.05118601769208908, + "union_independent": 0.10914582014083862, + "union_gap": -0.05795980244874954, + "mean_per_layer_hit_rate": 0.008835109882056713 + }, + "2": { + "rho_bar": 0.2646557688713074, + "union_observed": 0.08489388227462769, + "union_independent": 0.20116651058197021, + "union_gap": -0.11627262830734253, + "mean_per_layer_hit_rate": 0.017094017937779427 + }, + "5": { + "rho_bar": 0.35913243889808655, + "union_observed": 0.14107365906238556, + "union_independent": 0.3646147847175598, + "union_gap": -0.22354112565517426, + "mean_per_layer_hit_rate": 0.034188032150268555 + }, + "10": { + "rho_bar": 0.42004257440567017, + "union_observed": 0.1785268485546112, + "union_independent": 0.5107241868972778, + "union_gap": -0.3321973383426666, + "mean_per_layer_hit_rate": 0.05320272594690323 + }, + "20": { + "rho_bar": 0.48541563749313354, + "union_observed": 0.22596754133701324, + "union_independent": 0.6626188158988953, + "union_gap": -0.436651274561882, + "mean_per_layer_hit_rate": 0.0795159861445427 + }, + "50": { + "rho_bar": 0.5027045011520386, + "union_observed": 0.3096129894256592, + "union_independent": 0.789072573184967, + "union_gap": -0.47945958375930786, + "mean_per_layer_hit_rate": 0.1114952564239502 + }, + "100": { + "rho_bar": 0.5255336761474609, + "union_observed": 0.3770287036895752, + "union_independent": 0.8716679811477661, + "union_gap": -0.4946392774581909, + "mean_per_layer_hit_rate": 0.14433881640434265 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3861290620906012, + "union_gap_mean_over_k": -0.3058172900761877 + }, + "Github|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.32538506388664246, + "union_observed": 0.05118601769208908, + "union_independent": 0.14870107173919678, + "union_gap": -0.0975150540471077, + "mean_per_layer_hit_rate": 0.012292327359318733 + }, + "2": { + "rho_bar": 0.4460795819759369, + "union_observed": 0.07365792989730835, + "union_independent": 0.2572149634361267, + "union_gap": -0.18355703353881836, + "mean_per_layer_hit_rate": 0.02256794460117817 + }, + "5": { + "rho_bar": 0.47698915004730225, + "union_observed": 0.12734082341194153, + "union_independent": 0.4350022077560425, + "union_gap": -0.30766138434410095, + "mean_per_layer_hit_rate": 0.04283107817173004 + }, + "10": { + "rho_bar": 0.5392359495162964, + "union_observed": 0.1810237169265747, + "union_independent": 0.616897463798523, + "union_gap": -0.43587374687194824, + "mean_per_layer_hit_rate": 0.0706808790564537 + }, + "20": { + "rho_bar": 0.588172972202301, + "union_observed": 0.2297128587961197, + "union_independent": 0.7460485696792603, + "union_gap": -0.5163357108831406, + "mean_per_layer_hit_rate": 0.09901085495948792 + }, + "50": { + "rho_bar": 0.5818673372268677, + "union_observed": 0.3071160912513733, + "union_independent": 0.8567877411842346, + "union_gap": -0.5496716499328613, + "mean_per_layer_hit_rate": 0.13704024255275726 + }, + "100": { + "rho_bar": 0.5903440713882446, + "union_observed": 0.37827715277671814, + "union_independent": 0.922446072101593, + "union_gap": -0.5441689193248749, + "mean_per_layer_hit_rate": 0.17574185132980347 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.5068677323205131, + "union_gap_mean_over_k": -0.3763976427061217 + }, + "Github|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.28950533270835876, + "union_observed": 0.05742821469902992, + "union_independent": 0.14449554681777954, + "union_gap": -0.08706733211874962, + "mean_per_layer_hit_rate": 0.011908192187547684 + }, + "2": { + "rho_bar": 0.3797335624694824, + "union_observed": 0.08364544063806534, + "union_independent": 0.2427353858947754, + "union_gap": -0.15908994525671005, + "mean_per_layer_hit_rate": 0.021127434447407722 + }, + "5": { + "rho_bar": 0.4056326448917389, + "union_observed": 0.1373283416032791, + "union_independent": 0.4438266158103943, + "union_gap": -0.3064982742071152, + "mean_per_layer_hit_rate": 0.04398348182439804 + }, + "10": { + "rho_bar": 0.4610331654548645, + "union_observed": 0.193508118391037, + "union_independent": 0.6083557605743408, + "union_gap": -0.41484764218330383, + "mean_per_layer_hit_rate": 0.06924037635326385 + }, + "20": { + "rho_bar": 0.5091440081596375, + "union_observed": 0.2509363293647766, + "union_independent": 0.752478837966919, + "union_gap": -0.5015425086021423, + "mean_per_layer_hit_rate": 0.10112358629703522 + }, + "50": { + "rho_bar": 0.5382612943649292, + "union_observed": 0.3208489418029785, + "union_independent": 0.8597755432128906, + "union_gap": -0.5389266014099121, + "mean_per_layer_hit_rate": 0.1388648897409439 + }, + "100": { + "rho_bar": 0.5587391257286072, + "union_observed": 0.38826465606689453, + "union_independent": 0.9265493750572205, + "union_gap": -0.5382847189903259, + "mean_per_layer_hit_rate": 0.17977526783943176 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4488641619682312, + "union_gap_mean_over_k": -0.36375100325260845 + }, + "Github|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.3204428255558014, + "union_observed": 0.05742821469902992, + "union_independent": 0.14231586456298828, + "union_gap": -0.08488764986395836, + "mean_per_layer_hit_rate": 0.011716123670339584 + }, + "2": { + "rho_bar": 0.33477017283439636, + "union_observed": 0.08364544063806534, + "union_independent": 0.2173355221748352, + "union_gap": -0.13369008153676987, + "mean_per_layer_hit_rate": 0.018630556762218475 + }, + "5": { + "rho_bar": 0.3760511577129364, + "union_observed": 0.1523096114397049, + "union_independent": 0.4323565363883972, + "union_gap": -0.2800469249486923, + "mean_per_layer_hit_rate": 0.042446937412023544 + }, + "10": { + "rho_bar": 0.42499610781669617, + "union_observed": 0.20973782241344452, + "union_independent": 0.5790396928787231, + "union_gap": -0.3693018704652786, + "mean_per_layer_hit_rate": 0.06395851820707321 + }, + "20": { + "rho_bar": 0.46637412905693054, + "union_observed": 0.2634207308292389, + "union_independent": 0.7224311232566833, + "union_gap": -0.45901039242744446, + "mean_per_layer_hit_rate": 0.09296072274446487 + }, + "50": { + "rho_bar": 0.5267342329025269, + "union_observed": 0.33458176255226135, + "union_independent": 0.8422439098358154, + "union_gap": -0.5076621472835541, + "mean_per_layer_hit_rate": 0.13089407980442047 + }, + "100": { + "rho_bar": 0.5630066990852356, + "union_observed": 0.40449437499046326, + "union_independent": 0.9162571430206299, + "union_gap": -0.5117627680301666, + "mean_per_layer_hit_rate": 0.17132431268692017 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.430339332137789, + "union_gap_mean_over_k": -0.3351945477936949 + }, + "Github|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.2562882602214813, + "union_observed": 0.054931335151195526, + "union_independent": 0.1303151249885559, + "union_gap": -0.07538378983736038, + "mean_per_layer_hit_rate": 0.010659752413630486 + }, + "2": { + "rho_bar": 0.3233053684234619, + "union_observed": 0.07990012317895889, + "union_independent": 0.21918624639511108, + "union_gap": -0.1392861232161522, + "mean_per_layer_hit_rate": 0.018822625279426575 + }, + "5": { + "rho_bar": 0.34752950072288513, + "union_observed": 0.15855181217193604, + "union_independent": 0.40798163414001465, + "union_gap": -0.2494298219680786, + "mean_per_layer_hit_rate": 0.03937385603785515 + }, + "10": { + "rho_bar": 0.40266379714012146, + "union_observed": 0.20099875330924988, + "union_independent": 0.5512547492980957, + "union_gap": -0.3502559959888458, + "mean_per_layer_hit_rate": 0.05934888869524002 + }, + "20": { + "rho_bar": 0.4388294816017151, + "union_observed": 0.255930095911026, + "union_independent": 0.6896294951438904, + "union_gap": -0.4336993992328644, + "mean_per_layer_hit_rate": 0.08518198132514954 + }, + "50": { + "rho_bar": 0.47550278902053833, + "union_observed": 0.3358302116394043, + "union_independent": 0.8144738674163818, + "union_gap": -0.47864365577697754, + "mean_per_layer_hit_rate": 0.11965811997652054 + }, + "100": { + "rho_bar": 0.51559978723526, + "union_observed": 0.40199750661849976, + "union_independent": 0.8959868550300598, + "union_gap": -0.49398934841156006, + "mean_per_layer_hit_rate": 0.15720732510089874 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3942455691950662, + "union_gap_mean_over_k": -0.3172411620616913 + }, + "Github|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.21463428437709808, + "union_observed": 0.048689138144254684, + "union_independent": 0.10700589418411255, + "union_gap": -0.058316756039857864, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.30487748980522156, + "union_observed": 0.06866417080163956, + "union_independent": 0.18477433919906616, + "union_gap": -0.1161101683974266, + "mean_per_layer_hit_rate": 0.015557477250695229 + }, + "5": { + "rho_bar": 0.3752271831035614, + "union_observed": 0.14107365906238556, + "union_independent": 0.37713122367858887, + "union_gap": -0.2360575646162033, + "mean_per_layer_hit_rate": 0.035628542304039 + }, + "10": { + "rho_bar": 0.4009685516357422, + "union_observed": 0.19225966930389404, + "union_independent": 0.5130243301391602, + "union_gap": -0.3207646608352661, + "mean_per_layer_hit_rate": 0.053490832448005676 + }, + "20": { + "rho_bar": 0.46533945202827454, + "union_observed": 0.23845192790031433, + "union_independent": 0.6519105434417725, + "union_gap": -0.41345861554145813, + "mean_per_layer_hit_rate": 0.07711514085531235 + }, + "50": { + "rho_bar": 0.5075504779815674, + "union_observed": 0.31710362434387207, + "union_independent": 0.7885638475418091, + "union_gap": -0.471460223197937, + "mean_per_layer_hit_rate": 0.1111111119389534 + }, + "100": { + "rho_bar": 0.5337455868721008, + "union_observed": 0.39950063824653625, + "union_independent": 0.8841302990913391, + "union_gap": -0.48462966084480286, + "mean_per_layer_hit_rate": 0.15067704021930695 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.400334717971938, + "union_gap_mean_over_k": -0.3001139499247074 + }, + "Github|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.2229224592447281, + "union_observed": 0.047440700232982635, + "union_independent": 0.09790575504302979, + "union_gap": -0.05046505481004715, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.2784738540649414, + "union_observed": 0.0749063640832901, + "union_independent": 0.17856019735336304, + "union_gap": -0.10365383327007294, + "mean_per_layer_hit_rate": 0.01498127356171608 + }, + "5": { + "rho_bar": 0.346137672662735, + "union_observed": 0.13358302414417267, + "union_independent": 0.34745073318481445, + "union_gap": -0.21386770904064178, + "mean_per_layer_hit_rate": 0.03217132389545441 + }, + "10": { + "rho_bar": 0.3920159339904785, + "union_observed": 0.18227216601371765, + "union_independent": 0.4830501675605774, + "union_gap": -0.30077800154685974, + "mean_per_layer_hit_rate": 0.04916930943727493 + }, + "20": { + "rho_bar": 0.4724961817264557, + "union_observed": 0.22596754133701324, + "union_independent": 0.6391761302947998, + "union_gap": -0.41320858895778656, + "mean_per_layer_hit_rate": 0.0747143030166626 + }, + "50": { + "rho_bar": 0.5044876337051392, + "union_observed": 0.30087390542030334, + "union_independent": 0.7663223147392273, + "union_gap": -0.46544840931892395, + "mean_per_layer_hit_rate": 0.10467684268951416 + }, + "100": { + "rho_bar": 0.5250430107116699, + "union_observed": 0.3770287036895752, + "union_independent": 0.8611220121383667, + "union_gap": -0.4840933084487915, + "mean_per_layer_hit_rate": 0.1392490118741989 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3916538208723068, + "union_gap_mean_over_k": -0.2902164150561605 + }, + "Github|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.1609812080860138, + "union_observed": 0.04494382068514824, + "union_independent": 0.10009533166885376, + "union_gap": -0.05515151098370552, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.2958686649799347, + "union_observed": 0.06117353215813637, + "union_independent": 0.16899323463439941, + "union_gap": -0.10781970247626305, + "mean_per_layer_hit_rate": 0.014116968959569931 + }, + "5": { + "rho_bar": 0.3736826181411743, + "union_observed": 0.10486891120672226, + "union_independent": 0.3195127248764038, + "union_gap": -0.21464381366968155, + "mean_per_layer_hit_rate": 0.02909824438393116 + }, + "10": { + "rho_bar": 0.4289449155330658, + "union_observed": 0.14606741070747375, + "union_independent": 0.4542528986930847, + "union_gap": -0.30818548798561096, + "mean_per_layer_hit_rate": 0.04532795399427414 + }, + "20": { + "rho_bar": 0.4713175892829895, + "union_observed": 0.1785268485546112, + "union_independent": 0.5975439548492432, + "union_gap": -0.41901710629463196, + "mean_per_layer_hit_rate": 0.06722366064786911 + }, + "50": { + "rho_bar": 0.5251286625862122, + "union_observed": 0.24968789517879486, + "union_independent": 0.7456284761428833, + "union_gap": -0.49594058096408844, + "mean_per_layer_hit_rate": 0.09939499199390411 + }, + "100": { + "rho_bar": 0.5274698734283447, + "union_observed": 0.32958802580833435, + "union_independent": 0.8408411741256714, + "union_gap": -0.511253148317337, + "mean_per_layer_hit_rate": 0.13108614087104797 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3976276474339621, + "union_gap_mean_over_k": -0.30171590724161695 + }, + "Github|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.1558370739221573, + "union_observed": 0.041198503226041794, + "union_independent": 0.0748753547668457, + "union_gap": -0.03367685154080391, + "mean_per_layer_hit_rate": 0.005954096093773842 + }, + "2": { + "rho_bar": 0.2354121059179306, + "union_observed": 0.05867665261030197, + "union_independent": 0.151042640209198, + "union_gap": -0.09236598759889603, + "mean_per_layer_hit_rate": 0.012484394945204258 + }, + "5": { + "rho_bar": 0.34281057119369507, + "union_observed": 0.09862671792507172, + "union_independent": 0.2902594208717346, + "union_gap": -0.1916327029466629, + "mean_per_layer_hit_rate": 0.025929125025868416 + }, + "10": { + "rho_bar": 0.4086308479309082, + "union_observed": 0.13358302414417267, + "union_independent": 0.42580491304397583, + "union_gap": -0.29222188889980316, + "mean_per_layer_hit_rate": 0.04158264026045799 + }, + "20": { + "rho_bar": 0.4890742301940918, + "union_observed": 0.16479401290416718, + "union_independent": 0.5715222358703613, + "union_gap": -0.40672822296619415, + "mean_per_layer_hit_rate": 0.06271007657051086 + }, + "50": { + "rho_bar": 0.512877881526947, + "union_observed": 0.23845192790031433, + "union_independent": 0.7347158193588257, + "union_gap": -0.49626389145851135, + "mean_per_layer_hit_rate": 0.09641793370246887 + }, + "100": { + "rho_bar": 0.5370166301727295, + "union_observed": 0.32209739089012146, + "union_independent": 0.8374143838882446, + "union_gap": -0.5153169929981232, + "mean_per_layer_hit_rate": 0.12954960763454437 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.3830941915512085, + "union_gap_mean_over_k": -0.28974379120128496 + }, + "Github|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.14945922791957855, + "union_observed": 0.039950061589479446, + "union_independent": 0.08986616134643555, + "union_gap": -0.0499160997569561, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.2751263976097107, + "union_observed": 0.05867665261030197, + "union_independent": 0.16608691215515137, + "union_gap": -0.1074102595448494, + "mean_per_layer_hit_rate": 0.013828868046402931 + }, + "5": { + "rho_bar": 0.3393591046333313, + "union_observed": 0.09987515956163406, + "union_independent": 0.30223315954208374, + "union_gap": -0.20235799998044968, + "mean_per_layer_hit_rate": 0.027177568525075912 + }, + "10": { + "rho_bar": 0.4130626618862152, + "union_observed": 0.1373283416032791, + "union_independent": 0.44207513332366943, + "union_gap": -0.3047467917203903, + "mean_per_layer_hit_rate": 0.04359934851527214 + }, + "20": { + "rho_bar": 0.46896111965179443, + "union_observed": 0.17977528274059296, + "union_independent": 0.5990952253341675, + "union_gap": -0.4193199425935745, + "mean_per_layer_hit_rate": 0.06731969863176346 + }, + "50": { + "rho_bar": 0.5218468904495239, + "union_observed": 0.24594257771968842, + "union_independent": 0.7537180185317993, + "union_gap": -0.5077754408121109, + "mean_per_layer_hit_rate": 0.10112360119819641 + }, + "100": { + "rho_bar": 0.5169531106948853, + "union_observed": 0.323345810174942, + "union_independent": 0.8446699976921082, + "union_gap": -0.5213241875171661, + "mean_per_layer_hit_rate": 0.13195045292377472 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.38353835897786276, + "union_gap_mean_over_k": -0.30183581741792814 + }, + "Github|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.15097801387310028, + "union_observed": 0.04619225859642029, + "union_independent": 0.09796768426895142, + "union_gap": -0.05177542567253113, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.2382308393716812, + "union_observed": 0.07116104662418365, + "union_independent": 0.1705048680305481, + "union_gap": -0.09934382140636444, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.29199981689453125, + "union_observed": 0.10362047702074051, + "union_independent": 0.2917743921279907, + "union_gap": -0.1881539151072502, + "mean_per_layer_hit_rate": 0.026025163009762764 + }, + "10": { + "rho_bar": 0.40354490280151367, + "union_observed": 0.13857677578926086, + "union_independent": 0.43799257278442383, + "union_gap": -0.29941579699516296, + "mean_per_layer_hit_rate": 0.04302314668893814 + }, + "20": { + "rho_bar": 0.4660419523715973, + "union_observed": 0.17977528274059296, + "union_independent": 0.6090991497039795, + "union_gap": -0.42932386696338654, + "mean_per_layer_hit_rate": 0.06895226985216141 + }, + "50": { + "rho_bar": 0.5107771158218384, + "union_observed": 0.24469412863254547, + "union_independent": 0.7610747218132019, + "union_gap": -0.5163805931806564, + "mean_per_layer_hit_rate": 0.10294824093580246 + }, + "100": { + "rho_bar": 0.5054445266723633, + "union_observed": 0.32958802580833435, + "union_independent": 0.8466125726699829, + "union_gap": -0.5170245468616486, + "mean_per_layer_hit_rate": 0.13252665102481842 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3667167382580893, + "union_gap_mean_over_k": -0.3002025665981429 + }, + "Github|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.15849627554416656, + "union_observed": 0.04369537904858589, + "union_independent": 0.08535003662109375, + "union_gap": -0.04165465757250786, + "mean_per_layer_hit_rate": 0.0068184006959199905 + }, + "2": { + "rho_bar": 0.2472919374704361, + "union_observed": 0.06741572916507721, + "union_independent": 0.16100019216537476, + "union_gap": -0.09358446300029755, + "mean_per_layer_hit_rate": 0.013348699547350407 + }, + "5": { + "rho_bar": 0.2940620183944702, + "union_observed": 0.1061173528432846, + "union_independent": 0.30609261989593506, + "union_gap": -0.19997526705265045, + "mean_per_layer_hit_rate": 0.027561701834201813 + }, + "10": { + "rho_bar": 0.3719668686389923, + "union_observed": 0.14357054233551025, + "union_independent": 0.45177161693573, + "union_gap": -0.3082010746002197, + "mean_per_layer_hit_rate": 0.04484778642654419 + }, + "20": { + "rho_bar": 0.4426771402359009, + "union_observed": 0.1860174834728241, + "union_independent": 0.6131252646446228, + "union_gap": -0.4271077811717987, + "mean_per_layer_hit_rate": 0.06962450593709946 + }, + "50": { + "rho_bar": 0.47864723205566406, + "union_observed": 0.25468164682388306, + "union_independent": 0.7632878422737122, + "union_gap": -0.5086061954498291, + "mean_per_layer_hit_rate": 0.10342840850353241 + }, + "100": { + "rho_bar": 0.48335251212120056, + "union_observed": 0.34207239747047424, + "union_independent": 0.8537144660949707, + "union_gap": -0.5116420686244965, + "mean_per_layer_hit_rate": 0.13531163334846497 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3537848549229758, + "union_gap_mean_over_k": -0.29868164392454283 + }, + "Github|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.18993999063968658, + "union_observed": 0.04244694113731384, + "union_independent": 0.0945088267326355, + "union_gap": -0.052061885595321655, + "mean_per_layer_hit_rate": 0.007586670108139515 + }, + "2": { + "rho_bar": 0.23265236616134644, + "union_observed": 0.0699126124382019, + "union_independent": 0.17367023229599, + "union_gap": -0.10375761985778809, + "mean_per_layer_hit_rate": 0.01450110413134098 + }, + "5": { + "rho_bar": 0.2674643099308014, + "union_observed": 0.11485642939805984, + "union_independent": 0.3229016661643982, + "union_gap": -0.20804523676633835, + "mean_per_layer_hit_rate": 0.02938634529709816 + }, + "10": { + "rho_bar": 0.3507869839668274, + "union_observed": 0.1473158597946167, + "union_independent": 0.47530168294906616, + "union_gap": -0.32798582315444946, + "mean_per_layer_hit_rate": 0.048016902059316635 + }, + "20": { + "rho_bar": 0.4099232256412506, + "union_observed": 0.20848938822746277, + "union_independent": 0.6441651582717896, + "union_gap": -0.4356757700443268, + "mean_per_layer_hit_rate": 0.0754825696349144 + }, + "50": { + "rho_bar": 0.45239585638046265, + "union_observed": 0.2796504497528076, + "union_independent": 0.7830315232276917, + "union_gap": -0.503381073474884, + "mean_per_layer_hit_rate": 0.10928647220134735 + }, + "100": { + "rho_bar": 0.4609225392341614, + "union_observed": 0.36579275131225586, + "union_independent": 0.8697893619537354, + "union_gap": -0.5039966106414795, + "mean_per_layer_hit_rate": 0.1427062302827835 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3377264674220766, + "union_gap_mean_over_k": -0.30498628850494114 + }, + "Github|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.20132742822170258, + "union_observed": 0.04494382068514824, + "union_independent": 0.10257500410079956, + "union_gap": -0.05763118341565132, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.27092739939689636, + "union_observed": 0.07990012317895889, + "union_independent": 0.19751542806625366, + "union_gap": -0.11761530488729477, + "mean_per_layer_hit_rate": 0.016709882766008377 + }, + "5": { + "rho_bar": 0.30279266834259033, + "union_observed": 0.12484394758939743, + "union_independent": 0.3458961844444275, + "union_gap": -0.22105223685503006, + "mean_per_layer_hit_rate": 0.03197925537824631 + }, + "10": { + "rho_bar": 0.36898863315582275, + "union_observed": 0.17103619873523712, + "union_independent": 0.5157599449157715, + "union_gap": -0.34472374618053436, + "mean_per_layer_hit_rate": 0.05387496203184128 + }, + "20": { + "rho_bar": 0.4335120618343353, + "union_observed": 0.2197253406047821, + "union_independent": 0.6853923797607422, + "union_gap": -0.4656670391559601, + "mean_per_layer_hit_rate": 0.08422164618968964 + }, + "50": { + "rho_bar": 0.46018537878990173, + "union_observed": 0.31710362434387207, + "union_independent": 0.8136552572250366, + "union_gap": -0.49655163288116455, + "mean_per_layer_hit_rate": 0.11975415050983429 + }, + "100": { + "rho_bar": 0.48458245396614075, + "union_observed": 0.3895131051540375, + "union_independent": 0.8908275961875916, + "union_gap": -0.5013144910335541, + "mean_per_layer_hit_rate": 0.1542302817106247 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3603308605296271, + "union_gap_mean_over_k": -0.31493651920131277 + }, + "Github|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.21138343214988708, + "union_observed": 0.0387016236782074, + "union_independent": 0.10129296779632568, + "union_gap": -0.06259134411811829, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.3012640178203583, + "union_observed": 0.0749063640832901, + "union_independent": 0.20233893394470215, + "union_gap": -0.12743256986141205, + "mean_per_layer_hit_rate": 0.017190052196383476 + }, + "5": { + "rho_bar": 0.3401844799518585, + "union_observed": 0.11860174685716629, + "union_independent": 0.3664514422416687, + "union_gap": -0.2478496953845024, + "mean_per_layer_hit_rate": 0.03438010439276695 + }, + "10": { + "rho_bar": 0.38597947359085083, + "union_observed": 0.19225966930389404, + "union_independent": 0.5597814321517944, + "union_gap": -0.3675217628479004, + "mean_per_layer_hit_rate": 0.06078939139842987 + }, + "20": { + "rho_bar": 0.4433266520500183, + "union_observed": 0.24843944609165192, + "union_independent": 0.711606502532959, + "union_gap": -0.46316705644130707, + "mean_per_layer_hit_rate": 0.09036780893802643 + }, + "50": { + "rho_bar": 0.48080042004585266, + "union_observed": 0.3283395767211914, + "union_independent": 0.8387923240661621, + "union_gap": -0.5104527473449707, + "mean_per_layer_hit_rate": 0.12935753166675568 + }, + "100": { + "rho_bar": 0.4965972304344177, + "union_observed": 0.392009973526001, + "union_independent": 0.9043272733688354, + "union_gap": -0.5123172998428345, + "mean_per_layer_hit_rate": 0.16277728974819183 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3799336722918919, + "union_gap_mean_over_k": -0.32733321083443506 + }, + "Github|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.18657849729061127, + "union_observed": 0.04619225859642029, + "union_independent": 0.10122865438461304, + "union_gap": -0.05503639578819275, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.26175305247306824, + "union_observed": 0.08239700645208359, + "union_independent": 0.19702857732772827, + "union_gap": -0.11463157087564468, + "mean_per_layer_hit_rate": 0.016709882766008377 + }, + "5": { + "rho_bar": 0.3599683940410614, + "union_observed": 0.144818976521492, + "union_independent": 0.3587542176246643, + "union_gap": -0.2139352411031723, + "mean_per_layer_hit_rate": 0.033515796065330505 + }, + "10": { + "rho_bar": 0.4107145667076111, + "union_observed": 0.18976280093193054, + "union_independent": 0.5089519023895264, + "union_gap": -0.3191891014575958, + "mean_per_layer_hit_rate": 0.05301066115498543 + }, + "20": { + "rho_bar": 0.4668016731739044, + "union_observed": 0.24094881117343903, + "union_independent": 0.6694691181182861, + "union_gap": -0.4285203069448471, + "mean_per_layer_hit_rate": 0.08105253428220749 + }, + "50": { + "rho_bar": 0.510193943977356, + "union_observed": 0.30586767196655273, + "union_independent": 0.7980456948280334, + "union_gap": -0.4921780228614807, + "mean_per_layer_hit_rate": 0.11466435343027115 + }, + "100": { + "rho_bar": 0.5100613236427307, + "union_observed": 0.3982521891593933, + "union_independent": 0.8800550699234009, + "union_gap": -0.48180288076400757, + "mean_per_layer_hit_rate": 0.14875635504722595 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.38658163590090616, + "union_gap_mean_over_k": -0.300756217113563 + }, + "Github|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.1940666288137436, + "union_observed": 0.05368289723992348, + "union_independent": 0.10701113939285278, + "union_gap": -0.053328242152929306, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.22363613545894623, + "union_observed": 0.08364544063806534, + "union_independent": 0.19328218698501587, + "union_gap": -0.10963674634695053, + "mean_per_layer_hit_rate": 0.01632574573159218 + }, + "5": { + "rho_bar": 0.3012537956237793, + "union_observed": 0.12858925759792328, + "union_independent": 0.3504287004470825, + "union_gap": -0.22183944284915924, + "mean_per_layer_hit_rate": 0.032459426671266556 + }, + "10": { + "rho_bar": 0.3664822280406952, + "union_observed": 0.17103619873523712, + "union_independent": 0.48612159490585327, + "union_gap": -0.31508539617061615, + "mean_per_layer_hit_rate": 0.049553439021110535 + }, + "20": { + "rho_bar": 0.4553230106830597, + "union_observed": 0.227215975522995, + "union_independent": 0.6549758911132812, + "union_gap": -0.42775991559028625, + "mean_per_layer_hit_rate": 0.077787384390831 + }, + "50": { + "rho_bar": 0.5038652420043945, + "union_observed": 0.30087390542030334, + "union_independent": 0.7928393483161926, + "union_gap": -0.4919654428958893, + "mean_per_layer_hit_rate": 0.11255161464214325 + }, + "100": { + "rho_bar": 0.5054575204849243, + "union_observed": 0.39950063824653625, + "union_independent": 0.8786299228668213, + "union_gap": -0.47912928462028503, + "mean_per_layer_hit_rate": 0.1475079208612442 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3642977944442204, + "union_gap_mean_over_k": -0.29982063866087366 + }, + "Github|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.1398116797208786, + "union_observed": 0.054931335151195526, + "union_independent": 0.10254800319671631, + "union_gap": -0.04761666804552078, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.2208210676908493, + "union_observed": 0.08489388227462769, + "union_independent": 0.18604326248168945, + "union_gap": -0.10114938020706177, + "mean_per_layer_hit_rate": 0.015653511509299278 + }, + "5": { + "rho_bar": 0.2847001552581787, + "union_observed": 0.14107365906238556, + "union_independent": 0.3544195294380188, + "union_gap": -0.21334587037563324, + "mean_per_layer_hit_rate": 0.032939594238996506 + }, + "10": { + "rho_bar": 0.37462925910949707, + "union_observed": 0.18976280093193054, + "union_independent": 0.5051336884498596, + "union_gap": -0.3153708875179291, + "mean_per_layer_hit_rate": 0.05224239081144333 + }, + "20": { + "rho_bar": 0.4458251893520355, + "union_observed": 0.24843944609165192, + "union_independent": 0.6701756119728088, + "union_gap": -0.4217361658811569, + "mean_per_layer_hit_rate": 0.08086045831441879 + }, + "50": { + "rho_bar": 0.4910672903060913, + "union_observed": 0.33707866072654724, + "union_independent": 0.805548906326294, + "union_gap": -0.4684702455997467, + "mean_per_layer_hit_rate": 0.11658503115177155 + }, + "100": { + "rho_bar": 0.4949350655078888, + "union_observed": 0.40449437499046326, + "union_independent": 0.8792949318885803, + "union_gap": -0.47480055689811707, + "mean_per_layer_hit_rate": 0.1476999819278717 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3502556724207742, + "union_gap_mean_over_k": -0.2917842535035951 + }, + "Github|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.18383754789829254, + "union_observed": 0.041198503226041794, + "union_independent": 0.09670364856719971, + "union_gap": -0.05550514534115791, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.24843166768550873, + "union_observed": 0.06866417080163956, + "union_independent": 0.17765110731124878, + "union_gap": -0.10898693650960922, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2858069837093353, + "union_observed": 0.10986267030239105, + "union_independent": 0.30500584840774536, + "union_gap": -0.1951431781053543, + "mean_per_layer_hit_rate": 0.027465667575597763 + }, + "10": { + "rho_bar": 0.3788568377494812, + "union_observed": 0.16229712963104248, + "union_independent": 0.4716903567314148, + "union_gap": -0.3093932271003723, + "mean_per_layer_hit_rate": 0.04753673076629639 + }, + "20": { + "rho_bar": 0.44087785482406616, + "union_observed": 0.21348313987255096, + "union_independent": 0.6253684163093567, + "union_gap": -0.4118852764368057, + "mean_per_layer_hit_rate": 0.07192932069301605 + }, + "50": { + "rho_bar": 0.46360158920288086, + "union_observed": 0.2946317195892334, + "union_independent": 0.7666168808937073, + "union_gap": -0.4719851613044739, + "mean_per_layer_hit_rate": 0.10438874363899231 + }, + "100": { + "rho_bar": 0.4817040264606476, + "union_observed": 0.37078651785850525, + "union_independent": 0.8538050651550293, + "union_gap": -0.48301854729652405, + "mean_per_layer_hit_rate": 0.13559973239898682 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3547309296471732, + "union_gap_mean_over_k": -0.2908453531563282 + }, + "Github|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.14702406525611877, + "union_observed": 0.05368289723992348, + "union_independent": 0.10916000604629517, + "union_gap": -0.05547710880637169, + "mean_per_layer_hit_rate": 0.008835109882056713 + }, + "2": { + "rho_bar": 0.26403772830963135, + "union_observed": 0.08489388227462769, + "union_independent": 0.20110023021697998, + "union_gap": -0.1162063479423523, + "mean_per_layer_hit_rate": 0.017094017937779427 + }, + "5": { + "rho_bar": 0.3630230128765106, + "union_observed": 0.14606741070747375, + "union_independent": 0.3744984269142151, + "union_gap": -0.22843101620674133, + "mean_per_layer_hit_rate": 0.035340435802936554 + }, + "10": { + "rho_bar": 0.40290433168411255, + "union_observed": 0.18726591765880585, + "union_independent": 0.5178712606430054, + "union_gap": -0.3306053429841995, + "mean_per_layer_hit_rate": 0.054259102791547775 + }, + "20": { + "rho_bar": 0.48351746797561646, + "union_observed": 0.22846442461013794, + "union_independent": 0.6638240814208984, + "union_gap": -0.4353596568107605, + "mean_per_layer_hit_rate": 0.0797080546617508 + }, + "50": { + "rho_bar": 0.5123292207717896, + "union_observed": 0.3096129894256592, + "union_independent": 0.8006768226623535, + "union_gap": -0.49106383323669434, + "mean_per_layer_hit_rate": 0.11524055898189545 + }, + "100": { + "rho_bar": 0.5305647850036621, + "union_observed": 0.3820224702358246, + "union_independent": 0.8821985721588135, + "union_gap": -0.5001761019229889, + "mean_per_layer_hit_rate": 0.1498127281665802 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.38620008741106304, + "union_gap_mean_over_k": -0.3081884868443012 + }, + "Github|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.34431523084640503, + "union_observed": 0.04369537904858589, + "union_independent": 0.14112770557403564, + "union_gap": -0.09743232652544975, + "mean_per_layer_hit_rate": 0.01162009034305811 + }, + "2": { + "rho_bar": 0.4405376613140106, + "union_observed": 0.0774032473564148, + "union_independent": 0.24962157011032104, + "union_gap": -0.17221832275390625, + "mean_per_layer_hit_rate": 0.02179967425763607 + }, + "5": { + "rho_bar": 0.46040359139442444, + "union_observed": 0.13607990741729736, + "union_independent": 0.4382106065750122, + "union_gap": -0.30213069915771484, + "mean_per_layer_hit_rate": 0.04321521148085594 + }, + "10": { + "rho_bar": 0.5189365148544312, + "union_observed": 0.17727839946746826, + "union_independent": 0.6200904846191406, + "union_gap": -0.44281208515167236, + "mean_per_layer_hit_rate": 0.0712570771574974 + }, + "20": { + "rho_bar": 0.5750609040260315, + "union_observed": 0.23595505952835083, + "union_independent": 0.7546895146369934, + "union_gap": -0.5187344551086426, + "mean_per_layer_hit_rate": 0.10131566226482391 + }, + "50": { + "rho_bar": 0.584627091884613, + "union_observed": 0.3158551752567291, + "union_independent": 0.8666573166847229, + "union_gap": -0.5508021414279938, + "mean_per_layer_hit_rate": 0.1415538191795349 + }, + "100": { + "rho_bar": 0.5869897603988647, + "union_observed": 0.38826465606689453, + "union_independent": 0.9273321628570557, + "union_gap": -0.5390675067901611, + "mean_per_layer_hit_rate": 0.17958322167396545 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.5015529649598258, + "union_gap_mean_over_k": -0.37474250527364866 + }, + "Github|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.28614622354507446, + "union_observed": 0.05742821469902992, + "union_independent": 0.14118224382400513, + "union_gap": -0.0837540291249752, + "mean_per_layer_hit_rate": 0.01162009034305811 + }, + "2": { + "rho_bar": 0.38966256380081177, + "union_observed": 0.0774032473564148, + "union_independent": 0.23777461051940918, + "union_gap": -0.16037136316299438, + "mean_per_layer_hit_rate": 0.020647266879677773 + }, + "5": { + "rho_bar": 0.39836618304252625, + "union_observed": 0.1423220932483673, + "union_independent": 0.4399362802505493, + "union_gap": -0.297614187002182, + "mean_per_layer_hit_rate": 0.04350331053137779 + }, + "10": { + "rho_bar": 0.4563795328140259, + "union_observed": 0.1910112351179123, + "union_independent": 0.6053264141082764, + "union_gap": -0.4143151789903641, + "mean_per_layer_hit_rate": 0.06876020133495331 + }, + "20": { + "rho_bar": 0.5181804299354553, + "union_observed": 0.24094881117343903, + "union_independent": 0.744821310043335, + "union_gap": -0.5038724988698959, + "mean_per_layer_hit_rate": 0.09910689294338226 + }, + "50": { + "rho_bar": 0.5459930300712585, + "union_observed": 0.31460675597190857, + "union_independent": 0.8529186844825745, + "union_gap": -0.5383119285106659, + "mean_per_layer_hit_rate": 0.13598385453224182 + }, + "100": { + "rho_bar": 0.5510757565498352, + "union_observed": 0.3820224702358246, + "union_independent": 0.9214451313018799, + "union_gap": -0.5394226610660553, + "mean_per_layer_hit_rate": 0.17593392729759216 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.44940053139414105, + "union_gap_mean_over_k": -0.36252312096101896 + }, + "Github|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.3151436150074005, + "union_observed": 0.054931335151195526, + "union_independent": 0.14338374137878418, + "union_gap": -0.08845240622758865, + "mean_per_layer_hit_rate": 0.011812158860266209 + }, + "2": { + "rho_bar": 0.361470103263855, + "union_observed": 0.08239700645208359, + "union_independent": 0.22119677066802979, + "union_gap": -0.1387997642159462, + "mean_per_layer_hit_rate": 0.019014693796634674 + }, + "5": { + "rho_bar": 0.3727218210697174, + "union_observed": 0.15605492889881134, + "union_independent": 0.43586957454681396, + "union_gap": -0.2798146456480026, + "mean_per_layer_hit_rate": 0.04292710870504379 + }, + "10": { + "rho_bar": 0.4374198913574219, + "union_observed": 0.20474407076835632, + "union_independent": 0.5786488652229309, + "union_gap": -0.3739047944545746, + "mean_per_layer_hit_rate": 0.06395851075649261 + }, + "20": { + "rho_bar": 0.4678041636943817, + "union_observed": 0.25717851519584656, + "union_independent": 0.7121028304100037, + "union_gap": -0.4549243152141571, + "mean_per_layer_hit_rate": 0.09055987745523453 + }, + "50": { + "rho_bar": 0.5352407097816467, + "union_observed": 0.32709112763404846, + "union_independent": 0.8376895189285278, + "union_gap": -0.5105983912944794, + "mean_per_layer_hit_rate": 0.12916547060012817 + }, + "100": { + "rho_bar": 0.5525357723236084, + "union_observed": 0.4057428240776062, + "union_independent": 0.9124048948287964, + "union_gap": -0.5066620707511902, + "mean_per_layer_hit_rate": 0.16882742941379547 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.4346194394997188, + "union_gap_mean_over_k": -0.33616519825799124 + }, + "Github|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.23602938652038574, + "union_observed": 0.054931335151195526, + "union_independent": 0.13573414087295532, + "union_gap": -0.0808028057217598, + "mean_per_layer_hit_rate": 0.01113992091268301 + }, + "2": { + "rho_bar": 0.3235606253147125, + "union_observed": 0.08239700645208359, + "union_independent": 0.2221505045890808, + "union_gap": -0.13975349813699722, + "mean_per_layer_hit_rate": 0.019110728055238724 + }, + "5": { + "rho_bar": 0.3582729995250702, + "union_observed": 0.1498127281665802, + "union_independent": 0.4032447338104248, + "union_gap": -0.2534320056438446, + "mean_per_layer_hit_rate": 0.03879765421152115 + }, + "10": { + "rho_bar": 0.41617971658706665, + "union_observed": 0.19850187003612518, + "union_independent": 0.5443779826164246, + "union_gap": -0.3458761125802994, + "mean_per_layer_hit_rate": 0.05829251557588577 + }, + "20": { + "rho_bar": 0.45098188519477844, + "union_observed": 0.25468164682388306, + "union_independent": 0.6862110495567322, + "union_gap": -0.4315294027328491, + "mean_per_layer_hit_rate": 0.08450974524021149 + }, + "50": { + "rho_bar": 0.4859333038330078, + "union_observed": 0.3308364450931549, + "union_independent": 0.8084439039230347, + "union_gap": -0.47760745882987976, + "mean_per_layer_hit_rate": 0.11764141172170639 + }, + "100": { + "rho_bar": 0.5149073600769043, + "union_observed": 0.3957552909851074, + "union_independent": 0.8891096115112305, + "union_gap": -0.49335432052612305, + "mean_per_layer_hit_rate": 0.15317392349243164 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.39798075386456083, + "union_gap_mean_over_k": -0.31747937202453613 + }, + "Github|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.20746853947639465, + "union_observed": 0.05118601769208908, + "union_independent": 0.12038010358810425, + "union_gap": -0.06919408589601517, + "mean_per_layer_hit_rate": 0.009795447811484337 + }, + "2": { + "rho_bar": 0.2888428270816803, + "union_observed": 0.06741572916507721, + "union_independent": 0.18582618236541748, + "union_gap": -0.11841045320034027, + "mean_per_layer_hit_rate": 0.01565350964665413 + }, + "5": { + "rho_bar": 0.37725943326950073, + "union_observed": 0.13483145833015442, + "union_independent": 0.3712971806526184, + "union_gap": -0.236465722322464, + "mean_per_layer_hit_rate": 0.034956302493810654 + }, + "10": { + "rho_bar": 0.4037010967731476, + "union_observed": 0.1860174834728241, + "union_independent": 0.5115511417388916, + "union_gap": -0.3255336582660675, + "mean_per_layer_hit_rate": 0.05329875648021698 + }, + "20": { + "rho_bar": 0.4671609699726105, + "union_observed": 0.23470661044120789, + "union_independent": 0.6518798470497131, + "union_gap": -0.41717323660850525, + "mean_per_layer_hit_rate": 0.0772111788392067 + }, + "50": { + "rho_bar": 0.5124461650848389, + "union_observed": 0.3158551752567291, + "union_independent": 0.7837700247764587, + "union_gap": -0.4679148495197296, + "mean_per_layer_hit_rate": 0.10967060178518295 + }, + "100": { + "rho_bar": 0.5275471210479736, + "union_observed": 0.39700374007225037, + "union_independent": 0.8792721629142761, + "union_gap": -0.48226842284202576, + "mean_per_layer_hit_rate": 0.14818015694618225 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3977751646723066, + "union_gap_mean_over_k": -0.3024229183793068 + }, + "Github|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.23422953486442566, + "union_observed": 0.05243445560336113, + "union_independent": 0.10922425985336304, + "union_gap": -0.05678980425000191, + "mean_per_layer_hit_rate": 0.008835109882056713 + }, + "2": { + "rho_bar": 0.28704026341438293, + "union_observed": 0.0699126124382019, + "union_independent": 0.17537623643875122, + "union_gap": -0.10546362400054932, + "mean_per_layer_hit_rate": 0.014693171717226505 + }, + "5": { + "rho_bar": 0.3558940887451172, + "union_observed": 0.13483145833015442, + "union_independent": 0.3498649001121521, + "union_gap": -0.21503344178199768, + "mean_per_layer_hit_rate": 0.03245942294597626 + }, + "10": { + "rho_bar": 0.4101725220680237, + "union_observed": 0.17727839946746826, + "union_independent": 0.47738271951675415, + "union_gap": -0.3001043200492859, + "mean_per_layer_hit_rate": 0.048401035368442535 + }, + "20": { + "rho_bar": 0.47834762930870056, + "union_observed": 0.22347065806388855, + "union_independent": 0.6295530796051025, + "union_gap": -0.406082421541214, + "mean_per_layer_hit_rate": 0.07288965582847595 + }, + "50": { + "rho_bar": 0.5154850482940674, + "union_observed": 0.29588013887405396, + "union_independent": 0.7660923600196838, + "union_gap": -0.4702122211456299, + "mean_per_layer_hit_rate": 0.10467685014009476 + }, + "100": { + "rho_bar": 0.5310801267623901, + "union_observed": 0.37328338623046875, + "union_independent": 0.8581092357635498, + "union_gap": -0.48482584953308105, + "mean_per_layer_hit_rate": 0.13800057768821716 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.40174988763672964, + "union_gap_mean_over_k": -0.2912159546145371 + }, + "Github|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.2158379852771759, + "union_observed": 0.04494382068514824, + "union_independent": 0.11022794246673584, + "union_gap": -0.0652841217815876, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.34209731221199036, + "union_observed": 0.05742821469902992, + "union_independent": 0.18043529987335205, + "union_gap": -0.12300708517432213, + "mean_per_layer_hit_rate": 0.015173342078924179 + }, + "5": { + "rho_bar": 0.4115365743637085, + "union_observed": 0.10237203538417816, + "union_independent": 0.3323069214820862, + "union_gap": -0.22993488609790802, + "mean_per_layer_hit_rate": 0.03053875081241131 + }, + "10": { + "rho_bar": 0.4574041962623596, + "union_observed": 0.13607990741729736, + "union_independent": 0.4462418556213379, + "union_gap": -0.3101619482040405, + "mean_per_layer_hit_rate": 0.04427158460021019 + }, + "20": { + "rho_bar": 0.4771031141281128, + "union_observed": 0.1810237169265747, + "union_independent": 0.589601993560791, + "union_gap": -0.4085782766342163, + "mean_per_layer_hit_rate": 0.06587918847799301 + }, + "50": { + "rho_bar": 0.5285885334014893, + "union_observed": 0.24843944609165192, + "union_independent": 0.7408294081687927, + "union_gap": -0.4923899620771408, + "mean_per_layer_hit_rate": 0.09814655035734177 + }, + "100": { + "rho_bar": 0.5319090485572815, + "union_observed": 0.3308364450931549, + "union_independent": 0.8394662141799927, + "union_gap": -0.5086297690868378, + "mean_per_layer_hit_rate": 0.13060596585273743 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.42349668060030254, + "union_gap_mean_over_k": -0.305426578436579 + }, + "Github|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.18336991965770721, + "union_observed": 0.039950061589479446, + "union_independent": 0.08761739730834961, + "union_gap": -0.04766733571887016, + "mean_per_layer_hit_rate": 0.007010467816144228 + }, + "2": { + "rho_bar": 0.268719345331192, + "union_observed": 0.05742821469902992, + "union_independent": 0.15432816743850708, + "union_gap": -0.09689995273947716, + "mean_per_layer_hit_rate": 0.012772495858371258 + }, + "5": { + "rho_bar": 0.3618292808532715, + "union_observed": 0.09612983465194702, + "union_independent": 0.29567480087280273, + "union_gap": -0.1995449662208557, + "mean_per_layer_hit_rate": 0.026505330577492714 + }, + "10": { + "rho_bar": 0.41576310992240906, + "union_observed": 0.13483145833015442, + "union_independent": 0.4258425235748291, + "union_gap": -0.2910110652446747, + "mean_per_layer_hit_rate": 0.04158264026045799 + }, + "20": { + "rho_bar": 0.4961746335029602, + "union_observed": 0.15980024635791779, + "union_independent": 0.5698221921920776, + "union_gap": -0.41002194583415985, + "mean_per_layer_hit_rate": 0.062421973794698715 + }, + "50": { + "rho_bar": 0.5248589515686035, + "union_observed": 0.23595505952835083, + "union_independent": 0.7295899987220764, + "union_gap": -0.4936349391937256, + "mean_per_layer_hit_rate": 0.09507346153259277 + }, + "100": { + "rho_bar": 0.5417802929878235, + "union_observed": 0.3096129894256592, + "union_independent": 0.8302299976348877, + "union_gap": -0.5206170082092285, + "mean_per_layer_hit_rate": 0.12666860222816467 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.39892793340342386, + "union_gap_mean_over_k": -0.29419960188014166 + }, + "Github|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.1678943634033203, + "union_observed": 0.04494382068514824, + "union_independent": 0.10132598876953125, + "union_gap": -0.05638216808438301, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.2936429977416992, + "union_observed": 0.06117353215813637, + "union_independent": 0.17569661140441895, + "union_gap": -0.11452307924628258, + "mean_per_layer_hit_rate": 0.014693171717226505 + }, + "5": { + "rho_bar": 0.3398188650608063, + "union_observed": 0.10237203538417816, + "union_independent": 0.31120872497558594, + "union_gap": -0.20883668959140778, + "mean_per_layer_hit_rate": 0.028137903660535812 + }, + "10": { + "rho_bar": 0.41380470991134644, + "union_observed": 0.14107365906238556, + "union_independent": 0.45080137252807617, + "union_gap": -0.3097277134656906, + "mean_per_layer_hit_rate": 0.04475175589323044 + }, + "20": { + "rho_bar": 0.4914354681968689, + "union_observed": 0.17478151619434357, + "union_independent": 0.5991954207420349, + "union_gap": -0.42441390454769135, + "mean_per_layer_hit_rate": 0.06731969863176346 + }, + "50": { + "rho_bar": 0.5228752493858337, + "union_observed": 0.24094881117343903, + "union_independent": 0.7473941445350647, + "union_gap": -0.5064453333616257, + "mean_per_layer_hit_rate": 0.09939498454332352 + }, + "100": { + "rho_bar": 0.5172902345657349, + "union_observed": 0.3208489418029785, + "union_independent": 0.8397970199584961, + "union_gap": -0.5189480781555176, + "mean_per_layer_hit_rate": 0.12993374466896057 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3923945554665157, + "union_gap_mean_over_k": -0.30561099520751406 + }, + "Github|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1348678320646286, + "union_observed": 0.05243445560336113, + "union_independent": 0.1105772852897644, + "union_gap": -0.058142829686403275, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.2910207509994507, + "union_observed": 0.07116104662418365, + "union_independent": 0.17996615171432495, + "union_gap": -0.1088051050901413, + "mean_per_layer_hit_rate": 0.015077306888997555 + }, + "5": { + "rho_bar": 0.3074280321598053, + "union_observed": 0.10237203538417816, + "union_independent": 0.3081267476081848, + "union_gap": -0.20575471222400665, + "mean_per_layer_hit_rate": 0.027753770351409912 + }, + "10": { + "rho_bar": 0.3998560905456543, + "union_observed": 0.13607990741729736, + "union_independent": 0.44535505771636963, + "union_gap": -0.30927515029907227, + "mean_per_layer_hit_rate": 0.04398348554968834 + }, + "20": { + "rho_bar": 0.48295846581459045, + "union_observed": 0.17353308200836182, + "union_independent": 0.6107808351516724, + "union_gap": -0.43724775314331055, + "mean_per_layer_hit_rate": 0.06924037635326385 + }, + "50": { + "rho_bar": 0.5119057893753052, + "union_observed": 0.23970037698745728, + "union_independent": 0.7535594701766968, + "union_gap": -0.5138590931892395, + "mean_per_layer_hit_rate": 0.10083548724651337 + }, + "100": { + "rho_bar": 0.5062417984008789, + "union_observed": 0.3258427083492279, + "union_independent": 0.8392151594161987, + "union_gap": -0.5133724510669708, + "mean_per_layer_hit_rate": 0.12945356965065002 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.37632553705147337, + "union_gap_mean_over_k": -0.30663672781416346 + }, + "Github|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.14710623025894165, + "union_observed": 0.04619225859642029, + "union_independent": 0.09108638763427734, + "union_gap": -0.044894129037857056, + "mean_per_layer_hit_rate": 0.007298569194972515 + }, + "2": { + "rho_bar": 0.27010783553123474, + "union_observed": 0.06367041170597076, + "union_independent": 0.17048877477645874, + "union_gap": -0.10681836307048798, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.2988719940185547, + "union_observed": 0.10986267030239105, + "union_independent": 0.3105320334434509, + "union_gap": -0.20066936314105988, + "mean_per_layer_hit_rate": 0.028041871264576912 + }, + "10": { + "rho_bar": 0.37973785400390625, + "union_observed": 0.1423220932483673, + "union_independent": 0.4495950937271118, + "union_gap": -0.3072730004787445, + "mean_per_layer_hit_rate": 0.04455968365073204 + }, + "20": { + "rho_bar": 0.46288415789604187, + "union_observed": 0.18476903438568115, + "union_independent": 0.6114093065261841, + "union_gap": -0.42664027214050293, + "mean_per_layer_hit_rate": 0.0693364068865776 + }, + "50": { + "rho_bar": 0.47635582089424133, + "union_observed": 0.24968789517879486, + "union_independent": 0.751378059387207, + "union_gap": -0.5016901642084122, + "mean_per_layer_hit_rate": 0.10016325861215591 + }, + "100": { + "rho_bar": 0.4848443567752838, + "union_observed": 0.33957552909851074, + "union_independent": 0.8483709692955017, + "union_gap": -0.508795440196991, + "mean_per_layer_hit_rate": 0.13300682604312897 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3599868927683149, + "union_gap_mean_over_k": -0.29954010461057934 + }, + "Github|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.16946278512477875, + "union_observed": 0.04494382068514824, + "union_independent": 0.1002269983291626, + "union_gap": -0.05528317764401436, + "mean_per_layer_hit_rate": 0.00806683860719204 + }, + "2": { + "rho_bar": 0.24044036865234375, + "union_observed": 0.06866417080163956, + "union_independent": 0.18835413455963135, + "union_gap": -0.11968996375799179, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.2791528105735779, + "union_observed": 0.11235955357551575, + "union_independent": 0.32644498348236084, + "union_gap": -0.2140854299068451, + "mean_per_layer_hit_rate": 0.02977048046886921 + }, + "10": { + "rho_bar": 0.3655582666397095, + "union_observed": 0.1473158597946167, + "union_independent": 0.48290765285491943, + "union_gap": -0.33559179306030273, + "mean_per_layer_hit_rate": 0.04907327517867088 + }, + "20": { + "rho_bar": 0.4228166341781616, + "union_observed": 0.1960049867630005, + "union_independent": 0.6339192390441895, + "union_gap": -0.43791425228118896, + "mean_per_layer_hit_rate": 0.073561891913414 + }, + "50": { + "rho_bar": 0.46289587020874023, + "union_observed": 0.2746566832065582, + "union_independent": 0.773231029510498, + "union_gap": -0.4985743463039398, + "mean_per_layer_hit_rate": 0.10640545934438705 + }, + "100": { + "rho_bar": 0.4582499861717224, + "union_observed": 0.3570536971092224, + "union_independent": 0.8628525137901306, + "union_gap": -0.5057988166809082, + "mean_per_layer_hit_rate": 0.13953711092472076 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.34265381736414774, + "union_gap_mean_over_k": -0.3095625399478844 + }, + "Github|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.20292744040489197, + "union_observed": 0.04369537904858589, + "union_independent": 0.10480892658233643, + "union_gap": -0.061113547533750534, + "mean_per_layer_hit_rate": 0.008450974710285664 + }, + "2": { + "rho_bar": 0.2872655391693115, + "union_observed": 0.07365792989730835, + "union_independent": 0.1902828812599182, + "union_gap": -0.11662495136260986, + "mean_per_layer_hit_rate": 0.01603764481842518 + }, + "5": { + "rho_bar": 0.3191966414451599, + "union_observed": 0.11485642939805984, + "union_independent": 0.35597842931747437, + "union_gap": -0.24112199991941452, + "mean_per_layer_hit_rate": 0.033131662756204605 + }, + "10": { + "rho_bar": 0.37549760937690735, + "union_observed": 0.16604244709014893, + "union_independent": 0.5104039907455444, + "union_gap": -0.3443615436553955, + "mean_per_layer_hit_rate": 0.053106699138879776 + }, + "20": { + "rho_bar": 0.4386368691921234, + "union_observed": 0.21847690641880035, + "union_independent": 0.676193356513977, + "union_gap": -0.4577164500951767, + "mean_per_layer_hit_rate": 0.08230097591876984 + }, + "50": { + "rho_bar": 0.46631109714508057, + "union_observed": 0.30087390542030334, + "union_independent": 0.8030639886856079, + "union_gap": -0.5021900832653046, + "mean_per_layer_hit_rate": 0.11620090901851654 + }, + "100": { + "rho_bar": 0.4850912392139435, + "union_observed": 0.38576778769493103, + "union_independent": 0.882600724697113, + "union_gap": -0.496832937002182, + "mean_per_layer_hit_rate": 0.14990876615047455 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.367846633706774, + "union_gap_mean_over_k": -0.31713735897626194 + }, + "Github|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.22723309695720673, + "union_observed": 0.04369537904858589, + "union_independent": 0.10916531085968018, + "union_gap": -0.06546993181109428, + "mean_per_layer_hit_rate": 0.008835110813379288 + }, + "2": { + "rho_bar": 0.34593454003334045, + "union_observed": 0.07116104662418365, + "union_independent": 0.20528733730316162, + "union_gap": -0.13412629067897797, + "mean_per_layer_hit_rate": 0.017478151246905327 + }, + "5": { + "rho_bar": 0.33511146903038025, + "union_observed": 0.12109863013029099, + "union_independent": 0.3655647039413452, + "union_gap": -0.24446607381105423, + "mean_per_layer_hit_rate": 0.0342840701341629 + }, + "10": { + "rho_bar": 0.40106308460235596, + "union_observed": 0.17727839946746826, + "union_independent": 0.5445108413696289, + "union_gap": -0.36723244190216064, + "mean_per_layer_hit_rate": 0.05838855355978012 + }, + "20": { + "rho_bar": 0.43851906061172485, + "union_observed": 0.24469412863254547, + "union_independent": 0.702568531036377, + "union_gap": -0.4578744024038315, + "mean_per_layer_hit_rate": 0.08835109323263168 + }, + "50": { + "rho_bar": 0.4801308214664459, + "union_observed": 0.32709112763404846, + "union_independent": 0.8263999223709106, + "union_gap": -0.4993087947368622, + "mean_per_layer_hit_rate": 0.12455584108829498 + }, + "100": { + "rho_bar": 0.49222809076309204, + "union_observed": 0.3895131051540375, + "union_independent": 0.8995137810707092, + "union_gap": -0.5100006759166718, + "mean_per_layer_hit_rate": 0.15989629924297333 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3886028804949352, + "union_gap_mean_over_k": -0.3254969444658075 + }, + "Github|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.17799779772758484, + "union_observed": 0.04619225859642029, + "union_independent": 0.10686874389648438, + "union_gap": -0.06067648530006409, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.2861497700214386, + "union_observed": 0.0774032473564148, + "union_independent": 0.19391465187072754, + "union_gap": -0.11651140451431274, + "mean_per_layer_hit_rate": 0.016421781852841377 + }, + "5": { + "rho_bar": 0.37073564529418945, + "union_observed": 0.13233457505702972, + "union_independent": 0.35860657691955566, + "union_gap": -0.22627200186252594, + "mean_per_layer_hit_rate": 0.033515799790620804 + }, + "10": { + "rho_bar": 0.4227370321750641, + "union_observed": 0.18227216601371765, + "union_independent": 0.5042571425437927, + "union_gap": -0.3219849765300751, + "mean_per_layer_hit_rate": 0.05233842507004738 + }, + "20": { + "rho_bar": 0.477649450302124, + "union_observed": 0.23595505952835083, + "union_independent": 0.6654924154281616, + "union_gap": -0.4295373558998108, + "mean_per_layer_hit_rate": 0.08028426766395569 + }, + "50": { + "rho_bar": 0.5179688334465027, + "union_observed": 0.2971285879611969, + "union_independent": 0.7876909375190735, + "union_gap": -0.4905623495578766, + "mean_per_layer_hit_rate": 0.11139921844005585 + }, + "100": { + "rho_bar": 0.5106651186943054, + "union_observed": 0.3907615542411804, + "union_independent": 0.8730044364929199, + "union_gap": -0.4822428822517395, + "mean_per_layer_hit_rate": 0.14529913663864136 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3948433782373156, + "union_gap_mean_over_k": -0.3039696365594864 + }, + "Github|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.1794520616531372, + "union_observed": 0.056179776787757874, + "union_independent": 0.11376255750656128, + "union_gap": -0.057582780718803406, + "mean_per_layer_hit_rate": 0.009219245053827763 + }, + "2": { + "rho_bar": 0.25294458866119385, + "union_observed": 0.0774032473564148, + "union_independent": 0.19016844034194946, + "union_gap": -0.11276519298553467, + "mean_per_layer_hit_rate": 0.01603764481842518 + }, + "5": { + "rho_bar": 0.3126162886619568, + "union_observed": 0.12858925759792328, + "union_independent": 0.3537293076515198, + "union_gap": -0.2251400500535965, + "mean_per_layer_hit_rate": 0.032843563705682755 + }, + "10": { + "rho_bar": 0.3816777467727661, + "union_observed": 0.16604244709014893, + "union_independent": 0.48589617013931274, + "union_gap": -0.3198537230491638, + "mean_per_layer_hit_rate": 0.049553439021110535 + }, + "20": { + "rho_bar": 0.47180336713790894, + "union_observed": 0.22347065806388855, + "union_independent": 0.650321364402771, + "union_gap": -0.42685070633888245, + "mean_per_layer_hit_rate": 0.07692307978868484 + }, + "50": { + "rho_bar": 0.5100205540657043, + "union_observed": 0.30337077379226685, + "union_independent": 0.7854197025299072, + "union_gap": -0.4820489287376404, + "mean_per_layer_hit_rate": 0.11024679243564606 + }, + "100": { + "rho_bar": 0.5034611225128174, + "union_observed": 0.40449437499046326, + "union_independent": 0.8760829567909241, + "union_gap": -0.4715885818004608, + "mean_per_layer_hit_rate": 0.1463555246591568 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.37313938992364065, + "union_gap_mean_over_k": -0.2994042805262974 + }, + "Github|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.15249942243099213, + "union_observed": 0.05992509424686432, + "union_independent": 0.1160653829574585, + "union_gap": -0.05614028871059418, + "mean_per_layer_hit_rate": 0.009411312639713287 + }, + "2": { + "rho_bar": 0.25199586153030396, + "union_observed": 0.07865168899297714, + "union_independent": 0.19014370441436768, + "union_gap": -0.11149201542139053, + "mean_per_layer_hit_rate": 0.01603764481842518 + }, + "5": { + "rho_bar": 0.29201725125312805, + "union_observed": 0.14107365906238556, + "union_independent": 0.3552766442298889, + "union_gap": -0.21420298516750336, + "mean_per_layer_hit_rate": 0.033035628497600555 + }, + "10": { + "rho_bar": 0.3834494352340698, + "union_observed": 0.1810237169265747, + "union_independent": 0.5017324686050415, + "union_gap": -0.3207087516784668, + "mean_per_layer_hit_rate": 0.05176221951842308 + }, + "20": { + "rho_bar": 0.44274628162384033, + "union_observed": 0.24344569444656372, + "union_independent": 0.6606302261352539, + "union_gap": -0.4171845316886902, + "mean_per_layer_hit_rate": 0.07893978804349899 + }, + "50": { + "rho_bar": 0.49571701884269714, + "union_observed": 0.33957552909851074, + "union_independent": 0.8019561767578125, + "union_gap": -0.46238064765930176, + "mean_per_layer_hit_rate": 0.11543262749910355 + }, + "100": { + "rho_bar": 0.4865129888057709, + "union_observed": 0.40699127316474915, + "union_independent": 0.876678466796875, + "union_gap": -0.46968719363212585, + "mean_per_layer_hit_rate": 0.1463555246591568 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3578483228172575, + "union_gap_mean_over_k": -0.2931137734225818 + }, + "Github|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.19962181150913239, + "union_observed": 0.047440700232982635, + "union_independent": 0.10808247327804565, + "union_gap": -0.06064177304506302, + "mean_per_layer_hit_rate": 0.008739075623452663 + }, + "2": { + "rho_bar": 0.2719820737838745, + "union_observed": 0.06741572916507721, + "union_independent": 0.1787317991256714, + "union_gap": -0.11131606996059418, + "mean_per_layer_hit_rate": 0.01498127356171608 + }, + "5": { + "rho_bar": 0.3148719370365143, + "union_observed": 0.1086142286658287, + "union_independent": 0.31644874811172485, + "union_gap": -0.20783451944589615, + "mean_per_layer_hit_rate": 0.02871410734951496 + }, + "10": { + "rho_bar": 0.3979010581970215, + "union_observed": 0.15355804562568665, + "union_independent": 0.4720953702926636, + "union_gap": -0.31853732466697693, + "mean_per_layer_hit_rate": 0.047632768750190735 + }, + "20": { + "rho_bar": 0.4513193666934967, + "union_observed": 0.20349563658237457, + "union_independent": 0.6196032762527466, + "union_gap": -0.416107639670372, + "mean_per_layer_hit_rate": 0.07096897810697556 + }, + "50": { + "rho_bar": 0.4802101254463196, + "union_observed": 0.28838950395584106, + "union_independent": 0.762019157409668, + "union_gap": -0.4736296534538269, + "mean_per_layer_hit_rate": 0.10323633253574371 + }, + "100": { + "rho_bar": 0.4839017689228058, + "union_observed": 0.3695380687713623, + "union_independent": 0.8467788696289062, + "union_gap": -0.47724080085754395, + "mean_per_layer_hit_rate": 0.13262268900871277 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.3714011630841664, + "union_gap_mean_over_k": -0.29504396872861044 + }, + "Github|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.15084612369537354, + "union_observed": 0.05243445560336113, + "union_independent": 0.11475175619125366, + "union_gap": -0.06231730058789253, + "mean_per_layer_hit_rate": 0.009315279312431812 + }, + "2": { + "rho_bar": 0.2705700695514679, + "union_observed": 0.08614232391119003, + "union_independent": 0.20722103118896484, + "union_gap": -0.12107870727777481, + "mean_per_layer_hit_rate": 0.017670217901468277 + }, + "5": { + "rho_bar": 0.36956703662872314, + "union_observed": 0.1473158597946167, + "union_independent": 0.3850046396255493, + "union_gap": -0.23768877983093262, + "mean_per_layer_hit_rate": 0.0365888811647892 + }, + "10": { + "rho_bar": 0.429269939661026, + "union_observed": 0.18227216601371765, + "union_independent": 0.520346999168396, + "union_gap": -0.33807483315467834, + "mean_per_layer_hit_rate": 0.05464323237538338 + }, + "20": { + "rho_bar": 0.4973564147949219, + "union_observed": 0.22846442461013794, + "union_independent": 0.6661365032196045, + "union_gap": -0.43767207860946655, + "mean_per_layer_hit_rate": 0.0802842527627945 + }, + "50": { + "rho_bar": 0.506778359413147, + "union_observed": 0.3133583068847656, + "union_independent": 0.7950015068054199, + "union_gap": -0.4816431999206543, + "mean_per_layer_hit_rate": 0.1134159192442894 + }, + "100": { + "rho_bar": 0.5258435010910034, + "union_observed": 0.3820224702358246, + "union_independent": 0.875849187374115, + "union_gap": -0.4938267171382904, + "mean_per_layer_hit_rate": 0.14645154774188995 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3928902064050947, + "union_gap_mean_over_k": -0.31032880235995564 + }, + "Github|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.3179742991924286, + "union_observed": 0.05243445560336113, + "union_independent": 0.14762216806411743, + "union_gap": -0.0951877124607563, + "mean_per_layer_hit_rate": 0.012196293100714684 + }, + "2": { + "rho_bar": 0.44241616129875183, + "union_observed": 0.07615480571985245, + "union_independent": 0.2647804021835327, + "union_gap": -0.18862559646368027, + "mean_per_layer_hit_rate": 0.02333621308207512 + }, + "5": { + "rho_bar": 0.47369658946990967, + "union_observed": 0.13233457505702972, + "union_independent": 0.43731075525283813, + "union_gap": -0.3049761801958084, + "mean_per_layer_hit_rate": 0.04311918467283249 + }, + "10": { + "rho_bar": 0.5366672873497009, + "union_observed": 0.17727839946746826, + "union_independent": 0.6200312376022339, + "union_gap": -0.4427528381347656, + "mean_per_layer_hit_rate": 0.071257084608078 + }, + "20": { + "rho_bar": 0.5863531827926636, + "union_observed": 0.23096129298210144, + "union_independent": 0.7511342763900757, + "union_gap": -0.5201729834079742, + "mean_per_layer_hit_rate": 0.10035531967878342 + }, + "50": { + "rho_bar": 0.5863200426101685, + "union_observed": 0.3046192228794098, + "union_independent": 0.8580309748649597, + "union_gap": -0.5534117519855499, + "mean_per_layer_hit_rate": 0.13761644065380096 + }, + "100": { + "rho_bar": 0.5892564654350281, + "union_observed": 0.38077402114868164, + "union_independent": 0.9244929552078247, + "union_gap": -0.5437189340591431, + "mean_per_layer_hit_rate": 0.1773744374513626 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.5046691468783787, + "union_gap_mean_over_k": -0.3784065709582397 + }, + "OOD_News_2024|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.2396000623703003, + "union_observed": 0.05368289723992348, + "union_independent": 0.138147234916687, + "union_gap": -0.08446433767676353, + "mean_per_layer_hit_rate": 0.01133198942989111 + }, + "2": { + "rho_bar": 0.28104832768440247, + "union_observed": 0.08739075809717178, + "union_independent": 0.2196420431137085, + "union_gap": -0.1322512850165367, + "mean_per_layer_hit_rate": 0.018822625279426575 + }, + "5": { + "rho_bar": 0.34775352478027344, + "union_observed": 0.1398252248764038, + "union_independent": 0.41401737928390503, + "union_gap": -0.2741921544075012, + "mean_per_layer_hit_rate": 0.040046095848083496 + }, + "10": { + "rho_bar": 0.4387834966182709, + "union_observed": 0.1885143518447876, + "union_independent": 0.5743684768676758, + "union_gap": -0.3858541250228882, + "mean_per_layer_hit_rate": 0.06299817562103271 + }, + "20": { + "rho_bar": 0.4801396131515503, + "union_observed": 0.2634207308292389, + "union_independent": 0.7303110957145691, + "union_gap": -0.4668903648853302, + "mean_per_layer_hit_rate": 0.09468932449817657 + }, + "50": { + "rho_bar": 0.521639347076416, + "union_observed": 0.3545567989349365, + "union_independent": 0.863187849521637, + "union_gap": -0.5086310505867004, + "mean_per_layer_hit_rate": 0.1396331489086151 + }, + "100": { + "rho_bar": 0.5258630514144897, + "union_observed": 0.44569289684295654, + "union_independent": 0.9327012300491333, + "union_gap": -0.48700833320617676, + "mean_per_layer_hit_rate": 0.1840967983007431 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.404975346156529, + "union_gap_mean_over_k": -0.33418452154312817 + }, + "OOD_News_2024|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.19319990277290344, + "union_observed": 0.05118601769208908, + "union_independent": 0.12726271152496338, + "union_gap": -0.0760766938328743, + "mean_per_layer_hit_rate": 0.010371650569140911 + }, + "2": { + "rho_bar": 0.21472106873989105, + "union_observed": 0.07365792989730835, + "union_independent": 0.18196672201156616, + "union_gap": -0.10830879211425781, + "mean_per_layer_hit_rate": 0.01526937447488308 + }, + "5": { + "rho_bar": 0.27358683943748474, + "union_observed": 0.14107365906238556, + "union_independent": 0.36599141359329224, + "union_gap": -0.22491775453090668, + "mean_per_layer_hit_rate": 0.03418803587555885 + }, + "10": { + "rho_bar": 0.3691923916339874, + "union_observed": 0.1810237169265747, + "union_independent": 0.5203738212585449, + "union_gap": -0.3393501043319702, + "mean_per_layer_hit_rate": 0.054355137050151825 + }, + "20": { + "rho_bar": 0.41220465302467346, + "union_observed": 0.25218477845191956, + "union_independent": 0.6786249876022339, + "union_gap": -0.42644020915031433, + "mean_per_layer_hit_rate": 0.08239699900150299 + }, + "50": { + "rho_bar": 0.4821455478668213, + "union_observed": 0.3545567989349365, + "union_independent": 0.8249993920326233, + "union_gap": -0.47044259309768677, + "mean_per_layer_hit_rate": 0.12321136891841888 + }, + "100": { + "rho_bar": 0.5204368233680725, + "union_observed": 0.4319600462913513, + "union_independent": 0.9090772867202759, + "union_gap": -0.47711724042892456, + "mean_per_layer_hit_rate": 0.16498607397079468 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.35221246097769054, + "union_gap_mean_over_k": -0.30323619821241926 + }, + "OOD_News_2024|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.1686110496520996, + "union_observed": 0.04619225859642029, + "union_independent": 0.10591942071914673, + "union_gap": -0.05972716212272644, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.1748557686805725, + "union_observed": 0.08114856481552124, + "union_independent": 0.17364650964736938, + "union_gap": -0.09249794483184814, + "mean_per_layer_hit_rate": 0.014501105062663555 + }, + "5": { + "rho_bar": 0.23214633762836456, + "union_observed": 0.1423220932483673, + "union_independent": 0.34414803981781006, + "union_gap": -0.20182594656944275, + "mean_per_layer_hit_rate": 0.03169115632772446 + }, + "10": { + "rho_bar": 0.29353609681129456, + "union_observed": 0.18726591765880585, + "union_independent": 0.48436594009399414, + "union_gap": -0.2971000224351883, + "mean_per_layer_hit_rate": 0.04907327517867088 + }, + "20": { + "rho_bar": 0.3480207920074463, + "union_observed": 0.2584269642829895, + "union_independent": 0.647079586982727, + "union_gap": -0.38865262269973755, + "mean_per_layer_hit_rate": 0.07577066868543625 + }, + "50": { + "rho_bar": 0.4162154495716095, + "union_observed": 0.34706616401672363, + "union_independent": 0.798259973526001, + "union_gap": -0.45119380950927734, + "mean_per_layer_hit_rate": 0.11351195722818375 + }, + "100": { + "rho_bar": 0.4696568250656128, + "union_observed": 0.4369538128376007, + "union_independent": 0.8921111822128296, + "union_gap": -0.4551573693752289, + "mean_per_layer_hit_rate": 0.15394218266010284 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3004346170595714, + "union_gap_mean_over_k": -0.2780221253633499 + }, + "OOD_News_2024|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.14460012316703796, + "union_observed": 0.04369537904858589, + "union_independent": 0.08310246467590332, + "union_gap": -0.03940708562731743, + "mean_per_layer_hit_rate": 0.006626332178711891 + }, + "2": { + "rho_bar": 0.16445599496364594, + "union_observed": 0.072409488260746, + "union_independent": 0.14495015144348145, + "union_gap": -0.07254066318273544, + "mean_per_layer_hit_rate": 0.011908192187547684 + }, + "5": { + "rho_bar": 0.23897317051887512, + "union_observed": 0.12983770668506622, + "union_independent": 0.30838543176651, + "union_gap": -0.1785477250814438, + "mean_per_layer_hit_rate": 0.027753770351409912 + }, + "10": { + "rho_bar": 0.29607492685317993, + "union_observed": 0.1960049867630005, + "union_independent": 0.4566071629524231, + "union_gap": -0.2606021761894226, + "mean_per_layer_hit_rate": 0.04532795771956444 + }, + "20": { + "rho_bar": 0.3447027802467346, + "union_observed": 0.25717851519584656, + "union_independent": 0.6090715527534485, + "union_gap": -0.35189303755760193, + "mean_per_layer_hit_rate": 0.06866417080163956 + }, + "50": { + "rho_bar": 0.3918948471546173, + "union_observed": 0.34456929564476013, + "union_independent": 0.7686408162117004, + "union_gap": -0.4240715205669403, + "mean_per_layer_hit_rate": 0.10448478162288666 + }, + "100": { + "rho_bar": 0.4338008165359497, + "union_observed": 0.43320849537849426, + "union_independent": 0.8721137642860413, + "union_gap": -0.438905268907547, + "mean_per_layer_hit_rate": 0.14337848126888275 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.2877860942057201, + "union_gap_mean_over_k": -0.2522810681590012 + }, + "OOD_News_2024|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.09450346231460571, + "union_observed": 0.048689138144254684, + "union_independent": 0.07262396812438965, + "union_gap": -0.023934829980134964, + "mean_per_layer_hit_rate": 0.00576202804222703 + }, + "2": { + "rho_bar": 0.14528317749500275, + "union_observed": 0.07990012317895889, + "union_independent": 0.1426006555557251, + "union_gap": -0.0627005323767662, + "mean_per_layer_hit_rate": 0.011716123670339584 + }, + "5": { + "rho_bar": 0.23606407642364502, + "union_observed": 0.13607990741729736, + "union_independent": 0.30718469619750977, + "union_gap": -0.1711047887802124, + "mean_per_layer_hit_rate": 0.027657736092805862 + }, + "10": { + "rho_bar": 0.2894498109817505, + "union_observed": 0.1835206001996994, + "union_independent": 0.4472277760505676, + "union_gap": -0.2637071758508682, + "mean_per_layer_hit_rate": 0.04417555034160614 + }, + "20": { + "rho_bar": 0.3328314423561096, + "union_observed": 0.24469412863254547, + "union_independent": 0.5784026980400085, + "union_gap": -0.3337085694074631, + "mean_per_layer_hit_rate": 0.06357438117265701 + }, + "50": { + "rho_bar": 0.37044042348861694, + "union_observed": 0.3283395767211914, + "union_independent": 0.7361859083175659, + "union_gap": -0.4078463315963745, + "mean_per_layer_hit_rate": 0.09603380411863327 + }, + "100": { + "rho_bar": 0.3922746181488037, + "union_observed": 0.4307115972042084, + "union_independent": 0.8568605184555054, + "union_gap": -0.426148921251297, + "mean_per_layer_hit_rate": 0.13646402955055237 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2658352873155049, + "union_gap_mean_over_k": -0.2413073070347309 + }, + "OOD_News_2024|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.12449135631322861, + "union_observed": 0.04494382068514824, + "union_independent": 0.07478827238082886, + "union_gap": -0.02984445169568062, + "mean_per_layer_hit_rate": 0.005954096093773842 + }, + "2": { + "rho_bar": 0.1828407496213913, + "union_observed": 0.06367041170597076, + "union_independent": 0.13031268119812012, + "union_gap": -0.06664226949214935, + "mean_per_layer_hit_rate": 0.010659752413630486 + }, + "5": { + "rho_bar": 0.27592435479164124, + "union_observed": 0.10362047702074051, + "union_independent": 0.28398340940475464, + "union_gap": -0.18036293238401413, + "mean_per_layer_hit_rate": 0.025256890803575516 + }, + "10": { + "rho_bar": 0.2956033945083618, + "union_observed": 0.16104869544506073, + "union_independent": 0.4270746111869812, + "union_gap": -0.26602591574192047, + "mean_per_layer_hit_rate": 0.041678667068481445 + }, + "20": { + "rho_bar": 0.34006467461586, + "union_observed": 0.20724095404148102, + "union_independent": 0.5587591528892517, + "union_gap": -0.3515181988477707, + "mean_per_layer_hit_rate": 0.06050129979848862 + }, + "50": { + "rho_bar": 0.3509131371974945, + "union_observed": 0.31710362434387207, + "union_independent": 0.7261613607406616, + "union_gap": -0.40905773639678955, + "mean_per_layer_hit_rate": 0.09372898936271667 + }, + "100": { + "rho_bar": 0.3729168474674225, + "union_observed": 0.4032459557056427, + "union_independent": 0.8582744002342224, + "union_gap": -0.4550284445285797, + "mean_per_layer_hit_rate": 0.1371362805366516 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.27753635921648573, + "union_gap_mean_over_k": -0.25121142129812923 + }, + "OOD_News_2024|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.1437811702489853, + "union_observed": 0.041198503226041794, + "union_independent": 0.0748336911201477, + "union_gap": -0.03363518789410591, + "mean_per_layer_hit_rate": 0.005954096093773842 + }, + "2": { + "rho_bar": 0.20446108281612396, + "union_observed": 0.06117353215813637, + "union_independent": 0.1348438858985901, + "union_gap": -0.07367035374045372, + "mean_per_layer_hit_rate": 0.011043887585401535 + }, + "5": { + "rho_bar": 0.26766708493232727, + "union_observed": 0.1111111119389534, + "union_independent": 0.28695785999298096, + "union_gap": -0.17584674805402756, + "mean_per_layer_hit_rate": 0.025544991716742516 + }, + "10": { + "rho_bar": 0.31475239992141724, + "union_observed": 0.14856429398059845, + "union_independent": 0.40630006790161133, + "union_gap": -0.2577357739210129, + "mean_per_layer_hit_rate": 0.03898972272872925 + }, + "20": { + "rho_bar": 0.33842161297798157, + "union_observed": 0.20474407076835632, + "union_independent": 0.5614930391311646, + "union_gap": -0.3567489683628082, + "mean_per_layer_hit_rate": 0.060789402574300766 + }, + "50": { + "rho_bar": 0.3736056685447693, + "union_observed": 0.30586767196655273, + "union_independent": 0.7546579837799072, + "union_gap": -0.4487903118133545, + "mean_per_layer_hit_rate": 0.10064343363046646 + }, + "100": { + "rho_bar": 0.37875038385391235, + "union_observed": 0.4007490575313568, + "union_independent": 0.8737422227859497, + "union_gap": -0.4729931652545929, + "mean_per_layer_hit_rate": 0.14376260340213776 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2887770576136453, + "union_gap_mean_over_k": -0.25991721557719366 + }, + "OOD_News_2024|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.1557082086801529, + "union_observed": 0.0387016236782074, + "union_independent": 0.07135158777236938, + "union_gap": -0.03264996409416199, + "mean_per_layer_hit_rate": 0.005665994249284267 + }, + "2": { + "rho_bar": 0.17595399916172028, + "union_observed": 0.05867665261030197, + "union_independent": 0.12491041421890259, + "union_gap": -0.06623376160860062, + "mean_per_layer_hit_rate": 0.010179583914577961 + }, + "5": { + "rho_bar": 0.3131089508533478, + "union_observed": 0.09737827628850937, + "union_independent": 0.2665131092071533, + "union_gap": -0.16913483291864395, + "mean_per_layer_hit_rate": 0.02343224734067917 + }, + "10": { + "rho_bar": 0.34869086742401123, + "union_observed": 0.14107365906238556, + "union_independent": 0.4158136248588562, + "union_gap": -0.27473996579647064, + "mean_per_layer_hit_rate": 0.04014212638139725 + }, + "20": { + "rho_bar": 0.3557385802268982, + "union_observed": 0.21098627150058746, + "union_independent": 0.5865551829338074, + "union_gap": -0.3755689114332199, + "mean_per_layer_hit_rate": 0.06472678482532501 + }, + "50": { + "rho_bar": 0.3779834806919098, + "union_observed": 0.3121098577976227, + "union_independent": 0.7668907642364502, + "union_gap": -0.4547809064388275, + "mean_per_layer_hit_rate": 0.10390857607126236 + }, + "100": { + "rho_bar": 0.3714540898799896, + "union_observed": 0.41947564482688904, + "union_independent": 0.8831087350845337, + "union_gap": -0.46363309025764465, + "mean_per_layer_hit_rate": 0.14798809587955475 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29980545384543283, + "union_gap_mean_over_k": -0.26239163322108133 + }, + "OOD_News_2024|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1234273612499237, + "union_observed": 0.04244694113731384, + "union_independent": 0.07843929529190063, + "union_gap": -0.03599235415458679, + "mean_per_layer_hit_rate": 0.006242197472602129 + }, + "2": { + "rho_bar": 0.21687300503253937, + "union_observed": 0.06491885334253311, + "union_independent": 0.14702582359313965, + "union_gap": -0.08210697025060654, + "mean_per_layer_hit_rate": 0.012100258842110634 + }, + "5": { + "rho_bar": 0.27197837829589844, + "union_observed": 0.10486891120672226, + "union_independent": 0.2837708592414856, + "union_gap": -0.17890194803476334, + "mean_per_layer_hit_rate": 0.025160856544971466 + }, + "10": { + "rho_bar": 0.3520243763923645, + "union_observed": 0.1473158597946167, + "union_independent": 0.4254617691040039, + "union_gap": -0.2781459093093872, + "mean_per_layer_hit_rate": 0.041294537484645844 + }, + "20": { + "rho_bar": 0.36627453565597534, + "union_observed": 0.22097378969192505, + "union_independent": 0.5964610576629639, + "union_gap": -0.3754872679710388, + "mean_per_layer_hit_rate": 0.06635935604572296 + }, + "50": { + "rho_bar": 0.365669310092926, + "union_observed": 0.3283395767211914, + "union_independent": 0.7793881893157959, + "union_gap": -0.4510486125946045, + "mean_per_layer_hit_rate": 0.1072697639465332 + }, + "100": { + "rho_bar": 0.371764600276947, + "union_observed": 0.4282147288322449, + "union_independent": 0.8910364508628845, + "union_gap": -0.46282172203063965, + "mean_per_layer_hit_rate": 0.15202151238918304 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.29543022385665346, + "union_gap_mean_over_k": -0.26635782633508953 + }, + "OOD_News_2024|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.14704221487045288, + "union_observed": 0.039950061589479446, + "union_independent": 0.08998233079910278, + "union_gap": -0.05003226920962334, + "mean_per_layer_hit_rate": 0.00720253586769104 + }, + "2": { + "rho_bar": 0.1904725730419159, + "union_observed": 0.05992509424686432, + "union_independent": 0.14913588762283325, + "union_gap": -0.08921079337596893, + "mean_per_layer_hit_rate": 0.012292325496673584 + }, + "5": { + "rho_bar": 0.24889670312404633, + "union_observed": 0.10486891120672226, + "union_independent": 0.29004818201065063, + "union_gap": -0.18517927080392838, + "mean_per_layer_hit_rate": 0.025833092629909515 + }, + "10": { + "rho_bar": 0.33370932936668396, + "union_observed": 0.16729088127613068, + "union_independent": 0.45308852195739746, + "union_gap": -0.2857976406812668, + "mean_per_layer_hit_rate": 0.04484779015183449 + }, + "20": { + "rho_bar": 0.3431585729122162, + "union_observed": 0.23720349371433258, + "union_independent": 0.6092740297317505, + "union_gap": -0.3720705360174179, + "mean_per_layer_hit_rate": 0.0685681402683258 + }, + "50": { + "rho_bar": 0.34839776158332825, + "union_observed": 0.3595505654811859, + "union_independent": 0.7927910685539246, + "union_gap": -0.43324050307273865, + "mean_per_layer_hit_rate": 0.11120714247226715 + }, + "100": { + "rho_bar": 0.35772597789764404, + "union_observed": 0.45318353176116943, + "union_independent": 0.8944145441055298, + "union_gap": -0.44123101234436035, + "mean_per_layer_hit_rate": 0.1538461595773697 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2813433046851839, + "union_gap_mean_over_k": -0.26525171792932917 + }, + "OOD_News_2024|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.1896514892578125, + "union_observed": 0.041198503226041794, + "union_independent": 0.09228575229644775, + "union_gap": -0.05108724907040596, + "mean_per_layer_hit_rate": 0.007394603453576565 + }, + "2": { + "rho_bar": 0.19765880703926086, + "union_observed": 0.06367041170597076, + "union_independent": 0.1642337441444397, + "union_gap": -0.10056333243846893, + "mean_per_layer_hit_rate": 0.013636799529194832 + }, + "5": { + "rho_bar": 0.2919161915779114, + "union_observed": 0.10486891120672226, + "union_independent": 0.3189309239387512, + "union_gap": -0.21406201273202896, + "mean_per_layer_hit_rate": 0.02890617586672306 + }, + "10": { + "rho_bar": 0.351541131734848, + "union_observed": 0.17228464782238007, + "union_independent": 0.48811888694763184, + "union_gap": -0.31583423912525177, + "mean_per_layer_hit_rate": 0.04964947700500488 + }, + "20": { + "rho_bar": 0.3610723316669464, + "union_observed": 0.24719101190567017, + "union_independent": 0.6493135094642639, + "union_gap": -0.40212249755859375, + "mean_per_layer_hit_rate": 0.07615481317043304 + }, + "50": { + "rho_bar": 0.3602367341518402, + "union_observed": 0.36828964948654175, + "union_independent": 0.8138073682785034, + "union_gap": -0.44551771879196167, + "mean_per_layer_hit_rate": 0.11840967833995819 + }, + "100": { + "rho_bar": 0.37237799167633057, + "union_observed": 0.47440698742866516, + "union_independent": 0.9160795211791992, + "union_gap": -0.44167253375053406, + "mean_per_layer_hit_rate": 0.16815519332885742 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.3034935253007071, + "union_gap_mean_over_k": -0.2815513690667493 + }, + "OOD_News_2024|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.12356466054916382, + "union_observed": 0.05368289723992348, + "union_independent": 0.10703611373901367, + "union_gap": -0.053353216499090195, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.20167453587055206, + "union_observed": 0.07615480571985245, + "union_independent": 0.19667017459869385, + "union_gap": -0.1205153688788414, + "mean_per_layer_hit_rate": 0.016613848507404327 + }, + "5": { + "rho_bar": 0.3221238851547241, + "union_observed": 0.12234706431627274, + "union_independent": 0.36596453189849854, + "union_gap": -0.2436174675822258, + "mean_per_layer_hit_rate": 0.03418803587555885 + }, + "10": { + "rho_bar": 0.40790894627571106, + "union_observed": 0.18476903438568115, + "union_independent": 0.5299276113510132, + "union_gap": -0.34515857696533203, + "mean_per_layer_hit_rate": 0.055795636028051376 + }, + "20": { + "rho_bar": 0.3970091938972473, + "union_observed": 0.2746566832065582, + "union_independent": 0.6903782486915588, + "union_gap": -0.4157215654850006, + "mean_per_layer_hit_rate": 0.08498991280794144 + }, + "50": { + "rho_bar": 0.41169023513793945, + "union_observed": 0.37827715277671814, + "union_independent": 0.8377337455749512, + "union_gap": -0.45945659279823303, + "mean_per_layer_hit_rate": 0.12810909748077393 + }, + "100": { + "rho_bar": 0.4175008535385132, + "union_observed": 0.46691635251045227, + "union_independent": 0.9237990975379944, + "union_gap": -0.4568827450275421, + "mean_per_layer_hit_rate": 0.1756458282470703 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.32592461577483584, + "union_gap_mean_over_k": -0.29924364760518074 + }, + "OOD_News_2024|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.1648138463497162, + "union_observed": 0.047440700232982635, + "union_independent": 0.11725097894668579, + "union_gap": -0.06981027871370316, + "mean_per_layer_hit_rate": 0.009507346898317337 + }, + "2": { + "rho_bar": 0.22859610617160797, + "union_observed": 0.08863919973373413, + "union_independent": 0.20881152153015137, + "union_gap": -0.12017232179641724, + "mean_per_layer_hit_rate": 0.017766254022717476 + }, + "5": { + "rho_bar": 0.3451871871948242, + "union_observed": 0.13108614087104797, + "union_independent": 0.38031357526779175, + "union_gap": -0.24922743439674377, + "mean_per_layer_hit_rate": 0.03591664507985115 + }, + "10": { + "rho_bar": 0.42507368326187134, + "union_observed": 0.18476903438568115, + "union_independent": 0.5498743653297424, + "union_gap": -0.3651053309440613, + "mean_per_layer_hit_rate": 0.05896475538611412 + }, + "20": { + "rho_bar": 0.4370879530906677, + "union_observed": 0.255930095911026, + "union_independent": 0.6944540739059448, + "union_gap": -0.4385239779949188, + "mean_per_layer_hit_rate": 0.08595025539398193 + }, + "50": { + "rho_bar": 0.4503759741783142, + "union_observed": 0.3595505654811859, + "union_independent": 0.8439796566963196, + "union_gap": -0.48442909121513367, + "mean_per_layer_hit_rate": 0.13070200383663177 + }, + "100": { + "rho_bar": 0.44718679785728455, + "union_observed": 0.48689138889312744, + "union_independent": 0.9250003099441528, + "union_gap": -0.4381089210510254, + "mean_per_layer_hit_rate": 0.1773744374513626 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3569030783006123, + "union_gap_mean_over_k": -0.30933962230171475 + }, + "OOD_News_2024|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.08430826663970947, + "union_observed": 0.05243445560336113, + "union_independent": 0.09454536437988281, + "union_gap": -0.04211090877652168, + "mean_per_layer_hit_rate": 0.007586670573800802 + }, + "2": { + "rho_bar": 0.15992140769958496, + "union_observed": 0.08239700645208359, + "union_independent": 0.17043471336364746, + "union_gap": -0.08803770691156387, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.28404173254966736, + "union_observed": 0.12983770668506622, + "union_independent": 0.3335074186325073, + "union_gap": -0.2036697119474411, + "mean_per_layer_hit_rate": 0.03053874894976616 + }, + "10": { + "rho_bar": 0.35630232095718384, + "union_observed": 0.1885143518447876, + "union_independent": 0.48332685232162476, + "union_gap": -0.29481250047683716, + "mean_per_layer_hit_rate": 0.04907327517867088 + }, + "20": { + "rho_bar": 0.390473335981369, + "union_observed": 0.2584269642829895, + "union_independent": 0.6405532360076904, + "union_gap": -0.3821262717247009, + "mean_per_layer_hit_rate": 0.0747143030166626 + }, + "50": { + "rho_bar": 0.4009396731853485, + "union_observed": 0.36329588294029236, + "union_independent": 0.7926662564277649, + "union_gap": -0.42937037348747253, + "mean_per_layer_hit_rate": 0.1120714470744133 + }, + "100": { + "rho_bar": 0.4052664637565613, + "union_observed": 0.47940075397491455, + "union_independent": 0.8981875777244568, + "union_gap": -0.41878682374954224, + "mean_per_layer_hit_rate": 0.15826371312141418 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2973218858242035, + "union_gap_mean_over_k": -0.2655591852962971 + }, + "OOD_News_2024|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.1279090791940689, + "union_observed": 0.04619225859642029, + "union_independent": 0.09463077783584595, + "union_gap": -0.04843851923942566, + "mean_per_layer_hit_rate": 0.007586670573800802 + }, + "2": { + "rho_bar": 0.206797793507576, + "union_observed": 0.06866417080163956, + "union_independent": 0.16101789474487305, + "union_gap": -0.09235372394323349, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.2578069567680359, + "union_observed": 0.10986267030239105, + "union_independent": 0.31011146306991577, + "union_gap": -0.20024879276752472, + "mean_per_layer_hit_rate": 0.027945837005972862 + }, + "10": { + "rho_bar": 0.35275912284851074, + "union_observed": 0.16354556381702423, + "union_independent": 0.4567806124687195, + "union_gap": -0.29323504865169525, + "mean_per_layer_hit_rate": 0.04542398825287819 + }, + "20": { + "rho_bar": 0.3825586140155792, + "union_observed": 0.23845192790031433, + "union_independent": 0.606013298034668, + "union_gap": -0.36756137013435364, + "mean_per_layer_hit_rate": 0.06828003376722336 + }, + "50": { + "rho_bar": 0.39550966024398804, + "union_observed": 0.37078651785850525, + "union_independent": 0.7911385297775269, + "union_gap": -0.4203520119190216, + "mean_per_layer_hit_rate": 0.1114952489733696 + }, + "100": { + "rho_bar": 0.41062918305397034, + "union_observed": 0.4756554365158081, + "union_independent": 0.8960456252098083, + "union_gap": -0.42039018869400024, + "mean_per_layer_hit_rate": 0.1565350890159607 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3048529156616756, + "union_gap_mean_over_k": -0.26322566504989353 + }, + "OOD_News_2024|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.14269523322582245, + "union_observed": 0.05992509424686432, + "union_independent": 0.10495138168334961, + "union_gap": -0.04502628743648529, + "mean_per_layer_hit_rate": 0.008450974710285664 + }, + "2": { + "rho_bar": 0.1378655880689621, + "union_observed": 0.09737827628850937, + "union_independent": 0.1782054305076599, + "union_gap": -0.08082715421915054, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2685518264770508, + "union_observed": 0.15106117725372314, + "union_independent": 0.3579370975494385, + "union_gap": -0.20687592029571533, + "mean_per_layer_hit_rate": 0.033227697014808655 + }, + "10": { + "rho_bar": 0.352072536945343, + "union_observed": 0.2122347056865692, + "union_independent": 0.5108299255371094, + "union_gap": -0.29859521985054016, + "mean_per_layer_hit_rate": 0.05291462317109108 + }, + "20": { + "rho_bar": 0.35931289196014404, + "union_observed": 0.2946317195892334, + "union_independent": 0.6555936336517334, + "union_gap": -0.3609619140625, + "mean_per_layer_hit_rate": 0.0775953158736229 + }, + "50": { + "rho_bar": 0.36969658732414246, + "union_observed": 0.4057428240776062, + "union_independent": 0.8059245347976685, + "union_gap": -0.40018171072006226, + "mean_per_layer_hit_rate": 0.11600883305072784 + }, + "100": { + "rho_bar": 0.38508740067481995, + "union_observed": 0.4931335747241974, + "union_independent": 0.8946731686592102, + "union_gap": -0.4015395939350128, + "mean_per_layer_hit_rate": 0.15528665482997894 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.2878974378108978, + "union_gap_mean_over_k": -0.25628682864563807 + }, + "OOD_News_2024|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.12793242931365967, + "union_observed": 0.054931335151195526, + "union_independent": 0.10150516033172607, + "union_gap": -0.04657382518053055, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.15011098980903625, + "union_observed": 0.08114856481552124, + "union_independent": 0.16656678915023804, + "union_gap": -0.0854182243347168, + "mean_per_layer_hit_rate": 0.013828868046402931 + }, + "5": { + "rho_bar": 0.2616840898990631, + "union_observed": 0.13358302414417267, + "union_independent": 0.32266324758529663, + "union_gap": -0.18908022344112396, + "mean_per_layer_hit_rate": 0.02929030917584896 + }, + "10": { + "rho_bar": 0.3151290714740753, + "union_observed": 0.193508118391037, + "union_independent": 0.46932315826416016, + "union_gap": -0.27581503987312317, + "mean_per_layer_hit_rate": 0.04696052893996239 + }, + "20": { + "rho_bar": 0.32233625650405884, + "union_observed": 0.2734082341194153, + "union_independent": 0.6296670436859131, + "union_gap": -0.3562588095664978, + "mean_per_layer_hit_rate": 0.07231345772743225 + }, + "50": { + "rho_bar": 0.3816315233707428, + "union_observed": 0.3895131051540375, + "union_independent": 0.7934533357620239, + "union_gap": -0.40394023060798645, + "mean_per_layer_hit_rate": 0.1116873100399971 + }, + "100": { + "rho_bar": 0.39389413595199585, + "union_observed": 0.49188515543937683, + "union_independent": 0.8918494582176208, + "union_gap": -0.399964302778244, + "mean_per_layer_hit_rate": 0.1530778706073761 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.27895978518894743, + "union_gap_mean_over_k": -0.25100723654031754 + }, + "OOD_News_2024|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.10761655867099762, + "union_observed": 0.05243445560336113, + "union_independent": 0.09563523530960083, + "union_gap": -0.0432007797062397, + "mean_per_layer_hit_rate": 0.007682703901082277 + }, + "2": { + "rho_bar": 0.15792079269886017, + "union_observed": 0.08364544063806534, + "union_independent": 0.16095983982086182, + "union_gap": -0.07731439918279648, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.23155681788921356, + "union_observed": 0.14357054233551025, + "union_independent": 0.3261708617210388, + "union_gap": -0.18260031938552856, + "mean_per_layer_hit_rate": 0.02967444807291031 + }, + "10": { + "rho_bar": 0.3281974196434021, + "union_observed": 0.19850187003612518, + "union_independent": 0.4840041995048523, + "union_gap": -0.2855023294687271, + "mean_per_layer_hit_rate": 0.04907327517867088 + }, + "20": { + "rho_bar": 0.35834306478500366, + "union_observed": 0.2808988690376282, + "union_independent": 0.6389276385307312, + "union_gap": -0.358028769493103, + "mean_per_layer_hit_rate": 0.07423412799835205 + }, + "50": { + "rho_bar": 0.4125412702560425, + "union_observed": 0.3845193386077881, + "union_independent": 0.80085688829422, + "union_gap": -0.4163375496864319, + "mean_per_layer_hit_rate": 0.11408816277980804 + }, + "100": { + "rho_bar": 0.4422991871833801, + "union_observed": 0.4881398379802704, + "union_independent": 0.8971075415611267, + "union_gap": -0.4089677035808563, + "mean_per_layer_hit_rate": 0.15595890581607819 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2912107301609857, + "union_gap_mean_over_k": -0.2531359786433833 + }, + "OOD_News_2024|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2591981291770935, + "union_observed": 0.041198503226041794, + "union_independent": 0.1293594241142273, + "union_gap": -0.0881609208881855, + "mean_per_layer_hit_rate": 0.010563718155026436 + }, + "2": { + "rho_bar": 0.27622777223587036, + "union_observed": 0.0699126124382019, + "union_independent": 0.2036716341972351, + "union_gap": -0.1337590217590332, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.4035433530807495, + "union_observed": 0.12109863013029099, + "union_independent": 0.4111917018890381, + "union_gap": -0.2900930717587471, + "mean_per_layer_hit_rate": 0.039661962538957596 + }, + "10": { + "rho_bar": 0.4482872784137726, + "union_observed": 0.16229712963104248, + "union_independent": 0.5542581081390381, + "union_gap": -0.3919609785079956, + "mean_per_layer_hit_rate": 0.05963699147105217 + }, + "20": { + "rho_bar": 0.46007752418518066, + "union_observed": 0.2509363293647766, + "union_independent": 0.7216085195541382, + "union_gap": -0.4706721901893616, + "mean_per_layer_hit_rate": 0.09209641814231873 + }, + "50": { + "rho_bar": 0.4875405728816986, + "union_observed": 0.34706616401672363, + "union_independent": 0.8638386726379395, + "union_gap": -0.5167725086212158, + "mean_per_layer_hit_rate": 0.13857677578926086 + }, + "100": { + "rho_bar": 0.5031740069389343, + "union_observed": 0.4419475793838501, + "union_independent": 0.9359077215194702, + "union_gap": -0.4939601421356201, + "mean_per_layer_hit_rate": 0.18419283628463745 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4054355195590428, + "union_gap_mean_over_k": -0.3407684048371656 + }, + "OOD_News_2024|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.2487439215183258, + "union_observed": 0.05118601769208908, + "union_independent": 0.13598036766052246, + "union_gap": -0.08479434996843338, + "mean_per_layer_hit_rate": 0.01113992091268301 + }, + "2": { + "rho_bar": 0.27786508202552795, + "union_observed": 0.08614232391119003, + "union_independent": 0.22058337926864624, + "union_gap": -0.1344410553574562, + "mean_per_layer_hit_rate": 0.018918659538030624 + }, + "5": { + "rho_bar": 0.35148903727531433, + "union_observed": 0.14357054233551025, + "union_independent": 0.4123554825782776, + "union_gap": -0.26878494024276733, + "mean_per_layer_hit_rate": 0.039854031056165695 + }, + "10": { + "rho_bar": 0.45093926787376404, + "union_observed": 0.1860174834728241, + "union_independent": 0.5810834169387817, + "union_gap": -0.39506593346595764, + "mean_per_layer_hit_rate": 0.06415057927370071 + }, + "20": { + "rho_bar": 0.49402862787246704, + "union_observed": 0.260923832654953, + "union_independent": 0.7310606837272644, + "union_gap": -0.4701368510723114, + "mean_per_layer_hit_rate": 0.09488140791654587 + }, + "50": { + "rho_bar": 0.5209588408470154, + "union_observed": 0.36079901456832886, + "union_independent": 0.8659167289733887, + "union_gap": -0.5051177144050598, + "mean_per_layer_hit_rate": 0.1409776210784912 + }, + "100": { + "rho_bar": 0.5241564512252808, + "union_observed": 0.44818976521492004, + "union_independent": 0.9344409704208374, + "union_gap": -0.48625120520591736, + "mean_per_layer_hit_rate": 0.1858254224061966 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.40974017551967074, + "union_gap_mean_over_k": -0.3349417213882719 + }, + "OOD_News_2024|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.19105158746242523, + "union_observed": 0.05243445560336113, + "union_independent": 0.12943178415298462, + "union_gap": -0.07699732854962349, + "mean_per_layer_hit_rate": 0.010563718155026436 + }, + "2": { + "rho_bar": 0.1932244598865509, + "union_observed": 0.07865168899297714, + "union_independent": 0.1871652603149414, + "union_gap": -0.10851357132196426, + "mean_per_layer_hit_rate": 0.01574954390525818 + }, + "5": { + "rho_bar": 0.2819312810897827, + "union_observed": 0.1373283416032791, + "union_independent": 0.37096846103668213, + "union_gap": -0.23364011943340302, + "mean_per_layer_hit_rate": 0.03476423770189285 + }, + "10": { + "rho_bar": 0.36835119128227234, + "union_observed": 0.1760299652814865, + "union_independent": 0.5214784741401672, + "union_gap": -0.3454485088586807, + "mean_per_layer_hit_rate": 0.05454719811677933 + }, + "20": { + "rho_bar": 0.41631972789764404, + "union_observed": 0.25468164682388306, + "union_independent": 0.6804623603820801, + "union_gap": -0.425780713558197, + "mean_per_layer_hit_rate": 0.08278114348649979 + }, + "50": { + "rho_bar": 0.4818064868450165, + "union_observed": 0.35830211639404297, + "union_independent": 0.8307018876075745, + "union_gap": -0.4723997712135315, + "mean_per_layer_hit_rate": 0.12532411515712738 + }, + "100": { + "rho_bar": 0.511064887046814, + "union_observed": 0.44069913029670715, + "union_independent": 0.9096943140029907, + "union_gap": -0.46899518370628357, + "mean_per_layer_hit_rate": 0.16546624898910522 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3491070887872151, + "union_gap_mean_over_k": -0.3045393138059548 + }, + "OOD_News_2024|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.16674456000328064, + "union_observed": 0.04494382068514824, + "union_independent": 0.10479342937469482, + "union_gap": -0.059849608689546585, + "mean_per_layer_hit_rate": 0.008450973778963089 + }, + "2": { + "rho_bar": 0.17078149318695068, + "union_observed": 0.07990012317895889, + "union_independent": 0.17467927932739258, + "union_gap": -0.09477915614843369, + "mean_per_layer_hit_rate": 0.014597137458622456 + }, + "5": { + "rho_bar": 0.23907732963562012, + "union_observed": 0.14107365906238556, + "union_independent": 0.3474922180175781, + "union_gap": -0.20641855895519257, + "mean_per_layer_hit_rate": 0.032075293362140656 + }, + "10": { + "rho_bar": 0.2959747612476349, + "union_observed": 0.19225966930389404, + "union_independent": 0.4971224069595337, + "union_gap": -0.30486273765563965, + "mean_per_layer_hit_rate": 0.05089791491627693 + }, + "20": { + "rho_bar": 0.35136720538139343, + "union_observed": 0.25717851519584656, + "union_independent": 0.6491422653198242, + "union_gap": -0.39196375012397766, + "mean_per_layer_hit_rate": 0.07615481317043304 + }, + "50": { + "rho_bar": 0.41365349292755127, + "union_observed": 0.34956303238868713, + "union_independent": 0.7997530102729797, + "union_gap": -0.4501899778842926, + "mean_per_layer_hit_rate": 0.1139921247959137 + }, + "100": { + "rho_bar": 0.4757443070411682, + "union_observed": 0.43320849537849426, + "union_independent": 0.8918913006782532, + "union_gap": -0.4586828052997589, + "mean_per_layer_hit_rate": 0.15375012159347534 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3019061642033713, + "union_gap_mean_over_k": -0.28096379925097736 + }, + "OOD_News_2024|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.13891513645648956, + "union_observed": 0.039950061589479446, + "union_independent": 0.08076661825180054, + "union_gap": -0.04081655666232109, + "mean_per_layer_hit_rate": 0.006434264592826366 + }, + "2": { + "rho_bar": 0.1689622551202774, + "union_observed": 0.07365792989730835, + "union_independent": 0.14718222618103027, + "union_gap": -0.07352429628372192, + "mean_per_layer_hit_rate": 0.012100259773433208 + }, + "5": { + "rho_bar": 0.23733638226985931, + "union_observed": 0.13358302414417267, + "union_independent": 0.3127986788749695, + "union_gap": -0.17921565473079681, + "mean_per_layer_hit_rate": 0.028233937919139862 + }, + "10": { + "rho_bar": 0.3026185631752014, + "union_observed": 0.1910112351179123, + "union_independent": 0.4560477137565613, + "union_gap": -0.265036478638649, + "mean_per_layer_hit_rate": 0.04523192346096039 + }, + "20": { + "rho_bar": 0.3558604419231415, + "union_observed": 0.2534331977367401, + "union_independent": 0.6099916696548462, + "union_gap": -0.3565584719181061, + "mean_per_layer_hit_rate": 0.06885623931884766 + }, + "50": { + "rho_bar": 0.40077829360961914, + "union_observed": 0.34456929564476013, + "union_independent": 0.7713619470596313, + "union_gap": -0.4267926514148712, + "mean_per_layer_hit_rate": 0.10525305569171906 + }, + "100": { + "rho_bar": 0.43942442536354065, + "union_observed": 0.42696627974510193, + "union_independent": 0.8713490962982178, + "union_gap": -0.44438281655311584, + "mean_per_layer_hit_rate": 0.14299432933330536 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2919850711311613, + "union_gap_mean_over_k": -0.2551895608859403 + }, + "OOD_News_2024|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.10675693303346634, + "union_observed": 0.05243445560336113, + "union_independent": 0.07728612422943115, + "union_gap": -0.024851668626070023, + "mean_per_layer_hit_rate": 0.006146163679659367 + }, + "2": { + "rho_bar": 0.1472828984260559, + "union_observed": 0.0774032473564148, + "union_independent": 0.13821721076965332, + "union_gap": -0.060813963413238525, + "mean_per_layer_hit_rate": 0.01133198942989111 + }, + "5": { + "rho_bar": 0.22926929593086243, + "union_observed": 0.13857677578926086, + "union_independent": 0.31166785955429077, + "union_gap": -0.1730910837650299, + "mean_per_layer_hit_rate": 0.028137903660535812 + }, + "10": { + "rho_bar": 0.288073867559433, + "union_observed": 0.1835206001996994, + "union_independent": 0.45007020235061646, + "union_gap": -0.26654960215091705, + "mean_per_layer_hit_rate": 0.04455968365073204 + }, + "20": { + "rho_bar": 0.3266642093658447, + "union_observed": 0.24469412863254547, + "union_independent": 0.5773206949234009, + "union_gap": -0.3326265662908554, + "mean_per_layer_hit_rate": 0.06338231265544891 + }, + "50": { + "rho_bar": 0.37178415060043335, + "union_observed": 0.32958802580833435, + "union_independent": 0.7370159029960632, + "union_gap": -0.4074278771877289, + "mean_per_layer_hit_rate": 0.09622586518526077 + }, + "100": { + "rho_bar": 0.3956151604652405, + "union_observed": 0.42571786046028137, + "union_independent": 0.8562801480293274, + "union_gap": -0.430562287569046, + "mean_per_layer_hit_rate": 0.13627196848392487 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2664923593401909, + "union_gap_mean_over_k": -0.24227472128612654 + }, + "OOD_News_2024|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.11809961497783661, + "union_observed": 0.048689138144254684, + "union_independent": 0.07825899124145508, + "union_gap": -0.029569853097200394, + "mean_per_layer_hit_rate": 0.006242197472602129 + }, + "2": { + "rho_bar": 0.1813175082206726, + "union_observed": 0.06367041170597076, + "union_independent": 0.13032197952270508, + "union_gap": -0.06665156781673431, + "mean_per_layer_hit_rate": 0.010659752413630486 + }, + "5": { + "rho_bar": 0.2569834589958191, + "union_observed": 0.10986267030239105, + "union_independent": 0.2867661714553833, + "union_gap": -0.17690350115299225, + "mean_per_layer_hit_rate": 0.025544991716742516 + }, + "10": { + "rho_bar": 0.29682090878486633, + "union_observed": 0.15980024635791779, + "union_independent": 0.4277513027191162, + "union_gap": -0.2679510563611984, + "mean_per_layer_hit_rate": 0.04177470505237579 + }, + "20": { + "rho_bar": 0.34293869137763977, + "union_observed": 0.20724095404148102, + "union_independent": 0.557639479637146, + "union_gap": -0.350398525595665, + "mean_per_layer_hit_rate": 0.06030922755599022 + }, + "50": { + "rho_bar": 0.36394914984703064, + "union_observed": 0.3046192228794098, + "union_independent": 0.7198476791381836, + "union_gap": -0.4152284562587738, + "mean_per_layer_hit_rate": 0.09219244867563248 + }, + "100": { + "rho_bar": 0.38005298376083374, + "union_observed": 0.39700374007225037, + "union_independent": 0.8557330369949341, + "union_gap": -0.4587292969226837, + "mean_per_layer_hit_rate": 0.13607989251613617 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2771660451378141, + "union_gap_mean_over_k": -0.25220460817217827 + }, + "OOD_News_2024|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.15284323692321777, + "union_observed": 0.04494382068514824, + "union_independent": 0.07833373546600342, + "union_gap": -0.03338991478085518, + "mean_per_layer_hit_rate": 0.006242197472602129 + }, + "2": { + "rho_bar": 0.19314031302928925, + "union_observed": 0.06367041170597076, + "union_independent": 0.1337013840675354, + "union_gap": -0.07003097236156464, + "mean_per_layer_hit_rate": 0.010947853326797485 + }, + "5": { + "rho_bar": 0.26050516963005066, + "union_observed": 0.10986267030239105, + "union_independent": 0.2832796573638916, + "union_gap": -0.17341698706150055, + "mean_per_layer_hit_rate": 0.025160856544971466 + }, + "10": { + "rho_bar": 0.32614865899086, + "union_observed": 0.14856429398059845, + "union_independent": 0.4214746952056885, + "union_gap": -0.27291040122509, + "mean_per_layer_hit_rate": 0.040910400450229645 + }, + "20": { + "rho_bar": 0.3485461175441742, + "union_observed": 0.20099875330924988, + "union_independent": 0.5643774271011353, + "union_gap": -0.3633786737918854, + "mean_per_layer_hit_rate": 0.061269573867321014 + }, + "50": { + "rho_bar": 0.37299588322639465, + "union_observed": 0.31086140871047974, + "union_independent": 0.7533649206161499, + "union_gap": -0.44250351190567017, + "mean_per_layer_hit_rate": 0.10035533457994461 + }, + "100": { + "rho_bar": 0.38210007548332214, + "union_observed": 0.39950063824653625, + "union_independent": 0.8742184042930603, + "union_gap": -0.47471776604652405, + "mean_per_layer_hit_rate": 0.1440507024526596 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2908970649753298, + "union_gap_mean_over_k": -0.26147831816758427 + }, + "OOD_News_2024|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.15049295127391815, + "union_observed": 0.041198503226041794, + "union_independent": 0.07488119602203369, + "union_gap": -0.0336826927959919, + "mean_per_layer_hit_rate": 0.005954096093773842 + }, + "2": { + "rho_bar": 0.17352645099163055, + "union_observed": 0.05867665261030197, + "union_independent": 0.1260087490081787, + "union_gap": -0.06733209639787674, + "mean_per_layer_hit_rate": 0.010275617241859436 + }, + "5": { + "rho_bar": 0.31303057074546814, + "union_observed": 0.10112359374761581, + "union_independent": 0.2740424871444702, + "union_gap": -0.1729188933968544, + "mean_per_layer_hit_rate": 0.024200517684221268 + }, + "10": { + "rho_bar": 0.3374338150024414, + "union_observed": 0.13483145833015442, + "union_independent": 0.4087289571762085, + "union_gap": -0.2738974988460541, + "mean_per_layer_hit_rate": 0.039277829229831696 + }, + "20": { + "rho_bar": 0.3651266396045685, + "union_observed": 0.21098627150058746, + "union_independent": 0.5889551639556885, + "union_gap": -0.377968892455101, + "mean_per_layer_hit_rate": 0.06520695239305496 + }, + "50": { + "rho_bar": 0.38105544447898865, + "union_observed": 0.3133583068847656, + "union_independent": 0.7689703106880188, + "union_gap": -0.4556120038032532, + "mean_per_layer_hit_rate": 0.10458081215620041 + }, + "100": { + "rho_bar": 0.377023845911026, + "union_observed": 0.4219725430011749, + "union_independent": 0.884479284286499, + "union_gap": -0.4625067412853241, + "mean_per_layer_hit_rate": 0.14875635504722595 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.2996699597154345, + "union_gap_mean_over_k": -0.26341697414006504 + }, + "OOD_News_2024|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1337718963623047, + "union_observed": 0.04369537904858589, + "union_independent": 0.07956188917160034, + "union_gap": -0.03586651012301445, + "mean_per_layer_hit_rate": 0.006338230799883604 + }, + "2": { + "rho_bar": 0.21686452627182007, + "union_observed": 0.06491885334253311, + "union_independent": 0.14913713932037354, + "union_gap": -0.08421828597784042, + "mean_per_layer_hit_rate": 0.012292327359318733 + }, + "5": { + "rho_bar": 0.268298476934433, + "union_observed": 0.10486891120672226, + "union_independent": 0.2856241464614868, + "union_gap": -0.18075523525476456, + "mean_per_layer_hit_rate": 0.025352923199534416 + }, + "10": { + "rho_bar": 0.34513357281684875, + "union_observed": 0.14606741070747375, + "union_independent": 0.42832303047180176, + "union_gap": -0.282255619764328, + "mean_per_layer_hit_rate": 0.041678670793771744 + }, + "20": { + "rho_bar": 0.36734655499458313, + "union_observed": 0.2222222238779068, + "union_independent": 0.6022142171859741, + "union_gap": -0.3799919933080673, + "mean_per_layer_hit_rate": 0.06741572916507721 + }, + "50": { + "rho_bar": 0.3696453273296356, + "union_observed": 0.32709112763404846, + "union_independent": 0.7773494124412537, + "union_gap": -0.4502582848072052, + "mean_per_layer_hit_rate": 0.1066935583949089 + }, + "100": { + "rho_bar": 0.37867140769958496, + "union_observed": 0.4219725430011749, + "union_independent": 0.8888005614280701, + "union_gap": -0.46682801842689514, + "mean_per_layer_hit_rate": 0.15086911618709564 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.29710453748703003, + "union_gap_mean_over_k": -0.26859627823744503 + }, + "OOD_News_2024|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.13765858113765717, + "union_observed": 0.041198503226041794, + "union_independent": 0.09107023477554321, + "union_gap": -0.04987173154950142, + "mean_per_layer_hit_rate": 0.007298569660633802 + }, + "2": { + "rho_bar": 0.18530544638633728, + "union_observed": 0.06117353215813637, + "union_independent": 0.14585256576538086, + "union_gap": -0.08467903360724449, + "mean_per_layer_hit_rate": 0.012004225514829159 + }, + "5": { + "rho_bar": 0.25728681683540344, + "union_observed": 0.10237203538417816, + "union_independent": 0.29182374477386475, + "union_gap": -0.18945170938968658, + "mean_per_layer_hit_rate": 0.026025159284472466 + }, + "10": { + "rho_bar": 0.33944061398506165, + "union_observed": 0.16729088127613068, + "union_independent": 0.45730066299438477, + "union_gap": -0.2900097817182541, + "mean_per_layer_hit_rate": 0.04542398825287819 + }, + "20": { + "rho_bar": 0.3488689363002777, + "union_observed": 0.23720349371433258, + "union_independent": 0.6154340505599976, + "union_gap": -0.378230556845665, + "mean_per_layer_hit_rate": 0.0697205439209938 + }, + "50": { + "rho_bar": 0.3561560809612274, + "union_observed": 0.3458177149295807, + "union_independent": 0.7908572554588318, + "union_gap": -0.4450395405292511, + "mean_per_layer_hit_rate": 0.11063093692064285 + }, + "100": { + "rho_bar": 0.36281198263168335, + "union_observed": 0.46441948413848877, + "union_independent": 0.9002076387405396, + "union_gap": -0.4357881546020508, + "mean_per_layer_hit_rate": 0.1573033630847931 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.28393263689109255, + "union_gap_mean_over_k": -0.26758150117737906 + }, + "OOD_News_2024|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.14779077470302582, + "union_observed": 0.039950061589479446, + "union_independent": 0.08881539106369019, + "union_gap": -0.04886532947421074, + "mean_per_layer_hit_rate": 0.007106501143425703 + }, + "2": { + "rho_bar": 0.19726687669754028, + "union_observed": 0.06491885334253311, + "union_independent": 0.167432963848114, + "union_gap": -0.1025141105055809, + "mean_per_layer_hit_rate": 0.013924901373684406 + }, + "5": { + "rho_bar": 0.2994610369205475, + "union_observed": 0.1086142286658287, + "union_independent": 0.32409149408340454, + "union_gap": -0.21547726541757584, + "mean_per_layer_hit_rate": 0.02948237769305706 + }, + "10": { + "rho_bar": 0.34982261061668396, + "union_observed": 0.17228464782238007, + "union_independent": 0.49219560623168945, + "union_gap": -0.3199109584093094, + "mean_per_layer_hit_rate": 0.05022567883133888 + }, + "20": { + "rho_bar": 0.356327623128891, + "union_observed": 0.24968789517879486, + "union_independent": 0.6484031677246094, + "union_gap": -0.3987152725458145, + "mean_per_layer_hit_rate": 0.07596273720264435 + }, + "50": { + "rho_bar": 0.36806628108024597, + "union_observed": 0.36579275131225586, + "union_independent": 0.815674901008606, + "union_gap": -0.4498821496963501, + "mean_per_layer_hit_rate": 0.11898588389158249 + }, + "100": { + "rho_bar": 0.3739416301250458, + "union_observed": 0.47440698742866516, + "union_independent": 0.9156976938247681, + "union_gap": -0.4412907063961029, + "mean_per_layer_hit_rate": 0.16796311736106873 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.2989538333245686, + "union_gap_mean_over_k": -0.28237939892070635 + }, + "OOD_News_2024|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.15594574809074402, + "union_observed": 0.05118601769208908, + "union_independent": 0.11153632402420044, + "union_gap": -0.06035030633211136, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.2007448673248291, + "union_observed": 0.0774032473564148, + "union_independent": 0.19769537448883057, + "union_gap": -0.12029212713241577, + "mean_per_layer_hit_rate": 0.016709882766008377 + }, + "5": { + "rho_bar": 0.3251650333404541, + "union_observed": 0.11860174685716629, + "union_independent": 0.3609827160835266, + "union_gap": -0.24238096922636032, + "mean_per_layer_hit_rate": 0.033611830323934555 + }, + "10": { + "rho_bar": 0.4125388264656067, + "union_observed": 0.17727839946746826, + "union_independent": 0.5239388942718506, + "union_gap": -0.3466604948043823, + "mean_per_layer_hit_rate": 0.054931338876485825 + }, + "20": { + "rho_bar": 0.40376123785972595, + "union_observed": 0.2659175992012024, + "union_independent": 0.681676983833313, + "union_gap": -0.4157593846321106, + "mean_per_layer_hit_rate": 0.08306923508644104 + }, + "50": { + "rho_bar": 0.41226714849472046, + "union_observed": 0.3720349669456482, + "union_independent": 0.8355504274368286, + "union_gap": -0.4635154604911804, + "mean_per_layer_hit_rate": 0.12724478542804718 + }, + "100": { + "rho_bar": 0.41908952593803406, + "union_observed": 0.47940075397491455, + "union_independent": 0.9269580841064453, + "union_gap": -0.44755733013153076, + "mean_per_layer_hit_rate": 0.178142711520195 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.33278748393058777, + "union_gap_mean_over_k": -0.29950229610715595 + }, + "OOD_News_2024|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.16282257437705994, + "union_observed": 0.05118601769208908, + "union_independent": 0.12727224826812744, + "union_gap": -0.07608623057603836, + "mean_per_layer_hit_rate": 0.010371650569140911 + }, + "2": { + "rho_bar": 0.21878375113010406, + "union_observed": 0.08489388227462769, + "union_independent": 0.20567083358764648, + "union_gap": -0.1207769513130188, + "mean_per_layer_hit_rate": 0.017478153109550476 + }, + "5": { + "rho_bar": 0.3556031584739685, + "union_observed": 0.12734082341194153, + "union_independent": 0.37842637300491333, + "union_gap": -0.2510855495929718, + "mean_per_layer_hit_rate": 0.03572457656264305 + }, + "10": { + "rho_bar": 0.44019797444343567, + "union_observed": 0.1760299652814865, + "union_independent": 0.547209620475769, + "union_gap": -0.37117965519428253, + "mean_per_layer_hit_rate": 0.05858061835169792 + }, + "20": { + "rho_bar": 0.431436151266098, + "union_observed": 0.25967541337013245, + "union_independent": 0.6972612738609314, + "union_gap": -0.43758586049079895, + "mean_per_layer_hit_rate": 0.08652646094560623 + }, + "50": { + "rho_bar": 0.4428759813308716, + "union_observed": 0.3670412003993988, + "union_independent": 0.848351001739502, + "union_gap": -0.48130980134010315, + "mean_per_layer_hit_rate": 0.13262268900871277 + }, + "100": { + "rho_bar": 0.45290350914001465, + "union_observed": 0.47940075397491455, + "union_independent": 0.9254609942436218, + "union_gap": -0.4460602402687073, + "mean_per_layer_hit_rate": 0.1775665134191513 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3578033000230789, + "union_gap_mean_over_k": -0.31201204125370297 + }, + "OOD_News_2024|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.09520286321640015, + "union_observed": 0.05243445560336113, + "union_independent": 0.09681195020675659, + "union_gap": -0.04437749460339546, + "mean_per_layer_hit_rate": 0.007778738159686327 + }, + "2": { + "rho_bar": 0.14426633715629578, + "union_observed": 0.08114856481552124, + "union_independent": 0.16840630769729614, + "union_gap": -0.0872577428817749, + "mean_per_layer_hit_rate": 0.014020935632288456 + }, + "5": { + "rho_bar": 0.2858748435974121, + "union_observed": 0.13483145833015442, + "union_independent": 0.3368278741836548, + "union_gap": -0.20199641585350037, + "mean_per_layer_hit_rate": 0.030922885984182358 + }, + "10": { + "rho_bar": 0.36037561297416687, + "union_observed": 0.1910112351179123, + "union_independent": 0.487204909324646, + "union_gap": -0.2961936742067337, + "mean_per_layer_hit_rate": 0.04964947700500488 + }, + "20": { + "rho_bar": 0.3866472840309143, + "union_observed": 0.2534331977367401, + "union_independent": 0.6346215009689331, + "union_gap": -0.381188303232193, + "mean_per_layer_hit_rate": 0.073561891913414 + }, + "50": { + "rho_bar": 0.4028591215610504, + "union_observed": 0.35830211639404297, + "union_independent": 0.7927966117858887, + "union_gap": -0.4344944953918457, + "mean_per_layer_hit_rate": 0.11216748505830765 + }, + "100": { + "rho_bar": 0.4133354127407074, + "union_observed": 0.4756554365158081, + "union_independent": 0.8977349996566772, + "union_gap": -0.42207956314086914, + "mean_per_layer_hit_rate": 0.15787957608699799 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2983659250395639, + "union_gap_mean_over_k": -0.2667982413300446 + }, + "OOD_News_2024|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.13143274188041687, + "union_observed": 0.047440700232982635, + "union_independent": 0.09796935319900513, + "union_gap": -0.05052865296602249, + "mean_per_layer_hit_rate": 0.007874772883951664 + }, + "2": { + "rho_bar": 0.20281054079532623, + "union_observed": 0.07116104662418365, + "union_independent": 0.16422533988952637, + "union_gap": -0.09306429326534271, + "mean_per_layer_hit_rate": 0.013636799529194832 + }, + "5": { + "rho_bar": 0.2623729407787323, + "union_observed": 0.10986267030239105, + "union_independent": 0.3127574324607849, + "union_gap": -0.20289476215839386, + "mean_per_layer_hit_rate": 0.02823394164443016 + }, + "10": { + "rho_bar": 0.3602127134799957, + "union_observed": 0.16978776454925537, + "union_independent": 0.46451377868652344, + "union_gap": -0.29472601413726807, + "mean_per_layer_hit_rate": 0.04648036137223244 + }, + "20": { + "rho_bar": 0.37803542613983154, + "union_observed": 0.23720349371433258, + "union_independent": 0.6129740476608276, + "union_gap": -0.37577055394649506, + "mean_per_layer_hit_rate": 0.0695284754037857 + }, + "50": { + "rho_bar": 0.3989174962043762, + "union_observed": 0.36329588294029236, + "union_independent": 0.7911481261253357, + "union_gap": -0.42785224318504333, + "mean_per_layer_hit_rate": 0.1114952489733696 + }, + "100": { + "rho_bar": 0.4141371250152588, + "union_observed": 0.47191011905670166, + "union_independent": 0.8961734175682068, + "union_gap": -0.4242632985115051, + "mean_per_layer_hit_rate": 0.15663114190101624 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.30684556918484823, + "union_gap_mean_over_k": -0.2670142597385815 + }, + "OOD_News_2024|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.138764426112175, + "union_observed": 0.05742821469902992, + "union_independent": 0.10149431228637695, + "union_gap": -0.04406609758734703, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.12170255184173584, + "union_observed": 0.09612983465194702, + "union_independent": 0.1685854196548462, + "union_gap": -0.07245558500289917, + "mean_per_layer_hit_rate": 0.014020935632288456 + }, + "5": { + "rho_bar": 0.269098162651062, + "union_observed": 0.144818976521492, + "union_independent": 0.34942537546157837, + "union_gap": -0.20460639894008636, + "mean_per_layer_hit_rate": 0.032267358154058456 + }, + "10": { + "rho_bar": 0.3569103479385376, + "union_observed": 0.20099875330924988, + "union_independent": 0.5073392391204834, + "union_gap": -0.3063404858112335, + "mean_per_layer_hit_rate": 0.05243445932865143 + }, + "20": { + "rho_bar": 0.36622047424316406, + "union_observed": 0.28838950395584106, + "union_independent": 0.652538537979126, + "union_gap": -0.3641490340232849, + "mean_per_layer_hit_rate": 0.07692307978868484 + }, + "50": { + "rho_bar": 0.37646424770355225, + "union_observed": 0.39700374007225037, + "union_independent": 0.8045514822006226, + "union_gap": -0.4075477421283722, + "mean_per_layer_hit_rate": 0.1155286654829979 + }, + "100": { + "rho_bar": 0.39038926362991333, + "union_observed": 0.48938825726509094, + "union_independent": 0.8945827484130859, + "union_gap": -0.405194491147995, + "mean_per_layer_hit_rate": 0.15538270771503448 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.2885070677314486, + "union_gap_mean_over_k": -0.2577656906630312 + }, + "OOD_News_2024|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.11029350757598877, + "union_observed": 0.054931335151195526, + "union_independent": 0.09805792570114136, + "union_gap": -0.04312659054994583, + "mean_per_layer_hit_rate": 0.007874772883951664 + }, + "2": { + "rho_bar": 0.15408764779567719, + "union_observed": 0.07865168899297714, + "union_independent": 0.164361834526062, + "union_gap": -0.08571014553308487, + "mean_per_layer_hit_rate": 0.013636799529194832 + }, + "5": { + "rho_bar": 0.2797606885433197, + "union_observed": 0.12484394758939743, + "union_independent": 0.3214946389198303, + "union_gap": -0.1966506913304329, + "mean_per_layer_hit_rate": 0.02919427677989006 + }, + "10": { + "rho_bar": 0.31940630078315735, + "union_observed": 0.17977528274059296, + "union_independent": 0.46411240100860596, + "union_gap": -0.284337118268013, + "mean_per_layer_hit_rate": 0.04628829285502434 + }, + "20": { + "rho_bar": 0.32807865738868713, + "union_observed": 0.2684144675731659, + "union_independent": 0.6270930171012878, + "union_gap": -0.35867854952812195, + "mean_per_layer_hit_rate": 0.0718332827091217 + }, + "50": { + "rho_bar": 0.3789222538471222, + "union_observed": 0.3820224702358246, + "union_independent": 0.7886611819267273, + "union_gap": -0.4066387116909027, + "mean_per_layer_hit_rate": 0.11024680733680725 + }, + "100": { + "rho_bar": 0.3981396555900574, + "union_observed": 0.49687889218330383, + "union_independent": 0.8923110961914062, + "union_gap": -0.3954322040081024, + "mean_per_layer_hit_rate": 0.15336598455905914 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.28124124450342997, + "union_gap_mean_over_k": -0.2529391444155148 + }, + "OOD_News_2024|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.10721851885318756, + "union_observed": 0.05368289723992348, + "union_independent": 0.09675413370132446, + "union_gap": -0.043071236461400986, + "mean_per_layer_hit_rate": 0.007778738159686327 + }, + "2": { + "rho_bar": 0.15506133437156677, + "union_observed": 0.08489388227462769, + "union_independent": 0.16313880681991577, + "union_gap": -0.07824492454528809, + "mean_per_layer_hit_rate": 0.013540768064558506 + }, + "5": { + "rho_bar": 0.23711109161376953, + "union_observed": 0.14606741070747375, + "union_independent": 0.3321332335472107, + "union_gap": -0.18606582283973694, + "mean_per_layer_hit_rate": 0.03034668229520321 + }, + "10": { + "rho_bar": 0.32714250683784485, + "union_observed": 0.19850187003612518, + "union_independent": 0.4880530834197998, + "union_gap": -0.2895512133836746, + "mean_per_layer_hit_rate": 0.049649473279714584 + }, + "20": { + "rho_bar": 0.36219725012779236, + "union_observed": 0.26966291666030884, + "union_independent": 0.637265682220459, + "union_gap": -0.36760276556015015, + "mean_per_layer_hit_rate": 0.0739460289478302 + }, + "50": { + "rho_bar": 0.4122852385044098, + "union_observed": 0.3745318353176117, + "union_independent": 0.8011335730552673, + "union_gap": -0.42660173773765564, + "mean_per_layer_hit_rate": 0.11408814787864685 + }, + "100": { + "rho_bar": 0.44340986013412476, + "union_observed": 0.4931335747241974, + "union_independent": 0.8995237350463867, + "union_gap": -0.40639016032218933, + "mean_per_layer_hit_rate": 0.15739940106868744 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2920608286346708, + "union_gap_mean_over_k": -0.25678969440715654 + }, + "OOD_News_2024|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.25841206312179565, + "union_observed": 0.04244694113731384, + "union_independent": 0.12934815883636475, + "union_gap": -0.0869012176990509, + "mean_per_layer_hit_rate": 0.010563718155026436 + }, + "2": { + "rho_bar": 0.2740349769592285, + "union_observed": 0.06866417080163956, + "union_independent": 0.2056860327720642, + "union_gap": -0.13702186197042465, + "mean_per_layer_hit_rate": 0.017478151246905327 + }, + "5": { + "rho_bar": 0.40357115864753723, + "union_observed": 0.11985018849372864, + "union_independent": 0.41037189960479736, + "union_gap": -0.2905217111110687, + "mean_per_layer_hit_rate": 0.03956592455506325 + }, + "10": { + "rho_bar": 0.44455277919769287, + "union_observed": 0.16729088127613068, + "union_independent": 0.5590133666992188, + "union_gap": -0.3917224854230881, + "mean_per_layer_hit_rate": 0.06040526181459427 + }, + "20": { + "rho_bar": 0.4624442160129547, + "union_observed": 0.24968789517879486, + "union_independent": 0.7260100841522217, + "union_gap": -0.4763221889734268, + "mean_per_layer_hit_rate": 0.09315279126167297 + }, + "50": { + "rho_bar": 0.49438604712486267, + "union_observed": 0.33957552909851074, + "union_independent": 0.8628247976303101, + "union_gap": -0.5232492685317993, + "mean_per_layer_hit_rate": 0.13819263875484467 + }, + "100": { + "rho_bar": 0.5043213963508606, + "union_observed": 0.44818976521492004, + "union_independent": 0.9373366236686707, + "union_gap": -0.4891468584537506, + "mean_per_layer_hit_rate": 0.1856333315372467 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4059603767735617, + "union_gap_mean_over_k": -0.342126513166087 + }, + "OOD_News_2024|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.24683575332164764, + "union_observed": 0.05118601769208908, + "union_independent": 0.13706457614898682, + "union_gap": -0.08587855845689774, + "mean_per_layer_hit_rate": 0.011235954239964485 + }, + "2": { + "rho_bar": 0.2840560972690582, + "union_observed": 0.08489388227462769, + "union_independent": 0.2195976972579956, + "union_gap": -0.13470381498336792, + "mean_per_layer_hit_rate": 0.018822625279426575 + }, + "5": { + "rho_bar": 0.3444732129573822, + "union_observed": 0.14606741070747375, + "union_independent": 0.41625744104385376, + "union_gap": -0.27019003033638, + "mean_per_layer_hit_rate": 0.040334198623895645 + }, + "10": { + "rho_bar": 0.4395533800125122, + "union_observed": 0.18976280093193054, + "union_independent": 0.5750344395637512, + "union_gap": -0.3852716386318207, + "mean_per_layer_hit_rate": 0.06309421360492706 + }, + "20": { + "rho_bar": 0.47587645053863525, + "union_observed": 0.26966291666030884, + "union_independent": 0.7311489582061768, + "union_gap": -0.4614860415458679, + "mean_per_layer_hit_rate": 0.09488140046596527 + }, + "50": { + "rho_bar": 0.5153000354766846, + "union_observed": 0.3595505654811859, + "union_independent": 0.8626584410667419, + "union_gap": -0.503107875585556, + "mean_per_layer_hit_rate": 0.13934504985809326 + }, + "100": { + "rho_bar": 0.5218384265899658, + "union_observed": 0.45443195104599, + "union_independent": 0.9335053563117981, + "union_gap": -0.4790734052658081, + "mean_per_layer_hit_rate": 0.1848650723695755 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.4039904794522694, + "union_gap_mean_over_k": -0.3313873378293855 + }, + "OOD_News_2024|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.17977134883403778, + "union_observed": 0.05368289723992348, + "union_independent": 0.1294499635696411, + "union_gap": -0.07576706632971764, + "mean_per_layer_hit_rate": 0.010563718155026436 + }, + "2": { + "rho_bar": 0.1917484849691391, + "union_observed": 0.0774032473564148, + "union_independent": 0.1830156445503235, + "union_gap": -0.10561239719390869, + "mean_per_layer_hit_rate": 0.015365407802164555 + }, + "5": { + "rho_bar": 0.2728970944881439, + "union_observed": 0.13857677578926086, + "union_independent": 0.36435049772262573, + "union_gap": -0.22577372193336487, + "mean_per_layer_hit_rate": 0.033995967358350754 + }, + "10": { + "rho_bar": 0.3568284213542938, + "union_observed": 0.1785268485546112, + "union_independent": 0.5183006525039673, + "union_gap": -0.3397738039493561, + "mean_per_layer_hit_rate": 0.054067034274339676 + }, + "20": { + "rho_bar": 0.41248413920402527, + "union_observed": 0.25468164682388306, + "union_independent": 0.6764429211616516, + "union_gap": -0.42176127433776855, + "mean_per_layer_hit_rate": 0.08191683888435364 + }, + "50": { + "rho_bar": 0.4740007221698761, + "union_observed": 0.3570536971092224, + "union_independent": 0.827337384223938, + "union_gap": -0.4702836871147156, + "mean_per_layer_hit_rate": 0.12397964298725128 + }, + "100": { + "rho_bar": 0.5086728930473328, + "union_observed": 0.4419475793838501, + "union_independent": 0.9078397750854492, + "union_gap": -0.4658921957015991, + "mean_per_layer_hit_rate": 0.16402572393417358 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3423433005809784, + "union_gap_mean_over_k": -0.3006948780800615 + }, + "OOD_News_2024|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.1638810634613037, + "union_observed": 0.04619225859642029, + "union_independent": 0.10591942071914673, + "union_gap": -0.05972716212272644, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.1634574681520462, + "union_observed": 0.08114856481552124, + "union_independent": 0.1715019941329956, + "union_gap": -0.09035342931747437, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.22566655278205872, + "union_observed": 0.14357054233551025, + "union_independent": 0.34164959192276, + "union_gap": -0.19807904958724976, + "mean_per_layer_hit_rate": 0.031403057277202606 + }, + "10": { + "rho_bar": 0.2848415672779083, + "union_observed": 0.19475655257701874, + "union_independent": 0.49146366119384766, + "union_gap": -0.2967071086168289, + "mean_per_layer_hit_rate": 0.05003361031413078 + }, + "20": { + "rho_bar": 0.3432520925998688, + "union_observed": 0.25717851519584656, + "union_independent": 0.6430467963218689, + "union_gap": -0.38586828112602234, + "mean_per_layer_hit_rate": 0.0749063640832901 + }, + "50": { + "rho_bar": 0.4128769040107727, + "union_observed": 0.3508114814758301, + "union_independent": 0.7983824014663696, + "union_gap": -0.44757091999053955, + "mean_per_layer_hit_rate": 0.11351194977760315 + }, + "100": { + "rho_bar": 0.4703168570995331, + "union_observed": 0.4319600462913513, + "union_independent": 0.8897455930709839, + "union_gap": -0.45778554677963257, + "mean_per_layer_hit_rate": 0.1525016725063324 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.29489892934049877, + "union_gap_mean_over_k": -0.2765844996486391 + }, + "OOD_News_2024|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.15889666974544525, + "union_observed": 0.04369537904858589, + "union_independent": 0.08195638656616211, + "union_gap": -0.03826100751757622, + "mean_per_layer_hit_rate": 0.006530298851430416 + }, + "2": { + "rho_bar": 0.15769927203655243, + "union_observed": 0.07365792989730835, + "union_independent": 0.14396744966506958, + "union_gap": -0.07030951976776123, + "mean_per_layer_hit_rate": 0.011812158860266209 + }, + "5": { + "rho_bar": 0.24358490109443665, + "union_observed": 0.13233457505702972, + "union_independent": 0.31282883882522583, + "union_gap": -0.1804942637681961, + "mean_per_layer_hit_rate": 0.02823394164443016 + }, + "10": { + "rho_bar": 0.3014972507953644, + "union_observed": 0.18476903438568115, + "union_independent": 0.4488643407821655, + "union_gap": -0.2640953063964844, + "mean_per_layer_hit_rate": 0.04427158832550049 + }, + "20": { + "rho_bar": 0.34107065200805664, + "union_observed": 0.25468164682388306, + "union_independent": 0.6013356447219849, + "union_gap": -0.3466539978981018, + "mean_per_layer_hit_rate": 0.06722366064786911 + }, + "50": { + "rho_bar": 0.38893887400627136, + "union_observed": 0.3458177149295807, + "union_independent": 0.7695609331130981, + "union_gap": -0.42374321818351746, + "mean_per_layer_hit_rate": 0.10467684268951416 + }, + "100": { + "rho_bar": 0.4393210709095001, + "union_observed": 0.42696627974510193, + "union_independent": 0.868877649307251, + "union_gap": -0.44191136956214905, + "mean_per_layer_hit_rate": 0.1417458951473236 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.29014409865651813, + "union_gap_mean_over_k": -0.25220981187054087 + }, + "OOD_News_2024|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.10110773891210556, + "union_observed": 0.05243445560336113, + "union_independent": 0.07496482133865356, + "union_gap": -0.022530365735292435, + "mean_per_layer_hit_rate": 0.005954096093773842 + }, + "2": { + "rho_bar": 0.15267308056354523, + "union_observed": 0.07615480571985245, + "union_independent": 0.13605046272277832, + "union_gap": -0.05989565700292587, + "mean_per_layer_hit_rate": 0.01113992091268301 + }, + "5": { + "rho_bar": 0.23196259140968323, + "union_observed": 0.13483145833015442, + "union_independent": 0.3028267025947571, + "union_gap": -0.16799524426460266, + "mean_per_layer_hit_rate": 0.027177568525075912 + }, + "10": { + "rho_bar": 0.28386884927749634, + "union_observed": 0.18227216601371765, + "union_independent": 0.44295787811279297, + "union_gap": -0.2606857120990753, + "mean_per_layer_hit_rate": 0.04359934851527214 + }, + "20": { + "rho_bar": 0.31992676854133606, + "union_observed": 0.23845192790031433, + "union_independent": 0.5676220655441284, + "union_gap": -0.3291701376438141, + "mean_per_layer_hit_rate": 0.06174973398447037 + }, + "50": { + "rho_bar": 0.3637409806251526, + "union_observed": 0.3283395767211914, + "union_independent": 0.7295446991920471, + "union_gap": -0.4012051224708557, + "mean_per_layer_hit_rate": 0.09430519491434097 + }, + "100": { + "rho_bar": 0.39386317133903503, + "union_observed": 0.4232209622859955, + "union_independent": 0.8533531427383423, + "union_gap": -0.4301321804523468, + "mean_per_layer_hit_rate": 0.13492749631404877 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2638775972383363, + "union_gap_mean_over_k": -0.23880205995270185 + }, + "OOD_News_2024|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.12245900928974152, + "union_observed": 0.04619225859642029, + "union_independent": 0.07478678226470947, + "union_gap": -0.028594523668289185, + "mean_per_layer_hit_rate": 0.005954096093773842 + }, + "2": { + "rho_bar": 0.18715129792690277, + "union_observed": 0.06117353215813637, + "union_independent": 0.1248132586479187, + "union_gap": -0.06363972648978233, + "mean_per_layer_hit_rate": 0.010179582983255386 + }, + "5": { + "rho_bar": 0.24968597292900085, + "union_observed": 0.10736579447984695, + "union_independent": 0.2813147306442261, + "union_gap": -0.17394893616437912, + "mean_per_layer_hit_rate": 0.024968789890408516 + }, + "10": { + "rho_bar": 0.2916870713233948, + "union_observed": 0.15980024635791779, + "union_independent": 0.42174506187438965, + "union_gap": -0.26194481551647186, + "mean_per_layer_hit_rate": 0.04100643843412399 + }, + "20": { + "rho_bar": 0.3376184105873108, + "union_observed": 0.20349563658237457, + "union_independent": 0.5515364408493042, + "union_gap": -0.3480408042669296, + "mean_per_layer_hit_rate": 0.05934888869524002 + }, + "50": { + "rho_bar": 0.3578719198703766, + "union_observed": 0.3071160912513733, + "union_independent": 0.7174013257026672, + "union_gap": -0.41028523445129395, + "mean_per_layer_hit_rate": 0.09161624312400818 + }, + "100": { + "rho_bar": 0.377371221780777, + "union_observed": 0.39700374007225037, + "union_independent": 0.8547436594963074, + "union_gap": -0.457739919424057, + "mean_per_layer_hit_rate": 0.135599747300148 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.27483498624392916, + "union_gap_mean_over_k": -0.24917056571160043 + }, + "OOD_News_2024|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.18517190217971802, + "union_observed": 0.039950061589479446, + "union_independent": 0.07251495122909546, + "union_gap": -0.03256488963961601, + "mean_per_layer_hit_rate": 0.00576202804222703 + }, + "2": { + "rho_bar": 0.18772347271442413, + "union_observed": 0.062421973794698715, + "union_independent": 0.13046753406524658, + "union_gap": -0.06804556027054787, + "mean_per_layer_hit_rate": 0.01065975334495306 + }, + "5": { + "rho_bar": 0.252494752407074, + "union_observed": 0.1086142286658287, + "union_independent": 0.28149956464767456, + "union_gap": -0.17288533598184586, + "mean_per_layer_hit_rate": 0.024968789890408516 + }, + "10": { + "rho_bar": 0.31553640961647034, + "union_observed": 0.1498127281665802, + "union_independent": 0.4139258861541748, + "union_gap": -0.2641131579875946, + "mean_per_layer_hit_rate": 0.039950065314769745 + }, + "20": { + "rho_bar": 0.3399994671344757, + "union_observed": 0.19850187003612518, + "union_independent": 0.5567819476127625, + "union_gap": -0.35828007757663727, + "mean_per_layer_hit_rate": 0.060021135956048965 + }, + "50": { + "rho_bar": 0.37098821997642517, + "union_observed": 0.3133583068847656, + "union_independent": 0.7496581673622131, + "union_gap": -0.4362998604774475, + "mean_per_layer_hit_rate": 0.09929895401000977 + }, + "100": { + "rho_bar": 0.37922725081443787, + "union_observed": 0.4007490575313568, + "union_independent": 0.8701369762420654, + "union_gap": -0.4693879187107086, + "mean_per_layer_hit_rate": 0.1419379562139511 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2901630678347179, + "union_gap_mean_over_k": -0.25736811437777113 + }, + "OOD_News_2024|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.14398270845413208, + "union_observed": 0.03745318204164505, + "union_independent": 0.06785029172897339, + "union_gap": -0.03039710968732834, + "mean_per_layer_hit_rate": 0.00537789287045598 + }, + "2": { + "rho_bar": 0.1710761934518814, + "union_observed": 0.056179776787757874, + "union_independent": 0.12044990062713623, + "union_gap": -0.06427012383937836, + "mean_per_layer_hit_rate": 0.009795447811484337 + }, + "5": { + "rho_bar": 0.3039887845516205, + "union_observed": 0.10362047702074051, + "union_independent": 0.26748335361480713, + "union_gap": -0.16386287659406662, + "mean_per_layer_hit_rate": 0.02352828159928322 + }, + "10": { + "rho_bar": 0.3366309404373169, + "union_observed": 0.13857677578926086, + "union_independent": 0.4081331491470337, + "union_gap": -0.2695563733577728, + "mean_per_layer_hit_rate": 0.03918179124593735 + }, + "20": { + "rho_bar": 0.3509548306465149, + "union_observed": 0.20973782241344452, + "union_independent": 0.5796961188316345, + "union_gap": -0.36995829641819, + "mean_per_layer_hit_rate": 0.06357438117265701 + }, + "50": { + "rho_bar": 0.37215158343315125, + "union_observed": 0.3121098577976227, + "union_independent": 0.7635965943336487, + "union_gap": -0.451486736536026, + "mean_per_layer_hit_rate": 0.10294824093580246 + }, + "100": { + "rho_bar": 0.3814332187175751, + "union_observed": 0.41697877645492554, + "union_independent": 0.8825507164001465, + "union_gap": -0.46557193994522095, + "mean_per_layer_hit_rate": 0.14779601991176605 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29431689424174173, + "union_gap_mean_over_k": -0.2593004937682833 + }, + "OOD_News_2024|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1401083618402481, + "union_observed": 0.041198503226041794, + "union_independent": 0.07604151964187622, + "union_gap": -0.03484301641583443, + "mean_per_layer_hit_rate": 0.006050129421055317 + }, + "2": { + "rho_bar": 0.2210010141134262, + "union_observed": 0.062421973794698715, + "union_independent": 0.14484405517578125, + "union_gap": -0.08242208138108253, + "mean_per_layer_hit_rate": 0.011908192187547684 + }, + "5": { + "rho_bar": 0.2616875171661377, + "union_observed": 0.1061173528432846, + "union_independent": 0.2857249975204468, + "union_gap": -0.17960764467716217, + "mean_per_layer_hit_rate": 0.025352923199534416 + }, + "10": { + "rho_bar": 0.33980634808540344, + "union_observed": 0.15355804562568665, + "union_independent": 0.427768349647522, + "union_gap": -0.2742103040218353, + "mean_per_layer_hit_rate": 0.04158264026045799 + }, + "20": { + "rho_bar": 0.3648419976234436, + "union_observed": 0.21598002314567566, + "union_independent": 0.5930001139640808, + "union_gap": -0.37702009081840515, + "mean_per_layer_hit_rate": 0.06578315794467926 + }, + "50": { + "rho_bar": 0.3611651062965393, + "union_observed": 0.33208489418029785, + "union_independent": 0.7743131518363953, + "union_gap": -0.4422282576560974, + "mean_per_layer_hit_rate": 0.10582925379276276 + }, + "100": { + "rho_bar": 0.3739232122898102, + "union_observed": 0.4307115972042084, + "union_independent": 0.8906787633895874, + "union_gap": -0.45996716618537903, + "mean_per_layer_hit_rate": 0.15182945132255554 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2946476510592869, + "union_gap_mean_over_k": -0.26432836587939945 + }, + "OOD_News_2024|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.14963997900485992, + "union_observed": 0.039950061589479446, + "union_independent": 0.08880841732025146, + "union_gap": -0.04885835573077202, + "mean_per_layer_hit_rate": 0.007106501143425703 + }, + "2": { + "rho_bar": 0.17757102847099304, + "union_observed": 0.06367041170597076, + "union_independent": 0.14154624938964844, + "union_gap": -0.07787583768367767, + "mean_per_layer_hit_rate": 0.01162009034305811 + }, + "5": { + "rho_bar": 0.23532146215438843, + "union_observed": 0.1111111119389534, + "union_independent": 0.29003942012786865, + "union_gap": -0.17892830818891525, + "mean_per_layer_hit_rate": 0.025833092629909515 + }, + "10": { + "rho_bar": 0.3345799744129181, + "union_observed": 0.16479401290416718, + "union_independent": 0.44878894090652466, + "union_gap": -0.2839949280023575, + "mean_per_layer_hit_rate": 0.04427158460021019 + }, + "20": { + "rho_bar": 0.3417053818702698, + "union_observed": 0.23470661044120789, + "union_independent": 0.6114898920059204, + "union_gap": -0.3767832815647125, + "mean_per_layer_hit_rate": 0.068952277302742 + }, + "50": { + "rho_bar": 0.34758302569389343, + "union_observed": 0.3645443320274353, + "union_independent": 0.7949724793434143, + "union_gap": -0.430428147315979, + "mean_per_layer_hit_rate": 0.1118793785572052 + }, + "100": { + "rho_bar": 0.36244457960128784, + "union_observed": 0.4631710350513458, + "union_independent": 0.898500919342041, + "union_gap": -0.4353298842906952, + "mean_per_layer_hit_rate": 0.1561509668827057 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.27840649017265867, + "union_gap_mean_over_k": -0.261742677539587 + }, + "OOD_News_2024|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.16864129900932312, + "union_observed": 0.041198503226041794, + "union_independent": 0.09000086784362793, + "union_gap": -0.048802364617586136, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.18408365547657013, + "union_observed": 0.06491885334253311, + "union_independent": 0.15998512506484985, + "union_gap": -0.09506627172231674, + "mean_per_layer_hit_rate": 0.013252665288746357 + }, + "5": { + "rho_bar": 0.28269025683403015, + "union_observed": 0.1086142286658287, + "union_independent": 0.32080352306365967, + "union_gap": -0.21218929439783096, + "mean_per_layer_hit_rate": 0.02909824252128601 + }, + "10": { + "rho_bar": 0.34656068682670593, + "union_observed": 0.17103619873523712, + "union_independent": 0.4847807288169861, + "union_gap": -0.31374453008174896, + "mean_per_layer_hit_rate": 0.04916930943727493 + }, + "20": { + "rho_bar": 0.3506588637828827, + "union_observed": 0.24968789517879486, + "union_independent": 0.6470194458961487, + "union_gap": -0.3973315507173538, + "mean_per_layer_hit_rate": 0.0756746381521225 + }, + "50": { + "rho_bar": 0.3609773814678192, + "union_observed": 0.37078651785850525, + "union_independent": 0.8140044212341309, + "union_gap": -0.4432179033756256, + "mean_per_layer_hit_rate": 0.11840967833995819 + }, + "100": { + "rho_bar": 0.36924564838409424, + "union_observed": 0.47690385580062866, + "union_independent": 0.919208288192749, + "union_gap": -0.44230443239212036, + "mean_per_layer_hit_rate": 0.17046000063419342 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.29469397025448935, + "union_gap_mean_over_k": -0.2789509067577975 + }, + "OOD_News_2024|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.15449203550815582, + "union_observed": 0.05243445560336113, + "union_independent": 0.1126907467842102, + "union_gap": -0.060256291180849075, + "mean_per_layer_hit_rate": 0.009123211726546288 + }, + "2": { + "rho_bar": 0.22631517052650452, + "union_observed": 0.07865168899297714, + "union_independent": 0.197729229927063, + "union_gap": -0.11907754093408585, + "mean_per_layer_hit_rate": 0.016709882766008377 + }, + "5": { + "rho_bar": 0.32181069254875183, + "union_observed": 0.12484394758939743, + "union_independent": 0.3702446222305298, + "union_gap": -0.24540067464113235, + "mean_per_layer_hit_rate": 0.0346682034432888 + }, + "10": { + "rho_bar": 0.4063052237033844, + "union_observed": 0.17977528274059296, + "union_independent": 0.5255444049835205, + "union_gap": -0.34576912224292755, + "mean_per_layer_hit_rate": 0.055123403668403625 + }, + "20": { + "rho_bar": 0.3979916274547577, + "union_observed": 0.2746566832065582, + "union_independent": 0.6867663860321045, + "union_gap": -0.41210970282554626, + "mean_per_layer_hit_rate": 0.08412560820579529 + }, + "50": { + "rho_bar": 0.4148938059806824, + "union_observed": 0.3795256018638611, + "union_independent": 0.8396421074867249, + "union_gap": -0.46011650562286377, + "mean_per_layer_hit_rate": 0.12878133356571198 + }, + "100": { + "rho_bar": 0.41294553875923157, + "union_observed": 0.4756554365158081, + "union_independent": 0.9263557195663452, + "union_gap": -0.4507002830505371, + "mean_per_layer_hit_rate": 0.17766252160072327 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3335362992116383, + "union_gap_mean_over_k": -0.29906144578542027 + }, + "OOD_News_2024|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.15580633282661438, + "union_observed": 0.05118601769208908, + "union_independent": 0.12279552221298218, + "union_gap": -0.0716095045208931, + "mean_per_layer_hit_rate": 0.009987516328692436 + }, + "2": { + "rho_bar": 0.2216302752494812, + "union_observed": 0.08364544063806534, + "union_independent": 0.20370841026306152, + "union_gap": -0.12006296962499619, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.3421162962913513, + "union_observed": 0.13108614087104797, + "union_independent": 0.3818703293800354, + "union_gap": -0.2507841885089874, + "mean_per_layer_hit_rate": 0.03610870987176895 + }, + "10": { + "rho_bar": 0.41367897391319275, + "union_observed": 0.18726591765880585, + "union_independent": 0.5467813014984131, + "union_gap": -0.35951538383960724, + "mean_per_layer_hit_rate": 0.058484580367803574 + }, + "20": { + "rho_bar": 0.4328533113002777, + "union_observed": 0.26466915011405945, + "union_independent": 0.6972358226776123, + "union_gap": -0.43256667256355286, + "mean_per_layer_hit_rate": 0.08652646094560623 + }, + "50": { + "rho_bar": 0.4446321725845337, + "union_observed": 0.3645443320274353, + "union_independent": 0.8478770852088928, + "union_gap": -0.4833327531814575, + "mean_per_layer_hit_rate": 0.13243061304092407 + }, + "100": { + "rho_bar": 0.45077112317085266, + "union_observed": 0.47690385580062866, + "union_independent": 0.925064206123352, + "union_gap": -0.4481603503227234, + "mean_per_layer_hit_rate": 0.1771823614835739 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.35164121219090055, + "union_gap_mean_over_k": -0.30943311750888824 + }, + "OOD_News_2024|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.09520059078931808, + "union_observed": 0.05243445560336113, + "union_independent": 0.0956771969795227, + "union_gap": -0.043242741376161575, + "mean_per_layer_hit_rate": 0.007682703901082277 + }, + "2": { + "rho_bar": 0.1453193575143814, + "union_observed": 0.07990012317895889, + "union_independent": 0.16520464420318604, + "union_gap": -0.08530452102422714, + "mean_per_layer_hit_rate": 0.013732834719121456 + }, + "5": { + "rho_bar": 0.2750677466392517, + "union_observed": 0.12983770668506622, + "union_independent": 0.3300189971923828, + "union_gap": -0.2001812905073166, + "mean_per_layer_hit_rate": 0.03015461564064026 + }, + "10": { + "rho_bar": 0.36195966601371765, + "union_observed": 0.18976280093193054, + "union_independent": 0.484708309173584, + "union_gap": -0.29494550824165344, + "mean_per_layer_hit_rate": 0.049265339970588684 + }, + "20": { + "rho_bar": 0.38599249720573425, + "union_observed": 0.25717851519584656, + "union_independent": 0.6347371339797974, + "union_gap": -0.3775586187839508, + "mean_per_layer_hit_rate": 0.073561891913414 + }, + "50": { + "rho_bar": 0.4037458896636963, + "union_observed": 0.3620474338531494, + "union_independent": 0.7945230007171631, + "union_gap": -0.43247556686401367, + "mean_per_layer_hit_rate": 0.1126476526260376 + }, + "100": { + "rho_bar": 0.4129630923271179, + "union_observed": 0.4756554365158081, + "union_independent": 0.8972117304801941, + "union_gap": -0.421556293964386, + "mean_per_layer_hit_rate": 0.1574954241514206 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2971784057361739, + "union_gap_mean_over_k": -0.26503779153738705 + }, + "OOD_News_2024|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.1510949730873108, + "union_observed": 0.04494382068514824, + "union_independent": 0.09573620557785034, + "union_gap": -0.0507923848927021, + "mean_per_layer_hit_rate": 0.007682703901082277 + }, + "2": { + "rho_bar": 0.1984144002199173, + "union_observed": 0.0699126124382019, + "union_independent": 0.15680253505706787, + "union_gap": -0.08688992261886597, + "mean_per_layer_hit_rate": 0.012964563444256783 + }, + "5": { + "rho_bar": 0.2539021670818329, + "union_observed": 0.1111111119389534, + "union_independent": 0.31018728017807007, + "union_gap": -0.19907616823911667, + "mean_per_layer_hit_rate": 0.027945837005972862 + }, + "10": { + "rho_bar": 0.34393006563186646, + "union_observed": 0.16853933036327362, + "union_independent": 0.45449262857437134, + "union_gap": -0.2859532982110977, + "mean_per_layer_hit_rate": 0.04513588920235634 + }, + "20": { + "rho_bar": 0.3750867545604706, + "union_observed": 0.24094881117343903, + "union_independent": 0.6114449501037598, + "union_gap": -0.37049613893032074, + "mean_per_layer_hit_rate": 0.06924037635326385 + }, + "50": { + "rho_bar": 0.3855847120285034, + "union_observed": 0.37328338623046875, + "union_independent": 0.794431209564209, + "union_gap": -0.42114782333374023, + "mean_per_layer_hit_rate": 0.11255161464214325 + }, + "100": { + "rho_bar": 0.4131494164466858, + "union_observed": 0.47191011905670166, + "union_independent": 0.8959872126579285, + "union_gap": -0.4240770936012268, + "mean_per_layer_hit_rate": 0.15643906593322754 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3030232127223696, + "union_gap_mean_over_k": -0.26263326140386717 + }, + "OOD_News_2024|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.12306860089302063, + "union_observed": 0.05742821469902992, + "union_independent": 0.09696483612060547, + "union_gap": -0.039536621421575546, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.14430715143680573, + "union_observed": 0.09612983465194702, + "union_independent": 0.17183631658554077, + "union_gap": -0.07570648193359375, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.2611086666584015, + "union_observed": 0.1548064947128296, + "union_independent": 0.3513656258583069, + "union_gap": -0.1965591311454773, + "mean_per_layer_hit_rate": 0.03245942294597626 + }, + "10": { + "rho_bar": 0.3428482115268707, + "union_observed": 0.20973782241344452, + "union_independent": 0.5057753324508667, + "union_gap": -0.2960375100374222, + "mean_per_layer_hit_rate": 0.05214635655283928 + }, + "20": { + "rho_bar": 0.35865339636802673, + "union_observed": 0.2871410846710205, + "union_independent": 0.648388683795929, + "union_gap": -0.36124759912490845, + "mean_per_layer_hit_rate": 0.0760587751865387 + }, + "50": { + "rho_bar": 0.3722682595252991, + "union_observed": 0.4032459557056427, + "union_independent": 0.8043226003646851, + "union_gap": -0.40107664465904236, + "mean_per_layer_hit_rate": 0.11543262004852295 + }, + "100": { + "rho_bar": 0.38458582758903503, + "union_observed": 0.48938825726509094, + "union_independent": 0.8937287926673889, + "union_gap": -0.404340535402298, + "mean_per_layer_hit_rate": 0.15471045672893524 + } + }, + "n_degenerate_pairs": 45, + "rho_bar_mean_over_k": 0.28383430199963705, + "union_gap_mean_over_k": -0.25350064624633106 + }, + "OOD_News_2024|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.11384802311658859, + "union_observed": 0.05368289723992348, + "union_independent": 0.0980420708656311, + "union_gap": -0.044359173625707626, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.15553273260593414, + "union_observed": 0.07990012317895889, + "union_independent": 0.16123813390731812, + "union_gap": -0.08133801072835922, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.2584424316883087, + "union_observed": 0.13358302414417267, + "union_independent": 0.3236631751060486, + "union_gap": -0.19008015096187592, + "mean_per_layer_hit_rate": 0.02938634343445301 + }, + "10": { + "rho_bar": 0.3118427097797394, + "union_observed": 0.18976280093193054, + "union_independent": 0.4667283296585083, + "union_gap": -0.27696552872657776, + "mean_per_layer_hit_rate": 0.046576399356126785 + }, + "20": { + "rho_bar": 0.3093979060649872, + "union_observed": 0.2684144675731659, + "union_independent": 0.6252222657203674, + "union_gap": -0.35680779814720154, + "mean_per_layer_hit_rate": 0.0714491531252861 + }, + "50": { + "rho_bar": 0.3632693886756897, + "union_observed": 0.38077402114868164, + "union_independent": 0.7865606546401978, + "union_gap": -0.4057866334915161, + "mean_per_layer_hit_rate": 0.10947852581739426 + }, + "100": { + "rho_bar": 0.38943442702293396, + "union_observed": 0.49188515543937683, + "union_independent": 0.891877293586731, + "union_gap": -0.3999921381473541, + "mean_per_layer_hit_rate": 0.15298184752464294 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.27168108842202593, + "union_gap_mean_over_k": -0.2507613476897989 + }, + "OOD_News_2024|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.09810885787010193, + "union_observed": 0.05368289723992348, + "union_independent": 0.09449762105941772, + "union_gap": -0.04081472381949425, + "mean_per_layer_hit_rate": 0.007586670573800802 + }, + "2": { + "rho_bar": 0.1575622856616974, + "union_observed": 0.08489388227462769, + "union_independent": 0.16101205348968506, + "union_gap": -0.07611817121505737, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.22763384878635406, + "union_observed": 0.1498127281665802, + "union_independent": 0.33317720890045166, + "union_gap": -0.18336448073387146, + "mean_per_layer_hit_rate": 0.03044271469116211 + }, + "10": { + "rho_bar": 0.3137432932853699, + "union_observed": 0.20099875330924988, + "union_independent": 0.4807627201080322, + "union_gap": -0.27976396679878235, + "mean_per_layer_hit_rate": 0.048593103885650635 + }, + "20": { + "rho_bar": 0.3502918779850006, + "union_observed": 0.2784020006656647, + "union_independent": 0.6381889581680298, + "union_gap": -0.3597869575023651, + "mean_per_layer_hit_rate": 0.07404206693172455 + }, + "50": { + "rho_bar": 0.4040431082248688, + "union_observed": 0.38327091932296753, + "union_independent": 0.7999693155288696, + "union_gap": -0.4166983962059021, + "mean_per_layer_hit_rate": 0.11370401829481125 + }, + "100": { + "rho_bar": 0.4386097490787506, + "union_observed": 0.49188515543937683, + "union_independent": 0.8967873454093933, + "union_gap": -0.4049021899700165, + "mean_per_layer_hit_rate": 0.15567079186439514 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2842847172703062, + "union_gap_mean_over_k": -0.251635555177927 + }, + "OOD_News_2024|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.27755963802337646, + "union_observed": 0.04369537904858589, + "union_independent": 0.13046163320541382, + "union_gap": -0.08676625415682793, + "mean_per_layer_hit_rate": 0.010659752413630486 + }, + "2": { + "rho_bar": 0.28982481360435486, + "union_observed": 0.0699126124382019, + "union_independent": 0.20876753330230713, + "union_gap": -0.13885492086410522, + "mean_per_layer_hit_rate": 0.017766254022717476 + }, + "5": { + "rho_bar": 0.393001526594162, + "union_observed": 0.12234706431627274, + "union_independent": 0.40726566314697266, + "union_gap": -0.2849185988306999, + "mean_per_layer_hit_rate": 0.03918179124593735 + }, + "10": { + "rho_bar": 0.4360848367214203, + "union_observed": 0.16978776454925537, + "union_independent": 0.560531497001648, + "union_gap": -0.3907437324523926, + "mean_per_layer_hit_rate": 0.06059732660651207 + }, + "20": { + "rho_bar": 0.45521828532218933, + "union_observed": 0.2534331977367401, + "union_independent": 0.7243489027023315, + "union_gap": -0.47091570496559143, + "mean_per_layer_hit_rate": 0.09267262369394302 + }, + "50": { + "rho_bar": 0.49401572346687317, + "union_observed": 0.34706616401672363, + "union_independent": 0.8651926517486572, + "union_gap": -0.5181264877319336, + "mean_per_layer_hit_rate": 0.1392490267753601 + }, + "100": { + "rho_bar": 0.503531813621521, + "union_observed": 0.4469413161277771, + "union_independent": 0.9382604360580444, + "union_gap": -0.49131911993026733, + "mean_per_layer_hit_rate": 0.18630559742450714 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.40703380533627104, + "union_gap_mean_over_k": -0.3402349741331169 + }, + "OOD_arXiv_2023|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.2979109287261963, + "union_observed": 0.056179776787757874, + "union_independent": 0.1542300581932068, + "union_gap": -0.09805028140544891, + "mean_per_layer_hit_rate": 0.012772495858371258 + }, + "2": { + "rho_bar": 0.3426760733127594, + "union_observed": 0.08364544063806534, + "union_independent": 0.2479211688041687, + "union_gap": -0.16427572816610336, + "mean_per_layer_hit_rate": 0.02160760574042797 + }, + "5": { + "rho_bar": 0.36372068524360657, + "union_observed": 0.1423220932483673, + "union_independent": 0.41757988929748535, + "union_gap": -0.27525779604911804, + "mean_per_layer_hit_rate": 0.040430232882499695 + }, + "10": { + "rho_bar": 0.416423499584198, + "union_observed": 0.2122347056865692, + "union_independent": 0.5990462899208069, + "union_gap": -0.38681158423423767, + "mean_per_layer_hit_rate": 0.06722366809844971 + }, + "20": { + "rho_bar": 0.4459402561187744, + "union_observed": 0.28589263558387756, + "union_independent": 0.7518810033798218, + "union_gap": -0.4659883677959442, + "mean_per_layer_hit_rate": 0.10035533457994461 + }, + "50": { + "rho_bar": 0.48753562569618225, + "union_observed": 0.36828964948654175, + "union_independent": 0.8786784410476685, + "union_gap": -0.5103887915611267, + "mean_per_layer_hit_rate": 0.14779601991176605 + }, + "100": { + "rho_bar": 0.5136732459068298, + "union_observed": 0.45817726850509644, + "union_independent": 0.9427018761634827, + "union_gap": -0.48452460765838623, + "mean_per_layer_hit_rate": 0.19418033957481384 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.40969718779836384, + "union_gap_mean_over_k": -0.34075673669576645 + }, + "OOD_arXiv_2023|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.23797449469566345, + "union_observed": 0.05118601769208908, + "union_independent": 0.1271451711654663, + "union_gap": -0.07595915347337723, + "mean_per_layer_hit_rate": 0.010371650569140911 + }, + "2": { + "rho_bar": 0.321051687002182, + "union_observed": 0.08114856481552124, + "union_independent": 0.21974575519561768, + "union_gap": -0.13859719038009644, + "mean_per_layer_hit_rate": 0.018822625279426575 + }, + "5": { + "rho_bar": 0.31973856687545776, + "union_observed": 0.13483145833015442, + "union_independent": 0.37696224451065063, + "union_gap": -0.24213078618049622, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.33477747440338135, + "union_observed": 0.19850187003612518, + "union_independent": 0.5434728264808655, + "union_gap": -0.3449709564447403, + "mean_per_layer_hit_rate": 0.05781235173344612 + }, + "20": { + "rho_bar": 0.39092832803726196, + "union_observed": 0.2721598148345947, + "union_independent": 0.6998435258865356, + "union_gap": -0.4276837110519409, + "mean_per_layer_hit_rate": 0.08719869703054428 + }, + "50": { + "rho_bar": 0.46299293637275696, + "union_observed": 0.3670412003993988, + "union_independent": 0.8512963652610779, + "union_gap": -0.4842551648616791, + "mean_per_layer_hit_rate": 0.13425526022911072 + }, + "100": { + "rho_bar": 0.506251871585846, + "union_observed": 0.4444444477558136, + "union_independent": 0.9169275164604187, + "union_gap": -0.4724830687046051, + "mean_per_layer_hit_rate": 0.1708441525697708 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3676736227103642, + "union_gap_mean_over_k": -0.3122971472995622 + }, + "OOD_arXiv_2023|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.2003786563873291, + "union_observed": 0.05118601769208908, + "union_independent": 0.11936616897583008, + "union_gap": -0.068180151283741, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.2578708529472351, + "union_observed": 0.0774032473564148, + "union_independent": 0.19128870964050293, + "union_gap": -0.11388546228408813, + "mean_per_layer_hit_rate": 0.016133679077029228 + }, + "5": { + "rho_bar": 0.2867242693901062, + "union_observed": 0.13358302414417267, + "union_independent": 0.3356931805610657, + "union_gap": -0.202110156416893, + "mean_per_layer_hit_rate": 0.03073081746697426 + }, + "10": { + "rho_bar": 0.2864021956920624, + "union_observed": 0.20973782241344452, + "union_independent": 0.4991157650947571, + "union_gap": -0.28937794268131256, + "mean_per_layer_hit_rate": 0.05118602141737938 + }, + "20": { + "rho_bar": 0.3299718499183655, + "union_observed": 0.26966291666030884, + "union_independent": 0.6521937251091003, + "union_gap": -0.3825308084487915, + "mean_per_layer_hit_rate": 0.0768270418047905 + }, + "50": { + "rho_bar": 0.40937933325767517, + "union_observed": 0.3545567989349365, + "union_independent": 0.815596342086792, + "union_gap": -0.46103954315185547, + "mean_per_layer_hit_rate": 0.11975415050983429 + }, + "100": { + "rho_bar": 0.443575382232666, + "union_observed": 0.4444444477558136, + "union_independent": 0.9000635147094727, + "union_gap": -0.45561906695365906, + "mean_per_layer_hit_rate": 0.15883991122245789 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3163289342607771, + "union_gap_mean_over_k": -0.2818204473171915 + }, + "OOD_arXiv_2023|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.18850544095039368, + "union_observed": 0.04619225859642029, + "union_independent": 0.10483372211456299, + "union_gap": -0.0586414635181427, + "mean_per_layer_hit_rate": 0.008450974710285664 + }, + "2": { + "rho_bar": 0.24377292394638062, + "union_observed": 0.072409488260746, + "union_independent": 0.17043304443359375, + "union_gap": -0.09802355617284775, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.2813361585140228, + "union_observed": 0.13358302414417267, + "union_independent": 0.314935564994812, + "union_gap": -0.18135254085063934, + "mean_per_layer_hit_rate": 0.028426004573702812 + }, + "10": { + "rho_bar": 0.3017421066761017, + "union_observed": 0.19725343585014343, + "union_independent": 0.4689871668815613, + "union_gap": -0.27173373103141785, + "mean_per_layer_hit_rate": 0.04696052893996239 + }, + "20": { + "rho_bar": 0.332794189453125, + "union_observed": 0.2584269642829895, + "union_independent": 0.626266598701477, + "union_gap": -0.36783963441848755, + "mean_per_layer_hit_rate": 0.07173725217580795 + }, + "50": { + "rho_bar": 0.4028836190700531, + "union_observed": 0.3483146131038666, + "union_independent": 0.791314423084259, + "union_gap": -0.44299980998039246, + "mean_per_layer_hit_rate": 0.11139921098947525 + }, + "100": { + "rho_bar": 0.4374239444732666, + "union_observed": 0.4369538128376007, + "union_independent": 0.881463348865509, + "union_gap": -0.4445095360279083, + "mean_per_layer_hit_rate": 0.1482762098312378 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3126369118690491, + "union_gap_mean_over_k": -0.26644289599997656 + }, + "OOD_arXiv_2023|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.10427030175924301, + "union_observed": 0.048689138144254684, + "union_independent": 0.08650761842727661, + "union_gap": -0.03781848028302193, + "mean_per_layer_hit_rate": 0.006914434023201466 + }, + "2": { + "rho_bar": 0.2016116827726364, + "union_observed": 0.0774032473564148, + "union_independent": 0.1715390682220459, + "union_gap": -0.0941358208656311, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.2287355363368988, + "union_observed": 0.14107365906238556, + "union_independent": 0.2972562313079834, + "union_gap": -0.15618257224559784, + "mean_per_layer_hit_rate": 0.026601364836096764 + }, + "10": { + "rho_bar": 0.3055395483970642, + "union_observed": 0.18976280093193054, + "union_independent": 0.45425862073898315, + "union_gap": -0.2644958198070526, + "mean_per_layer_hit_rate": 0.04513589292764664 + }, + "20": { + "rho_bar": 0.32376495003700256, + "union_observed": 0.2584269642829895, + "union_independent": 0.6010241508483887, + "union_gap": -0.34259718656539917, + "mean_per_layer_hit_rate": 0.06741572916507721 + }, + "50": { + "rho_bar": 0.38602790236473083, + "union_observed": 0.3458177149295807, + "union_independent": 0.7677325010299683, + "union_gap": -0.4219147861003876, + "mean_per_layer_hit_rate": 0.10467684268951416 + }, + "100": { + "rho_bar": 0.4203423261642456, + "union_observed": 0.4369538128376007, + "union_independent": 0.8650792837142944, + "union_gap": -0.4281254708766937, + "mean_per_layer_hit_rate": 0.1404014229774475 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.28147032111883163, + "union_gap_mean_over_k": -0.249324305249112 + }, + "OOD_arXiv_2023|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.13556388020515442, + "union_observed": 0.03745318204164505, + "union_independent": 0.07128214836120605, + "union_gap": -0.033828966319561005, + "mean_per_layer_hit_rate": 0.005665994714945555 + }, + "2": { + "rho_bar": 0.20806029438972473, + "union_observed": 0.06616728752851486, + "union_independent": 0.15526753664016724, + "union_gap": -0.08910024911165237, + "mean_per_layer_hit_rate": 0.012868530116975307 + }, + "5": { + "rho_bar": 0.2471144050359726, + "union_observed": 0.11985018849372864, + "union_independent": 0.2911139130592346, + "union_gap": -0.17126372456550598, + "mean_per_layer_hit_rate": 0.026025159284472466 + }, + "10": { + "rho_bar": 0.31608834862709045, + "union_observed": 0.1548064947128296, + "union_independent": 0.4351688623428345, + "union_gap": -0.2803623676300049, + "mean_per_layer_hit_rate": 0.04273504391312599 + }, + "20": { + "rho_bar": 0.351447731256485, + "union_observed": 0.2197253406047821, + "union_independent": 0.5873602628707886, + "union_gap": -0.36763492226600647, + "mean_per_layer_hit_rate": 0.06530298292636871 + }, + "50": { + "rho_bar": 0.39232274889945984, + "union_observed": 0.29338327050209045, + "union_independent": 0.7464247345924377, + "union_gap": -0.4530414640903473, + "mean_per_layer_hit_rate": 0.09929895401000977 + }, + "100": { + "rho_bar": 0.4033620357513428, + "union_observed": 0.4057428240776062, + "union_independent": 0.8539777994155884, + "union_gap": -0.4482349753379822, + "mean_per_layer_hit_rate": 0.13588783144950867 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29342277773789, + "union_gap_mean_over_k": -0.26335238133158 + }, + "OOD_arXiv_2023|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.11400352418422699, + "union_observed": 0.0387016236782074, + "union_independent": 0.0702521800994873, + "union_gap": -0.03155055642127991, + "mean_per_layer_hit_rate": 0.005569960456341505 + }, + "2": { + "rho_bar": 0.1698450893163681, + "union_observed": 0.06491885334253311, + "union_independent": 0.1392468810081482, + "union_gap": -0.07432802766561508, + "mean_per_layer_hit_rate": 0.011428022757172585 + }, + "5": { + "rho_bar": 0.231515571475029, + "union_observed": 0.11485642939805984, + "union_independent": 0.2731589078903198, + "union_gap": -0.15830247849225998, + "mean_per_layer_hit_rate": 0.024104485288262367 + }, + "10": { + "rho_bar": 0.2966378927230835, + "union_observed": 0.15980024635791779, + "union_independent": 0.4135500192642212, + "union_gap": -0.2537497729063034, + "mean_per_layer_hit_rate": 0.039950065314769745 + }, + "20": { + "rho_bar": 0.34138038754463196, + "union_observed": 0.20724095404148102, + "union_independent": 0.5723626613616943, + "union_gap": -0.3651217073202133, + "mean_per_layer_hit_rate": 0.06261403858661652 + }, + "50": { + "rho_bar": 0.39734718203544617, + "union_observed": 0.2971285879611969, + "union_independent": 0.7350764274597168, + "union_gap": -0.4379478394985199, + "mean_per_layer_hit_rate": 0.09603380411863327 + }, + "100": { + "rho_bar": 0.41739165782928467, + "union_observed": 0.39950063824653625, + "union_independent": 0.8608711361885071, + "union_gap": -0.4613704979419708, + "mean_per_layer_hit_rate": 0.1386728137731552 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.28116018644401003, + "union_gap_mean_over_k": -0.2546244114637375 + }, + "OOD_arXiv_2023|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.13350233435630798, + "union_observed": 0.03745318204164505, + "union_independent": 0.07367730140686035, + "union_gap": -0.0362241193652153, + "mean_per_layer_hit_rate": 0.005858062766492367 + }, + "2": { + "rho_bar": 0.17036966979503632, + "union_observed": 0.06616728752851486, + "union_independent": 0.14899438619613647, + "union_gap": -0.08282709866762161, + "mean_per_layer_hit_rate": 0.012292326427996159 + }, + "5": { + "rho_bar": 0.2393454760313034, + "union_observed": 0.10237203538417816, + "union_independent": 0.2721596360206604, + "union_gap": -0.16978760063648224, + "mean_per_layer_hit_rate": 0.024008452892303467 + }, + "10": { + "rho_bar": 0.30389878153800964, + "union_observed": 0.1473158597946167, + "union_independent": 0.4134947657585144, + "union_gap": -0.2661789059638977, + "mean_per_layer_hit_rate": 0.0398540273308754 + }, + "20": { + "rho_bar": 0.373063325881958, + "union_observed": 0.21348313987255096, + "union_independent": 0.5953490734100342, + "union_gap": -0.3818659335374832, + "mean_per_layer_hit_rate": 0.06635936349630356 + }, + "50": { + "rho_bar": 0.41668251156806946, + "union_observed": 0.3096129894256592, + "union_independent": 0.7747787833213806, + "union_gap": -0.46516579389572144, + "mean_per_layer_hit_rate": 0.10659752786159515 + }, + "100": { + "rho_bar": 0.412411630153656, + "union_observed": 0.43570536375045776, + "union_independent": 0.8824788928031921, + "union_gap": -0.4467735290527344, + "mean_per_layer_hit_rate": 0.1486603319644928 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29275338990347727, + "union_gap_mean_over_k": -0.264117568731308 + }, + "OOD_arXiv_2023|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.15063630044460297, + "union_observed": 0.048689138144254684, + "union_independent": 0.1002340316772461, + "union_gap": -0.05154489353299141, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.1712307333946228, + "union_observed": 0.0699126124382019, + "union_independent": 0.15556353330612183, + "union_gap": -0.08565092086791992, + "mean_per_layer_hit_rate": 0.012868530116975307 + }, + "5": { + "rho_bar": 0.21848666667938232, + "union_observed": 0.1086142286658287, + "union_independent": 0.2712234854698181, + "union_gap": -0.1626092568039894, + "mean_per_layer_hit_rate": 0.023912418633699417 + }, + "10": { + "rho_bar": 0.3007064461708069, + "union_observed": 0.16853933036327362, + "union_independent": 0.4460970163345337, + "union_gap": -0.27755768597126007, + "mean_per_layer_hit_rate": 0.04398348182439804 + }, + "20": { + "rho_bar": 0.34991174936294556, + "union_observed": 0.23845192790031433, + "union_independent": 0.6042071580886841, + "union_gap": -0.36575523018836975, + "mean_per_layer_hit_rate": 0.06779986619949341 + }, + "50": { + "rho_bar": 0.4001484513282776, + "union_observed": 0.3283395767211914, + "union_independent": 0.7902132868766785, + "union_gap": -0.46187371015548706, + "mean_per_layer_hit_rate": 0.11101508140563965 + }, + "100": { + "rho_bar": 0.4070236384868622, + "union_observed": 0.4182272255420685, + "union_independent": 0.8857036232948303, + "union_gap": -0.46747639775276184, + "mean_per_layer_hit_rate": 0.14990876615047455 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2854491408382143, + "union_gap_mean_over_k": -0.26749544218182564 + }, + "OOD_arXiv_2023|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.19581574201583862, + "union_observed": 0.04619225859642029, + "union_independent": 0.0921858549118042, + "union_gap": -0.04599359631538391, + "mean_per_layer_hit_rate": 0.00739460252225399 + }, + "2": { + "rho_bar": 0.2585342526435852, + "union_observed": 0.06616728752851486, + "union_independent": 0.1608506441116333, + "union_gap": -0.09468335658311844, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.25211650133132935, + "union_observed": 0.10986267030239105, + "union_independent": 0.2974897027015686, + "union_gap": -0.18762703239917755, + "mean_per_layer_hit_rate": 0.026601364836096764 + }, + "10": { + "rho_bar": 0.3318297863006592, + "union_observed": 0.17103619873523712, + "union_independent": 0.45843589305877686, + "union_gap": -0.28739969432353973, + "mean_per_layer_hit_rate": 0.04561605677008629 + }, + "20": { + "rho_bar": 0.344114750623703, + "union_observed": 0.23845192790031433, + "union_independent": 0.6226578950881958, + "union_gap": -0.38420596718788147, + "mean_per_layer_hit_rate": 0.07116104662418365 + }, + "50": { + "rho_bar": 0.38017019629478455, + "union_observed": 0.35830211639404297, + "union_independent": 0.8037490844726562, + "union_gap": -0.4454469680786133, + "mean_per_layer_hit_rate": 0.11514453589916229 + }, + "100": { + "rho_bar": 0.40668055415153503, + "union_observed": 0.44069913029670715, + "union_independent": 0.9045753479003906, + "union_gap": -0.46387621760368347, + "mean_per_layer_hit_rate": 0.16056853532791138 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.309894540480205, + "union_gap_mean_over_k": -0.2727475474987711 + }, + "OOD_arXiv_2023|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.23916344344615936, + "union_observed": 0.047440700232982635, + "union_independent": 0.09453034400939941, + "union_gap": -0.04708964377641678, + "mean_per_layer_hit_rate": 0.0075866710394620895 + }, + "2": { + "rho_bar": 0.2765285074710846, + "union_observed": 0.07865168899297714, + "union_independent": 0.17468678951263428, + "union_gap": -0.09603510051965714, + "mean_per_layer_hit_rate": 0.01459713838994503 + }, + "5": { + "rho_bar": 0.2798495292663574, + "union_observed": 0.13607990741729736, + "union_independent": 0.3406127691268921, + "union_gap": -0.20453286170959473, + "mean_per_layer_hit_rate": 0.03130701929330826 + }, + "10": { + "rho_bar": 0.32385891675949097, + "union_observed": 0.1960049867630005, + "union_independent": 0.4999855160713196, + "union_gap": -0.3039805293083191, + "mean_per_layer_hit_rate": 0.05137808620929718 + }, + "20": { + "rho_bar": 0.34027597308158875, + "union_observed": 0.26217228174209595, + "union_independent": 0.6610239744186401, + "union_gap": -0.3988516926765442, + "mean_per_layer_hit_rate": 0.07865168154239655 + }, + "50": { + "rho_bar": 0.38246604800224304, + "union_observed": 0.3770287036895752, + "union_independent": 0.836139976978302, + "union_gap": -0.4591112732887268, + "mean_per_layer_hit_rate": 0.12705272436141968 + }, + "100": { + "rho_bar": 0.39748290181159973, + "union_observed": 0.4681648015975952, + "union_independent": 0.9143375754356384, + "union_gap": -0.4461727738380432, + "mean_per_layer_hit_rate": 0.16757899522781372 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.3199464742626463, + "union_gap_mean_over_k": -0.27939626787390026 + }, + "OOD_arXiv_2023|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.19519741833209991, + "union_observed": 0.04993757978081703, + "union_independent": 0.10702371597290039, + "union_gap": -0.05708613619208336, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.2647334337234497, + "union_observed": 0.07865168899297714, + "union_independent": 0.20363777875900269, + "union_gap": -0.12498608976602554, + "mean_per_layer_hit_rate": 0.017286084592342377 + }, + "5": { + "rho_bar": 0.3075826168060303, + "union_observed": 0.13358302414417267, + "union_independent": 0.36904680728912354, + "union_gap": -0.23546378314495087, + "mean_per_layer_hit_rate": 0.03457216918468475 + }, + "10": { + "rho_bar": 0.33231866359710693, + "union_observed": 0.20724095404148102, + "union_independent": 0.5417599678039551, + "union_gap": -0.33451901376247406, + "mean_per_layer_hit_rate": 0.05771631747484207 + }, + "20": { + "rho_bar": 0.36722201108932495, + "union_observed": 0.2709113657474518, + "union_independent": 0.7101038694381714, + "union_gap": -0.4391925036907196, + "mean_per_layer_hit_rate": 0.08969557285308838 + }, + "50": { + "rho_bar": 0.4001564383506775, + "union_observed": 0.37328338623046875, + "union_independent": 0.8562796115875244, + "union_gap": -0.48299622535705566, + "mean_per_layer_hit_rate": 0.1363680064678192 + }, + "100": { + "rho_bar": 0.42068079113960266, + "union_observed": 0.46441948413848877, + "union_independent": 0.9242216348648071, + "union_gap": -0.45980215072631836, + "mean_per_layer_hit_rate": 0.17593392729759216 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.326841624719756, + "union_gap_mean_over_k": -0.30486370037708965 + }, + "OOD_arXiv_2023|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.19361451268196106, + "union_observed": 0.056179776787757874, + "union_independent": 0.12709039449691772, + "union_gap": -0.07091061770915985, + "mean_per_layer_hit_rate": 0.010371650569140911 + }, + "2": { + "rho_bar": 0.27920547127723694, + "union_observed": 0.07865168899297714, + "union_independent": 0.21562981605529785, + "union_gap": -0.1369781270623207, + "mean_per_layer_hit_rate": 0.018438490107655525 + }, + "5": { + "rho_bar": 0.31504952907562256, + "union_observed": 0.1423220932483673, + "union_independent": 0.3860861659049988, + "union_gap": -0.24376407265663147, + "mean_per_layer_hit_rate": 0.0365888811647892 + }, + "10": { + "rho_bar": 0.35584792494773865, + "union_observed": 0.21098627150058746, + "union_independent": 0.5627336502075195, + "union_gap": -0.35174737870693207, + "mean_per_layer_hit_rate": 0.06107750162482262 + }, + "20": { + "rho_bar": 0.36444351077079773, + "union_observed": 0.30586767196655273, + "union_independent": 0.7303845286369324, + "union_gap": -0.42451685667037964, + "mean_per_layer_hit_rate": 0.09468933194875717 + }, + "50": { + "rho_bar": 0.42483001947402954, + "union_observed": 0.36329588294029236, + "union_independent": 0.8635504841804504, + "union_gap": -0.5002546012401581, + "mean_per_layer_hit_rate": 0.13982520997524261 + }, + "100": { + "rho_bar": 0.4621695578098297, + "union_observed": 0.4469413161277771, + "union_independent": 0.9281570315361023, + "union_gap": -0.4812157154083252, + "mean_per_layer_hit_rate": 0.17996734380722046 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.34216578943388803, + "union_gap_mean_over_k": -0.3156267670648439 + }, + "OOD_arXiv_2023|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.16865862905979156, + "union_observed": 0.04494382068514824, + "union_independent": 0.09788471460342407, + "union_gap": -0.05294089391827583, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.2417343109846115, + "union_observed": 0.0749063640832901, + "union_independent": 0.17959898710250854, + "union_gap": -0.10469262301921844, + "mean_per_layer_hit_rate": 0.01507730782032013 + }, + "5": { + "rho_bar": 0.29352834820747375, + "union_observed": 0.1423220932483673, + "union_independent": 0.3341720700263977, + "union_gap": -0.1918499767780304, + "mean_per_layer_hit_rate": 0.03063478320837021 + }, + "10": { + "rho_bar": 0.3416799008846283, + "union_observed": 0.19975031912326813, + "union_independent": 0.49994999170303345, + "union_gap": -0.3001996725797653, + "mean_per_layer_hit_rate": 0.05147411674261093 + }, + "20": { + "rho_bar": 0.3834021985530853, + "union_observed": 0.26217228174209595, + "union_independent": 0.6531320214271545, + "union_gap": -0.3909597396850586, + "mean_per_layer_hit_rate": 0.07740325480699539 + }, + "50": { + "rho_bar": 0.43721702694892883, + "union_observed": 0.3595505654811859, + "union_independent": 0.8185684680938721, + "union_gap": -0.45901790261268616, + "mean_per_layer_hit_rate": 0.12129069864749908 + }, + "100": { + "rho_bar": 0.44373926520347595, + "union_observed": 0.4619225859642029, + "union_independent": 0.901580274105072, + "union_gap": -0.43965768814086914, + "mean_per_layer_hit_rate": 0.16018438339233398 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3299942399774279, + "union_gap_mean_over_k": -0.27704549953341484 + }, + "OOD_arXiv_2023|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.18463687598705292, + "union_observed": 0.05243445560336113, + "union_independent": 0.10582500696182251, + "union_gap": -0.05339055135846138, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.24690879881381989, + "union_observed": 0.08489388227462769, + "union_independent": 0.18703687191009521, + "union_gap": -0.10214298963546753, + "mean_per_layer_hit_rate": 0.01574954390525818 + }, + "5": { + "rho_bar": 0.26372143626213074, + "union_observed": 0.1398252248764038, + "union_independent": 0.3328061103820801, + "union_gap": -0.19298088550567627, + "mean_per_layer_hit_rate": 0.03044271469116211 + }, + "10": { + "rho_bar": 0.3278941512107849, + "union_observed": 0.20349563658237457, + "union_independent": 0.5056674480438232, + "union_gap": -0.30217181146144867, + "mean_per_layer_hit_rate": 0.05224239081144333 + }, + "20": { + "rho_bar": 0.3746553659439087, + "union_observed": 0.27715355157852173, + "union_independent": 0.6572104096412659, + "union_gap": -0.38005685806274414, + "mean_per_layer_hit_rate": 0.07807548344135284 + }, + "50": { + "rho_bar": 0.41074028611183167, + "union_observed": 0.3645443320274353, + "union_independent": 0.8134465217590332, + "union_gap": -0.4489021897315979, + "mean_per_layer_hit_rate": 0.11898589134216309 + }, + "100": { + "rho_bar": 0.44163385033607483, + "union_observed": 0.4606741666793823, + "union_independent": 0.9049670696258545, + "union_gap": -0.44429290294647217, + "mean_per_layer_hit_rate": 0.16200901567935944 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.32145582352365765, + "union_gap_mean_over_k": -0.27484831267169546 + }, + "OOD_arXiv_2023|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.1733258217573166, + "union_observed": 0.05243445560336113, + "union_independent": 0.10599082708358765, + "union_gap": -0.05355637148022652, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.22911027073860168, + "union_observed": 0.0774032473564148, + "union_independent": 0.17887067794799805, + "union_gap": -0.10146743059158325, + "mean_per_layer_hit_rate": 0.014981274493038654 + }, + "5": { + "rho_bar": 0.2572697401046753, + "union_observed": 0.15730337798595428, + "union_independent": 0.33279949426651, + "union_gap": -0.17549611628055573, + "mean_per_layer_hit_rate": 0.03044271469116211 + }, + "10": { + "rho_bar": 0.29691603779792786, + "union_observed": 0.21847690641880035, + "union_independent": 0.4976559281349182, + "union_gap": -0.27917902171611786, + "mean_per_layer_hit_rate": 0.05108998715877533 + }, + "20": { + "rho_bar": 0.33116036653518677, + "union_observed": 0.3096129894256592, + "union_independent": 0.6742691993713379, + "union_gap": -0.3646562099456787, + "mean_per_layer_hit_rate": 0.08153270184993744 + }, + "50": { + "rho_bar": 0.39801323413848877, + "union_observed": 0.40449437499046326, + "union_independent": 0.8306719660758972, + "union_gap": -0.42617759108543396, + "mean_per_layer_hit_rate": 0.12561222910881042 + }, + "100": { + "rho_bar": 0.40606191754341125, + "union_observed": 0.483146071434021, + "union_independent": 0.9033401012420654, + "union_gap": -0.42019402980804443, + "mean_per_layer_hit_rate": 0.16104871034622192 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29883676980222973, + "union_gap_mean_over_k": -0.2601038244153772 + }, + "OOD_arXiv_2023|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.15067079663276672, + "union_observed": 0.04369537904858589, + "union_independent": 0.08308368921279907, + "union_gap": -0.03938831016421318, + "mean_per_layer_hit_rate": 0.0066263326443731785 + }, + "2": { + "rho_bar": 0.20021618902683258, + "union_observed": 0.07365792989730835, + "union_independent": 0.16721266508102417, + "union_gap": -0.09355473518371582, + "mean_per_layer_hit_rate": 0.013924901373684406 + }, + "5": { + "rho_bar": 0.22850951552391052, + "union_observed": 0.13233457505702972, + "union_independent": 0.29913902282714844, + "union_gap": -0.1668044477701187, + "mean_per_layer_hit_rate": 0.026793431490659714 + }, + "10": { + "rho_bar": 0.2738385796546936, + "union_observed": 0.20973782241344452, + "union_independent": 0.47541356086730957, + "union_gap": -0.26567573845386505, + "mean_per_layer_hit_rate": 0.047920867800712585 + }, + "20": { + "rho_bar": 0.2999449372291565, + "union_observed": 0.2746566832065582, + "union_independent": 0.625732421875, + "union_gap": -0.3510757386684418, + "mean_per_layer_hit_rate": 0.07154519110918045 + }, + "50": { + "rho_bar": 0.35518884658813477, + "union_observed": 0.3770287036895752, + "union_independent": 0.7917694449424744, + "union_gap": -0.41474074125289917, + "mean_per_layer_hit_rate": 0.11159128695726395 + }, + "100": { + "rho_bar": 0.39439657330513, + "union_observed": 0.4656679034233093, + "union_independent": 0.8882299661636353, + "union_gap": -0.4225620627403259, + "mean_per_layer_hit_rate": 0.1517334133386612 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.271823633994375, + "union_gap_mean_over_k": -0.2505431106047971 + }, + "OOD_arXiv_2023|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.14080539345741272, + "union_observed": 0.05243445560336113, + "union_independent": 0.10027819871902466, + "union_gap": -0.04784374311566353, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.23768039047718048, + "union_observed": 0.0749063640832901, + "union_independent": 0.17349082231521606, + "union_gap": -0.09858445823192596, + "mean_per_layer_hit_rate": 0.01450110413134098 + }, + "5": { + "rho_bar": 0.29149186611175537, + "union_observed": 0.1373283416032791, + "union_independent": 0.33269309997558594, + "union_gap": -0.19536475837230682, + "mean_per_layer_hit_rate": 0.03044271655380726 + }, + "10": { + "rho_bar": 0.31329527497291565, + "union_observed": 0.19975031912326813, + "union_independent": 0.4860547184944153, + "union_gap": -0.28630439937114716, + "mean_per_layer_hit_rate": 0.04945740848779678 + }, + "20": { + "rho_bar": 0.36529621481895447, + "union_observed": 0.2684144675731659, + "union_independent": 0.6495789885520935, + "union_gap": -0.3811645209789276, + "mean_per_layer_hit_rate": 0.0764429122209549 + }, + "50": { + "rho_bar": 0.4011337459087372, + "union_observed": 0.40449437499046326, + "union_independent": 0.8223198056221008, + "union_gap": -0.4178254306316376, + "mean_per_layer_hit_rate": 0.12177086621522903 + }, + "100": { + "rho_bar": 0.443296879529953, + "union_observed": 0.48189762234687805, + "union_independent": 0.9074821472167969, + "union_gap": -0.4255845248699188, + "mean_per_layer_hit_rate": 0.16306540369987488 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3132856807538441, + "union_gap_mean_over_k": -0.26466740508164677 + }, + "OOD_arXiv_2023|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2759765386581421, + "union_observed": 0.04494382068514824, + "union_independent": 0.12486934661865234, + "union_gap": -0.0799255259335041, + "mean_per_layer_hit_rate": 0.010179583914577961 + }, + "2": { + "rho_bar": 0.3208664059638977, + "union_observed": 0.072409488260746, + "union_independent": 0.21859532594680786, + "union_gap": -0.14618583768606186, + "mean_per_layer_hit_rate": 0.018726591020822525 + }, + "5": { + "rho_bar": 0.3709951639175415, + "union_observed": 0.12359550595283508, + "union_independent": 0.38920557498931885, + "union_gap": -0.26561006903648376, + "mean_per_layer_hit_rate": 0.0369730181992054 + }, + "10": { + "rho_bar": 0.45108047127723694, + "union_observed": 0.1860174834728241, + "union_independent": 0.601198136806488, + "union_gap": -0.41518065333366394, + "mean_per_layer_hit_rate": 0.06751176714897156 + }, + "20": { + "rho_bar": 0.5120678544044495, + "union_observed": 0.24344569444656372, + "union_independent": 0.7522080540657043, + "union_gap": -0.5087623596191406, + "mean_per_layer_hit_rate": 0.10006722062826157 + }, + "50": { + "rho_bar": 0.5294856429100037, + "union_observed": 0.33707866072654724, + "union_independent": 0.8734526038169861, + "union_gap": -0.5363739430904388, + "mean_per_layer_hit_rate": 0.14376260340213776 + }, + "100": { + "rho_bar": 0.5393359065055847, + "union_observed": 0.4344569146633148, + "union_independent": 0.939167857170105, + "union_gap": -0.5047109425067902, + "mean_per_layer_hit_rate": 0.1885143667459488 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.428543997662408, + "union_gap_mean_over_k": -0.35096419017229763 + }, + "OOD_arXiv_2023|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.2989172339439392, + "union_observed": 0.05742821469902992, + "union_independent": 0.15420681238174438, + "union_gap": -0.09677859768271446, + "mean_per_layer_hit_rate": 0.012772495858371258 + }, + "2": { + "rho_bar": 0.34521201252937317, + "union_observed": 0.08489388227462769, + "union_independent": 0.2537508010864258, + "union_gap": -0.1688569188117981, + "mean_per_layer_hit_rate": 0.02218380756676197 + }, + "5": { + "rho_bar": 0.3672531247138977, + "union_observed": 0.1398252248764038, + "union_independent": 0.4198836088180542, + "union_gap": -0.2800583839416504, + "mean_per_layer_hit_rate": 0.040718335658311844 + }, + "10": { + "rho_bar": 0.4126165211200714, + "union_observed": 0.2122347056865692, + "union_independent": 0.5979989767074585, + "union_gap": -0.3857642710208893, + "mean_per_layer_hit_rate": 0.06703159213066101 + }, + "20": { + "rho_bar": 0.45036664605140686, + "union_observed": 0.2846441864967346, + "union_independent": 0.7528349757194519, + "union_gap": -0.4681907892227173, + "mean_per_layer_hit_rate": 0.10064343363046646 + }, + "50": { + "rho_bar": 0.49813657999038696, + "union_observed": 0.37078651785850525, + "union_independent": 0.8791159987449646, + "union_gap": -0.5083294808864594, + "mean_per_layer_hit_rate": 0.14798809587955475 + }, + "100": { + "rho_bar": 0.5156357884407043, + "union_observed": 0.45817726850509644, + "union_independent": 0.9426326751708984, + "union_gap": -0.484455406665802, + "mean_per_layer_hit_rate": 0.1940843164920807 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4125911295413971, + "union_gap_mean_over_k": -0.34177626403314726 + }, + "OOD_arXiv_2023|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.23849797248840332, + "union_observed": 0.05368289723992348, + "union_independent": 0.1315469741821289, + "union_gap": -0.07786407694220543, + "mean_per_layer_hit_rate": 0.010755786672234535 + }, + "2": { + "rho_bar": 0.3206897974014282, + "union_observed": 0.07990012317895889, + "union_independent": 0.21676301956176758, + "union_gap": -0.13686289638280869, + "mean_per_layer_hit_rate": 0.018534524366259575 + }, + "5": { + "rho_bar": 0.3103169798851013, + "union_observed": 0.13607990741729736, + "union_independent": 0.376861572265625, + "union_gap": -0.24078166484832764, + "mean_per_layer_hit_rate": 0.0354364737868309 + }, + "10": { + "rho_bar": 0.33506569266319275, + "union_observed": 0.19725343585014343, + "union_independent": 0.5361638069152832, + "union_gap": -0.33891037106513977, + "mean_per_layer_hit_rate": 0.056659944355487823 + }, + "20": { + "rho_bar": 0.4097154438495636, + "union_observed": 0.26966291666030884, + "union_independent": 0.7060476541519165, + "union_gap": -0.43638473749160767, + "mean_per_layer_hit_rate": 0.08863919228315353 + }, + "50": { + "rho_bar": 0.4744545817375183, + "union_observed": 0.36079901456832886, + "union_independent": 0.8494012355804443, + "union_gap": -0.4886022210121155, + "mean_per_layer_hit_rate": 0.13339096307754517 + }, + "100": { + "rho_bar": 0.5047382712364197, + "union_observed": 0.449438214302063, + "union_independent": 0.9184288382530212, + "union_gap": -0.46899062395095825, + "mean_per_layer_hit_rate": 0.17190052568912506 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.37049696275166105, + "union_gap_mean_over_k": -0.3126280845275947 + }, + "OOD_arXiv_2023|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.19799214601516724, + "union_observed": 0.05243445560336113, + "union_independent": 0.12270480394363403, + "union_gap": -0.0702703483402729, + "mean_per_layer_hit_rate": 0.009987516328692436 + }, + "2": { + "rho_bar": 0.261928528547287, + "union_observed": 0.0774032473564148, + "union_independent": 0.19029617309570312, + "union_gap": -0.11289292573928833, + "mean_per_layer_hit_rate": 0.01603764481842518 + }, + "5": { + "rho_bar": 0.28439992666244507, + "union_observed": 0.13483145833015442, + "union_independent": 0.33495962619781494, + "union_gap": -0.20012816786766052, + "mean_per_layer_hit_rate": 0.03063478320837021 + }, + "10": { + "rho_bar": 0.27763715386390686, + "union_observed": 0.20474407076835632, + "union_independent": 0.4972763657569885, + "union_gap": -0.2925322949886322, + "mean_per_layer_hit_rate": 0.05089791864156723 + }, + "20": { + "rho_bar": 0.3348904252052307, + "union_observed": 0.26217228174209595, + "union_independent": 0.652600884437561, + "union_gap": -0.3904286026954651, + "mean_per_layer_hit_rate": 0.07692307978868484 + }, + "50": { + "rho_bar": 0.4139351546764374, + "union_observed": 0.3570536971092224, + "union_independent": 0.8159977197647095, + "union_gap": -0.45894402265548706, + "mean_per_layer_hit_rate": 0.11985018849372864 + }, + "100": { + "rho_bar": 0.4455963969230652, + "union_observed": 0.449438214302063, + "union_independent": 0.9010480046272278, + "union_gap": -0.4516097903251648, + "mean_per_layer_hit_rate": 0.15932008624076843 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31662567598479135, + "union_gap_mean_over_k": -0.2824008789445673 + }, + "OOD_arXiv_2023|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.19263149797916412, + "union_observed": 0.04369537904858589, + "union_independent": 0.10366451740264893, + "union_gap": -0.059969138354063034, + "mean_per_layer_hit_rate": 0.008354940451681614 + }, + "2": { + "rho_bar": 0.2374313324689865, + "union_observed": 0.06741572916507721, + "union_independent": 0.16202688217163086, + "union_gap": -0.09461115300655365, + "mean_per_layer_hit_rate": 0.013444732874631882 + }, + "5": { + "rho_bar": 0.29644182324409485, + "union_observed": 0.13108614087104797, + "union_independent": 0.312324583530426, + "union_gap": -0.18123844265937805, + "mean_per_layer_hit_rate": 0.028137903660535812 + }, + "10": { + "rho_bar": 0.3003702759742737, + "union_observed": 0.19975031912326813, + "union_independent": 0.4668055772781372, + "union_gap": -0.2670552581548691, + "mean_per_layer_hit_rate": 0.046672429889440536 + }, + "20": { + "rho_bar": 0.3315211236476898, + "union_observed": 0.255930095911026, + "union_independent": 0.629685640335083, + "union_gap": -0.373755544424057, + "mean_per_layer_hit_rate": 0.072409488260746 + }, + "50": { + "rho_bar": 0.40613335371017456, + "union_observed": 0.34706616401672363, + "union_independent": 0.7904515266418457, + "union_gap": -0.44338536262512207, + "mean_per_layer_hit_rate": 0.1111111119389534 + }, + "100": { + "rho_bar": 0.44407570362091064, + "union_observed": 0.4294631779193878, + "union_independent": 0.880052387714386, + "union_gap": -0.45058920979499817, + "mean_per_layer_hit_rate": 0.1475079208612442 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3155150158064706, + "union_gap_mean_over_k": -0.2672291584312916 + }, + "OOD_arXiv_2023|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.12222056835889816, + "union_observed": 0.048689138144254684, + "union_independent": 0.09222674369812012, + "union_gap": -0.04353760555386543, + "mean_per_layer_hit_rate": 0.00739460252225399 + }, + "2": { + "rho_bar": 0.19751498103141785, + "union_observed": 0.0774032473564148, + "union_independent": 0.17156553268432617, + "union_gap": -0.09416228532791138, + "mean_per_layer_hit_rate": 0.014309035614132881 + }, + "5": { + "rho_bar": 0.23942582309246063, + "union_observed": 0.14107365906238556, + "union_independent": 0.3017807602882385, + "union_gap": -0.16070710122585297, + "mean_per_layer_hit_rate": 0.027081530541181564 + }, + "10": { + "rho_bar": 0.30550798773765564, + "union_observed": 0.1910112351179123, + "union_independent": 0.45787858963012695, + "union_gap": -0.26686735451221466, + "mean_per_layer_hit_rate": 0.04561605677008629 + }, + "20": { + "rho_bar": 0.32780125737190247, + "union_observed": 0.25967541337013245, + "union_independent": 0.60102379322052, + "union_gap": -0.3413483798503876, + "mean_per_layer_hit_rate": 0.06741572916507721 + }, + "50": { + "rho_bar": 0.3922233581542969, + "union_observed": 0.34706616401672363, + "union_independent": 0.7683183550834656, + "union_gap": -0.42125219106674194, + "mean_per_layer_hit_rate": 0.10486890375614166 + }, + "100": { + "rho_bar": 0.41947442293167114, + "union_observed": 0.4369538128376007, + "union_independent": 0.863146960735321, + "union_gap": -0.42619314789772034, + "mean_per_layer_hit_rate": 0.13944107294082642 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.28630977123975754, + "union_gap_mean_over_k": -0.2505811522049563 + }, + "OOD_arXiv_2023|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.1426522582769394, + "union_observed": 0.04244694113731384, + "union_independent": 0.07827669382095337, + "union_gap": -0.035829752683639526, + "mean_per_layer_hit_rate": 0.006242197472602129 + }, + "2": { + "rho_bar": 0.1995636373758316, + "union_observed": 0.072409488260746, + "union_independent": 0.16384917497634888, + "union_gap": -0.09143968671560287, + "mean_per_layer_hit_rate": 0.013636799529194832 + }, + "5": { + "rho_bar": 0.262408584356308, + "union_observed": 0.11985018849372864, + "union_independent": 0.29387396574020386, + "union_gap": -0.17402377724647522, + "mean_per_layer_hit_rate": 0.026313263922929764 + }, + "10": { + "rho_bar": 0.31530481576919556, + "union_observed": 0.15355804562568665, + "union_independent": 0.4300665259361267, + "union_gap": -0.27650848031044006, + "mean_per_layer_hit_rate": 0.04206281155347824 + }, + "20": { + "rho_bar": 0.35844650864601135, + "union_observed": 0.21598002314567566, + "union_independent": 0.5908800959587097, + "union_gap": -0.37490007281303406, + "mean_per_layer_hit_rate": 0.06587918847799301 + }, + "50": { + "rho_bar": 0.39401140809059143, + "union_observed": 0.29088640213012695, + "union_independent": 0.7442266941070557, + "union_gap": -0.4533402919769287, + "mean_per_layer_hit_rate": 0.09872274845838547 + }, + "100": { + "rho_bar": 0.4023149609565735, + "union_observed": 0.4032459557056427, + "union_independent": 0.8539496660232544, + "union_gap": -0.4507037103176117, + "mean_per_layer_hit_rate": 0.13588783144950867 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29638602478163584, + "union_gap_mean_over_k": -0.2652493960091046 + }, + "OOD_arXiv_2023|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.13024964928627014, + "union_observed": 0.036204744130373, + "union_independent": 0.06555747985839844, + "union_gap": -0.029352735728025436, + "mean_per_layer_hit_rate": 0.0051858252845704556 + }, + "2": { + "rho_bar": 0.17127569019794464, + "union_observed": 0.06367041170597076, + "union_independent": 0.14033228158950806, + "union_gap": -0.07666186988353729, + "mean_per_layer_hit_rate": 0.011524057015776634 + }, + "5": { + "rho_bar": 0.23570530116558075, + "union_observed": 0.1136079877614975, + "union_independent": 0.2703363299369812, + "union_gap": -0.1567283421754837, + "mean_per_layer_hit_rate": 0.023816384375095367 + }, + "10": { + "rho_bar": 0.30080533027648926, + "union_observed": 0.1523096114397049, + "union_independent": 0.41447919607162476, + "union_gap": -0.26216958463191986, + "mean_per_layer_hit_rate": 0.040046095848083496 + }, + "20": { + "rho_bar": 0.3455411493778229, + "union_observed": 0.20724095404148102, + "union_independent": 0.5748839378356934, + "union_gap": -0.36764298379421234, + "mean_per_layer_hit_rate": 0.06299817562103271 + }, + "50": { + "rho_bar": 0.4020763635635376, + "union_observed": 0.29338327050209045, + "union_independent": 0.7384030818939209, + "union_gap": -0.44501981139183044, + "mean_per_layer_hit_rate": 0.09689810872077942 + }, + "100": { + "rho_bar": 0.42186570167541504, + "union_observed": 0.3895131051540375, + "union_independent": 0.8621118068695068, + "union_gap": -0.47259870171546936, + "mean_per_layer_hit_rate": 0.1392490118741989 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.28678845507758005, + "union_gap_mean_over_k": -0.2585962899029255 + }, + "OOD_arXiv_2023|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.142715722322464, + "union_observed": 0.03745318204164505, + "union_independent": 0.07249295711517334, + "union_gap": -0.03503977507352829, + "mean_per_layer_hit_rate": 0.00576202804222703 + }, + "2": { + "rho_bar": 0.1827279031276703, + "union_observed": 0.06491885334253311, + "union_independent": 0.15007686614990234, + "union_gap": -0.08515801280736923, + "mean_per_layer_hit_rate": 0.012388359755277634 + }, + "5": { + "rho_bar": 0.26082929968833923, + "union_observed": 0.10362047702074051, + "union_independent": 0.2824280261993408, + "union_gap": -0.1788075491786003, + "mean_per_layer_hit_rate": 0.025064824149012566 + }, + "10": { + "rho_bar": 0.3172808885574341, + "union_observed": 0.14606741070747375, + "union_independent": 0.4172589182853699, + "union_gap": -0.2711915075778961, + "mean_per_layer_hit_rate": 0.040334198623895645 + }, + "20": { + "rho_bar": 0.3838922679424286, + "union_observed": 0.21348313987255096, + "union_independent": 0.6003917455673218, + "union_gap": -0.3869086056947708, + "mean_per_layer_hit_rate": 0.06722366064786911 + }, + "50": { + "rho_bar": 0.41355466842651367, + "union_observed": 0.3121098577976227, + "union_independent": 0.7739756107330322, + "union_gap": -0.46186575293540955, + "mean_per_layer_hit_rate": 0.1063094213604927 + }, + "100": { + "rho_bar": 0.41896337270736694, + "union_observed": 0.42696627974510193, + "union_independent": 0.8805131316184998, + "union_gap": -0.4535468518733978, + "mean_per_layer_hit_rate": 0.14760397374629974 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.30285201753888813, + "union_gap_mean_over_k": -0.26750257930585314 + }, + "OOD_arXiv_2023|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.1473328173160553, + "union_observed": 0.048689138144254684, + "union_independent": 0.09914135932922363, + "union_gap": -0.05045222118496895, + "mean_per_layer_hit_rate": 0.007970805279910564 + }, + "2": { + "rho_bar": 0.18811091780662537, + "union_observed": 0.06741572916507721, + "union_independent": 0.15339308977127075, + "union_gap": -0.08597736060619354, + "mean_per_layer_hit_rate": 0.012676462531089783 + }, + "5": { + "rho_bar": 0.2258693426847458, + "union_observed": 0.10986267030239105, + "union_independent": 0.274128258228302, + "union_gap": -0.16426558792591095, + "mean_per_layer_hit_rate": 0.024200517684221268 + }, + "10": { + "rho_bar": 0.3135521113872528, + "union_observed": 0.16604244709014893, + "union_independent": 0.44102704524993896, + "union_gap": -0.27498459815979004, + "mean_per_layer_hit_rate": 0.04331124573945999 + }, + "20": { + "rho_bar": 0.3548358380794525, + "union_observed": 0.23595505952835083, + "union_independent": 0.609754204750061, + "union_gap": -0.3737991452217102, + "mean_per_layer_hit_rate": 0.06876020133495331 + }, + "50": { + "rho_bar": 0.40679091215133667, + "union_observed": 0.33208489418029785, + "union_independent": 0.7955732941627502, + "union_gap": -0.4634883999824524, + "mean_per_layer_hit_rate": 0.1126476600766182 + }, + "100": { + "rho_bar": 0.4083138406276703, + "union_observed": 0.41947564482688904, + "union_independent": 0.8872689008712769, + "union_gap": -0.4677932560443878, + "mean_per_layer_hit_rate": 0.1507730633020401 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2921151114361627, + "union_gap_mean_over_k": -0.2686800813036306 + }, + "OOD_arXiv_2023|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.20109635591506958, + "union_observed": 0.04619225859642029, + "union_independent": 0.09334385395050049, + "union_gap": -0.0471515953540802, + "mean_per_layer_hit_rate": 0.00749063678085804 + }, + "2": { + "rho_bar": 0.27323415875434875, + "union_observed": 0.06491885334253311, + "union_independent": 0.16299182176589966, + "union_gap": -0.09807296842336655, + "mean_per_layer_hit_rate": 0.013540768064558506 + }, + "5": { + "rho_bar": 0.25261491537094116, + "union_observed": 0.10986267030239105, + "union_independent": 0.2974703907966614, + "union_gap": -0.18760772049427032, + "mean_per_layer_hit_rate": 0.026601364836096764 + }, + "10": { + "rho_bar": 0.32344505190849304, + "union_observed": 0.1760299652814865, + "union_independent": 0.46490025520324707, + "union_gap": -0.28887028992176056, + "mean_per_layer_hit_rate": 0.04648036137223244 + }, + "20": { + "rho_bar": 0.3543788194656372, + "union_observed": 0.23470661044120789, + "union_independent": 0.6205463409423828, + "union_gap": -0.3858397305011749, + "mean_per_layer_hit_rate": 0.07077690958976746 + }, + "50": { + "rho_bar": 0.3883472681045532, + "union_observed": 0.3533083498477936, + "union_independent": 0.8058081865310669, + "union_gap": -0.4524998366832733, + "mean_per_layer_hit_rate": 0.11581676453351974 + }, + "100": { + "rho_bar": 0.410366952419281, + "union_observed": 0.44069913029670715, + "union_independent": 0.9029061794281006, + "union_gap": -0.46220704913139343, + "mean_per_layer_hit_rate": 0.15951214730739594 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.3147833602769034, + "union_gap_mean_over_k": -0.27460702721561703 + }, + "OOD_arXiv_2023|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.2506396472454071, + "union_observed": 0.04619225859642029, + "union_independent": 0.09684151411056519, + "union_gap": -0.0506492555141449, + "mean_per_layer_hit_rate": 0.007778738159686327 + }, + "2": { + "rho_bar": 0.2853373885154724, + "union_observed": 0.07990012317895889, + "union_independent": 0.18306010961532593, + "union_gap": -0.10315998643636703, + "mean_per_layer_hit_rate": 0.015365410596132278 + }, + "5": { + "rho_bar": 0.2887900471687317, + "union_observed": 0.13358302414417267, + "union_independent": 0.3405809998512268, + "union_gap": -0.20699797570705414, + "mean_per_layer_hit_rate": 0.03130701929330826 + }, + "10": { + "rho_bar": 0.3371187448501587, + "union_observed": 0.19225966930389404, + "union_independent": 0.5032086372375488, + "union_gap": -0.3109489679336548, + "mean_per_layer_hit_rate": 0.05185825005173683 + }, + "20": { + "rho_bar": 0.34686246514320374, + "union_observed": 0.26466915011405945, + "union_independent": 0.6639025211334229, + "union_gap": -0.3992333710193634, + "mean_per_layer_hit_rate": 0.07922788709402084 + }, + "50": { + "rho_bar": 0.38864827156066895, + "union_observed": 0.3720349669456482, + "union_independent": 0.8354371190071106, + "union_gap": -0.4634021520614624, + "mean_per_layer_hit_rate": 0.12676462531089783 + }, + "100": { + "rho_bar": 0.4025208652019501, + "union_observed": 0.46441948413848877, + "union_independent": 0.9162458777427673, + "union_gap": -0.45182639360427856, + "mean_per_layer_hit_rate": 0.16882742941379547 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.3285596328122275, + "union_gap_mean_over_k": -0.2837454431823322 + }, + "OOD_arXiv_2023|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.20415398478507996, + "union_observed": 0.04993757978081703, + "union_independent": 0.10590332746505737, + "union_gap": -0.05596574768424034, + "mean_per_layer_hit_rate": 0.008547008037567139 + }, + "2": { + "rho_bar": 0.2607869505882263, + "union_observed": 0.07990012317895889, + "union_independent": 0.2026275396347046, + "union_gap": -0.1227274164557457, + "mean_per_layer_hit_rate": 0.017190052196383476 + }, + "5": { + "rho_bar": 0.31369802355766296, + "union_observed": 0.13233457505702972, + "union_independent": 0.3691006302833557, + "union_gap": -0.236766055226326, + "mean_per_layer_hit_rate": 0.03457216918468475 + }, + "10": { + "rho_bar": 0.3358682096004486, + "union_observed": 0.20224718749523163, + "union_independent": 0.5363824367523193, + "union_gap": -0.3341352492570877, + "mean_per_layer_hit_rate": 0.056852009147405624 + }, + "20": { + "rho_bar": 0.36798399686813354, + "union_observed": 0.2746566832065582, + "union_independent": 0.7150721549987793, + "union_gap": -0.44041547179222107, + "mean_per_layer_hit_rate": 0.09084797650575638 + }, + "50": { + "rho_bar": 0.40715688467025757, + "union_observed": 0.3720349669456482, + "union_independent": 0.8551724553108215, + "union_gap": -0.48313748836517334, + "mean_per_layer_hit_rate": 0.13588783144950867 + }, + "100": { + "rho_bar": 0.4230040907859802, + "union_observed": 0.4631710350513458, + "union_independent": 0.9255151748657227, + "union_gap": -0.46234413981437683, + "mean_per_layer_hit_rate": 0.17689426243305206 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3303788772651127, + "union_gap_mean_over_k": -0.30507022408502443 + }, + "OOD_arXiv_2023|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.1951703131198883, + "union_observed": 0.05243445560336113, + "union_independent": 0.12154579162597656, + "union_gap": -0.06911133602261543, + "mean_per_layer_hit_rate": 0.009891481138765812 + }, + "2": { + "rho_bar": 0.28817635774612427, + "union_observed": 0.07365792989730835, + "union_independent": 0.21169918775558472, + "union_gap": -0.13804125785827637, + "mean_per_layer_hit_rate": 0.018054354935884476 + }, + "5": { + "rho_bar": 0.31669679284095764, + "union_observed": 0.1398252248764038, + "union_independent": 0.38444024324417114, + "union_gap": -0.24461501836776733, + "mean_per_layer_hit_rate": 0.0363968089222908 + }, + "10": { + "rho_bar": 0.3622422218322754, + "union_observed": 0.20848938822746277, + "union_independent": 0.5622509121894836, + "union_gap": -0.3537615239620209, + "mean_per_layer_hit_rate": 0.06098146736621857 + }, + "20": { + "rho_bar": 0.37799057364463806, + "union_observed": 0.30087390542030334, + "union_independent": 0.7353396415710449, + "union_gap": -0.4344657361507416, + "mean_per_layer_hit_rate": 0.09593776613473892 + }, + "50": { + "rho_bar": 0.4307365417480469, + "union_observed": 0.36329588294029236, + "union_independent": 0.8627686500549316, + "union_gap": -0.4994727671146393, + "mean_per_layer_hit_rate": 0.13944107294082642 + }, + "100": { + "rho_bar": 0.46631118655204773, + "union_observed": 0.4469413161277771, + "union_independent": 0.9276657700538635, + "union_gap": -0.4807244539260864, + "mean_per_layer_hit_rate": 0.17958322167396545 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.34818914106913973, + "union_gap_mean_over_k": -0.3171702990574496 + }, + "OOD_arXiv_2023|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.1628390997648239, + "union_observed": 0.04369537904858589, + "union_independent": 0.09445178508758545, + "union_gap": -0.05075640603899956, + "mean_per_layer_hit_rate": 0.007586670573800802 + }, + "2": { + "rho_bar": 0.23430706560611725, + "union_observed": 0.07615480571985245, + "union_independent": 0.1816837191581726, + "union_gap": -0.10552891343832016, + "mean_per_layer_hit_rate": 0.01526937447488308 + }, + "5": { + "rho_bar": 0.30215075612068176, + "union_observed": 0.144818976521492, + "union_independent": 0.3385156989097595, + "union_gap": -0.19369672238826752, + "mean_per_layer_hit_rate": 0.031114952638745308 + }, + "10": { + "rho_bar": 0.3472515046596527, + "union_observed": 0.19725343585014343, + "union_independent": 0.5000420808792114, + "union_gap": -0.302788645029068, + "mean_per_layer_hit_rate": 0.05147411674261093 + }, + "20": { + "rho_bar": 0.38998347520828247, + "union_observed": 0.260923832654953, + "union_independent": 0.6548391580581665, + "union_gap": -0.3939153254032135, + "mean_per_layer_hit_rate": 0.07769135385751724 + }, + "50": { + "rho_bar": 0.4424842298030853, + "union_observed": 0.35580524802207947, + "union_independent": 0.8170589208602905, + "union_gap": -0.46125367283821106, + "mean_per_layer_hit_rate": 0.12071449309587479 + }, + "100": { + "rho_bar": 0.4449164569377899, + "union_observed": 0.4631710350513458, + "union_independent": 0.9026719927787781, + "union_gap": -0.43950095772743225, + "mean_per_layer_hit_rate": 0.16085661947727203 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3319903697286333, + "union_gap_mean_over_k": -0.2782058061233589 + }, + "OOD_arXiv_2023|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.1875845193862915, + "union_observed": 0.05118601769208908, + "union_independent": 0.10471028089523315, + "union_gap": -0.053524263203144073, + "mean_per_layer_hit_rate": 0.008450975641608238 + }, + "2": { + "rho_bar": 0.27856919169425964, + "union_observed": 0.07990012317895889, + "union_independent": 0.18713289499282837, + "union_gap": -0.10723277181386948, + "mean_per_layer_hit_rate": 0.01574954390525818 + }, + "5": { + "rho_bar": 0.25765442848205566, + "union_observed": 0.13857677578926086, + "union_independent": 0.33105719089508057, + "union_gap": -0.1924804151058197, + "mean_per_layer_hit_rate": 0.03025064989924431 + }, + "10": { + "rho_bar": 0.33477783203125, + "union_observed": 0.20349563658237457, + "union_independent": 0.5075951218605042, + "union_gap": -0.3040994852781296, + "mean_per_layer_hit_rate": 0.05253048986196518 + }, + "20": { + "rho_bar": 0.3785794675350189, + "union_observed": 0.27590513229370117, + "union_independent": 0.6568136215209961, + "union_gap": -0.3809084892272949, + "mean_per_layer_hit_rate": 0.0779794454574585 + }, + "50": { + "rho_bar": 0.41799864172935486, + "union_observed": 0.3620474338531494, + "union_independent": 0.8139357566833496, + "union_gap": -0.4518883228302002, + "mean_per_layer_hit_rate": 0.11917794495820999 + }, + "100": { + "rho_bar": 0.44515618681907654, + "union_observed": 0.45817726850509644, + "union_independent": 0.9060439467430115, + "union_gap": -0.44786667823791504, + "mean_per_layer_hit_rate": 0.16268125176429749 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.32861718109675814, + "union_gap_mean_over_k": -0.27685720367091043 + }, + "OOD_arXiv_2023|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.1788448542356491, + "union_observed": 0.056179776787757874, + "union_independent": 0.1116178035736084, + "union_gap": -0.055438026785850525, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.22604748606681824, + "union_observed": 0.07990012317895889, + "union_independent": 0.17573761940002441, + "union_gap": -0.09583749622106552, + "mean_per_layer_hit_rate": 0.014693171717226505 + }, + "5": { + "rho_bar": 0.2705124318599701, + "union_observed": 0.1523096114397049, + "union_independent": 0.3328395485877991, + "union_gap": -0.18052993714809418, + "mean_per_layer_hit_rate": 0.03044271469116211 + }, + "10": { + "rho_bar": 0.31370192766189575, + "union_observed": 0.22097378969192505, + "union_independent": 0.49773693084716797, + "union_gap": -0.2767631411552429, + "mean_per_layer_hit_rate": 0.05108998343348503 + }, + "20": { + "rho_bar": 0.33287695050239563, + "union_observed": 0.30586767196655273, + "union_independent": 0.6802128553390503, + "union_gap": -0.37434518337249756, + "mean_per_layer_hit_rate": 0.08287716656923294 + }, + "50": { + "rho_bar": 0.39987829327583313, + "union_observed": 0.4057428240776062, + "union_independent": 0.829728901386261, + "union_gap": -0.4239860773086548, + "mean_per_layer_hit_rate": 0.12522807717323303 + }, + "100": { + "rho_bar": 0.4074057340621948, + "union_observed": 0.483146071434021, + "union_independent": 0.9030347466468811, + "union_gap": -0.4198886752128601, + "mean_per_layer_hit_rate": 0.16085661947727203 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.304181096809251, + "union_gap_mean_over_k": -0.2609697910291808 + }, + "OOD_arXiv_2023|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.15141095221042633, + "union_observed": 0.04369537904858589, + "union_independent": 0.08193033933639526, + "union_gap": -0.03823496028780937, + "mean_per_layer_hit_rate": 0.006530298851430416 + }, + "2": { + "rho_bar": 0.20393414795398712, + "union_observed": 0.0699126124382019, + "union_independent": 0.16188859939575195, + "union_gap": -0.09197598695755005, + "mean_per_layer_hit_rate": 0.013444732874631882 + }, + "5": { + "rho_bar": 0.2363196164369583, + "union_observed": 0.13607990741729736, + "union_independent": 0.3046320080757141, + "union_gap": -0.16855210065841675, + "mean_per_layer_hit_rate": 0.027369635179638863 + }, + "10": { + "rho_bar": 0.2740851044654846, + "union_observed": 0.20848938822746277, + "union_independent": 0.47267764806747437, + "union_gap": -0.2641882598400116, + "mean_per_layer_hit_rate": 0.04753673076629639 + }, + "20": { + "rho_bar": 0.3077247738838196, + "union_observed": 0.2784020006656647, + "union_independent": 0.632843017578125, + "union_gap": -0.3544410169124603, + "mean_per_layer_hit_rate": 0.07288965582847595 + }, + "50": { + "rho_bar": 0.3603242337703705, + "union_observed": 0.3695380687713623, + "union_independent": 0.7898989319801331, + "union_gap": -0.42036086320877075, + "mean_per_layer_hit_rate": 0.1109190434217453 + }, + "100": { + "rho_bar": 0.3990284502506256, + "union_observed": 0.4606741666793823, + "union_independent": 0.8878800272941589, + "union_gap": -0.4272058606147766, + "mean_per_layer_hit_rate": 0.151349276304245 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.27611818271023886, + "union_gap_mean_over_k": -0.2521370069256851 + }, + "OOD_arXiv_2023|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.15596117079257965, + "union_observed": 0.05243445560336113, + "union_independent": 0.10137981176376343, + "union_gap": -0.0489453561604023, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.25501108169555664, + "union_observed": 0.072409488260746, + "union_independent": 0.1703277826309204, + "union_gap": -0.09791829437017441, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.291689932346344, + "union_observed": 0.13233457505702972, + "union_independent": 0.32669544219970703, + "union_gap": -0.1943608671426773, + "mean_per_layer_hit_rate": 0.02977048046886921 + }, + "10": { + "rho_bar": 0.31567567586898804, + "union_observed": 0.1910112351179123, + "union_independent": 0.48070991039276123, + "union_gap": -0.28969867527484894, + "mean_per_layer_hit_rate": 0.048689134418964386 + }, + "20": { + "rho_bar": 0.3643527030944824, + "union_observed": 0.2721598148345947, + "union_independent": 0.6540096998214722, + "union_gap": -0.38184988498687744, + "mean_per_layer_hit_rate": 0.07730721682310104 + }, + "50": { + "rho_bar": 0.41092008352279663, + "union_observed": 0.40199750661849976, + "union_independent": 0.8246206045150757, + "union_gap": -0.4226230978965759, + "mean_per_layer_hit_rate": 0.12263517081737518 + }, + "100": { + "rho_bar": 0.4427233934402466, + "union_observed": 0.47690385580062866, + "union_independent": 0.9057826399803162, + "union_gap": -0.4288787841796875, + "mean_per_layer_hit_rate": 0.16181696951389313 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31947629153728485, + "union_gap_mean_over_k": -0.26632499428732054 + }, + "OOD_arXiv_2023|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.27245262265205383, + "union_observed": 0.04494382068514824, + "union_independent": 0.12152057886123657, + "union_gap": -0.07657675817608833, + "mean_per_layer_hit_rate": 0.009891481138765812 + }, + "2": { + "rho_bar": 0.332255095243454, + "union_observed": 0.07116104662418365, + "union_independent": 0.21863943338394165, + "union_gap": -0.147478386759758, + "mean_per_layer_hit_rate": 0.018726591020822525 + }, + "5": { + "rho_bar": 0.3790756165981293, + "union_observed": 0.12234706431627274, + "union_independent": 0.38917917013168335, + "union_gap": -0.2668321058154106, + "mean_per_layer_hit_rate": 0.0369730181992054 + }, + "10": { + "rho_bar": 0.45858505368232727, + "union_observed": 0.17977528274059296, + "union_independent": 0.6006407141685486, + "union_gap": -0.4208654314279556, + "mean_per_layer_hit_rate": 0.06741572916507721 + }, + "20": { + "rho_bar": 0.5129172801971436, + "union_observed": 0.24094881117343903, + "union_independent": 0.750799834728241, + "union_gap": -0.5098510235548019, + "mean_per_layer_hit_rate": 0.09968309104442596 + }, + "50": { + "rho_bar": 0.526788055896759, + "union_observed": 0.3408239781856537, + "union_independent": 0.8748115301132202, + "union_gap": -0.5339875519275665, + "mean_per_layer_hit_rate": 0.1444348394870758 + }, + "100": { + "rho_bar": 0.539971649646759, + "union_observed": 0.43320849537849426, + "union_independent": 0.9386972784996033, + "union_gap": -0.505488783121109, + "mean_per_layer_hit_rate": 0.1881302297115326 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.43172076770237516, + "union_gap_mean_over_k": -0.35158286296895574 + }, + "OOD_arXiv_2023|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.2895313799381256, + "union_observed": 0.054931335151195526, + "union_independent": 0.1542147994041443, + "union_gap": -0.09928346425294876, + "mean_per_layer_hit_rate": 0.012772495858371258 + }, + "2": { + "rho_bar": 0.35939979553222656, + "union_observed": 0.08239700645208359, + "union_independent": 0.2488841414451599, + "union_gap": -0.16648713499307632, + "mean_per_layer_hit_rate": 0.02170363999903202 + }, + "5": { + "rho_bar": 0.367302268743515, + "union_observed": 0.14357054233551025, + "union_independent": 0.4206888675689697, + "union_gap": -0.2771183252334595, + "mean_per_layer_hit_rate": 0.0408143624663353 + }, + "10": { + "rho_bar": 0.39968016743659973, + "union_observed": 0.2147315889596939, + "union_independent": 0.6002329587936401, + "union_gap": -0.38550136983394623, + "mean_per_layer_hit_rate": 0.06741572916507721 + }, + "20": { + "rho_bar": 0.45084595680236816, + "union_observed": 0.28339576721191406, + "union_independent": 0.7487568855285645, + "union_gap": -0.4653611183166504, + "mean_per_layer_hit_rate": 0.09949101507663727 + }, + "50": { + "rho_bar": 0.4821005165576935, + "union_observed": 0.3745318353176117, + "union_independent": 0.8779515027999878, + "union_gap": -0.5034196674823761, + "mean_per_layer_hit_rate": 0.1475079208612442 + }, + "100": { + "rho_bar": 0.5109114050865173, + "union_observed": 0.45443195104599, + "union_independent": 0.941804051399231, + "union_gap": -0.48737210035324097, + "mean_per_layer_hit_rate": 0.1933160424232483 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.40853878429957796, + "union_gap_mean_over_k": -0.340649025780814 + }, + "OOD_arXiv_2023|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.25860321521759033, + "union_observed": 0.048689138144254684, + "union_independent": 0.128229022026062, + "union_gap": -0.07953988388180733, + "mean_per_layer_hit_rate": 0.010467683896422386 + }, + "2": { + "rho_bar": 0.3468072712421417, + "union_observed": 0.0774032473564148, + "union_independent": 0.21763497591018677, + "union_gap": -0.14023172855377197, + "mean_per_layer_hit_rate": 0.018630558624863625 + }, + "5": { + "rho_bar": 0.30874934792518616, + "union_observed": 0.1373283416032791, + "union_independent": 0.3720051050186157, + "union_gap": -0.2346767634153366, + "mean_per_layer_hit_rate": 0.0348602719604969 + }, + "10": { + "rho_bar": 0.32175493240356445, + "union_observed": 0.20099875330924988, + "union_independent": 0.5374359488487244, + "union_gap": -0.3364371955394745, + "mean_per_layer_hit_rate": 0.056852009147405624 + }, + "20": { + "rho_bar": 0.39316537976264954, + "union_observed": 0.27590513229370117, + "union_independent": 0.6991113424301147, + "union_gap": -0.4232062101364136, + "mean_per_layer_hit_rate": 0.08700662106275558 + }, + "50": { + "rho_bar": 0.4559418559074402, + "union_observed": 0.37078651785850525, + "union_independent": 0.8479862809181213, + "union_gap": -0.4771997630596161, + "mean_per_layer_hit_rate": 0.13271871209144592 + }, + "100": { + "rho_bar": 0.5031102299690247, + "union_observed": 0.449438214302063, + "union_independent": 0.9158225059509277, + "union_gap": -0.46638429164886475, + "mean_per_layer_hit_rate": 0.16988380253314972 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3697331760610853, + "union_gap_mean_over_k": -0.30823940517646925 + }, + "OOD_arXiv_2023|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.20244356989860535, + "union_observed": 0.05368289723992348, + "union_independent": 0.12381547689437866, + "union_gap": -0.07013257965445518, + "mean_per_layer_hit_rate": 0.010083549655973911 + }, + "2": { + "rho_bar": 0.26741650700569153, + "union_observed": 0.07865168899297714, + "union_independent": 0.19127702713012695, + "union_gap": -0.11262533813714981, + "mean_per_layer_hit_rate": 0.016133679077029228 + }, + "5": { + "rho_bar": 0.2816047966480255, + "union_observed": 0.13483145833015442, + "union_independent": 0.33329248428344727, + "union_gap": -0.19846102595329285, + "mean_per_layer_hit_rate": 0.03044271655380726 + }, + "10": { + "rho_bar": 0.28245359659194946, + "union_observed": 0.20973782241344452, + "union_independent": 0.500049352645874, + "union_gap": -0.2903115302324295, + "mean_per_layer_hit_rate": 0.05128205195069313 + }, + "20": { + "rho_bar": 0.331288605928421, + "union_observed": 0.26217228174209595, + "union_independent": 0.6512031555175781, + "union_gap": -0.3890308737754822, + "mean_per_layer_hit_rate": 0.0766349732875824 + }, + "50": { + "rho_bar": 0.40872034430503845, + "union_observed": 0.34706616401672363, + "union_independent": 0.8085615634918213, + "union_gap": -0.46149539947509766, + "mean_per_layer_hit_rate": 0.11725728213787079 + }, + "100": { + "rho_bar": 0.44546350836753845, + "union_observed": 0.44818976521492004, + "union_independent": 0.8994348049163818, + "union_gap": -0.4512450397014618, + "mean_per_layer_hit_rate": 0.15835973620414734 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31705584696360994, + "union_gap_mean_over_k": -0.2819002552756241 + }, + "OOD_arXiv_2023|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.17556194961071014, + "union_observed": 0.04494382068514824, + "union_independent": 0.10141968727111816, + "union_gap": -0.056475866585969925, + "mean_per_layer_hit_rate": 0.00816287286579609 + }, + "2": { + "rho_bar": 0.2406940758228302, + "union_observed": 0.07116104662418365, + "union_independent": 0.165155291557312, + "union_gap": -0.09399424493312836, + "mean_per_layer_hit_rate": 0.013732833787798882 + }, + "5": { + "rho_bar": 0.2856670916080475, + "union_observed": 0.13108614087104797, + "union_independent": 0.30783265829086304, + "union_gap": -0.17674651741981506, + "mean_per_layer_hit_rate": 0.027657736092805862 + }, + "10": { + "rho_bar": 0.2976262867450714, + "union_observed": 0.1910112351179123, + "union_independent": 0.4634671211242676, + "union_gap": -0.2724558860063553, + "mean_per_layer_hit_rate": 0.04619226232171059 + }, + "20": { + "rho_bar": 0.33245599269866943, + "union_observed": 0.255930095911026, + "union_independent": 0.6164183616638184, + "union_gap": -0.36048826575279236, + "mean_per_layer_hit_rate": 0.06981658190488815 + }, + "50": { + "rho_bar": 0.40493500232696533, + "union_observed": 0.3458177149295807, + "union_independent": 0.7864772081375122, + "union_gap": -0.4406594932079315, + "mean_per_layer_hit_rate": 0.10986267775297165 + }, + "100": { + "rho_bar": 0.43934768438339233, + "union_observed": 0.42696627974510193, + "union_independent": 0.876395583152771, + "union_gap": -0.44942930340766907, + "mean_per_layer_hit_rate": 0.14549121260643005 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31089829759938375, + "union_gap_mean_over_k": -0.26432136818766594 + }, + "OOD_arXiv_2023|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.12314537167549133, + "union_observed": 0.04993757978081703, + "union_independent": 0.08994460105895996, + "union_gap": -0.04000702127814293, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.20951175689697266, + "union_observed": 0.0774032473564148, + "union_independent": 0.174655020236969, + "union_gap": -0.0972517728805542, + "mean_per_layer_hit_rate": 0.01459713838994503 + }, + "5": { + "rho_bar": 0.2314079999923706, + "union_observed": 0.1373283416032791, + "union_independent": 0.29634183645248413, + "union_gap": -0.15901349484920502, + "mean_per_layer_hit_rate": 0.026505330577492714 + }, + "10": { + "rho_bar": 0.30440473556518555, + "union_observed": 0.18476903438568115, + "union_independent": 0.44716525077819824, + "union_gap": -0.2623962163925171, + "mean_per_layer_hit_rate": 0.04417555034160614 + }, + "20": { + "rho_bar": 0.3329618275165558, + "union_observed": 0.2534331977367401, + "union_independent": 0.5950753688812256, + "union_gap": -0.3416421711444855, + "mean_per_layer_hit_rate": 0.06635935604572296 + }, + "50": { + "rho_bar": 0.3888239860534668, + "union_observed": 0.34706616401672363, + "union_independent": 0.7669342756271362, + "union_gap": -0.4198681116104126, + "mean_per_layer_hit_rate": 0.10448478162288666 + }, + "100": { + "rho_bar": 0.4152117371559143, + "union_observed": 0.44569289684295654, + "union_independent": 0.8617973923683167, + "union_gap": -0.4161044955253601, + "mean_per_layer_hit_rate": 0.13876883685588837 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.2864953449794224, + "union_gap_mean_over_k": -0.24804046909723962 + }, + "OOD_arXiv_2023|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.15747196972370148, + "union_observed": 0.041198503226041794, + "union_independent": 0.08291876316070557, + "union_gap": -0.04172025993466377, + "mean_per_layer_hit_rate": 0.006626332178711891 + }, + "2": { + "rho_bar": 0.194795623421669, + "union_observed": 0.06866417080163956, + "union_independent": 0.15739405155181885, + "union_gap": -0.08872988075017929, + "mean_per_layer_hit_rate": 0.013060597702860832 + }, + "5": { + "rho_bar": 0.2560214400291443, + "union_observed": 0.12234706431627274, + "union_independent": 0.29746049642562866, + "union_gap": -0.17511343210935593, + "mean_per_layer_hit_rate": 0.026697397232055664 + }, + "10": { + "rho_bar": 0.3263847827911377, + "union_observed": 0.15106117725372314, + "union_independent": 0.43523573875427246, + "union_gap": -0.2841745615005493, + "mean_per_layer_hit_rate": 0.04273504391312599 + }, + "20": { + "rho_bar": 0.3608410358428955, + "union_observed": 0.21098627150058746, + "union_independent": 0.5806050300598145, + "union_gap": -0.369618758559227, + "mean_per_layer_hit_rate": 0.06415057927370071 + }, + "50": { + "rho_bar": 0.3884500563144684, + "union_observed": 0.29588013887405396, + "union_independent": 0.743252694606781, + "union_gap": -0.44737255573272705, + "mean_per_layer_hit_rate": 0.09843464940786362 + }, + "100": { + "rho_bar": 0.4015830457210541, + "union_observed": 0.4032459557056427, + "union_independent": 0.8501712679862976, + "union_gap": -0.4469253122806549, + "mean_per_layer_hit_rate": 0.13425526022911072 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.2979354219777243, + "union_gap_mean_over_k": -0.26480782298105104 + }, + "OOD_arXiv_2023|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.10600778460502625, + "union_observed": 0.0387016236782074, + "union_independent": 0.06904399394989014, + "union_gap": -0.03034237027168274, + "mean_per_layer_hit_rate": 0.005473926663398743 + }, + "2": { + "rho_bar": 0.17916662991046906, + "union_observed": 0.06491885334253311, + "union_independent": 0.1413823366165161, + "union_gap": -0.076463483273983, + "mean_per_layer_hit_rate": 0.01162009034305811 + }, + "5": { + "rho_bar": 0.23485973477363586, + "union_observed": 0.1136079877614975, + "union_independent": 0.27122795581817627, + "union_gap": -0.15761996805667877, + "mean_per_layer_hit_rate": 0.023912416771054268 + }, + "10": { + "rho_bar": 0.29526811838150024, + "union_observed": 0.15355804562568665, + "union_independent": 0.4106488823890686, + "union_gap": -0.25709083676338196, + "mean_per_layer_hit_rate": 0.03956592455506325 + }, + "20": { + "rho_bar": 0.3433406352996826, + "union_observed": 0.20599250495433807, + "union_independent": 0.5660617351531982, + "union_gap": -0.36006923019886017, + "mean_per_layer_hit_rate": 0.061557672917842865 + }, + "50": { + "rho_bar": 0.3988712728023529, + "union_observed": 0.29837703704833984, + "union_independent": 0.7321699857711792, + "union_gap": -0.43379294872283936, + "mean_per_layer_hit_rate": 0.09526553750038147 + }, + "100": { + "rho_bar": 0.4264105260372162, + "union_observed": 0.3957552909851074, + "union_independent": 0.857683002948761, + "union_gap": -0.46192771196365356, + "mean_per_layer_hit_rate": 0.13723231852054596 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.283417814544269, + "union_gap_mean_over_k": -0.2539009356072971 + }, + "OOD_arXiv_2023|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.131770059466362, + "union_observed": 0.036204744130373, + "union_independent": 0.06782478094100952, + "union_gap": -0.03162003681063652, + "mean_per_layer_hit_rate": 0.00537789287045598 + }, + "2": { + "rho_bar": 0.17535115778446198, + "union_observed": 0.06616728752851486, + "union_independent": 0.14904111623764038, + "union_gap": -0.08287382870912552, + "mean_per_layer_hit_rate": 0.012292327359318733 + }, + "5": { + "rho_bar": 0.24209240078926086, + "union_observed": 0.10237203538417816, + "union_independent": 0.2711957097053528, + "union_gap": -0.16882367432117462, + "mean_per_layer_hit_rate": 0.023912418633699417 + }, + "10": { + "rho_bar": 0.30941176414489746, + "union_observed": 0.1473158597946167, + "union_independent": 0.4134845733642578, + "union_gap": -0.2661687135696411, + "mean_per_layer_hit_rate": 0.039854031056165695 + }, + "20": { + "rho_bar": 0.3688875436782837, + "union_observed": 0.21098627150058746, + "union_independent": 0.5917111039161682, + "union_gap": -0.38072483241558075, + "mean_per_layer_hit_rate": 0.06568712741136551 + }, + "50": { + "rho_bar": 0.4112648665904999, + "union_observed": 0.3133583068847656, + "union_independent": 0.7702784538269043, + "union_gap": -0.45692014694213867, + "mean_per_layer_hit_rate": 0.10525305569171906 + }, + "100": { + "rho_bar": 0.4146459400653839, + "union_observed": 0.4244694113731384, + "union_independent": 0.8803529739379883, + "union_gap": -0.45588356256484985, + "mean_per_layer_hit_rate": 0.14760395884513855 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.2933462475027357, + "union_gap_mean_over_k": -0.2632878279047353 + }, + "OOD_arXiv_2023|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.16933000087738037, + "union_observed": 0.04494382068514824, + "union_independent": 0.08993428945541382, + "union_gap": -0.04499046877026558, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.18678277730941772, + "union_observed": 0.07116104662418365, + "union_independent": 0.15770018100738525, + "union_gap": -0.0865391343832016, + "mean_per_layer_hit_rate": 0.013060597702860832 + }, + "5": { + "rho_bar": 0.2251194268465042, + "union_observed": 0.10736579447984695, + "union_independent": 0.2693746089935303, + "union_gap": -0.16200881451368332, + "mean_per_layer_hit_rate": 0.023720350116491318 + }, + "10": { + "rho_bar": 0.2945025861263275, + "union_observed": 0.16729088127613068, + "union_independent": 0.4403219223022461, + "union_gap": -0.2730310410261154, + "mean_per_layer_hit_rate": 0.04321521148085594 + }, + "20": { + "rho_bar": 0.3534596860408783, + "union_observed": 0.23720349371433258, + "union_independent": 0.6080260872840881, + "union_gap": -0.37082259356975555, + "mean_per_layer_hit_rate": 0.06847210228443146 + }, + "50": { + "rho_bar": 0.40154707431793213, + "union_observed": 0.33208489418029785, + "union_independent": 0.7904906868934631, + "union_gap": -0.4584057927131653, + "mean_per_layer_hit_rate": 0.11101508140563965 + }, + "100": { + "rho_bar": 0.40833520889282227, + "union_observed": 0.41947564482688904, + "union_independent": 0.8855603337287903, + "union_gap": -0.46608468890190125, + "mean_per_layer_hit_rate": 0.1498127281665802 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.2912966800587518, + "union_gap_mean_over_k": -0.26598321912544115 + }, + "OOD_arXiv_2023|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.20063139498233795, + "union_observed": 0.04494382068514824, + "union_independent": 0.0887458324432373, + "union_gap": -0.043802011758089066, + "mean_per_layer_hit_rate": 0.007106502074748278 + }, + "2": { + "rho_bar": 0.2660681903362274, + "union_observed": 0.06491885334253311, + "union_independent": 0.16086846590042114, + "union_gap": -0.09594961255788803, + "mean_per_layer_hit_rate": 0.013348698616027832 + }, + "5": { + "rho_bar": 0.25118952989578247, + "union_observed": 0.10986267030239105, + "union_independent": 0.2938855290412903, + "union_gap": -0.18402285873889923, + "mean_per_layer_hit_rate": 0.026217225939035416 + }, + "10": { + "rho_bar": 0.32957300543785095, + "union_observed": 0.16729088127613068, + "union_independent": 0.44668930768966675, + "union_gap": -0.27939842641353607, + "mean_per_layer_hit_rate": 0.04407951608300209 + }, + "20": { + "rho_bar": 0.3431408703327179, + "union_observed": 0.23345817625522614, + "union_independent": 0.6174142956733704, + "union_gap": -0.3839561194181442, + "mean_per_layer_hit_rate": 0.07020071148872375 + }, + "50": { + "rho_bar": 0.39476415514945984, + "union_observed": 0.3483146131038666, + "union_independent": 0.802627682685852, + "union_gap": -0.4543130695819855, + "mean_per_layer_hit_rate": 0.11485642939805984 + }, + "100": { + "rho_bar": 0.4050328731536865, + "union_observed": 0.4419475793838501, + "union_independent": 0.9028011560440063, + "union_gap": -0.46085357666015625, + "mean_per_layer_hit_rate": 0.15960818529129028 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.3129142884697233, + "union_gap_mean_over_k": -0.2717565250183855 + }, + "OOD_arXiv_2023|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.23781509697437286, + "union_observed": 0.048689138144254684, + "union_independent": 0.10024416446685791, + "union_gap": -0.051555026322603226, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.28259819746017456, + "union_observed": 0.0749063640832901, + "union_independent": 0.17254793643951416, + "union_gap": -0.09764157235622406, + "mean_per_layer_hit_rate": 0.01440507173538208 + }, + "5": { + "rho_bar": 0.2862538695335388, + "union_observed": 0.13358302414417267, + "union_independent": 0.34311240911483765, + "union_gap": -0.20952938497066498, + "mean_per_layer_hit_rate": 0.03159511834383011 + }, + "10": { + "rho_bar": 0.3226683437824249, + "union_observed": 0.20224718749523163, + "union_independent": 0.5026361346244812, + "union_gap": -0.3003889471292496, + "mean_per_layer_hit_rate": 0.05176221951842308 + }, + "20": { + "rho_bar": 0.33526208996772766, + "union_observed": 0.2684144675731659, + "union_independent": 0.6656394600868225, + "union_gap": -0.3972249925136566, + "mean_per_layer_hit_rate": 0.07961202412843704 + }, + "50": { + "rho_bar": 0.38428831100463867, + "union_observed": 0.37827715277671814, + "union_independent": 0.8356185555458069, + "union_gap": -0.45734140276908875, + "mean_per_layer_hit_rate": 0.12695668637752533 + }, + "100": { + "rho_bar": 0.40264856815338135, + "union_observed": 0.4656679034233093, + "union_independent": 0.9141950607299805, + "union_gap": -0.44852715730667114, + "mean_per_layer_hit_rate": 0.16748295724391937 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.32164778241089415, + "union_gap_mean_over_k": -0.28031549762402264 + }, + "OOD_arXiv_2023|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.20299477875232697, + "union_observed": 0.05118601769208908, + "union_independent": 0.11151373386383057, + "union_gap": -0.060327716171741486, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.2645740807056427, + "union_observed": 0.07865168899297714, + "union_independent": 0.20159167051315308, + "union_gap": -0.12293998152017593, + "mean_per_layer_hit_rate": 0.017094016075134277 + }, + "5": { + "rho_bar": 0.3053916096687317, + "union_observed": 0.13857677578926086, + "union_independent": 0.3716058135032654, + "union_gap": -0.23302903771400452, + "mean_per_layer_hit_rate": 0.0348602719604969 + }, + "10": { + "rho_bar": 0.3382556438446045, + "union_observed": 0.1960049867630005, + "union_independent": 0.5348132848739624, + "union_gap": -0.3388082981109619, + "mean_per_layer_hit_rate": 0.056659940630197525 + }, + "20": { + "rho_bar": 0.3646260201931, + "union_observed": 0.2684144675731659, + "union_independent": 0.7105098962783813, + "union_gap": -0.44209542870521545, + "mean_per_layer_hit_rate": 0.08979160338640213 + }, + "50": { + "rho_bar": 0.40398168563842773, + "union_observed": 0.36329588294029236, + "union_independent": 0.8529102802276611, + "union_gap": -0.4896143972873688, + "mean_per_layer_hit_rate": 0.13492749631404877 + }, + "100": { + "rho_bar": 0.4241020083427429, + "union_observed": 0.4619225859642029, + "union_independent": 0.924064040184021, + "union_gap": -0.4621414542198181, + "mean_per_layer_hit_rate": 0.17593392729759216 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3291322610207966, + "union_gap_mean_over_k": -0.30699375910418375 + }, + "OOD_arXiv_2023|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.20554645359516144, + "union_observed": 0.05368289723992348, + "union_independent": 0.12379485368728638, + "union_gap": -0.0701119564473629, + "mean_per_layer_hit_rate": 0.010083549655973911 + }, + "2": { + "rho_bar": 0.2971438765525818, + "union_observed": 0.07365792989730835, + "union_independent": 0.20660090446472168, + "union_gap": -0.13294297456741333, + "mean_per_layer_hit_rate": 0.017574185505509377 + }, + "5": { + "rho_bar": 0.3067058324813843, + "union_observed": 0.1423220932483673, + "union_independent": 0.38602346181869507, + "union_gap": -0.24370136857032776, + "mean_per_layer_hit_rate": 0.0365888811647892 + }, + "10": { + "rho_bar": 0.3564191162586212, + "union_observed": 0.2172284573316574, + "union_independent": 0.5650091171264648, + "union_gap": -0.34778065979480743, + "mean_per_layer_hit_rate": 0.06146163493394852 + }, + "20": { + "rho_bar": 0.37088119983673096, + "union_observed": 0.2996254563331604, + "union_independent": 0.7326775193214417, + "union_gap": -0.43305206298828125, + "mean_per_layer_hit_rate": 0.09526553750038147 + }, + "50": { + "rho_bar": 0.43024200201034546, + "union_observed": 0.3595505654811859, + "union_independent": 0.8603701591491699, + "union_gap": -0.500819593667984, + "mean_per_layer_hit_rate": 0.13838471472263336 + }, + "100": { + "rho_bar": 0.4625047445297241, + "union_observed": 0.43570536375045776, + "union_independent": 0.9252974390983582, + "union_gap": -0.4895920753479004, + "mean_per_layer_hit_rate": 0.17766253650188446 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3470633178949356, + "union_gap_mean_over_k": -0.31685724162629675 + }, + "OOD_arXiv_2023|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.18390658497810364, + "union_observed": 0.039950061589479446, + "union_independent": 0.09787338972091675, + "union_gap": -0.0579233281314373, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.228485569357872, + "union_observed": 0.0749063640832901, + "union_independent": 0.1816854476928711, + "union_gap": -0.106779083609581, + "mean_per_layer_hit_rate": 0.015269375406205654 + }, + "5": { + "rho_bar": 0.2929428517818451, + "union_observed": 0.144818976521492, + "union_independent": 0.3377273678779602, + "union_gap": -0.1929083913564682, + "mean_per_layer_hit_rate": 0.03101891651749611 + }, + "10": { + "rho_bar": 0.3448677361011505, + "union_observed": 0.19725343585014343, + "union_independent": 0.4953913688659668, + "union_gap": -0.29813793301582336, + "mean_per_layer_hit_rate": 0.05080188065767288 + }, + "20": { + "rho_bar": 0.3859823942184448, + "union_observed": 0.25967541337013245, + "union_independent": 0.6504412889480591, + "union_gap": -0.39076587557792664, + "mean_per_layer_hit_rate": 0.0768270492553711 + }, + "50": { + "rho_bar": 0.4420010447502136, + "union_observed": 0.35830211639404297, + "union_independent": 0.8174375891685486, + "union_gap": -0.4591354727745056, + "mean_per_layer_hit_rate": 0.12090655416250229 + }, + "100": { + "rho_bar": 0.44411760568618774, + "union_observed": 0.4631710350513458, + "union_independent": 0.9008926749229431, + "union_gap": -0.4377216398715973, + "mean_per_layer_hit_rate": 0.15970420837402344 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3317576838391168, + "union_gap_mean_over_k": -0.2776245320481913 + }, + "OOD_arXiv_2023|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.18444915115833282, + "union_observed": 0.05243445560336113, + "union_independent": 0.10355252027511597, + "union_gap": -0.05111806467175484, + "mean_per_layer_hit_rate": 0.008354940451681614 + }, + "2": { + "rho_bar": 0.2731142044067383, + "union_observed": 0.07990012317895889, + "union_independent": 0.182935893535614, + "union_gap": -0.10303577035665512, + "mean_per_layer_hit_rate": 0.01536540873348713 + }, + "5": { + "rho_bar": 0.2607738971710205, + "union_observed": 0.13857677578926086, + "union_independent": 0.3301633596420288, + "union_gap": -0.19158658385276794, + "mean_per_layer_hit_rate": 0.03015461377799511 + }, + "10": { + "rho_bar": 0.3300674855709076, + "union_observed": 0.20599250495433807, + "union_independent": 0.5023148655891418, + "union_gap": -0.29632236063480377, + "mean_per_layer_hit_rate": 0.05176222324371338 + }, + "20": { + "rho_bar": 0.37120047211647034, + "union_observed": 0.27590513229370117, + "union_independent": 0.6521880030632019, + "union_gap": -0.37628287076950073, + "mean_per_layer_hit_rate": 0.0770191103219986 + }, + "50": { + "rho_bar": 0.41257673501968384, + "union_observed": 0.3620474338531494, + "union_independent": 0.8113763332366943, + "union_gap": -0.4493288993835449, + "mean_per_layer_hit_rate": 0.11821761727333069 + }, + "100": { + "rho_bar": 0.4413061738014221, + "union_observed": 0.46691635251045227, + "union_independent": 0.9033883213996887, + "union_gap": -0.43647196888923645, + "mean_per_layer_hit_rate": 0.16095265746116638 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.32478401703493937, + "union_gap_mean_over_k": -0.2720209312226091 + }, + "OOD_arXiv_2023|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.23018404841423035, + "union_observed": 0.05368289723992348, + "union_independent": 0.1093829870223999, + "union_gap": -0.055700089782476425, + "mean_per_layer_hit_rate": 0.008835108950734138 + }, + "2": { + "rho_bar": 0.23686878383159637, + "union_observed": 0.07865168899297714, + "union_independent": 0.1778581738471985, + "union_gap": -0.09920648485422134, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2658492624759674, + "union_observed": 0.1548064947128296, + "union_independent": 0.3302221894264221, + "union_gap": -0.17541569471359253, + "mean_per_layer_hit_rate": 0.03015461564064026 + }, + "10": { + "rho_bar": 0.29943418502807617, + "union_observed": 0.22097378969192505, + "union_independent": 0.4983832836151123, + "union_gap": -0.27740949392318726, + "mean_per_layer_hit_rate": 0.05118601769208908 + }, + "20": { + "rho_bar": 0.33408123254776, + "union_observed": 0.30586767196655273, + "union_independent": 0.6714199781417847, + "union_gap": -0.36555230617523193, + "mean_per_layer_hit_rate": 0.08095649629831314 + }, + "50": { + "rho_bar": 0.3965980112552643, + "union_observed": 0.39950063824653625, + "union_independent": 0.8248190879821777, + "union_gap": -0.4253184497356415, + "mean_per_layer_hit_rate": 0.12340343743562698 + }, + "100": { + "rho_bar": 0.4096488356590271, + "union_observed": 0.47940075397491455, + "union_independent": 0.9000039100646973, + "union_gap": -0.4206031560897827, + "mean_per_layer_hit_rate": 0.1590319722890854 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.31038062274456024, + "union_gap_mean_over_k": -0.25988652503916193 + }, + "OOD_arXiv_2023|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.14870630204677582, + "union_observed": 0.04244694113731384, + "union_independent": 0.07961791753768921, + "union_gap": -0.037170976400375366, + "mean_per_layer_hit_rate": 0.006338230799883604 + }, + "2": { + "rho_bar": 0.20390979945659637, + "union_observed": 0.0699126124382019, + "union_independent": 0.16296237707138062, + "union_gap": -0.09304976463317871, + "mean_per_layer_hit_rate": 0.013540768064558506 + }, + "5": { + "rho_bar": 0.23452243208885193, + "union_observed": 0.13108614087104797, + "union_independent": 0.2964886426925659, + "union_gap": -0.16540250182151794, + "mean_per_layer_hit_rate": 0.026505330577492714 + }, + "10": { + "rho_bar": 0.2823970913887024, + "union_observed": 0.20848938822746277, + "union_independent": 0.476277232170105, + "union_gap": -0.2677878439426422, + "mean_per_layer_hit_rate": 0.048016905784606934 + }, + "20": { + "rho_bar": 0.30815091729164124, + "union_observed": 0.2721598148345947, + "union_independent": 0.6289320588111877, + "union_gap": -0.356772243976593, + "mean_per_layer_hit_rate": 0.07212138921022415 + }, + "50": { + "rho_bar": 0.36138051748275757, + "union_observed": 0.36828964948654175, + "union_independent": 0.7869169116020203, + "union_gap": -0.4186272621154785, + "mean_per_layer_hit_rate": 0.11005473136901855 + }, + "100": { + "rho_bar": 0.3977898359298706, + "union_observed": 0.45318353176116943, + "union_independent": 0.8845309615135193, + "union_gap": -0.43134742975234985, + "mean_per_layer_hit_rate": 0.14971670508384705 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.2766938422407423, + "union_gap_mean_over_k": -0.2528797175203051 + }, + "OOD_arXiv_2023|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.14953924715518951, + "union_observed": 0.05243445560336113, + "union_independent": 0.10027402639389038, + "union_gap": -0.04783957079052925, + "mean_per_layer_hit_rate": 0.008066839538514614 + }, + "2": { + "rho_bar": 0.25019124150276184, + "union_observed": 0.07615480571985245, + "union_independent": 0.1766034960746765, + "union_gap": -0.10044869035482407, + "mean_per_layer_hit_rate": 0.01478920504450798 + }, + "5": { + "rho_bar": 0.2920864224433899, + "union_observed": 0.13358302414417267, + "union_independent": 0.328458309173584, + "union_gap": -0.19487528502941132, + "mean_per_layer_hit_rate": 0.02996254898607731 + }, + "10": { + "rho_bar": 0.3151260316371918, + "union_observed": 0.19475655257701874, + "union_independent": 0.4833582043647766, + "union_gap": -0.2886016517877579, + "mean_per_layer_hit_rate": 0.04907327517867088 + }, + "20": { + "rho_bar": 0.3603859841823578, + "union_observed": 0.26716604828834534, + "union_independent": 0.6476263999938965, + "union_gap": -0.38046035170555115, + "mean_per_layer_hit_rate": 0.0760587751865387 + }, + "50": { + "rho_bar": 0.4080427289009094, + "union_observed": 0.39450687170028687, + "union_independent": 0.817560076713562, + "union_gap": -0.42305320501327515, + "mean_per_layer_hit_rate": 0.12004225701093674 + }, + "100": { + "rho_bar": 0.4413553476333618, + "union_observed": 0.4781523048877716, + "union_independent": 0.9051913022994995, + "union_gap": -0.4270389974117279, + "mean_per_layer_hit_rate": 0.16143281757831573 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31667528620788027, + "union_gap_mean_over_k": -0.26604539315615383 + }, + "OOD_arXiv_2023|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2508279085159302, + "union_observed": 0.04619225859642029, + "union_independent": 0.12262296676635742, + "union_gap": -0.07643070816993713, + "mean_per_layer_hit_rate": 0.009987514466047287 + }, + "2": { + "rho_bar": 0.3237899839878082, + "union_observed": 0.072409488260746, + "union_independent": 0.215582013130188, + "union_gap": -0.14317252486944199, + "mean_per_layer_hit_rate": 0.018438490107655525 + }, + "5": { + "rho_bar": 0.3914882242679596, + "union_observed": 0.12234706431627274, + "union_independent": 0.3939577341079712, + "union_gap": -0.27161066979169846, + "mean_per_layer_hit_rate": 0.0375492163002491 + }, + "10": { + "rho_bar": 0.4662352204322815, + "union_observed": 0.1760299652814865, + "union_independent": 0.5995625257492065, + "union_gap": -0.42353256046772003, + "mean_per_layer_hit_rate": 0.06722366064786911 + }, + "20": { + "rho_bar": 0.5163217782974243, + "union_observed": 0.24219726026058197, + "union_independent": 0.7494534254074097, + "union_gap": -0.5072561651468277, + "mean_per_layer_hit_rate": 0.09939499199390411 + }, + "50": { + "rho_bar": 0.5236332416534424, + "union_observed": 0.34207239747047424, + "union_independent": 0.873281717300415, + "union_gap": -0.5312093198299408, + "mean_per_layer_hit_rate": 0.1436665654182434 + }, + "100": { + "rho_bar": 0.5363065004348755, + "union_observed": 0.44069913029670715, + "union_independent": 0.9389047622680664, + "union_gap": -0.49820563197135925, + "mean_per_layer_hit_rate": 0.1883222907781601 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4298004082271031, + "union_gap_mean_over_k": -0.35020251146384646 + }, + "OOD_CommonPile|s0|real": { + "per_k": { + "1": { + "rho_bar": 0.2850554883480072, + "union_observed": 0.048689138144254684, + "union_independent": 0.13579314947128296, + "union_gap": -0.08710401132702827, + "mean_per_layer_hit_rate": 0.01113992091268301 + }, + "2": { + "rho_bar": 0.35633817315101624, + "union_observed": 0.08364544063806534, + "union_independent": 0.2528747320175171, + "union_gap": -0.16922929137945175, + "mean_per_layer_hit_rate": 0.02208777330815792 + }, + "5": { + "rho_bar": 0.39388108253479004, + "union_observed": 0.13483145833015442, + "union_independent": 0.4244070053100586, + "union_gap": -0.2895755469799042, + "mean_per_layer_hit_rate": 0.041294537484645844 + }, + "10": { + "rho_bar": 0.46026304364204407, + "union_observed": 0.20349563658237457, + "union_independent": 0.6174055337905884, + "union_gap": -0.4139098972082138, + "mean_per_layer_hit_rate": 0.0706808865070343 + }, + "20": { + "rho_bar": 0.5091291666030884, + "union_observed": 0.2734082341194153, + "union_independent": 0.7654199600219727, + "union_gap": -0.4920117259025574, + "mean_per_layer_hit_rate": 0.1043887510895729 + }, + "50": { + "rho_bar": 0.5392228364944458, + "union_observed": 0.3458177149295807, + "union_independent": 0.8768677115440369, + "union_gap": -0.5310499966144562, + "mean_per_layer_hit_rate": 0.14702776074409485 + }, + "100": { + "rho_bar": 0.5621992945671082, + "union_observed": 0.43820226192474365, + "union_independent": 0.9430235028266907, + "union_gap": -0.504821240901947, + "mean_per_layer_hit_rate": 0.19475653767585754 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.44372701219149996, + "union_gap_mean_over_k": -0.35538595861622263 + }, + "OOD_CommonPile|s0|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.2581786513328552, + "union_observed": 0.056179776787757874, + "union_independent": 0.1348462700843811, + "union_gap": -0.07866649329662323, + "mean_per_layer_hit_rate": 0.011043887585401535 + }, + "2": { + "rho_bar": 0.279427707195282, + "union_observed": 0.08239700645208359, + "union_independent": 0.22483569383621216, + "union_gap": -0.14243868738412857, + "mean_per_layer_hit_rate": 0.019302794709801674 + }, + "5": { + "rho_bar": 0.32345226407051086, + "union_observed": 0.144818976521492, + "union_independent": 0.3917454481124878, + "union_gap": -0.2469264715909958, + "mean_per_layer_hit_rate": 0.03726111352443695 + }, + "10": { + "rho_bar": 0.36967575550079346, + "union_observed": 0.21098627150058746, + "union_independent": 0.5662095546722412, + "union_gap": -0.35522328317165375, + "mean_per_layer_hit_rate": 0.06165369972586632 + }, + "20": { + "rho_bar": 0.4249456226825714, + "union_observed": 0.2746566832065582, + "union_independent": 0.71771240234375, + "union_gap": -0.4430557191371918, + "mean_per_layer_hit_rate": 0.09152022004127502 + }, + "50": { + "rho_bar": 0.49631813168525696, + "union_observed": 0.3570536971092224, + "union_independent": 0.8525834083557129, + "union_gap": -0.4955297112464905, + "mean_per_layer_hit_rate": 0.13492749631404877 + }, + "100": { + "rho_bar": 0.5347961187362671, + "union_observed": 0.45318353176116943, + "union_independent": 0.9244077205657959, + "union_gap": -0.47122418880462646, + "mean_per_layer_hit_rate": 0.17699028551578522 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3838277501719339, + "union_gap_mean_over_k": -0.3190092220902443 + }, + "OOD_CommonPile|s0|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.22196821868419647, + "union_observed": 0.05118601769208908, + "union_independent": 0.1193951964378357, + "union_gap": -0.06820917874574661, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.2981780469417572, + "union_observed": 0.08239700645208359, + "union_independent": 0.20151656866073608, + "union_gap": -0.1191195622086525, + "mean_per_layer_hit_rate": 0.017094017937779427 + }, + "5": { + "rho_bar": 0.2815571427345276, + "union_observed": 0.144818976521492, + "union_independent": 0.35091692209243774, + "union_gap": -0.20609794557094574, + "mean_per_layer_hit_rate": 0.032459426671266556 + }, + "10": { + "rho_bar": 0.30436840653419495, + "union_observed": 0.20724095404148102, + "union_independent": 0.5151808261871338, + "union_gap": -0.30793987214565277, + "mean_per_layer_hit_rate": 0.05358685925602913 + }, + "20": { + "rho_bar": 0.3672811985015869, + "union_observed": 0.2659175992012024, + "union_independent": 0.670522928237915, + "union_gap": -0.40460532903671265, + "mean_per_layer_hit_rate": 0.08076442033052444 + }, + "50": { + "rho_bar": 0.43093642592430115, + "union_observed": 0.3533083498477936, + "union_independent": 0.8223458528518677, + "union_gap": -0.4690375030040741, + "mean_per_layer_hit_rate": 0.12244310975074768 + }, + "100": { + "rho_bar": 0.477662056684494, + "union_observed": 0.44069913029670715, + "union_independent": 0.9054229259490967, + "union_gap": -0.4647237956523895, + "mean_per_layer_hit_rate": 0.16277728974819183 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.34027878514357973, + "union_gap_mean_over_k": -0.291390455194882 + }, + "OOD_CommonPile|s0|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.20971672236919403, + "union_observed": 0.04619225859642029, + "union_independent": 0.11048102378845215, + "union_gap": -0.06428876519203186, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.23450446128845215, + "union_observed": 0.07365792989730835, + "union_independent": 0.17253756523132324, + "union_gap": -0.09887963533401489, + "mean_per_layer_hit_rate": 0.014405070804059505 + }, + "5": { + "rho_bar": 0.2890399992465973, + "union_observed": 0.144818976521492, + "union_independent": 0.3348867893218994, + "union_gap": -0.1900678128004074, + "mean_per_layer_hit_rate": 0.03063478320837021 + }, + "10": { + "rho_bar": 0.3000718057155609, + "union_observed": 0.20474407076835632, + "union_independent": 0.49340641498565674, + "union_gap": -0.2886623442173004, + "mean_per_layer_hit_rate": 0.05041774362325668 + }, + "20": { + "rho_bar": 0.36177587509155273, + "union_observed": 0.2659175992012024, + "union_independent": 0.6541324853897095, + "union_gap": -0.3882148861885071, + "mean_per_layer_hit_rate": 0.07730720937252045 + }, + "50": { + "rho_bar": 0.4094041585922241, + "union_observed": 0.35830211639404297, + "union_independent": 0.8045812845230103, + "union_gap": -0.4462791681289673, + "mean_per_layer_hit_rate": 0.1159127950668335 + }, + "100": { + "rho_bar": 0.46891990303993225, + "union_observed": 0.4319600462913513, + "union_independent": 0.8877073526382446, + "union_gap": -0.4557473063468933, + "mean_per_layer_hit_rate": 0.15202149748802185 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3247761321919305, + "union_gap_mean_over_k": -0.27601998831544605 + }, + "OOD_CommonPile|s0|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.14766019582748413, + "union_observed": 0.05243445560336113, + "union_independent": 0.09570211172103882, + "union_gap": -0.04326765611767769, + "mean_per_layer_hit_rate": 0.007682704366743565 + }, + "2": { + "rho_bar": 0.20064383745193481, + "union_observed": 0.08239700645208359, + "union_independent": 0.17159104347229004, + "union_gap": -0.08919403702020645, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.2533787786960602, + "union_observed": 0.14357054233551025, + "union_independent": 0.3194236755371094, + "union_gap": -0.17585313320159912, + "mean_per_layer_hit_rate": 0.02900220826268196 + }, + "10": { + "rho_bar": 0.3016599118709564, + "union_observed": 0.1960049867630005, + "union_independent": 0.4746537208557129, + "union_gap": -0.2786487340927124, + "mean_per_layer_hit_rate": 0.047920867800712585 + }, + "20": { + "rho_bar": 0.3540078103542328, + "union_observed": 0.26466915011405945, + "union_independent": 0.6291441917419434, + "union_gap": -0.3644750416278839, + "mean_per_layer_hit_rate": 0.07269758731126785 + }, + "50": { + "rho_bar": 0.4127795994281769, + "union_observed": 0.3408239781856537, + "union_independent": 0.7828409671783447, + "union_gap": -0.44201698899269104, + "mean_per_layer_hit_rate": 0.10928647220134735 + }, + "100": { + "rho_bar": 0.45474115014076233, + "union_observed": 0.4282147288322449, + "union_independent": 0.8774771094322205, + "union_gap": -0.4492623805999756, + "mean_per_layer_hit_rate": 0.1467396467924118 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.30355304053851534, + "union_gap_mean_over_k": -0.26324542452182087 + }, + "OOD_CommonPile|s0|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.18861091136932373, + "union_observed": 0.04369537904858589, + "union_independent": 0.1023937463760376, + "union_gap": -0.058698367327451706, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.20798934996128082, + "union_observed": 0.07116104662418365, + "union_independent": 0.1702837347984314, + "union_gap": -0.09912268817424774, + "mean_per_layer_hit_rate": 0.014213002286851406 + }, + "5": { + "rho_bar": 0.27005910873413086, + "union_observed": 0.12609238922595978, + "union_independent": 0.30210399627685547, + "union_gap": -0.1760116070508957, + "mean_per_layer_hit_rate": 0.027177564799785614 + }, + "10": { + "rho_bar": 0.326444536447525, + "union_observed": 0.17353308200836182, + "union_independent": 0.4620998501777649, + "union_gap": -0.2885667681694031, + "mean_per_layer_hit_rate": 0.04628828912973404 + }, + "20": { + "rho_bar": 0.36600905656814575, + "union_observed": 0.22347065806388855, + "union_independent": 0.6007770895957947, + "union_gap": -0.37730643153190613, + "mean_per_layer_hit_rate": 0.06760779768228531 + }, + "50": { + "rho_bar": 0.4090317487716675, + "union_observed": 0.29837703704833984, + "union_independent": 0.7662197947502136, + "union_gap": -0.4678427577018738, + "mean_per_layer_hit_rate": 0.10467685014009476 + }, + "100": { + "rho_bar": 0.44219300150871277, + "union_observed": 0.3932584226131439, + "union_independent": 0.8663167357444763, + "union_gap": -0.4730583131313324, + "mean_per_layer_hit_rate": 0.1415538191795349 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3157625304801123, + "union_gap_mean_over_k": -0.2772295618695872 + }, + "OOD_CommonPile|s0|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.17364980280399323, + "union_observed": 0.04369537904858589, + "union_independent": 0.08300787210464478, + "union_gap": -0.039312493056058884, + "mean_per_layer_hit_rate": 0.0066263326443731785 + }, + "2": { + "rho_bar": 0.19320949912071228, + "union_observed": 0.06491885334253311, + "union_independent": 0.14460545778274536, + "union_gap": -0.07968660444021225, + "mean_per_layer_hit_rate": 0.011908192187547684 + }, + "5": { + "rho_bar": 0.2585236132144928, + "union_observed": 0.1111111119389534, + "union_independent": 0.2803571820259094, + "union_gap": -0.16924607008695602, + "mean_per_layer_hit_rate": 0.024872753769159317 + }, + "10": { + "rho_bar": 0.3263663351535797, + "union_observed": 0.16354556381702423, + "union_independent": 0.4474055767059326, + "union_gap": -0.2838600128889084, + "mean_per_layer_hit_rate": 0.04427158832550049 + }, + "20": { + "rho_bar": 0.38346025347709656, + "union_observed": 0.20973782241344452, + "union_independent": 0.6027196645736694, + "union_gap": -0.3929818421602249, + "mean_per_layer_hit_rate": 0.06779985874891281 + }, + "50": { + "rho_bar": 0.41007134318351746, + "union_observed": 0.28838950395584106, + "union_independent": 0.7558502554893494, + "union_gap": -0.4674607515335083, + "mean_per_layer_hit_rate": 0.10150773078203201 + }, + "100": { + "rho_bar": 0.4575594961643219, + "union_observed": 0.37078651785850525, + "union_independent": 0.8667235970497131, + "union_gap": -0.4959370791912079, + "mean_per_layer_hit_rate": 0.14145779609680176 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.31469147758824484, + "union_gap_mean_over_k": -0.2754978361938681 + }, + "OOD_CommonPile|s0|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.1566881686449051, + "union_observed": 0.0387016236782074, + "union_independent": 0.08526372909545898, + "union_gap": -0.04656210541725159, + "mean_per_layer_hit_rate": 0.006818399764597416 + }, + "2": { + "rho_bar": 0.2139115035533905, + "union_observed": 0.05867665261030197, + "union_independent": 0.16190749406814575, + "union_gap": -0.10323084145784378, + "mean_per_layer_hit_rate": 0.013444732874631882 + }, + "5": { + "rho_bar": 0.2863321602344513, + "union_observed": 0.10362047702074051, + "union_independent": 0.30103182792663574, + "union_gap": -0.19741135090589523, + "mean_per_layer_hit_rate": 0.026985498145222664 + }, + "10": { + "rho_bar": 0.32837003469467163, + "union_observed": 0.1498127281665802, + "union_independent": 0.44523531198501587, + "union_gap": -0.29542258381843567, + "mean_per_layer_hit_rate": 0.04388745129108429 + }, + "20": { + "rho_bar": 0.3779492676258087, + "union_observed": 0.2172284573316574, + "union_independent": 0.6161891222000122, + "union_gap": -0.3989606648683548, + "mean_per_layer_hit_rate": 0.07000864297151566 + }, + "50": { + "rho_bar": 0.43823355436325073, + "union_observed": 0.289637953042984, + "union_independent": 0.7715167999267578, + "union_gap": -0.4818788468837738, + "mean_per_layer_hit_rate": 0.1055411547422409 + }, + "100": { + "rho_bar": 0.4367612302303314, + "union_observed": 0.4032459557056427, + "union_independent": 0.877608060836792, + "union_gap": -0.4743621051311493, + "mean_per_layer_hit_rate": 0.14625948667526245 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3197494170495442, + "union_gap_mean_over_k": -0.2854040712118149 + }, + "OOD_CommonPile|s0|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.15693677961826324, + "union_observed": 0.04619225859642029, + "union_independent": 0.08647644519805908, + "union_gap": -0.040284186601638794, + "mean_per_layer_hit_rate": 0.006914434023201466 + }, + "2": { + "rho_bar": 0.19978737831115723, + "union_observed": 0.06741572916507721, + "union_independent": 0.16627871990203857, + "union_gap": -0.09886299073696136, + "mean_per_layer_hit_rate": 0.013828868977725506 + }, + "5": { + "rho_bar": 0.24156051874160767, + "union_observed": 0.1111111119389534, + "union_independent": 0.3000105023384094, + "union_gap": -0.18889939039945602, + "mean_per_layer_hit_rate": 0.026889465749263763 + }, + "10": { + "rho_bar": 0.32454317808151245, + "union_observed": 0.16229712963104248, + "union_independent": 0.46137547492980957, + "union_gap": -0.2990783452987671, + "mean_per_layer_hit_rate": 0.04600019007921219 + }, + "20": { + "rho_bar": 0.35173872113227844, + "union_observed": 0.2222222238779068, + "union_independent": 0.6252578496932983, + "union_gap": -0.40303562581539154, + "mean_per_layer_hit_rate": 0.0716412216424942 + }, + "50": { + "rho_bar": 0.4228096008300781, + "union_observed": 0.3158551752567291, + "union_independent": 0.791528582572937, + "union_gap": -0.4756734073162079, + "mean_per_layer_hit_rate": 0.11139921098947525 + }, + "100": { + "rho_bar": 0.4327804446220398, + "union_observed": 0.3957552909851074, + "union_independent": 0.8838998079299927, + "union_gap": -0.48814451694488525, + "mean_per_layer_hit_rate": 0.14894843101501465 + } + }, + "n_degenerate_pairs": 35, + "rho_bar_mean_over_k": 0.30430808876241955, + "union_gap_mean_over_k": -0.284854066159044 + }, + "OOD_CommonPile|s0|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.20634765923023224, + "union_observed": 0.041198503226041794, + "union_independent": 0.09108614921569824, + "union_gap": -0.04988764598965645, + "mean_per_layer_hit_rate": 0.007298569194972515 + }, + "2": { + "rho_bar": 0.2523465156555176, + "union_observed": 0.06616728752851486, + "union_independent": 0.16733568906784058, + "union_gap": -0.10116840153932571, + "mean_per_layer_hit_rate": 0.013924901373684406 + }, + "5": { + "rho_bar": 0.25934988260269165, + "union_observed": 0.10362047702074051, + "union_independent": 0.30443620681762695, + "union_gap": -0.20081572979688644, + "mean_per_layer_hit_rate": 0.027369637042284012 + }, + "10": { + "rho_bar": 0.3353814482688904, + "union_observed": 0.15855181217193604, + "union_independent": 0.4617295265197754, + "union_gap": -0.30317771434783936, + "mean_per_layer_hit_rate": 0.04609622433781624 + }, + "20": { + "rho_bar": 0.3640452027320862, + "union_observed": 0.23595505952835083, + "union_independent": 0.6426929235458374, + "union_gap": -0.4067378640174866, + "mean_per_layer_hit_rate": 0.07500240206718445 + }, + "50": { + "rho_bar": 0.39775070548057556, + "union_observed": 0.3283395767211914, + "union_independent": 0.8066251277923584, + "union_gap": -0.478285551071167, + "mean_per_layer_hit_rate": 0.11629693955183029 + }, + "100": { + "rho_bar": 0.4229295253753662, + "union_observed": 0.42571786046028137, + "union_independent": 0.8980945348739624, + "union_gap": -0.47237667441368103, + "mean_per_layer_hit_rate": 0.15701526403427124 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.31973584847790854, + "union_gap_mean_over_k": -0.28749279731086325 + }, + "OOD_CommonPile|s0|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.23471853137016296, + "union_observed": 0.04619225859642029, + "union_independent": 0.09227269887924194, + "union_gap": -0.046080440282821655, + "mean_per_layer_hit_rate": 0.007394603453576565 + }, + "2": { + "rho_bar": 0.2785228192806244, + "union_observed": 0.06866417080163956, + "union_independent": 0.17580103874206543, + "union_gap": -0.10713686794042587, + "mean_per_layer_hit_rate": 0.01469317264854908 + }, + "5": { + "rho_bar": 0.3112674653530121, + "union_observed": 0.11735330522060394, + "union_independent": 0.34060680866241455, + "union_gap": -0.2232535034418106, + "mean_per_layer_hit_rate": 0.03130701929330826 + }, + "10": { + "rho_bar": 0.35252878069877625, + "union_observed": 0.1810237169265747, + "union_independent": 0.5062777996063232, + "union_gap": -0.32525408267974854, + "mean_per_layer_hit_rate": 0.05233842134475708 + }, + "20": { + "rho_bar": 0.35277676582336426, + "union_observed": 0.25717851519584656, + "union_independent": 0.6753746271133423, + "union_gap": -0.4181961119174957, + "mean_per_layer_hit_rate": 0.08172477036714554 + }, + "50": { + "rho_bar": 0.3959802985191345, + "union_observed": 0.352059930562973, + "union_independent": 0.8280541896820068, + "union_gap": -0.4759942591190338, + "mean_per_layer_hit_rate": 0.12417171895503998 + }, + "100": { + "rho_bar": 0.41603147983551025, + "union_observed": 0.43570536375045776, + "union_independent": 0.9086553454399109, + "union_gap": -0.4729499816894531, + "mean_per_layer_hit_rate": 0.16383367776870728 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.3345465915543692, + "union_gap_mean_over_k": -0.2955521781529699 + }, + "OOD_CommonPile|s0|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.2391311526298523, + "union_observed": 0.048689138144254684, + "union_independent": 0.10821127891540527, + "union_gap": -0.05952214077115059, + "mean_per_layer_hit_rate": 0.008739076554775238 + }, + "2": { + "rho_bar": 0.2864241302013397, + "union_observed": 0.072409488260746, + "union_independent": 0.19545406103134155, + "union_gap": -0.12304457277059555, + "mean_per_layer_hit_rate": 0.016517816111445427 + }, + "5": { + "rho_bar": 0.33829623460769653, + "union_observed": 0.12359550595283508, + "union_independent": 0.3615887761116028, + "union_gap": -0.2379932701587677, + "mean_per_layer_hit_rate": 0.033707864582538605 + }, + "10": { + "rho_bar": 0.36798295378685, + "union_observed": 0.18476903438568115, + "union_independent": 0.5488767027854919, + "union_gap": -0.3641076683998108, + "mean_per_layer_hit_rate": 0.05886872112751007 + }, + "20": { + "rho_bar": 0.390697181224823, + "union_observed": 0.25468164682388306, + "union_independent": 0.706882894039154, + "union_gap": -0.452201247215271, + "mean_per_layer_hit_rate": 0.08902333676815033 + }, + "50": { + "rho_bar": 0.426179438829422, + "union_observed": 0.352059930562973, + "union_independent": 0.8475518822669983, + "union_gap": -0.49549195170402527, + "mean_per_layer_hit_rate": 0.13262268900871277 + }, + "100": { + "rho_bar": 0.4511435031890869, + "union_observed": 0.4469413161277771, + "union_independent": 0.9224165678024292, + "union_gap": -0.4754752516746521, + "mean_per_layer_hit_rate": 0.1750696301460266 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3571220849241529, + "union_gap_mean_over_k": -0.3154051575277533 + }, + "OOD_CommonPile|s0|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.2019181251525879, + "union_observed": 0.05243445560336113, + "union_independent": 0.12719672918319702, + "union_gap": -0.07476227357983589, + "mean_per_layer_hit_rate": 0.010371650569140911 + }, + "2": { + "rho_bar": 0.2837565839290619, + "union_observed": 0.07990012317895889, + "union_independent": 0.220744788646698, + "union_gap": -0.1408446654677391, + "mean_per_layer_hit_rate": 0.018918659538030624 + }, + "5": { + "rho_bar": 0.32783281803131104, + "union_observed": 0.13483145833015442, + "union_independent": 0.3829335570335388, + "union_gap": -0.2481020987033844, + "mean_per_layer_hit_rate": 0.0362047478556633 + }, + "10": { + "rho_bar": 0.3792576491832733, + "union_observed": 0.19225966930389404, + "union_independent": 0.5508759021759033, + "union_gap": -0.3586162328720093, + "mean_per_layer_hit_rate": 0.05915682017803192 + }, + "20": { + "rho_bar": 0.4326230585575104, + "union_observed": 0.2709113657474518, + "union_independent": 0.7293993234634399, + "union_gap": -0.45848795771598816, + "mean_per_layer_hit_rate": 0.09449725598096848 + }, + "50": { + "rho_bar": 0.47610464692115784, + "union_observed": 0.34956303238868713, + "union_independent": 0.8640363216400146, + "union_gap": -0.5144732892513275, + "mean_per_layer_hit_rate": 0.1402093470096588 + }, + "100": { + "rho_bar": 0.49834662675857544, + "union_observed": 0.4232209622859955, + "union_independent": 0.9252254366874695, + "union_gap": -0.502004474401474, + "mean_per_layer_hit_rate": 0.17785461246967316 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3714056440762111, + "union_gap_mean_over_k": -0.32818442742739407 + }, + "OOD_CommonPile|s0|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.166306272149086, + "union_observed": 0.047440700232982635, + "union_independent": 0.0967254638671875, + "union_gap": -0.049284763634204865, + "mean_per_layer_hit_rate": 0.007778738625347614 + }, + "2": { + "rho_bar": 0.22583220899105072, + "union_observed": 0.0774032473564148, + "union_independent": 0.1796383261680603, + "union_gap": -0.10223507881164551, + "mean_per_layer_hit_rate": 0.015077306888997555 + }, + "5": { + "rho_bar": 0.2921185791492462, + "union_observed": 0.14606741070747375, + "union_independent": 0.33674412965774536, + "union_gap": -0.1906767189502716, + "mean_per_layer_hit_rate": 0.030922885984182358 + }, + "10": { + "rho_bar": 0.3712012469768524, + "union_observed": 0.20724095404148102, + "union_independent": 0.5185882449150085, + "union_gap": -0.3113472908735275, + "mean_per_layer_hit_rate": 0.054259102791547775 + }, + "20": { + "rho_bar": 0.41432616114616394, + "union_observed": 0.26716604828834534, + "union_independent": 0.6610519289970398, + "union_gap": -0.39388588070869446, + "mean_per_layer_hit_rate": 0.07903582602739334 + }, + "50": { + "rho_bar": 0.4855870008468628, + "union_observed": 0.3433208465576172, + "union_independent": 0.8204681277275085, + "union_gap": -0.47714728116989136, + "mean_per_layer_hit_rate": 0.12215500324964523 + }, + "100": { + "rho_bar": 0.49450409412384033, + "union_observed": 0.45068663358688354, + "union_independent": 0.9035934209823608, + "union_gap": -0.4529067873954773, + "mean_per_layer_hit_rate": 0.16210506856441498 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3499822233404432, + "union_gap_mean_over_k": -0.2824976859348161 + }, + "OOD_CommonPile|s0|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.20018641650676727, + "union_observed": 0.05243445560336113, + "union_independent": 0.1115342378616333, + "union_gap": -0.05909978225827217, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.24972306191921234, + "union_observed": 0.08364544063806534, + "union_independent": 0.1893165111541748, + "union_gap": -0.10567107051610947, + "mean_per_layer_hit_rate": 0.01594161055982113 + }, + "5": { + "rho_bar": 0.2758778929710388, + "union_observed": 0.13233457505702972, + "union_independent": 0.3344677686691284, + "union_gap": -0.2021331936120987, + "mean_per_layer_hit_rate": 0.03063478134572506 + }, + "10": { + "rho_bar": 0.36461877822875977, + "union_observed": 0.1960049867630005, + "union_independent": 0.5119620561599731, + "union_gap": -0.31595706939697266, + "mean_per_layer_hit_rate": 0.05320272594690323 + }, + "20": { + "rho_bar": 0.4040047526359558, + "union_observed": 0.2584269642829895, + "union_independent": 0.6612697839736938, + "union_gap": -0.40284281969070435, + "mean_per_layer_hit_rate": 0.07893978804349899 + }, + "50": { + "rho_bar": 0.42872321605682373, + "union_observed": 0.35830211639404297, + "union_independent": 0.8236929178237915, + "union_gap": -0.46539080142974854, + "mean_per_layer_hit_rate": 0.12292326241731644 + }, + "100": { + "rho_bar": 0.46617674827575684, + "union_observed": 0.44569289684295654, + "union_independent": 0.9056962132453918, + "union_gap": -0.4600033164024353, + "mean_per_layer_hit_rate": 0.16268126666545868 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3413301237991878, + "union_gap_mean_over_k": -0.28729972190090586 + }, + "OOD_CommonPile|s0|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.1911187767982483, + "union_observed": 0.056179776787757874, + "union_independent": 0.11279857158660889, + "union_gap": -0.05661879479885101, + "mean_per_layer_hit_rate": 0.009123211726546288 + }, + "2": { + "rho_bar": 0.2329436093568802, + "union_observed": 0.08239700645208359, + "union_independent": 0.1914304494857788, + "union_gap": -0.10903344303369522, + "mean_per_layer_hit_rate": 0.016133679077029228 + }, + "5": { + "rho_bar": 0.29777956008911133, + "union_observed": 0.14357054233551025, + "union_independent": 0.3387640118598938, + "union_gap": -0.19519346952438354, + "mean_per_layer_hit_rate": 0.031114952638745308 + }, + "10": { + "rho_bar": 0.319375216960907, + "union_observed": 0.2172284573316574, + "union_independent": 0.5130245685577393, + "union_gap": -0.29579611122608185, + "mean_per_layer_hit_rate": 0.05339479446411133 + }, + "20": { + "rho_bar": 0.363664835691452, + "union_observed": 0.29338327050209045, + "union_independent": 0.6828277111053467, + "union_gap": -0.3894444406032562, + "mean_per_layer_hit_rate": 0.08345337957143784 + }, + "50": { + "rho_bar": 0.4120890200138092, + "union_observed": 0.3907615542411804, + "union_independent": 0.8313865661621094, + "union_gap": -0.44062501192092896, + "mean_per_layer_hit_rate": 0.12599635124206543 + }, + "100": { + "rho_bar": 0.4384806752204895, + "union_observed": 0.4681648015975952, + "union_independent": 0.9093751907348633, + "union_gap": -0.44121038913726807, + "mean_per_layer_hit_rate": 0.16546624898910522 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.3222073848758425, + "union_gap_mean_over_k": -0.27541738003492355 + }, + "OOD_CommonPile|s0|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.15922386944293976, + "union_observed": 0.047440700232982635, + "union_independent": 0.08646655082702637, + "union_gap": -0.03902585059404373, + "mean_per_layer_hit_rate": 0.006914434488862753 + }, + "2": { + "rho_bar": 0.18727965652942657, + "union_observed": 0.07990012317895889, + "union_independent": 0.17453253269195557, + "union_gap": -0.09463240951299667, + "mean_per_layer_hit_rate": 0.01459713838994503 + }, + "5": { + "rho_bar": 0.2463911920785904, + "union_observed": 0.13483145833015442, + "union_independent": 0.31505173444747925, + "union_gap": -0.18022027611732483, + "mean_per_layer_hit_rate": 0.02852204255759716 + }, + "10": { + "rho_bar": 0.3148120939731598, + "union_observed": 0.1910112351179123, + "union_independent": 0.4786316156387329, + "union_gap": -0.2876203805208206, + "mean_per_layer_hit_rate": 0.048401039093732834 + }, + "20": { + "rho_bar": 0.3412413001060486, + "union_observed": 0.27715355157852173, + "union_independent": 0.6483592391014099, + "union_gap": -0.3712056875228882, + "mean_per_layer_hit_rate": 0.07615481317043304 + }, + "50": { + "rho_bar": 0.4024190604686737, + "union_observed": 0.34456929564476013, + "union_independent": 0.7925570607185364, + "union_gap": -0.44798776507377625, + "mean_per_layer_hit_rate": 0.1120714470744133 + }, + "100": { + "rho_bar": 0.4361259341239929, + "union_observed": 0.4444444477558136, + "union_independent": 0.8927351832389832, + "union_gap": -0.44829073548316956, + "mean_per_layer_hit_rate": 0.15471045672893524 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.29821330096040455, + "union_gap_mean_over_k": -0.26699758640357424 + }, + "OOD_CommonPile|s0|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.13980545103549957, + "union_observed": 0.05243445560336113, + "union_independent": 0.09909462928771973, + "union_gap": -0.0466601736843586, + "mean_per_layer_hit_rate": 0.007970805279910564 + }, + "2": { + "rho_bar": 0.21998557448387146, + "union_observed": 0.08364544063806534, + "union_independent": 0.18178552389144897, + "union_gap": -0.09814008325338364, + "mean_per_layer_hit_rate": 0.015269375406205654 + }, + "5": { + "rho_bar": 0.29350680112838745, + "union_observed": 0.1373283416032791, + "union_independent": 0.34188205003738403, + "union_gap": -0.20455370843410492, + "mean_per_layer_hit_rate": 0.03149908781051636 + }, + "10": { + "rho_bar": 0.33709582686424255, + "union_observed": 0.19475655257701874, + "union_independent": 0.5003986954689026, + "union_gap": -0.30564214289188385, + "mean_per_layer_hit_rate": 0.05157015100121498 + }, + "20": { + "rho_bar": 0.39660775661468506, + "union_observed": 0.2584269642829895, + "union_independent": 0.6606860160827637, + "union_gap": -0.40225905179977417, + "mean_per_layer_hit_rate": 0.07884375751018524 + }, + "50": { + "rho_bar": 0.44625136256217957, + "union_observed": 0.35830211639404297, + "union_independent": 0.8160774111747742, + "union_gap": -0.4577752947807312, + "mean_per_layer_hit_rate": 0.12004224956035614 + }, + "100": { + "rho_bar": 0.48365411162376404, + "union_observed": 0.45817726850509644, + "union_independent": 0.9028021097183228, + "union_gap": -0.4446248412132263, + "mean_per_layer_hit_rate": 0.16066455841064453 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3309866977589471, + "union_gap_mean_over_k": -0.2799507565796375 + }, + "OOD_CommonPile|s0|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2756688594818115, + "union_observed": 0.041198503226041794, + "union_independent": 0.11817079782485962, + "union_gap": -0.07697229459881783, + "mean_per_layer_hit_rate": 0.009603380225598812 + }, + "2": { + "rho_bar": 0.33708301186561584, + "union_observed": 0.07116104662418365, + "union_independent": 0.23255807161331177, + "union_gap": -0.1613970249891281, + "mean_per_layer_hit_rate": 0.020071066915988922 + }, + "5": { + "rho_bar": 0.3656005263328552, + "union_observed": 0.11985018849372864, + "union_independent": 0.39962929487228394, + "union_gap": -0.2797791063785553, + "mean_per_layer_hit_rate": 0.03822145611047745 + }, + "10": { + "rho_bar": 0.4456540644168854, + "union_observed": 0.1760299652814865, + "union_independent": 0.5957303047180176, + "union_gap": -0.41970033943653107, + "mean_per_layer_hit_rate": 0.06655143201351166 + }, + "20": { + "rho_bar": 0.5122010111808777, + "union_observed": 0.24219726026058197, + "union_independent": 0.7532117366790771, + "union_gap": -0.5110144764184952, + "mean_per_layer_hit_rate": 0.10045135766267776 + }, + "50": { + "rho_bar": 0.5399116277694702, + "union_observed": 0.33458176255226135, + "union_independent": 0.8756718635559082, + "union_gap": -0.5410901010036469, + "mean_per_layer_hit_rate": 0.1450110375881195 + }, + "100": { + "rho_bar": 0.5543181896209717, + "union_observed": 0.41448190808296204, + "union_independent": 0.9379331469535828, + "union_gap": -0.5234512388706207, + "mean_per_layer_hit_rate": 0.18784213066101074 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.4329196129526411, + "union_gap_mean_over_k": -0.35905779738511356 + }, + "OOD_CommonPile|s1|real": { + "per_k": { + "1": { + "rho_bar": 0.28845587372779846, + "union_observed": 0.05368289723992348, + "union_independent": 0.14670521020889282, + "union_gap": -0.09302231296896935, + "mean_per_layer_hit_rate": 0.012100259773433208 + }, + "2": { + "rho_bar": 0.35632795095443726, + "union_observed": 0.08614232391119003, + "union_independent": 0.2538294792175293, + "union_gap": -0.16768715530633926, + "mean_per_layer_hit_rate": 0.02218380756676197 + }, + "5": { + "rho_bar": 0.391033411026001, + "union_observed": 0.13483145833015442, + "union_independent": 0.4154089093208313, + "union_gap": -0.2805774509906769, + "mean_per_layer_hit_rate": 0.040142133831977844 + }, + "10": { + "rho_bar": 0.4339073896408081, + "union_observed": 0.20724095404148102, + "union_independent": 0.6046003699302673, + "union_gap": -0.3973594158887863, + "mean_per_layer_hit_rate": 0.06828003376722336 + }, + "20": { + "rho_bar": 0.49066680669784546, + "union_observed": 0.2746566832065582, + "union_independent": 0.7627031207084656, + "union_gap": -0.48804643750190735, + "mean_per_layer_hit_rate": 0.10352443903684616 + }, + "50": { + "rho_bar": 0.5325659513473511, + "union_observed": 0.34706616401672363, + "union_independent": 0.8749187588691711, + "union_gap": -0.5278525948524475, + "mean_per_layer_hit_rate": 0.14587536454200745 + }, + "100": { + "rho_bar": 0.547735869884491, + "union_observed": 0.44069913029670715, + "union_independent": 0.9404991865158081, + "union_gap": -0.49980005621910095, + "mean_per_layer_hit_rate": 0.1917795091867447 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4343847504683903, + "union_gap_mean_over_k": -0.3506207748183182 + }, + "OOD_CommonPile|s1|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.23791296780109406, + "union_observed": 0.056179776787757874, + "union_independent": 0.13163578510284424, + "union_gap": -0.07545600831508636, + "mean_per_layer_hit_rate": 0.010755786672234535 + }, + "2": { + "rho_bar": 0.31613489985466003, + "union_observed": 0.08114856481552124, + "union_independent": 0.22474676370620728, + "union_gap": -0.14359819889068604, + "mean_per_layer_hit_rate": 0.019302794709801674 + }, + "5": { + "rho_bar": 0.32286009192466736, + "union_observed": 0.1473158597946167, + "union_independent": 0.39107733964920044, + "union_gap": -0.24376147985458374, + "mean_per_layer_hit_rate": 0.0371650829911232 + }, + "10": { + "rho_bar": 0.3478657007217407, + "union_observed": 0.21348313987255096, + "union_independent": 0.5622425079345703, + "union_gap": -0.34875936806201935, + "mean_per_layer_hit_rate": 0.06088542938232422 + }, + "20": { + "rho_bar": 0.40499284863471985, + "union_observed": 0.27590513229370117, + "union_independent": 0.7129491567611694, + "union_gap": -0.43704402446746826, + "mean_per_layer_hit_rate": 0.09027177840471268 + }, + "50": { + "rho_bar": 0.4875505864620209, + "union_observed": 0.36329588294029236, + "union_independent": 0.8480671048164368, + "union_gap": -0.4847712218761444, + "mean_per_layer_hit_rate": 0.13271872699260712 + }, + "100": { + "rho_bar": 0.5199795961380005, + "union_observed": 0.4519350826740265, + "union_independent": 0.9184530377388, + "union_gap": -0.46651795506477356, + "mean_per_layer_hit_rate": 0.17190051078796387 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3767566702195576, + "union_gap_mean_over_k": -0.3142726080758231 + }, + "OOD_CommonPile|s1|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.2087487429380417, + "union_observed": 0.04993757978081703, + "union_independent": 0.11156678199768066, + "union_gap": -0.06162920221686363, + "mean_per_layer_hit_rate": 0.009027177467942238 + }, + "2": { + "rho_bar": 0.2684968411922455, + "union_observed": 0.07865168899297714, + "union_independent": 0.19641989469528198, + "union_gap": -0.11776820570230484, + "mean_per_layer_hit_rate": 0.016613848507404327 + }, + "5": { + "rho_bar": 0.2770654857158661, + "union_observed": 0.14357054233551025, + "union_independent": 0.34259289503097534, + "union_gap": -0.1990223526954651, + "mean_per_layer_hit_rate": 0.03149908781051636 + }, + "10": { + "rho_bar": 0.2828659117221832, + "union_observed": 0.20724095404148102, + "union_independent": 0.5043717622756958, + "union_gap": -0.2971308082342148, + "mean_per_layer_hit_rate": 0.05195428431034088 + }, + "20": { + "rho_bar": 0.3351365029811859, + "union_observed": 0.260923832654953, + "union_independent": 0.656096339225769, + "union_gap": -0.39517250657081604, + "mean_per_layer_hit_rate": 0.0775953084230423 + }, + "50": { + "rho_bar": 0.41834837198257446, + "union_observed": 0.34956303238868713, + "union_independent": 0.8127727508544922, + "union_gap": -0.46320971846580505, + "mean_per_layer_hit_rate": 0.11869777739048004 + }, + "100": { + "rho_bar": 0.4674093723297119, + "union_observed": 0.43570536375045776, + "union_independent": 0.9007159471511841, + "union_gap": -0.4650105834007263, + "mean_per_layer_hit_rate": 0.15932005643844604 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.32258160412311554, + "union_gap_mean_over_k": -0.28556333961231367 + }, + "OOD_CommonPile|s1|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.17874230444431305, + "union_observed": 0.04619225859642029, + "union_independent": 0.10598123073577881, + "union_gap": -0.05978897213935852, + "mean_per_layer_hit_rate": 0.008547008968889713 + }, + "2": { + "rho_bar": 0.2358728051185608, + "union_observed": 0.0749063640832901, + "union_independent": 0.17359739542007446, + "union_gap": -0.09869103133678436, + "mean_per_layer_hit_rate": 0.014501105062663555 + }, + "5": { + "rho_bar": 0.2895868718624115, + "union_observed": 0.13607990741729736, + "union_independent": 0.3236311674118042, + "union_gap": -0.18755125999450684, + "mean_per_layer_hit_rate": 0.02938634343445301 + }, + "10": { + "rho_bar": 0.31105199456214905, + "union_observed": 0.20349563658237457, + "union_independent": 0.4826684594154358, + "union_gap": -0.2791728228330612, + "mean_per_layer_hit_rate": 0.048881206661462784 + }, + "20": { + "rho_bar": 0.3343571424484253, + "union_observed": 0.2634207308292389, + "union_independent": 0.6392343044281006, + "union_gap": -0.3758135735988617, + "mean_per_layer_hit_rate": 0.07423412799835205 + }, + "50": { + "rho_bar": 0.4001351594924927, + "union_observed": 0.35830211639404297, + "union_independent": 0.7979031801223755, + "union_gap": -0.4396010637283325, + "mean_per_layer_hit_rate": 0.11351194977760315 + }, + "100": { + "rho_bar": 0.4482232630252838, + "union_observed": 0.4307115972042084, + "union_independent": 0.8825404047966003, + "union_gap": -0.45182880759239197, + "mean_per_layer_hit_rate": 0.1488524079322815 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.31399564870766233, + "union_gap_mean_over_k": -0.27034964731761385 + }, + "OOD_CommonPile|s1|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.13906680047512054, + "union_observed": 0.04993757978081703, + "union_independent": 0.09789717197418213, + "union_gap": -0.0479595921933651, + "mean_per_layer_hit_rate": 0.00787477195262909 + }, + "2": { + "rho_bar": 0.20622023940086365, + "union_observed": 0.08489388227462769, + "union_independent": 0.17786186933517456, + "union_gap": -0.09296798706054688, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2509259283542633, + "union_observed": 0.14357054233551025, + "union_independent": 0.3124346137046814, + "union_gap": -0.16886407136917114, + "mean_per_layer_hit_rate": 0.02823394164443016 + }, + "10": { + "rho_bar": 0.298616498708725, + "union_observed": 0.19475655257701874, + "union_independent": 0.45850813388824463, + "union_gap": -0.2637515813112259, + "mean_per_layer_hit_rate": 0.04571209102869034 + }, + "20": { + "rho_bar": 0.3372364640235901, + "union_observed": 0.2584269642829895, + "union_independent": 0.6148737668991089, + "union_gap": -0.3564468026161194, + "mean_per_layer_hit_rate": 0.07000864297151566 + }, + "50": { + "rho_bar": 0.39859139919281006, + "union_observed": 0.3458177149295807, + "union_independent": 0.7785931825637817, + "union_gap": -0.43277546763420105, + "mean_per_layer_hit_rate": 0.10794199258089066 + }, + "100": { + "rho_bar": 0.4232986569404602, + "union_observed": 0.43320849537849426, + "union_independent": 0.871360182762146, + "union_gap": -0.43815168738365173, + "mean_per_layer_hit_rate": 0.14337846636772156 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.29342228387083324, + "union_gap_mean_over_k": -0.2572738842240402 + }, + "OOD_CommonPile|s1|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.15367355942726135, + "union_observed": 0.041198503226041794, + "union_independent": 0.08412319421768188, + "union_gap": -0.04292469099164009, + "mean_per_layer_hit_rate": 0.006722366437315941 + }, + "2": { + "rho_bar": 0.19933713972568512, + "union_observed": 0.06866417080163956, + "union_independent": 0.16496950387954712, + "union_gap": -0.09630533307790756, + "mean_per_layer_hit_rate": 0.013732834719121456 + }, + "5": { + "rho_bar": 0.2690054476261139, + "union_observed": 0.12359550595283508, + "union_independent": 0.3021286129951477, + "union_gap": -0.17853310704231262, + "mean_per_layer_hit_rate": 0.027177564799785614 + }, + "10": { + "rho_bar": 0.3199223279953003, + "union_observed": 0.16354556381702423, + "union_independent": 0.450025737285614, + "union_gap": -0.2864801734685898, + "mean_per_layer_hit_rate": 0.04465571790933609 + }, + "20": { + "rho_bar": 0.3616560399532318, + "union_observed": 0.2247191071510315, + "union_independent": 0.5878668427467346, + "union_gap": -0.3631477355957031, + "mean_per_layer_hit_rate": 0.06530298292636871 + }, + "50": { + "rho_bar": 0.3958788216114044, + "union_observed": 0.29837703704833984, + "union_independent": 0.757197380065918, + "union_gap": -0.4588203430175781, + "mean_per_layer_hit_rate": 0.10208393633365631 + }, + "100": { + "rho_bar": 0.41618165373802185, + "union_observed": 0.387016236782074, + "union_independent": 0.855051577091217, + "union_gap": -0.46803534030914307, + "mean_per_layer_hit_rate": 0.13617593050003052 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.30223642715385984, + "union_gap_mean_over_k": -0.27060667478612493 + }, + "OOD_CommonPile|s1|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.16370968520641327, + "union_observed": 0.04244694113731384, + "union_independent": 0.08184272050857544, + "union_gap": -0.0393957793712616, + "mean_per_layer_hit_rate": 0.006530298851430416 + }, + "2": { + "rho_bar": 0.18916358053684235, + "union_observed": 0.06367041170597076, + "union_independent": 0.1391347050666809, + "union_gap": -0.07546429336071014, + "mean_per_layer_hit_rate": 0.01142802368849516 + }, + "5": { + "rho_bar": 0.2625550925731659, + "union_observed": 0.10736579447984695, + "union_independent": 0.27566713094711304, + "union_gap": -0.16830133646726608, + "mean_per_layer_hit_rate": 0.024392586201429367 + }, + "10": { + "rho_bar": 0.31190910935401917, + "union_observed": 0.16479401290416718, + "union_independent": 0.43799424171447754, + "union_gap": -0.27320022881031036, + "mean_per_layer_hit_rate": 0.04302314668893814 + }, + "20": { + "rho_bar": 0.37372466921806335, + "union_observed": 0.2122347056865692, + "union_independent": 0.5940691232681274, + "union_gap": -0.3818344175815582, + "mean_per_layer_hit_rate": 0.06626332551240921 + }, + "50": { + "rho_bar": 0.403110533952713, + "union_observed": 0.289637953042984, + "union_independent": 0.7492924928665161, + "union_gap": -0.4596545398235321, + "mean_per_layer_hit_rate": 0.09968309104442596 + }, + "100": { + "rho_bar": 0.4449828565120697, + "union_observed": 0.36828964948654175, + "union_independent": 0.861545205116272, + "union_gap": -0.4932555556297302, + "mean_per_layer_hit_rate": 0.1388648897409439 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.3070222181933267, + "union_gap_mean_over_k": -0.27015802157776697 + }, + "OOD_CommonPile|s1|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.16012859344482422, + "union_observed": 0.041198503226041794, + "union_independent": 0.0887068510055542, + "union_gap": -0.047508347779512405, + "mean_per_layer_hit_rate": 0.007106501143425703 + }, + "2": { + "rho_bar": 0.2045062631368637, + "union_observed": 0.06117353215813637, + "union_independent": 0.15974509716033936, + "union_gap": -0.09857156500220299, + "mean_per_layer_hit_rate": 0.013252665288746357 + }, + "5": { + "rho_bar": 0.26247599720954895, + "union_observed": 0.09987515956163406, + "union_independent": 0.28174370527267456, + "union_gap": -0.1818685457110405, + "mean_per_layer_hit_rate": 0.024968789890408516 + }, + "10": { + "rho_bar": 0.32459136843681335, + "union_observed": 0.15106117725372314, + "union_independent": 0.4460325837135315, + "union_gap": -0.29497140645980835, + "mean_per_layer_hit_rate": 0.04398348182439804 + }, + "20": { + "rho_bar": 0.37953701615333557, + "union_observed": 0.21098627150058746, + "union_independent": 0.6027369499206543, + "union_gap": -0.39175067842006683, + "mean_per_layer_hit_rate": 0.06760779768228531 + }, + "50": { + "rho_bar": 0.42009299993515015, + "union_observed": 0.30586767196655273, + "union_independent": 0.7712242007255554, + "union_gap": -0.4653565287590027, + "mean_per_layer_hit_rate": 0.10534907877445221 + }, + "100": { + "rho_bar": 0.4251883029937744, + "union_observed": 0.4082396924495697, + "union_independent": 0.8760344386100769, + "union_gap": -0.4677947461605072, + "mean_per_layer_hit_rate": 0.145203098654747 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3109315059014729, + "union_gap_mean_over_k": -0.27826025975602015 + }, + "OOD_CommonPile|s1|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.12793129682540894, + "union_observed": 0.04494382068514824, + "union_independent": 0.08413457870483398, + "union_gap": -0.039190758019685745, + "mean_per_layer_hit_rate": 0.0067223659716546535 + }, + "2": { + "rho_bar": 0.20440150797367096, + "union_observed": 0.06741572916507721, + "union_independent": 0.16418296098709106, + "union_gap": -0.09676723182201385, + "mean_per_layer_hit_rate": 0.013636801391839981 + }, + "5": { + "rho_bar": 0.235126331448555, + "union_observed": 0.11235955357551575, + "union_independent": 0.2965102791786194, + "union_gap": -0.18415072560310364, + "mean_per_layer_hit_rate": 0.026505328714847565 + }, + "10": { + "rho_bar": 0.3102831244468689, + "union_observed": 0.16604244709014893, + "union_independent": 0.44947487115859985, + "union_gap": -0.2834324240684509, + "mean_per_layer_hit_rate": 0.04446365311741829 + }, + "20": { + "rho_bar": 0.3471316695213318, + "union_observed": 0.227215975522995, + "union_independent": 0.6152830123901367, + "union_gap": -0.3880670368671417, + "mean_per_layer_hit_rate": 0.06981657445430756 + }, + "50": { + "rho_bar": 0.40458059310913086, + "union_observed": 0.32209739089012146, + "union_independent": 0.7869554162025452, + "union_gap": -0.4648580253124237, + "mean_per_layer_hit_rate": 0.1099587008357048 + }, + "100": { + "rho_bar": 0.42348793148994446, + "union_observed": 0.4057428240776062, + "union_independent": 0.8830155730247498, + "union_gap": -0.47727274894714355, + "mean_per_layer_hit_rate": 0.1484682559967041 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2932774935449873, + "union_gap_mean_over_k": -0.27624842151999474 + }, + "OOD_CommonPile|s1|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.2086673378944397, + "union_observed": 0.04494382068514824, + "union_independent": 0.0922439694404602, + "union_gap": -0.047300148755311966, + "mean_per_layer_hit_rate": 0.007394603453576565 + }, + "2": { + "rho_bar": 0.2576918601989746, + "union_observed": 0.06616728752851486, + "union_independent": 0.16621798276901245, + "union_gap": -0.10005069524049759, + "mean_per_layer_hit_rate": 0.013828867115080357 + }, + "5": { + "rho_bar": 0.25242677330970764, + "union_observed": 0.1061173528432846, + "union_independent": 0.29737788438796997, + "union_gap": -0.19126053154468536, + "mean_per_layer_hit_rate": 0.026601364836096764 + }, + "10": { + "rho_bar": 0.3387037217617035, + "union_observed": 0.15355804562568665, + "union_independent": 0.45752841234207153, + "union_gap": -0.3039703667163849, + "mean_per_layer_hit_rate": 0.04552002251148224 + }, + "20": { + "rho_bar": 0.35593608021736145, + "union_observed": 0.23720349371433258, + "union_independent": 0.6276431679725647, + "union_gap": -0.3904396742582321, + "mean_per_layer_hit_rate": 0.07212138921022415 + }, + "50": { + "rho_bar": 0.394610732793808, + "union_observed": 0.3283395767211914, + "union_independent": 0.804349422454834, + "union_gap": -0.4760098457336426, + "mean_per_layer_hit_rate": 0.11543262749910355 + }, + "100": { + "rho_bar": 0.4111804962158203, + "union_observed": 0.43570536375045776, + "union_independent": 0.900101363658905, + "union_gap": -0.46439599990844727, + "mean_per_layer_hit_rate": 0.15816767513751984 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.3170310003416879, + "union_gap_mean_over_k": -0.2819181803081717 + }, + "OOD_CommonPile|s1|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.25767770409584045, + "union_observed": 0.04369537904858589, + "union_independent": 0.09228575229644775, + "union_gap": -0.04859037324786186, + "mean_per_layer_hit_rate": 0.007394603453576565 + }, + "2": { + "rho_bar": 0.2733512222766876, + "union_observed": 0.0699126124382019, + "union_independent": 0.177906334400177, + "union_gap": -0.1079937219619751, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.29610151052474976, + "union_observed": 0.12359550595283508, + "union_independent": 0.33208805322647095, + "union_gap": -0.20849254727363586, + "mean_per_layer_hit_rate": 0.03034668229520321 + }, + "10": { + "rho_bar": 0.3400006890296936, + "union_observed": 0.1885143518447876, + "union_independent": 0.49984586238861084, + "union_gap": -0.31133151054382324, + "mean_per_layer_hit_rate": 0.05137808620929718 + }, + "20": { + "rho_bar": 0.35211968421936035, + "union_observed": 0.24968789517879486, + "union_independent": 0.6611182689666748, + "union_gap": -0.41143037378787994, + "mean_per_layer_hit_rate": 0.0787477120757103 + }, + "50": { + "rho_bar": 0.39174896478652954, + "union_observed": 0.35830211639404297, + "union_independent": 0.8292127847671509, + "union_gap": -0.4709106683731079, + "mean_per_layer_hit_rate": 0.12465187907218933 + }, + "100": { + "rho_bar": 0.4144333600997925, + "union_observed": 0.4419475793838501, + "union_independent": 0.9094531536102295, + "union_gap": -0.4675055742263794, + "mean_per_layer_hit_rate": 0.16431382298469543 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.3322047335760934, + "union_gap_mean_over_k": -0.28946496705923763 + }, + "OOD_CommonPile|s1|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.23031476140022278, + "union_observed": 0.04619225859642029, + "union_independent": 0.10365968942642212, + "union_gap": -0.05746743083000183, + "mean_per_layer_hit_rate": 0.008354941383004189 + }, + "2": { + "rho_bar": 0.3076391816139221, + "union_observed": 0.06616728752851486, + "union_independent": 0.19042199850082397, + "union_gap": -0.12425471097230911, + "mean_per_layer_hit_rate": 0.01603764481842518 + }, + "5": { + "rho_bar": 0.32370850443840027, + "union_observed": 0.12609238922595978, + "union_independent": 0.3541327118873596, + "union_gap": -0.22804032266139984, + "mean_per_layer_hit_rate": 0.032843563705682755 + }, + "10": { + "rho_bar": 0.3638256788253784, + "union_observed": 0.19225966930389404, + "union_independent": 0.5435602068901062, + "union_gap": -0.35130053758621216, + "mean_per_layer_hit_rate": 0.05800442025065422 + }, + "20": { + "rho_bar": 0.38447946310043335, + "union_observed": 0.2634207308292389, + "union_independent": 0.7062965631484985, + "union_gap": -0.44287583231925964, + "mean_per_layer_hit_rate": 0.08883126825094223 + }, + "50": { + "rho_bar": 0.4220235347747803, + "union_observed": 0.3508114814758301, + "union_independent": 0.8493312001228333, + "union_gap": -0.4985197186470032, + "mean_per_layer_hit_rate": 0.13339096307754517 + }, + "100": { + "rho_bar": 0.44745561480522156, + "union_observed": 0.4444444477558136, + "union_independent": 0.9216777086257935, + "union_gap": -0.47723326086997986, + "mean_per_layer_hit_rate": 0.17439739406108856 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.35420667699405123, + "union_gap_mean_over_k": -0.3113845448408808 + }, + "OOD_CommonPile|s1|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.19415488839149475, + "union_observed": 0.05243445560336113, + "union_independent": 0.1193612813949585, + "union_gap": -0.06692682579159737, + "mean_per_layer_hit_rate": 0.009699414484202862 + }, + "2": { + "rho_bar": 0.2978459894657135, + "union_observed": 0.0774032473564148, + "union_independent": 0.21676850318908691, + "union_gap": -0.13936525583267212, + "mean_per_layer_hit_rate": 0.018534522503614426 + }, + "5": { + "rho_bar": 0.3361656367778778, + "union_observed": 0.13233457505702972, + "union_independent": 0.38535112142562866, + "union_gap": -0.25301654636859894, + "mean_per_layer_hit_rate": 0.03649284690618515 + }, + "10": { + "rho_bar": 0.36418458819389343, + "union_observed": 0.19475655257701874, + "union_independent": 0.549119234085083, + "union_gap": -0.35436268150806427, + "mean_per_layer_hit_rate": 0.05886872112751007 + }, + "20": { + "rho_bar": 0.4146001636981964, + "union_observed": 0.2734082341194153, + "union_independent": 0.7224593162536621, + "union_gap": -0.4490510821342468, + "mean_per_layer_hit_rate": 0.09276865422725677 + }, + "50": { + "rho_bar": 0.46547895669937134, + "union_observed": 0.34956303238868713, + "union_independent": 0.861155092716217, + "union_gap": -0.5115920603275299, + "mean_per_layer_hit_rate": 0.1388648897409439 + }, + "100": { + "rho_bar": 0.494029700756073, + "union_observed": 0.4244694113731384, + "union_independent": 0.9233909845352173, + "union_gap": -0.49892157316207886, + "mean_per_layer_hit_rate": 0.176222026348114 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3666371319975172, + "union_gap_mean_over_k": -0.3247480035892555 + }, + "OOD_CommonPile|s1|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.17139975726604462, + "union_observed": 0.04369537904858589, + "union_independent": 0.09099966287612915, + "union_gap": -0.04730428382754326, + "mean_per_layer_hit_rate": 0.007298569194972515 + }, + "2": { + "rho_bar": 0.21535339951515198, + "union_observed": 0.07865168899297714, + "union_independent": 0.17760425806045532, + "union_gap": -0.09895256906747818, + "mean_per_layer_hit_rate": 0.014885240234434605 + }, + "5": { + "rho_bar": 0.2848062217235565, + "union_observed": 0.14606741070747375, + "union_independent": 0.3342459201812744, + "union_gap": -0.18817850947380066, + "mean_per_layer_hit_rate": 0.03063478320837021 + }, + "10": { + "rho_bar": 0.3523904085159302, + "union_observed": 0.20599250495433807, + "union_independent": 0.5046185255050659, + "union_gap": -0.29862602055072784, + "mean_per_layer_hit_rate": 0.05214635655283928 + }, + "20": { + "rho_bar": 0.4045777916908264, + "union_observed": 0.26217228174209595, + "union_independent": 0.6476197838783264, + "union_gap": -0.38544750213623047, + "mean_per_layer_hit_rate": 0.0762508362531662 + }, + "50": { + "rho_bar": 0.4646684229373932, + "union_observed": 0.34706616401672363, + "union_independent": 0.8181198239326477, + "union_gap": -0.4710536599159241, + "mean_per_layer_hit_rate": 0.12119465321302414 + }, + "100": { + "rho_bar": 0.4810679852962494, + "union_observed": 0.45318353176116943, + "union_independent": 0.9012269973754883, + "union_gap": -0.44804346561431885, + "mean_per_layer_hit_rate": 0.16037645936012268 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.33918056956359316, + "union_gap_mean_over_k": -0.2768008586551462 + }, + "OOD_CommonPile|s1|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.1878119558095932, + "union_observed": 0.05243445560336113, + "union_independent": 0.1092645525932312, + "union_gap": -0.05683009698987007, + "mean_per_layer_hit_rate": 0.008835109882056713 + }, + "2": { + "rho_bar": 0.2695253789424896, + "union_observed": 0.07990012317895889, + "union_independent": 0.18616366386413574, + "union_gap": -0.10626354068517685, + "mean_per_layer_hit_rate": 0.01565350964665413 + }, + "5": { + "rho_bar": 0.28441041707992554, + "union_observed": 0.13607990741729736, + "union_independent": 0.3370171785354614, + "union_gap": -0.20093727111816406, + "mean_per_layer_hit_rate": 0.030922885984182358 + }, + "10": { + "rho_bar": 0.34765487909317017, + "union_observed": 0.20099875330924988, + "union_independent": 0.5049227476119995, + "union_gap": -0.30392399430274963, + "mean_per_layer_hit_rate": 0.05214635282754898 + }, + "20": { + "rho_bar": 0.3905981481075287, + "union_observed": 0.25967541337013245, + "union_independent": 0.6508545875549316, + "union_gap": -0.3911791741847992, + "mean_per_layer_hit_rate": 0.0768270418047905 + }, + "50": { + "rho_bar": 0.4158846437931061, + "union_observed": 0.3620474338531494, + "union_independent": 0.820968508720398, + "union_gap": -0.45892107486724854, + "mean_per_layer_hit_rate": 0.12177086621522903 + }, + "100": { + "rho_bar": 0.4518171548843384, + "union_observed": 0.45318353176116943, + "union_independent": 0.9030106663703918, + "union_gap": -0.4498271346092224, + "mean_per_layer_hit_rate": 0.16076058149337769 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.33538608253002167, + "union_gap_mean_over_k": -0.2811260409653187 + }, + "OOD_CommonPile|s1|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.15053503215312958, + "union_observed": 0.054931335151195526, + "union_independent": 0.11052972078323364, + "union_gap": -0.055598385632038116, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.2551826536655426, + "union_observed": 0.07865168899297714, + "union_independent": 0.18828392028808594, + "union_gap": -0.1096322312951088, + "mean_per_layer_hit_rate": 0.01584557816386223 + }, + "5": { + "rho_bar": 0.28167858719825745, + "union_observed": 0.15605492889881134, + "union_independent": 0.3499237298965454, + "union_gap": -0.19386880099773407, + "mean_per_layer_hit_rate": 0.032363392412662506 + }, + "10": { + "rho_bar": 0.3179243505001068, + "union_observed": 0.2172284573316574, + "union_independent": 0.5033160448074341, + "union_gap": -0.2860875874757767, + "mean_per_layer_hit_rate": 0.05195428803563118 + }, + "20": { + "rho_bar": 0.3585165739059448, + "union_observed": 0.29088640213012695, + "union_independent": 0.6769610643386841, + "union_gap": -0.38607466220855713, + "mean_per_layer_hit_rate": 0.08220493793487549 + }, + "50": { + "rho_bar": 0.4052083194255829, + "union_observed": 0.3957552909851074, + "union_independent": 0.8301417827606201, + "union_gap": -0.4343864917755127, + "mean_per_layer_hit_rate": 0.12542015314102173 + }, + "100": { + "rho_bar": 0.4292927384376526, + "union_observed": 0.46441948413848877, + "union_independent": 0.9048882126808167, + "union_gap": -0.4404687285423279, + "mean_per_layer_hit_rate": 0.16239316761493683 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.31404832218374523, + "union_gap_mean_over_k": -0.2723024125610079 + }, + "OOD_CommonPile|s1|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.15568867325782776, + "union_observed": 0.04619225859642029, + "union_independent": 0.08417743444442749, + "union_gap": -0.0379851758480072, + "mean_per_layer_hit_rate": 0.006722366437315941 + }, + "2": { + "rho_bar": 0.19270868599414825, + "union_observed": 0.0774032473564148, + "union_independent": 0.17140543460845947, + "union_gap": -0.09400218725204468, + "mean_per_layer_hit_rate": 0.01430903747677803 + }, + "5": { + "rho_bar": 0.23559178411960602, + "union_observed": 0.13857677578926086, + "union_independent": 0.31080538034439087, + "union_gap": -0.17222860455513, + "mean_per_layer_hit_rate": 0.028041871264576912 + }, + "10": { + "rho_bar": 0.30520492792129517, + "union_observed": 0.1910112351179123, + "union_independent": 0.46823006868362427, + "union_gap": -0.277218833565712, + "mean_per_layer_hit_rate": 0.04696052893996239 + }, + "20": { + "rho_bar": 0.32601043581962585, + "union_observed": 0.260923832654953, + "union_independent": 0.6282398700714111, + "union_gap": -0.36731603741645813, + "mean_per_layer_hit_rate": 0.0722174197435379 + }, + "50": { + "rho_bar": 0.37828895449638367, + "union_observed": 0.3458177149295807, + "union_independent": 0.7856903076171875, + "union_gap": -0.4398725926876068, + "mean_per_layer_hit_rate": 0.1097666397690773 + }, + "100": { + "rho_bar": 0.42074400186538696, + "union_observed": 0.44569289684295654, + "union_independent": 0.8869518637657166, + "union_gap": -0.44125896692276, + "mean_per_layer_hit_rate": 0.1511572152376175 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.2877482090677534, + "union_gap_mean_over_k": -0.26141177117824554 + }, + "OOD_CommonPile|s1|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.1258850246667862, + "union_observed": 0.05243445560336113, + "union_independent": 0.09111142158508301, + "union_gap": -0.03867696598172188, + "mean_per_layer_hit_rate": 0.007298569194972515 + }, + "2": { + "rho_bar": 0.239544078707695, + "union_observed": 0.07865168899297714, + "union_independent": 0.18175643682479858, + "union_gap": -0.10310474783182144, + "mean_per_layer_hit_rate": 0.015269375406205654 + }, + "5": { + "rho_bar": 0.3042930066585541, + "union_observed": 0.13483145833015442, + "union_independent": 0.33592963218688965, + "union_gap": -0.20109817385673523, + "mean_per_layer_hit_rate": 0.03082684986293316 + }, + "10": { + "rho_bar": 0.32595333456993103, + "union_observed": 0.1910112351179123, + "union_independent": 0.49055910110473633, + "union_gap": -0.29954786598682404, + "mean_per_layer_hit_rate": 0.05012964457273483 + }, + "20": { + "rho_bar": 0.3727933466434479, + "union_observed": 0.255930095911026, + "union_independent": 0.6481114625930786, + "union_gap": -0.3921813666820526, + "mean_per_layer_hit_rate": 0.0762508362531662 + }, + "50": { + "rho_bar": 0.42554426193237305, + "union_observed": 0.3620474338531494, + "union_independent": 0.8102773427963257, + "union_gap": -0.44822990894317627, + "mean_per_layer_hit_rate": 0.11773744225502014 + }, + "100": { + "rho_bar": 0.4582584500312805, + "union_observed": 0.46441948413848877, + "union_independent": 0.8988120555877686, + "union_gap": -0.4343925714492798, + "mean_per_layer_hit_rate": 0.15787957608699799 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.3217530718871525, + "union_gap_mean_over_k": -0.2738902286759445 + }, + "OOD_CommonPile|s1|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2658415734767914, + "union_observed": 0.04244694113731384, + "union_independent": 0.12154483795166016, + "union_gap": -0.07909789681434631, + "mean_per_layer_hit_rate": 0.009891482070088387 + }, + "2": { + "rho_bar": 0.3270757496356964, + "union_observed": 0.06741572916507721, + "union_independent": 0.22262299060821533, + "union_gap": -0.15520726144313812, + "mean_per_layer_hit_rate": 0.019110726192593575 + }, + "5": { + "rho_bar": 0.3775079548358917, + "union_observed": 0.12109863013029099, + "union_independent": 0.39960765838623047, + "union_gap": -0.2785090282559395, + "mean_per_layer_hit_rate": 0.03822145611047745 + }, + "10": { + "rho_bar": 0.44236722588539124, + "union_observed": 0.1760299652814865, + "union_independent": 0.5935304164886475, + "union_gap": -0.41750045120716095, + "mean_per_layer_hit_rate": 0.06616728752851486 + }, + "20": { + "rho_bar": 0.5170838236808777, + "union_observed": 0.24344569444656372, + "union_independent": 0.7516524791717529, + "union_gap": -0.5082067847251892, + "mean_per_layer_hit_rate": 0.09997118264436722 + }, + "50": { + "rho_bar": 0.5306070446968079, + "union_observed": 0.3308364450931549, + "union_independent": 0.8732764720916748, + "union_gap": -0.5424400269985199, + "mean_per_layer_hit_rate": 0.1436665803194046 + }, + "100": { + "rho_bar": 0.5418928265571594, + "union_observed": 0.41947564482688904, + "union_independent": 0.9366778135299683, + "union_gap": -0.5172021687030792, + "mean_per_layer_hit_rate": 0.18649765849113464 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.4289108855383737, + "union_gap_mean_over_k": -0.35688051687819616 + }, + "OOD_CommonPile|s2|real": { + "per_k": { + "1": { + "rho_bar": 0.27266937494277954, + "union_observed": 0.04993757978081703, + "union_independent": 0.13473129272460938, + "union_gap": -0.08479371294379234, + "mean_per_layer_hit_rate": 0.011043887585401535 + }, + "2": { + "rho_bar": 0.3346019983291626, + "union_observed": 0.08114856481552124, + "union_independent": 0.23926901817321777, + "union_gap": -0.15812045335769653, + "mean_per_layer_hit_rate": 0.020743301138281822 + }, + "5": { + "rho_bar": 0.39173272252082825, + "union_observed": 0.12983770668506622, + "union_independent": 0.41321295499801636, + "union_gap": -0.28337524831295013, + "mean_per_layer_hit_rate": 0.039854031056165695 + }, + "10": { + "rho_bar": 0.45480966567993164, + "union_observed": 0.20224718749523163, + "union_independent": 0.6015549898147583, + "union_gap": -0.3993078023195267, + "mean_per_layer_hit_rate": 0.06770383566617966 + }, + "20": { + "rho_bar": 0.5065693855285645, + "union_observed": 0.2721598148345947, + "union_independent": 0.7583746910095215, + "union_gap": -0.48621487617492676, + "mean_per_layer_hit_rate": 0.10227599740028381 + }, + "50": { + "rho_bar": 0.5363987684249878, + "union_observed": 0.3483146131038666, + "union_independent": 0.8744803667068481, + "union_gap": -0.5261657536029816, + "mean_per_layer_hit_rate": 0.14568327367305756 + }, + "100": { + "rho_bar": 0.5559464693069458, + "union_observed": 0.4394506812095642, + "union_independent": 0.9405314326286316, + "union_gap": -0.5010807514190674, + "mean_per_layer_hit_rate": 0.19187553226947784 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.4361040549618857, + "union_gap_mean_over_k": -0.34843694259013447 + }, + "OOD_CommonPile|s2|cyc1": { + "per_k": { + "1": { + "rho_bar": 0.25358742475509644, + "union_observed": 0.054931335151195526, + "union_independent": 0.13045978546142578, + "union_gap": -0.07552845031023026, + "mean_per_layer_hit_rate": 0.010659752413630486 + }, + "2": { + "rho_bar": 0.32374584674835205, + "union_observed": 0.08114856481552124, + "union_independent": 0.2297123670578003, + "union_gap": -0.14856380224227905, + "mean_per_layer_hit_rate": 0.019782964140176773 + }, + "5": { + "rho_bar": 0.3334234654903412, + "union_observed": 0.1398252248764038, + "union_independent": 0.38223737478256226, + "union_gap": -0.24241214990615845, + "mean_per_layer_hit_rate": 0.03610870987176895 + }, + "10": { + "rho_bar": 0.3591296374797821, + "union_observed": 0.20848938822746277, + "union_independent": 0.5566310286521912, + "union_gap": -0.3481416404247284, + "mean_per_layer_hit_rate": 0.06002112478017807 + }, + "20": { + "rho_bar": 0.422211617231369, + "union_observed": 0.2784020006656647, + "union_independent": 0.7098901271820068, + "union_gap": -0.43148812651634216, + "mean_per_layer_hit_rate": 0.08959953486919403 + }, + "50": { + "rho_bar": 0.49751752614974976, + "union_observed": 0.3570536971092224, + "union_independent": 0.8483724594116211, + "union_gap": -0.4913187623023987, + "mean_per_layer_hit_rate": 0.13300681114196777 + }, + "100": { + "rho_bar": 0.5260457992553711, + "union_observed": 0.45068663358688354, + "union_independent": 0.9214531183242798, + "union_gap": -0.47076648473739624, + "mean_per_layer_hit_rate": 0.17439739406108856 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.38795161673000883, + "union_gap_mean_over_k": -0.315459916634219 + }, + "OOD_CommonPile|s2|cyc2": { + "per_k": { + "1": { + "rho_bar": 0.21979635953903198, + "union_observed": 0.048689138144254684, + "union_independent": 0.11272090673446655, + "union_gap": -0.06403176859021187, + "mean_per_layer_hit_rate": 0.009123211726546288 + }, + "2": { + "rho_bar": 0.29235512018203735, + "union_observed": 0.08114856481552124, + "union_independent": 0.19647961854934692, + "union_gap": -0.11533105373382568, + "mean_per_layer_hit_rate": 0.016613848507404327 + }, + "5": { + "rho_bar": 0.2804850935935974, + "union_observed": 0.144818976521492, + "union_independent": 0.3475196957588196, + "union_gap": -0.20270071923732758, + "mean_per_layer_hit_rate": 0.03207528963685036 + }, + "10": { + "rho_bar": 0.2915247976779938, + "union_observed": 0.20474407076835632, + "union_independent": 0.5016108155250549, + "union_gap": -0.2968667447566986, + "mean_per_layer_hit_rate": 0.05157015472650528 + }, + "20": { + "rho_bar": 0.34940412640571594, + "union_observed": 0.26217228174209595, + "union_independent": 0.6580427885055542, + "union_gap": -0.39587050676345825, + "mean_per_layer_hit_rate": 0.07807548344135284 + }, + "50": { + "rho_bar": 0.42616599798202515, + "union_observed": 0.34956303238868713, + "union_independent": 0.8129059672355652, + "union_gap": -0.46334293484687805, + "mean_per_layer_hit_rate": 0.11888985335826874 + }, + "100": { + "rho_bar": 0.4797358512878418, + "union_observed": 0.4307115972042084, + "union_independent": 0.9016619324684143, + "union_gap": -0.47095033526420593, + "mean_per_layer_hit_rate": 0.16018438339233398 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3342096209526062, + "union_gap_mean_over_k": -0.2870134375989437 + }, + "OOD_CommonPile|s2|cyc3": { + "per_k": { + "1": { + "rho_bar": 0.1871696561574936, + "union_observed": 0.04369537904858589, + "union_independent": 0.10143256187438965, + "union_gap": -0.05773718282580376, + "mean_per_layer_hit_rate": 0.008162873797118664 + }, + "2": { + "rho_bar": 0.24481460452079773, + "union_observed": 0.07116104662418365, + "union_independent": 0.1736081838607788, + "union_gap": -0.10244713723659515, + "mean_per_layer_hit_rate": 0.01450110413134098 + }, + "5": { + "rho_bar": 0.2782422602176666, + "union_observed": 0.14107365906238556, + "union_independent": 0.3245142102241516, + "union_gap": -0.18344055116176605, + "mean_per_layer_hit_rate": 0.02948237769305706 + }, + "10": { + "rho_bar": 0.2961865961551666, + "union_observed": 0.20224718749523163, + "union_independent": 0.48316264152526855, + "union_gap": -0.2809154540300369, + "mean_per_layer_hit_rate": 0.048977237194776535 + }, + "20": { + "rho_bar": 0.338147908449173, + "union_observed": 0.2684144675731659, + "union_independent": 0.6473063230514526, + "union_gap": -0.37889185547828674, + "mean_per_layer_hit_rate": 0.0758667066693306 + }, + "50": { + "rho_bar": 0.4012849032878876, + "union_observed": 0.36079901456832886, + "union_independent": 0.799034833908081, + "union_gap": -0.4382358193397522, + "mean_per_layer_hit_rate": 0.1139921247959137 + }, + "100": { + "rho_bar": 0.45645639300346375, + "union_observed": 0.4307115972042084, + "union_independent": 0.8843393325805664, + "union_gap": -0.45362773537635803, + "mean_per_layer_hit_rate": 0.1500048041343689 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.314614617398807, + "union_gap_mean_over_k": -0.2707565336355141 + }, + "OOD_CommonPile|s2|cyc4": { + "per_k": { + "1": { + "rho_bar": 0.15538658201694489, + "union_observed": 0.048689138144254684, + "union_independent": 0.09223246574401855, + "union_gap": -0.04354332759976387, + "mean_per_layer_hit_rate": 0.00739460252225399 + }, + "2": { + "rho_bar": 0.24606676399707794, + "union_observed": 0.08489388227462769, + "union_independent": 0.18101495504379272, + "union_gap": -0.09612107276916504, + "mean_per_layer_hit_rate": 0.015173341147601604 + }, + "5": { + "rho_bar": 0.2529734969139099, + "union_observed": 0.14856429398059845, + "union_independent": 0.3238227963447571, + "union_gap": -0.17525850236415863, + "mean_per_layer_hit_rate": 0.02948237769305706 + }, + "10": { + "rho_bar": 0.29478415846824646, + "union_observed": 0.193508118391037, + "union_independent": 0.4628734588623047, + "union_gap": -0.2693653404712677, + "mean_per_layer_hit_rate": 0.04628829285502434 + }, + "20": { + "rho_bar": 0.3452955186367035, + "union_observed": 0.26466915011405945, + "union_independent": 0.6240973472595215, + "union_gap": -0.35942819714546204, + "mean_per_layer_hit_rate": 0.07173725217580795 + }, + "50": { + "rho_bar": 0.41031724214553833, + "union_observed": 0.34706616401672363, + "union_independent": 0.7852626442909241, + "union_gap": -0.43819648027420044, + "mean_per_layer_hit_rate": 0.1099587008357048 + }, + "100": { + "rho_bar": 0.4362022280693054, + "union_observed": 0.4319600462913513, + "union_independent": 0.8736964464187622, + "union_gap": -0.4417364001274109, + "mean_per_layer_hit_rate": 0.1446269154548645 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3058608557496752, + "union_gap_mean_over_k": -0.2605213315359184 + }, + "OOD_CommonPile|s2|cyc5": { + "per_k": { + "1": { + "rho_bar": 0.14582033455371857, + "union_observed": 0.04244694113731384, + "union_independent": 0.0898810625076294, + "union_gap": -0.04743412137031555, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.19585728645324707, + "union_observed": 0.07116104662418365, + "union_independent": 0.16505062580108643, + "union_gap": -0.09388957917690277, + "mean_per_layer_hit_rate": 0.013732834719121456 + }, + "5": { + "rho_bar": 0.2625081241130829, + "union_observed": 0.12734082341194153, + "union_independent": 0.30126261711120605, + "union_gap": -0.17392179369926453, + "mean_per_layer_hit_rate": 0.027081530541181564 + }, + "10": { + "rho_bar": 0.30276617407798767, + "union_observed": 0.17228464782238007, + "union_independent": 0.45153653621673584, + "union_gap": -0.2792518883943558, + "mean_per_layer_hit_rate": 0.04484778642654419 + }, + "20": { + "rho_bar": 0.36355575919151306, + "union_observed": 0.2222222238779068, + "union_independent": 0.5982816219329834, + "union_gap": -0.3760593980550766, + "mean_per_layer_hit_rate": 0.06712763011455536 + }, + "50": { + "rho_bar": 0.3973204791545868, + "union_observed": 0.3021223545074463, + "union_independent": 0.7646831274032593, + "union_gap": -0.462560772895813, + "mean_per_layer_hit_rate": 0.10419667512178421 + }, + "100": { + "rho_bar": 0.4230895936489105, + "union_observed": 0.39450687170028687, + "union_independent": 0.8653804659843445, + "union_gap": -0.4708735942840576, + "mean_per_layer_hit_rate": 0.1409776210784912 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.29870253588472095, + "union_gap_mean_over_k": -0.27199873541082653 + }, + "OOD_CommonPile|s2|cyc6": { + "per_k": { + "1": { + "rho_bar": 0.1565588414669037, + "union_observed": 0.04369537904858589, + "union_independent": 0.08185607194900513, + "union_gap": -0.038160692900419235, + "mean_per_layer_hit_rate": 0.006530298851430416 + }, + "2": { + "rho_bar": 0.19977539777755737, + "union_observed": 0.06367041170597076, + "union_independent": 0.13918960094451904, + "union_gap": -0.07551918923854828, + "mean_per_layer_hit_rate": 0.011428022757172585 + }, + "5": { + "rho_bar": 0.2632857859134674, + "union_observed": 0.1086142286658287, + "union_independent": 0.2785518169403076, + "union_gap": -0.1699375882744789, + "mean_per_layer_hit_rate": 0.024680687114596367 + }, + "10": { + "rho_bar": 0.31498685479164124, + "union_observed": 0.15855181217193604, + "union_independent": 0.4402284622192383, + "union_gap": -0.28167665004730225, + "mean_per_layer_hit_rate": 0.04331124573945999 + }, + "20": { + "rho_bar": 0.37440893054008484, + "union_observed": 0.20848938822746277, + "union_independent": 0.5956859588623047, + "union_gap": -0.3871965706348419, + "mean_per_layer_hit_rate": 0.06655142456293106 + }, + "50": { + "rho_bar": 0.4029318392276764, + "union_observed": 0.28838950395584106, + "union_independent": 0.7531768083572388, + "union_gap": -0.4647873044013977, + "mean_per_layer_hit_rate": 0.10073946416378021 + }, + "100": { + "rho_bar": 0.4410809874534607, + "union_observed": 0.37578028440475464, + "union_independent": 0.8653215169906616, + "union_gap": -0.489541232585907, + "mean_per_layer_hit_rate": 0.14068952202796936 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.30757551959582735, + "union_gap_mean_over_k": -0.272402746868985 + }, + "OOD_CommonPile|s2|cyc7": { + "per_k": { + "1": { + "rho_bar": 0.1762276440858841, + "union_observed": 0.041198503226041794, + "union_independent": 0.08526945114135742, + "union_gap": -0.04407094791531563, + "mean_per_layer_hit_rate": 0.006818399764597416 + }, + "2": { + "rho_bar": 0.19950683414936066, + "union_observed": 0.05867665261030197, + "union_independent": 0.15653347969055176, + "union_gap": -0.09785682708024979, + "mean_per_layer_hit_rate": 0.012964563444256783 + }, + "5": { + "rho_bar": 0.28737810254096985, + "union_observed": 0.10237203538417816, + "union_independent": 0.2937503457069397, + "union_gap": -0.19137831032276154, + "mean_per_layer_hit_rate": 0.026217229664325714 + }, + "10": { + "rho_bar": 0.3241123855113983, + "union_observed": 0.15355804562568665, + "union_independent": 0.44615811109542847, + "union_gap": -0.2926000654697418, + "mean_per_layer_hit_rate": 0.04398348182439804 + }, + "20": { + "rho_bar": 0.37838584184646606, + "union_observed": 0.21598002314567566, + "union_independent": 0.611605167388916, + "union_gap": -0.39562514424324036, + "mean_per_layer_hit_rate": 0.0691443383693695 + }, + "50": { + "rho_bar": 0.4231281280517578, + "union_observed": 0.29588013887405396, + "union_independent": 0.7683480978012085, + "union_gap": -0.47246795892715454, + "mean_per_layer_hit_rate": 0.10458081215620041 + }, + "100": { + "rho_bar": 0.4274608790874481, + "union_observed": 0.40699127316474915, + "union_independent": 0.8771957755088806, + "union_gap": -0.47020450234413147, + "mean_per_layer_hit_rate": 0.1459713876247406 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.31659997361046927, + "union_gap_mean_over_k": -0.28060053661465645 + }, + "OOD_CommonPile|s2|cyc8": { + "per_k": { + "1": { + "rho_bar": 0.14317485690116882, + "union_observed": 0.04619225859642029, + "union_independent": 0.08761394023895264, + "union_gap": -0.04142168164253235, + "mean_per_layer_hit_rate": 0.007010467816144228 + }, + "2": { + "rho_bar": 0.20170491933822632, + "union_observed": 0.06741572916507721, + "union_independent": 0.1630975604057312, + "union_gap": -0.09568183124065399, + "mean_per_layer_hit_rate": 0.013540765270590782 + }, + "5": { + "rho_bar": 0.2343396097421646, + "union_observed": 0.11235955357551575, + "union_independent": 0.29461848735809326, + "union_gap": -0.18225893378257751, + "mean_per_layer_hit_rate": 0.026313263922929764 + }, + "10": { + "rho_bar": 0.3134956955909729, + "union_observed": 0.15980024635791779, + "union_independent": 0.4555806517601013, + "union_gap": -0.29578040540218353, + "mean_per_layer_hit_rate": 0.04523191973567009 + }, + "20": { + "rho_bar": 0.3473069369792938, + "union_observed": 0.2247191071510315, + "union_independent": 0.6212063431739807, + "union_gap": -0.3964872360229492, + "mean_per_layer_hit_rate": 0.0708729475736618 + }, + "50": { + "rho_bar": 0.40096548199653625, + "union_observed": 0.31960049271583557, + "union_independent": 0.7902579307556152, + "union_gap": -0.47065743803977966, + "mean_per_layer_hit_rate": 0.1109190359711647 + }, + "100": { + "rho_bar": 0.42734596133232117, + "union_observed": 0.3982521891593933, + "union_independent": 0.8812361359596252, + "union_gap": -0.48298394680023193, + "mean_per_layer_hit_rate": 0.14760395884513855 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.2954762088400977, + "union_gap_mean_over_k": -0.2807530675615583 + }, + "OOD_CommonPile|s2|cyc9": { + "per_k": { + "1": { + "rho_bar": 0.21395130455493927, + "union_observed": 0.041198503226041794, + "union_independent": 0.08415812253952026, + "union_gap": -0.04295961931347847, + "mean_per_layer_hit_rate": 0.0067223659716546535 + }, + "2": { + "rho_bar": 0.25245583057403564, + "union_observed": 0.06741572916507721, + "union_independent": 0.16732746362686157, + "union_gap": -0.09991173446178436, + "mean_per_layer_hit_rate": 0.013924902305006981 + }, + "5": { + "rho_bar": 0.2528533935546875, + "union_observed": 0.10362047702074051, + "union_independent": 0.29912883043289185, + "union_gap": -0.19550835341215134, + "mean_per_layer_hit_rate": 0.026793431490659714 + }, + "10": { + "rho_bar": 0.33455410599708557, + "union_observed": 0.15730337798595428, + "union_independent": 0.4610663056373596, + "union_gap": -0.30376292765140533, + "mean_per_layer_hit_rate": 0.04600019007921219 + }, + "20": { + "rho_bar": 0.365840882062912, + "union_observed": 0.22596754133701324, + "union_independent": 0.625149130821228, + "union_gap": -0.3991815894842148, + "mean_per_layer_hit_rate": 0.0716412216424942 + }, + "50": { + "rho_bar": 0.3957130014896393, + "union_observed": 0.32459425926208496, + "union_independent": 0.8005361557006836, + "union_gap": -0.47594189643859863, + "mean_per_layer_hit_rate": 0.1141841858625412 + }, + "100": { + "rho_bar": 0.41693970561027527, + "union_observed": 0.4294631779193878, + "union_independent": 0.8963931798934937, + "union_gap": -0.46693000197410583, + "mean_per_layer_hit_rate": 0.1557668298482895 + } + }, + "n_degenerate_pairs": 46, + "rho_bar_mean_over_k": 0.31890117483479635, + "union_gap_mean_over_k": -0.2834565889622484 + }, + "OOD_CommonPile|s2|cyc10": { + "per_k": { + "1": { + "rho_bar": 0.24186837673187256, + "union_observed": 0.04494382068514824, + "union_independent": 0.09226095676422119, + "union_gap": -0.04731713607907295, + "mean_per_layer_hit_rate": 0.00739460252225399 + }, + "2": { + "rho_bar": 0.2779136002063751, + "union_observed": 0.06616728752851486, + "union_independent": 0.17046207189559937, + "union_gap": -0.1042947843670845, + "mean_per_layer_hit_rate": 0.014213004149496555 + }, + "5": { + "rho_bar": 0.2980543375015259, + "union_observed": 0.11735330522060394, + "union_independent": 0.33125579357147217, + "union_gap": -0.21390248835086823, + "mean_per_layer_hit_rate": 0.03025064989924431 + }, + "10": { + "rho_bar": 0.3568553328514099, + "union_observed": 0.17977528274059296, + "union_independent": 0.4985496401786804, + "union_gap": -0.31877435743808746, + "mean_per_layer_hit_rate": 0.05118601769208908 + }, + "20": { + "rho_bar": 0.36065179109573364, + "union_observed": 0.24469412863254547, + "union_independent": 0.6638875007629395, + "union_gap": -0.419193372130394, + "mean_per_layer_hit_rate": 0.07932392507791519 + }, + "50": { + "rho_bar": 0.38939642906188965, + "union_observed": 0.3508114814758301, + "union_independent": 0.8245624899864197, + "union_gap": -0.4737510085105896, + "mean_per_layer_hit_rate": 0.12282724678516388 + }, + "100": { + "rho_bar": 0.4089703857898712, + "union_observed": 0.4394506812095642, + "union_independent": 0.9053125381469727, + "union_gap": -0.46586185693740845, + "mean_per_layer_hit_rate": 0.16162489354610443 + } + }, + "n_degenerate_pairs": 56, + "rho_bar_mean_over_k": 0.33338717903409687, + "union_gap_mean_over_k": -0.29187071483050075 + }, + "OOD_CommonPile|s2|cyc11": { + "per_k": { + "1": { + "rho_bar": 0.2456721067428589, + "union_observed": 0.047440700232982635, + "union_independent": 0.10707336664199829, + "union_gap": -0.059632666409015656, + "mean_per_layer_hit_rate": 0.008643042296171188 + }, + "2": { + "rho_bar": 0.295483261346817, + "union_observed": 0.06741572916507721, + "union_independent": 0.1893162727355957, + "union_gap": -0.1219005435705185, + "mean_per_layer_hit_rate": 0.015941612422466278 + }, + "5": { + "rho_bar": 0.33868643641471863, + "union_observed": 0.11985018849372864, + "union_independent": 0.35572683811187744, + "union_gap": -0.2358766496181488, + "mean_per_layer_hit_rate": 0.033035628497600555 + }, + "10": { + "rho_bar": 0.3692794740200043, + "union_observed": 0.1835206001996994, + "union_independent": 0.5410963296890259, + "union_gap": -0.3575757294893265, + "mean_per_layer_hit_rate": 0.05762028321623802 + }, + "20": { + "rho_bar": 0.3880563974380493, + "union_observed": 0.25468164682388306, + "union_independent": 0.6965361833572388, + "union_gap": -0.4418545365333557, + "mean_per_layer_hit_rate": 0.08662249147891998 + }, + "50": { + "rho_bar": 0.42532357573509216, + "union_observed": 0.34956303238868713, + "union_independent": 0.8451427221298218, + "union_gap": -0.49557968974113464, + "mean_per_layer_hit_rate": 0.13156631588935852 + }, + "100": { + "rho_bar": 0.4516395032405853, + "union_observed": 0.4369538128376007, + "union_independent": 0.9181597828865051, + "union_gap": -0.4812059700489044, + "mean_per_layer_hit_rate": 0.17170843482017517 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.3591629649911608, + "union_gap_mean_over_k": -0.31337511220148634 + }, + "OOD_CommonPile|s2|cyc12": { + "per_k": { + "1": { + "rho_bar": 0.196871817111969, + "union_observed": 0.05243445560336113, + "union_independent": 0.12826770544052124, + "union_gap": -0.07583324983716011, + "mean_per_layer_hit_rate": 0.01046768482774496 + }, + "2": { + "rho_bar": 0.28681913018226624, + "union_observed": 0.0774032473564148, + "union_independent": 0.21876007318496704, + "union_gap": -0.14135682582855225, + "mean_per_layer_hit_rate": 0.018726591020822525 + }, + "5": { + "rho_bar": 0.3332957327365875, + "union_observed": 0.12734082341194153, + "union_independent": 0.37561464309692383, + "union_gap": -0.2482738196849823, + "mean_per_layer_hit_rate": 0.03534043952822685 + }, + "10": { + "rho_bar": 0.36182641983032227, + "union_observed": 0.18976280093193054, + "union_independent": 0.5407342314720154, + "union_gap": -0.35097143054008484, + "mean_per_layer_hit_rate": 0.05752424895763397 + }, + "20": { + "rho_bar": 0.42965584993362427, + "union_observed": 0.26966291666030884, + "union_independent": 0.7229305505752563, + "union_gap": -0.4532676339149475, + "mean_per_layer_hit_rate": 0.09286469966173172 + }, + "50": { + "rho_bar": 0.4720202684402466, + "union_observed": 0.3458177149295807, + "union_independent": 0.8589557409286499, + "union_gap": -0.5131380259990692, + "mean_per_layer_hit_rate": 0.13780850172042847 + }, + "100": { + "rho_bar": 0.49521225690841675, + "union_observed": 0.4182272255420685, + "union_independent": 0.9212234020233154, + "union_gap": -0.502996176481247, + "mean_per_layer_hit_rate": 0.17439739406108856 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.36795735359191895, + "union_gap_mean_over_k": -0.3265481660408633 + }, + "OOD_CommonPile|s2|shuf0": { + "per_k": { + "1": { + "rho_bar": 0.1670876443386078, + "union_observed": 0.04494382068514824, + "union_independent": 0.09217631816864014, + "union_gap": -0.0472324974834919, + "mean_per_layer_hit_rate": 0.00739460252225399 + }, + "2": { + "rho_bar": 0.24426090717315674, + "union_observed": 0.07865168899297714, + "union_independent": 0.175512433052063, + "union_gap": -0.09686074405908585, + "mean_per_layer_hit_rate": 0.01469317264854908 + }, + "5": { + "rho_bar": 0.29774123430252075, + "union_observed": 0.144818976521492, + "union_independent": 0.33674830198287964, + "union_gap": -0.19192932546138763, + "mean_per_layer_hit_rate": 0.030922885984182358 + }, + "10": { + "rho_bar": 0.3522934019565582, + "union_observed": 0.19850187003612518, + "union_independent": 0.503912091255188, + "union_gap": -0.3054102212190628, + "mean_per_layer_hit_rate": 0.05205032601952553 + }, + "20": { + "rho_bar": 0.40958940982818604, + "union_observed": 0.2634207308292389, + "union_independent": 0.6517583131790161, + "union_gap": -0.3883375823497772, + "mean_per_layer_hit_rate": 0.07711514830589294 + }, + "50": { + "rho_bar": 0.47340890765190125, + "union_observed": 0.33957552909851074, + "union_independent": 0.8168998956680298, + "union_gap": -0.47732436656951904, + "mean_per_layer_hit_rate": 0.12081053853034973 + }, + "100": { + "rho_bar": 0.48911064863204956, + "union_observed": 0.45318353176116943, + "union_independent": 0.8993616700172424, + "union_gap": -0.446178138256073, + "mean_per_layer_hit_rate": 0.1592240333557129 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3476417362689972, + "union_gap_mean_over_k": -0.2790389821997711 + }, + "OOD_CommonPile|s2|shuf1": { + "per_k": { + "1": { + "rho_bar": 0.2077232450246811, + "union_observed": 0.047440700232982635, + "union_independent": 0.10248792171478271, + "union_gap": -0.05504722148180008, + "mean_per_layer_hit_rate": 0.008258907124400139 + }, + "2": { + "rho_bar": 0.257625550031662, + "union_observed": 0.08364544063806534, + "union_independent": 0.1893267035484314, + "union_gap": -0.10568126291036606, + "mean_per_layer_hit_rate": 0.01594161055982113 + }, + "5": { + "rho_bar": 0.2773153483867645, + "union_observed": 0.13108614087104797, + "union_independent": 0.33186793327331543, + "union_gap": -0.20078179240226746, + "mean_per_layer_hit_rate": 0.03034668229520321 + }, + "10": { + "rho_bar": 0.34825336933135986, + "union_observed": 0.1960049867630005, + "union_independent": 0.5055220127105713, + "union_gap": -0.3095170259475708, + "mean_per_layer_hit_rate": 0.05224239081144333 + }, + "20": { + "rho_bar": 0.39680588245391846, + "union_observed": 0.26466915011405945, + "union_independent": 0.6538053750991821, + "union_gap": -0.3891362249851227, + "mean_per_layer_hit_rate": 0.0774032399058342 + }, + "50": { + "rho_bar": 0.418093740940094, + "union_observed": 0.35830211639404297, + "union_independent": 0.8200947046279907, + "union_gap": -0.46179258823394775, + "mean_per_layer_hit_rate": 0.12148275971412659 + }, + "100": { + "rho_bar": 0.4578881859779358, + "union_observed": 0.44818976521492004, + "union_independent": 0.9033047556877136, + "union_gap": -0.4551149904727936, + "mean_per_layer_hit_rate": 0.16104871034622192 + } + }, + "n_degenerate_pairs": 24, + "rho_bar_mean_over_k": 0.3376721888780594, + "union_gap_mean_over_k": -0.28243872949055265 + }, + "OOD_CommonPile|s2|shuf2": { + "per_k": { + "1": { + "rho_bar": 0.16398069262504578, + "union_observed": 0.054931335151195526, + "union_independent": 0.11052674055099487, + "union_gap": -0.05559540539979935, + "mean_per_layer_hit_rate": 0.008931144140660763 + }, + "2": { + "rho_bar": 0.2515166699886322, + "union_observed": 0.08114856481552124, + "union_independent": 0.19031327962875366, + "union_gap": -0.10916471481323242, + "mean_per_layer_hit_rate": 0.01603764481842518 + }, + "5": { + "rho_bar": 0.2895846664905548, + "union_observed": 0.1473158597946167, + "union_independent": 0.34142202138900757, + "union_gap": -0.19410616159439087, + "mean_per_layer_hit_rate": 0.03140305355191231 + }, + "10": { + "rho_bar": 0.3194611370563507, + "union_observed": 0.21598002314567566, + "union_independent": 0.5007278323173523, + "union_gap": -0.28474780917167664, + "mean_per_layer_hit_rate": 0.05157015100121498 + }, + "20": { + "rho_bar": 0.365191787481308, + "union_observed": 0.28589263558387756, + "union_independent": 0.6757470369338989, + "union_gap": -0.38985440135002136, + "mean_per_layer_hit_rate": 0.08191683888435364 + }, + "50": { + "rho_bar": 0.4018927812576294, + "union_observed": 0.3907615542411804, + "union_independent": 0.8289302587509155, + "union_gap": -0.4381687045097351, + "mean_per_layer_hit_rate": 0.12493997067213058 + }, + "100": { + "rho_bar": 0.43322744965553284, + "union_observed": 0.4594257175922394, + "union_independent": 0.9055771231651306, + "union_gap": -0.44615140557289124, + "mean_per_layer_hit_rate": 0.16287332773208618 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.31783645493643625, + "union_gap_mean_over_k": -0.2739698003445353 + }, + "OOD_CommonPile|s2|shuf3": { + "per_k": { + "1": { + "rho_bar": 0.15640364587306976, + "union_observed": 0.04619225859642029, + "union_independent": 0.08647239208221436, + "union_gap": -0.04028013348579407, + "mean_per_layer_hit_rate": 0.006914434023201466 + }, + "2": { + "rho_bar": 0.18440814316272736, + "union_observed": 0.07990012317895889, + "union_independent": 0.17554926872253418, + "union_gap": -0.09564914554357529, + "mean_per_layer_hit_rate": 0.014693171717226505 + }, + "5": { + "rho_bar": 0.2412773221731186, + "union_observed": 0.13233457505702972, + "union_independent": 0.3098410964012146, + "union_gap": -0.17750652134418488, + "mean_per_layer_hit_rate": 0.02794583886861801 + }, + "10": { + "rho_bar": 0.30785468220710754, + "union_observed": 0.18976280093193054, + "union_independent": 0.47034770250320435, + "union_gap": -0.2805849015712738, + "mean_per_layer_hit_rate": 0.04724862799048424 + }, + "20": { + "rho_bar": 0.34150978922843933, + "union_observed": 0.25717851519584656, + "union_independent": 0.6323174238204956, + "union_gap": -0.37513890862464905, + "mean_per_layer_hit_rate": 0.0729856938123703 + }, + "50": { + "rho_bar": 0.38839414715766907, + "union_observed": 0.34706616401672363, + "union_independent": 0.7909684777259827, + "union_gap": -0.44390231370925903, + "mean_per_layer_hit_rate": 0.1114952489733696 + }, + "100": { + "rho_bar": 0.42460325360298157, + "union_observed": 0.44319599866867065, + "union_independent": 0.8890475034713745, + "union_gap": -0.44585150480270386, + "mean_per_layer_hit_rate": 0.1525016874074936 + } + }, + "n_degenerate_pairs": 23, + "rho_bar_mean_over_k": 0.2920644262007305, + "union_gap_mean_over_k": -0.2655590612973486 + }, + "OOD_CommonPile|s2|shuf4": { + "per_k": { + "1": { + "rho_bar": 0.1306002289056778, + "union_observed": 0.048689138144254684, + "union_independent": 0.08997398614883423, + "union_gap": -0.041284848004579544, + "mean_per_layer_hit_rate": 0.007202535402029753 + }, + "2": { + "rho_bar": 0.23350070416927338, + "union_observed": 0.07865168899297714, + "union_independent": 0.17552030086517334, + "union_gap": -0.0968686118721962, + "mean_per_layer_hit_rate": 0.014693171717226505 + }, + "5": { + "rho_bar": 0.2879793345928192, + "union_observed": 0.1423220932483673, + "union_independent": 0.335973858833313, + "union_gap": -0.19365176558494568, + "mean_per_layer_hit_rate": 0.03082684800028801 + }, + "10": { + "rho_bar": 0.3223574757575989, + "union_observed": 0.19475655257701874, + "union_independent": 0.49004387855529785, + "union_gap": -0.2952873259782791, + "mean_per_layer_hit_rate": 0.05003361403942108 + }, + "20": { + "rho_bar": 0.3882887363433838, + "union_observed": 0.25717851519584656, + "union_independent": 0.649753212928772, + "union_gap": -0.3925746977329254, + "mean_per_layer_hit_rate": 0.0766349732875824 + }, + "50": { + "rho_bar": 0.44001153111457825, + "union_observed": 0.3533083498477936, + "union_independent": 0.81256103515625, + "union_gap": -0.4592526853084564, + "mean_per_layer_hit_rate": 0.11869777739048004 + }, + "100": { + "rho_bar": 0.4771496653556824, + "union_observed": 0.45318353176116943, + "union_independent": 0.8984131217002869, + "union_gap": -0.44522958993911743, + "mean_per_layer_hit_rate": 0.15797559916973114 + } + }, + "n_degenerate_pairs": 0, + "rho_bar_mean_over_k": 0.32569823946271625, + "union_gap_mean_over_k": -0.27487850348864284 + }, + "OOD_CommonPile|s2|bandconst": { + "per_k": { + "1": { + "rho_bar": 0.2656002640724182, + "union_observed": 0.041198503226041794, + "union_independent": 0.11821842193603516, + "union_gap": -0.07701991870999336, + "mean_per_layer_hit_rate": 0.009603380225598812 + }, + "2": { + "rho_bar": 0.3352871835231781, + "union_observed": 0.06491885334253311, + "union_independent": 0.22271084785461426, + "union_gap": -0.15779199451208115, + "mean_per_layer_hit_rate": 0.019110726192593575 + }, + "5": { + "rho_bar": 0.3737242817878723, + "union_observed": 0.11735330522060394, + "union_independent": 0.4013864994049072, + "union_gap": -0.2840331941843033, + "mean_per_layer_hit_rate": 0.03841352090239525 + }, + "10": { + "rho_bar": 0.4317055642604828, + "union_observed": 0.17727839946746826, + "union_independent": 0.589261531829834, + "union_gap": -0.4119831323623657, + "mean_per_layer_hit_rate": 0.06539902091026306 + }, + "20": { + "rho_bar": 0.5024529099464417, + "union_observed": 0.24094881117343903, + "union_independent": 0.7460492253303528, + "union_gap": -0.5051004141569138, + "mean_per_layer_hit_rate": 0.09843464940786362 + }, + "50": { + "rho_bar": 0.5352194309234619, + "union_observed": 0.33458176255226135, + "union_independent": 0.8723506331443787, + "union_gap": -0.5377688705921173, + "mean_per_layer_hit_rate": 0.14318640530109406 + }, + "100": { + "rho_bar": 0.5430091619491577, + "union_observed": 0.4232209622859955, + "union_independent": 0.9362653493881226, + "union_gap": -0.5130443871021271, + "mean_per_layer_hit_rate": 0.1860174834728241 + } + }, + "n_degenerate_pairs": 12, + "rho_bar_mean_over_k": 0.4267141137804304, + "union_gap_mean_over_k": -0.3552488445171288 + } + }, + "per_layer_admitted_mean": { + "Wikipedia_en|s0|real": [ + 0.030625, + 0.041137, + 0.040028, + 0.078451, + 0.092194, + 0.105163, + 0.104392, + 0.112584, + 0.140501, + 0.136119, + 0.146992, + 0.146694, + 0.149346 + ], + "Wikipedia_en|s0|cyc1": [ + 0.039948, + 0.040236, + 0.055406, + 0.06907, + 0.099101, + 0.090474, + 0.091317, + 0.100877, + 0.128892, + 0.137235, + 0.149768, + 0.152652, + 0.056356 + ], + "Wikipedia_en|s0|cyc2": [ + 0.03913, + 0.046293, + 0.037172, + 0.083906, + 0.088956, + 0.082646, + 0.090289, + 0.082848, + 0.127011, + 0.145377, + 0.158985, + 0.051142, + 0.071245 + ], + "Wikipedia_en|s0|cyc3": [ + 0.050305, + 0.041145, + 0.038888, + 0.072233, + 0.074357, + 0.084657, + 0.075349, + 0.079019, + 0.137228, + 0.154731, + 0.075269, + 0.061923, + 0.088716 + ], + "Wikipedia_en|s0|cyc4": [ + 0.044384, + 0.036016, + 0.031884, + 0.060007, + 0.083032, + 0.071605, + 0.071617, + 0.094854, + 0.140855, + 0.059534, + 0.09223, + 0.082908, + 0.104693 + ], + "Wikipedia_en|s0|cyc5": [ + 0.03341, + 0.038266, + 0.018542, + 0.058135, + 0.072188, + 0.066566, + 0.085443, + 0.101841, + 0.059172, + 0.088298, + 0.107122, + 0.111199, + 0.108445 + ], + "Wikipedia_en|s0|cyc6": [ + 0.029841, + 0.02474, + 0.01802, + 0.036246, + 0.067729, + 0.072056, + 0.092631, + 0.05699, + 0.081881, + 0.103308, + 0.126309, + 0.118691, + 0.117303 + ], + "Wikipedia_en|s0|cyc7": [ + 0.016134, + 0.021235, + 0.012394, + 0.029715, + 0.072561, + 0.081096, + 0.068875, + 0.071408, + 0.095721, + 0.1227, + 0.132729, + 0.12102, + 0.112835 + ], + "Wikipedia_en|s0|cyc8": [ + 0.012574, + 0.016785, + 0.008013, + 0.031006, + 0.079268, + 0.069305, + 0.086109, + 0.080134, + 0.111602, + 0.126264, + 0.145507, + 0.116904, + 0.109104 + ], + "Wikipedia_en|s0|cyc9": [ + 0.013456, + 0.01006, + 0.007849, + 0.036429, + 0.060545, + 0.076597, + 0.087795, + 0.097764, + 0.116147, + 0.13844, + 0.141077, + 0.114685, + 0.111073 + ], + "Wikipedia_en|s0|cyc10": [ + 0.006922, + 0.012046, + 0.009272, + 0.05161, + 0.071751, + 0.082253, + 0.090784, + 0.102655, + 0.136933, + 0.145359, + 0.143036, + 0.125647, + 0.101452 + ], + "Wikipedia_en|s0|cyc11": [ + 0.007466, + 0.014157, + 0.030155, + 0.062697, + 0.071035, + 0.088265, + 0.104455, + 0.119671, + 0.150842, + 0.141244, + 0.14208, + 0.115482, + 0.121237 + ], + "Wikipedia_en|s0|cyc12": [ + 0.009479, + 0.030273, + 0.045077, + 0.062225, + 0.082538, + 0.096196, + 0.11561, + 0.111385, + 0.137929, + 0.141694, + 0.134159, + 0.134202, + 0.131233 + ], + "Wikipedia_en|s0|shuf0": [ + 0.029841, + 0.030273, + 0.037172, + 0.058135, + 0.079268, + 0.082253, + 0.085443, + 0.119671, + 0.127011, + 0.141244, + 0.09223, + 0.115482, + 0.104693 + ], + "Wikipedia_en|s0|shuf1": [ + 0.044384, + 0.01006, + 0.045077, + 0.031006, + 0.099101, + 0.084657, + 0.087795, + 0.05699, + 0.111602, + 0.154731, + 0.141077, + 0.114685, + 0.101452 + ], + "Wikipedia_en|s0|shuf2": [ + 0.012574, + 0.012046, + 0.030155, + 0.062697, + 0.099101, + 0.066566, + 0.075349, + 0.111385, + 0.137929, + 0.126264, + 0.158985, + 0.111199, + 0.088716 + ], + "Wikipedia_en|s0|shuf3": [ + 0.03913, + 0.012046, + 0.012394, + 0.072233, + 0.067729, + 0.076597, + 0.092631, + 0.102655, + 0.137929, + 0.059534, + 0.145507, + 0.125647, + 0.104693 + ], + "Wikipedia_en|s0|shuf4": [ + 0.03913, + 0.041145, + 0.055406, + 0.031006, + 0.072188, + 0.090474, + 0.086109, + 0.079019, + 0.136933, + 0.141244, + 0.158985, + 0.051142, + 0.111073 + ], + "Wikipedia_en|s0|bandconst": [ + 0.025611, + 0.034936, + 0.028056, + 0.074172, + 0.093913, + 0.098819, + 0.111494, + 0.114174, + 0.156527, + 0.160587, + 0.155541, + 0.144599, + 0.128409 + ], + "Wikipedia_en|s1|real": [ + 0.03216, + 0.043661, + 0.039724, + 0.078241, + 0.090403, + 0.10575, + 0.10241, + 0.113642, + 0.142356, + 0.13942, + 0.14837, + 0.149159, + 0.148141 + ], + "Wikipedia_en|s1|cyc1": [ + 0.039475, + 0.039898, + 0.054905, + 0.069631, + 0.096768, + 0.089626, + 0.09042, + 0.101281, + 0.128527, + 0.138265, + 0.151275, + 0.153802, + 0.056608 + ], + "Wikipedia_en|s1|cyc2": [ + 0.039151, + 0.044419, + 0.036912, + 0.07931, + 0.084824, + 0.083623, + 0.092098, + 0.083637, + 0.129107, + 0.145612, + 0.158942, + 0.052277, + 0.072421 + ], + "Wikipedia_en|s1|cyc3": [ + 0.049786, + 0.041186, + 0.038331, + 0.069521, + 0.074958, + 0.085806, + 0.076112, + 0.079528, + 0.137584, + 0.15523, + 0.07821, + 0.064802, + 0.087566 + ], + "Wikipedia_en|s1|cyc4": [ + 0.041898, + 0.034939, + 0.031215, + 0.059968, + 0.082741, + 0.07209, + 0.07347, + 0.095135, + 0.141039, + 0.06099, + 0.095657, + 0.082359, + 0.103664 + ], + "Wikipedia_en|s1|cyc5": [ + 0.031992, + 0.037525, + 0.018424, + 0.05684, + 0.072486, + 0.067797, + 0.085962, + 0.102399, + 0.062138, + 0.088956, + 0.110038, + 0.111095, + 0.10784 + ], + "Wikipedia_en|s1|cyc6": [ + 0.0291, + 0.025, + 0.01787, + 0.035352, + 0.069228, + 0.072873, + 0.093108, + 0.059826, + 0.082842, + 0.10209, + 0.125008, + 0.118871, + 0.116886 + ], + "Wikipedia_en|s1|cyc7": [ + 0.016397, + 0.022131, + 0.012912, + 0.030306, + 0.073046, + 0.080691, + 0.07165, + 0.072633, + 0.096004, + 0.121955, + 0.133771, + 0.120717, + 0.112805 + ], + "Wikipedia_en|s1|cyc8": [ + 0.012496, + 0.016303, + 0.007826, + 0.030414, + 0.079341, + 0.068415, + 0.085619, + 0.080797, + 0.112254, + 0.124379, + 0.144681, + 0.114537, + 0.11058 + ], + "Wikipedia_en|s1|cyc9": [ + 0.012601, + 0.010319, + 0.007849, + 0.035871, + 0.063147, + 0.076849, + 0.089513, + 0.096808, + 0.116405, + 0.135762, + 0.140834, + 0.112843, + 0.110188 + ], + "Wikipedia_en|s1|cyc10": [ + 0.00637, + 0.011307, + 0.008974, + 0.052088, + 0.072562, + 0.080748, + 0.090524, + 0.101698, + 0.137729, + 0.148173, + 0.145056, + 0.125684, + 0.102256 + ], + "Wikipedia_en|s1|cyc11": [ + 0.007168, + 0.014382, + 0.030503, + 0.06423, + 0.073651, + 0.088234, + 0.103612, + 0.116302, + 0.151523, + 0.14342, + 0.141562, + 0.117313, + 0.122215 + ], + "Wikipedia_en|s1|cyc12": [ + 0.009479, + 0.029637, + 0.047818, + 0.063331, + 0.084044, + 0.095008, + 0.113197, + 0.110883, + 0.137171, + 0.144455, + 0.136858, + 0.137343, + 0.13242 + ], + "Wikipedia_en|s1|shuf0": [ + 0.0291, + 0.029637, + 0.036912, + 0.05684, + 0.079341, + 0.080748, + 0.085962, + 0.116302, + 0.129107, + 0.14342, + 0.095657, + 0.117313, + 0.103664 + ], + "Wikipedia_en|s1|shuf1": [ + 0.041898, + 0.010319, + 0.047818, + 0.030414, + 0.096768, + 0.085806, + 0.089513, + 0.059826, + 0.112254, + 0.15523, + 0.140834, + 0.112843, + 0.102256 + ], + "Wikipedia_en|s1|shuf2": [ + 0.012496, + 0.011307, + 0.030503, + 0.06423, + 0.096768, + 0.067797, + 0.076112, + 0.110883, + 0.137171, + 0.124379, + 0.158942, + 0.111095, + 0.087566 + ], + "Wikipedia_en|s1|shuf3": [ + 0.039151, + 0.011307, + 0.012912, + 0.069521, + 0.069228, + 0.076849, + 0.093108, + 0.101698, + 0.137171, + 0.06099, + 0.144681, + 0.125684, + 0.103664 + ], + "Wikipedia_en|s1|shuf4": [ + 0.039151, + 0.041186, + 0.054905, + 0.030414, + 0.072486, + 0.089626, + 0.085619, + 0.079528, + 0.137729, + 0.14342, + 0.158942, + 0.052277, + 0.110188 + ], + "Wikipedia_en|s1|bandconst": [ + 0.027324, + 0.034974, + 0.028873, + 0.072392, + 0.093879, + 0.099566, + 0.11119, + 0.114461, + 0.155512, + 0.160318, + 0.157376, + 0.14281, + 0.128272 + ], + "Wikipedia_en|s2|real": [ + 0.030415, + 0.040078, + 0.036462, + 0.07475, + 0.089796, + 0.104385, + 0.103784, + 0.111544, + 0.142918, + 0.136862, + 0.147657, + 0.147946, + 0.149273 + ], + "Wikipedia_en|s2|cyc1": [ + 0.038188, + 0.035666, + 0.0503, + 0.068125, + 0.097142, + 0.091607, + 0.093624, + 0.100314, + 0.128673, + 0.138956, + 0.15029, + 0.154009, + 0.057001 + ], + "Wikipedia_en|s2|cyc2": [ + 0.034341, + 0.042055, + 0.033422, + 0.081238, + 0.088583, + 0.0837, + 0.092225, + 0.083765, + 0.127308, + 0.146094, + 0.159964, + 0.051215, + 0.072924 + ], + "Wikipedia_en|s2|cyc3": [ + 0.047708, + 0.038341, + 0.037403, + 0.070339, + 0.074436, + 0.08728, + 0.076499, + 0.078971, + 0.138149, + 0.155856, + 0.077869, + 0.064157, + 0.08826 + ], + "Wikipedia_en|s2|cyc4": [ + 0.04301, + 0.036307, + 0.031624, + 0.060344, + 0.08426, + 0.07227, + 0.073134, + 0.096451, + 0.141132, + 0.060999, + 0.094749, + 0.078495, + 0.104656 + ], + "Wikipedia_en|s2|cyc5": [ + 0.031472, + 0.038044, + 0.017822, + 0.0585, + 0.073005, + 0.066831, + 0.086055, + 0.103473, + 0.064485, + 0.090616, + 0.108373, + 0.10853, + 0.108431 + ], + "Wikipedia_en|s2|cyc6": [ + 0.02903, + 0.02526, + 0.017535, + 0.03628, + 0.06858, + 0.072129, + 0.092814, + 0.059054, + 0.08335, + 0.103064, + 0.12544, + 0.115453, + 0.117719 + ], + "Wikipedia_en|s2|cyc7": [ + 0.016767, + 0.021565, + 0.011982, + 0.030383, + 0.072489, + 0.080432, + 0.070558, + 0.071393, + 0.09559, + 0.121282, + 0.133171, + 0.119524, + 0.113957 + ], + "Wikipedia_en|s2|cyc8": [ + 0.012568, + 0.016228, + 0.008605, + 0.030674, + 0.079334, + 0.071747, + 0.087273, + 0.077448, + 0.111315, + 0.125757, + 0.145215, + 0.117544, + 0.110459 + ], + "Wikipedia_en|s2|cyc9": [ + 0.012971, + 0.009987, + 0.008108, + 0.036041, + 0.061843, + 0.076978, + 0.087288, + 0.096241, + 0.118769, + 0.136199, + 0.141115, + 0.115869, + 0.111785 + ], + "Wikipedia_en|s2|cyc10": [ + 0.00663, + 0.011567, + 0.008974, + 0.051545, + 0.070876, + 0.079909, + 0.090267, + 0.100999, + 0.136174, + 0.147836, + 0.143818, + 0.127031, + 0.102575 + ], + "Wikipedia_en|s2|cyc11": [ + 0.007836, + 0.014382, + 0.029075, + 0.062079, + 0.069371, + 0.087936, + 0.102524, + 0.116608, + 0.150085, + 0.139936, + 0.143026, + 0.117831, + 0.121129 + ], + "Wikipedia_en|s2|cyc12": [ + 0.009479, + 0.02899, + 0.045335, + 0.059223, + 0.082524, + 0.094687, + 0.113103, + 0.111905, + 0.136726, + 0.14315, + 0.136989, + 0.135424, + 0.131032 + ], + "Wikipedia_en|s2|shuf0": [ + 0.02903, + 0.02899, + 0.033422, + 0.0585, + 0.079334, + 0.079909, + 0.086055, + 0.116608, + 0.127308, + 0.139936, + 0.094749, + 0.117831, + 0.104656 + ], + "Wikipedia_en|s2|shuf1": [ + 0.04301, + 0.009987, + 0.045335, + 0.030674, + 0.097142, + 0.08728, + 0.087288, + 0.059054, + 0.111315, + 0.155856, + 0.141115, + 0.115869, + 0.102575 + ], + "Wikipedia_en|s2|shuf2": [ + 0.012568, + 0.011567, + 0.029075, + 0.062079, + 0.097142, + 0.066831, + 0.076499, + 0.111905, + 0.136726, + 0.125757, + 0.159964, + 0.10853, + 0.08826 + ], + "Wikipedia_en|s2|shuf3": [ + 0.034341, + 0.011567, + 0.011982, + 0.070339, + 0.06858, + 0.076978, + 0.092814, + 0.100999, + 0.136726, + 0.060999, + 0.145215, + 0.127031, + 0.104656 + ], + "Wikipedia_en|s2|shuf4": [ + 0.034341, + 0.038341, + 0.0503, + 0.030674, + 0.073005, + 0.091607, + 0.087273, + 0.078971, + 0.136174, + 0.139936, + 0.159964, + 0.051215, + 0.111785 + ], + "Wikipedia_en|s2|bandconst": [ + 0.026504, + 0.034423, + 0.027426, + 0.072354, + 0.094283, + 0.098809, + 0.110825, + 0.114144, + 0.154508, + 0.160953, + 0.157242, + 0.143593, + 0.127694 + ], + "USPTO_Backgrounds|s0|real": [ + 0.045554, + 0.06603, + 0.060205, + 0.087298, + 0.110665, + 0.122178, + 0.116951, + 0.126378, + 0.146504, + 0.147942, + 0.146355, + 0.149303, + 0.150726 + ], + "USPTO_Backgrounds|s0|cyc1": [ + 0.060233, + 0.058705, + 0.065658, + 0.091268, + 0.106517, + 0.10779, + 0.11571, + 0.11045, + 0.137264, + 0.138271, + 0.154242, + 0.155277, + 0.056102 + ], + "USPTO_Backgrounds|s0|cyc2": [ + 0.057685, + 0.059872, + 0.055231, + 0.083948, + 0.095438, + 0.104095, + 0.104012, + 0.092216, + 0.130554, + 0.146624, + 0.15979, + 0.048848, + 0.069361 + ], + "USPTO_Backgrounds|s0|cyc3": [ + 0.056221, + 0.05738, + 0.042731, + 0.080843, + 0.089083, + 0.095288, + 0.087414, + 0.079474, + 0.141907, + 0.157887, + 0.081502, + 0.056041, + 0.09061 + ], + "USPTO_Backgrounds|s0|cyc4": [ + 0.049486, + 0.05172, + 0.041273, + 0.072419, + 0.085365, + 0.082418, + 0.077778, + 0.099202, + 0.144281, + 0.060424, + 0.099529, + 0.07771, + 0.099511 + ], + "USPTO_Backgrounds|s0|cyc5": [ + 0.037944, + 0.054899, + 0.030739, + 0.066242, + 0.076755, + 0.070084, + 0.088078, + 0.103933, + 0.053053, + 0.087603, + 0.120009, + 0.10266, + 0.115169 + ], + "USPTO_Backgrounds|s0|cyc6": [ + 0.041693, + 0.033767, + 0.0235, + 0.044912, + 0.068784, + 0.0751, + 0.096845, + 0.063754, + 0.075604, + 0.102637, + 0.116717, + 0.123011, + 0.119082 + ], + "USPTO_Backgrounds|s0|cyc7": [ + 0.028039, + 0.031687, + 0.017302, + 0.029934, + 0.074203, + 0.08097, + 0.082275, + 0.068032, + 0.089047, + 0.111852, + 0.143159, + 0.132522, + 0.125759 + ], + "USPTO_Backgrounds|s0|cyc8": [ + 0.019611, + 0.02304, + 0.009153, + 0.035116, + 0.080027, + 0.084225, + 0.088228, + 0.073033, + 0.10139, + 0.137368, + 0.148767, + 0.129955, + 0.118269 + ], + "USPTO_Backgrounds|s0|cyc9": [ + 0.018492, + 0.011937, + 0.008178, + 0.041941, + 0.074499, + 0.087172, + 0.089937, + 0.096112, + 0.115212, + 0.141856, + 0.148048, + 0.121082, + 0.115117 + ], + "USPTO_Backgrounds|s0|cyc10": [ + 0.008171, + 0.012401, + 0.009925, + 0.06526, + 0.079704, + 0.09034, + 0.109355, + 0.109371, + 0.14103, + 0.149853, + 0.144257, + 0.126913, + 0.11348 + ], + "USPTO_Backgrounds|s0|cyc11": [ + 0.008645, + 0.016631, + 0.047561, + 0.072087, + 0.081189, + 0.107238, + 0.118493, + 0.123366, + 0.15315, + 0.140899, + 0.146649, + 0.125792, + 0.126398 + ], + "USPTO_Backgrounds|s0|cyc12": [ + 0.010132, + 0.04842, + 0.065008, + 0.074526, + 0.098655, + 0.116587, + 0.12388, + 0.121163, + 0.142243, + 0.145608, + 0.143171, + 0.135777, + 0.134616 + ], + "USPTO_Backgrounds|s0|shuf0": [ + 0.041693, + 0.04842, + 0.055231, + 0.066242, + 0.080027, + 0.09034, + 0.088078, + 0.123366, + 0.130554, + 0.140899, + 0.099529, + 0.125792, + 0.099511 + ], + "USPTO_Backgrounds|s0|shuf1": [ + 0.049486, + 0.011937, + 0.065008, + 0.035116, + 0.106517, + 0.095288, + 0.089937, + 0.063754, + 0.10139, + 0.157887, + 0.148048, + 0.121082, + 0.11348 + ], + "USPTO_Backgrounds|s0|shuf2": [ + 0.019611, + 0.012401, + 0.047561, + 0.072087, + 0.106517, + 0.070084, + 0.087414, + 0.121163, + 0.142243, + 0.137368, + 0.15979, + 0.10266, + 0.09061 + ], + "USPTO_Backgrounds|s0|shuf3": [ + 0.057685, + 0.012401, + 0.017302, + 0.080843, + 0.068784, + 0.087172, + 0.096845, + 0.109371, + 0.142243, + 0.060424, + 0.148767, + 0.126913, + 0.099511 + ], + "USPTO_Backgrounds|s0|shuf4": [ + 0.057685, + 0.05738, + 0.065658, + 0.035116, + 0.076755, + 0.10779, + 0.088228, + 0.079474, + 0.14103, + 0.140899, + 0.15979, + 0.048848, + 0.115117 + ], + "USPTO_Backgrounds|s0|bandconst": [ + 0.033752, + 0.041746, + 0.036534, + 0.080631, + 0.1006, + 0.111698, + 0.121065, + 0.11772, + 0.155844, + 0.16528, + 0.161381, + 0.150656, + 0.135905 + ], + "USPTO_Backgrounds|s1|real": [ + 0.042512, + 0.064916, + 0.058676, + 0.087269, + 0.109459, + 0.123265, + 0.116178, + 0.126613, + 0.149536, + 0.147639, + 0.1472, + 0.150022, + 0.150487 + ], + "USPTO_Backgrounds|s1|cyc1": [ + 0.059755, + 0.058706, + 0.064856, + 0.089158, + 0.104397, + 0.103389, + 0.117534, + 0.111594, + 0.138009, + 0.139804, + 0.153542, + 0.157722, + 0.057715 + ], + "USPTO_Backgrounds|s1|cyc2": [ + 0.056467, + 0.0582, + 0.053486, + 0.081943, + 0.094269, + 0.102903, + 0.103759, + 0.093389, + 0.132116, + 0.146831, + 0.161327, + 0.05157, + 0.069663 + ], + "USPTO_Backgrounds|s1|cyc3": [ + 0.05529, + 0.05593, + 0.042323, + 0.079657, + 0.089965, + 0.096419, + 0.088681, + 0.079584, + 0.142211, + 0.158249, + 0.082644, + 0.056653, + 0.092525 + ], + "USPTO_Backgrounds|s1|cyc4": [ + 0.04824, + 0.050567, + 0.040778, + 0.074111, + 0.085209, + 0.083771, + 0.079665, + 0.097923, + 0.145142, + 0.061076, + 0.098892, + 0.079238, + 0.099398 + ], + "USPTO_Backgrounds|s1|cyc5": [ + 0.03743, + 0.056717, + 0.030007, + 0.065611, + 0.075841, + 0.070939, + 0.087333, + 0.104124, + 0.054486, + 0.086382, + 0.120225, + 0.104623, + 0.117261 + ], + "USPTO_Backgrounds|s1|cyc6": [ + 0.040145, + 0.032981, + 0.023278, + 0.04424, + 0.06834, + 0.074401, + 0.096111, + 0.062024, + 0.078203, + 0.101471, + 0.11654, + 0.122599, + 0.120181 + ], + "USPTO_Backgrounds|s1|cyc7": [ + 0.027669, + 0.031656, + 0.016751, + 0.02975, + 0.073943, + 0.081042, + 0.080644, + 0.068455, + 0.087676, + 0.110972, + 0.141339, + 0.1322, + 0.128046 + ], + "USPTO_Backgrounds|s1|cyc8": [ + 0.018689, + 0.023487, + 0.009191, + 0.034375, + 0.079969, + 0.080325, + 0.086366, + 0.071825, + 0.101877, + 0.136308, + 0.149999, + 0.128226, + 0.118338 + ], + "USPTO_Backgrounds|s1|cyc9": [ + 0.01879, + 0.012235, + 0.008105, + 0.040865, + 0.070925, + 0.086627, + 0.089088, + 0.094879, + 0.115663, + 0.141466, + 0.149125, + 0.123801, + 0.11651 + ], + "USPTO_Backgrounds|s1|cyc10": [ + 0.007905, + 0.012182, + 0.01033, + 0.062014, + 0.077839, + 0.088614, + 0.1054, + 0.10926, + 0.141808, + 0.148467, + 0.147066, + 0.127871, + 0.11283 + ], + "USPTO_Backgrounds|s1|cyc11": [ + 0.008639, + 0.016371, + 0.045887, + 0.070748, + 0.077654, + 0.10577, + 0.118055, + 0.123961, + 0.15573, + 0.142847, + 0.14563, + 0.125124, + 0.127181 + ], + "USPTO_Backgrounds|s1|cyc12": [ + 0.009872, + 0.046573, + 0.064639, + 0.072055, + 0.096278, + 0.113679, + 0.122506, + 0.120424, + 0.14256, + 0.147779, + 0.144702, + 0.137298, + 0.136474 + ], + "USPTO_Backgrounds|s1|shuf0": [ + 0.040145, + 0.046573, + 0.053486, + 0.065611, + 0.079969, + 0.088614, + 0.087333, + 0.123961, + 0.132116, + 0.142847, + 0.098892, + 0.125124, + 0.099398 + ], + "USPTO_Backgrounds|s1|shuf1": [ + 0.04824, + 0.012235, + 0.064639, + 0.034375, + 0.104397, + 0.096419, + 0.089088, + 0.062024, + 0.101877, + 0.158249, + 0.149125, + 0.123801, + 0.11283 + ], + "USPTO_Backgrounds|s1|shuf2": [ + 0.018689, + 0.012182, + 0.045887, + 0.070748, + 0.104397, + 0.070939, + 0.088681, + 0.120424, + 0.14256, + 0.136308, + 0.161327, + 0.104623, + 0.092525 + ], + "USPTO_Backgrounds|s1|shuf3": [ + 0.056467, + 0.012182, + 0.016751, + 0.079657, + 0.06834, + 0.086627, + 0.096111, + 0.10926, + 0.14256, + 0.061076, + 0.149999, + 0.127871, + 0.099398 + ], + "USPTO_Backgrounds|s1|shuf4": [ + 0.056467, + 0.05593, + 0.064856, + 0.034375, + 0.075841, + 0.103389, + 0.086366, + 0.079584, + 0.141808, + 0.142847, + 0.161327, + 0.05157, + 0.11651 + ], + "USPTO_Backgrounds|s1|bandconst": [ + 0.033157, + 0.041746, + 0.035755, + 0.080521, + 0.100874, + 0.111103, + 0.122857, + 0.117901, + 0.155596, + 0.164374, + 0.160586, + 0.151431, + 0.136845 + ], + "USPTO_Backgrounds|s2|real": [ + 0.045336, + 0.066735, + 0.061115, + 0.087937, + 0.112734, + 0.124214, + 0.115762, + 0.128375, + 0.147788, + 0.147164, + 0.146199, + 0.150458, + 0.150855 + ], + "USPTO_Backgrounds|s2|cyc1": [ + 0.060131, + 0.059782, + 0.066516, + 0.091303, + 0.107982, + 0.105421, + 0.117315, + 0.112487, + 0.139299, + 0.138355, + 0.154924, + 0.156942, + 0.056811 + ], + "USPTO_Backgrounds|s2|cyc2": [ + 0.058917, + 0.060581, + 0.057443, + 0.082673, + 0.095124, + 0.103642, + 0.105069, + 0.093902, + 0.130571, + 0.146696, + 0.161882, + 0.048091, + 0.071421 + ], + "USPTO_Backgrounds|s2|cyc3": [ + 0.056037, + 0.059073, + 0.043998, + 0.080846, + 0.090103, + 0.0978, + 0.090921, + 0.079571, + 0.143153, + 0.158176, + 0.078506, + 0.055103, + 0.089925 + ], + "USPTO_Backgrounds|s2|cyc4": [ + 0.049281, + 0.051975, + 0.041764, + 0.076708, + 0.085687, + 0.084333, + 0.079295, + 0.097618, + 0.144527, + 0.061062, + 0.100076, + 0.076945, + 0.099123 + ], + "USPTO_Backgrounds|s2|cyc5": [ + 0.038767, + 0.055512, + 0.032148, + 0.067578, + 0.076883, + 0.071271, + 0.087818, + 0.103196, + 0.052297, + 0.088127, + 0.119411, + 0.105153, + 0.115731 + ], + "USPTO_Backgrounds|s2|cyc6": [ + 0.042187, + 0.033507, + 0.024538, + 0.045129, + 0.067966, + 0.07527, + 0.096515, + 0.062417, + 0.07633, + 0.100858, + 0.117926, + 0.125884, + 0.11914 + ], + "USPTO_Backgrounds|s2|cyc7": [ + 0.029376, + 0.033435, + 0.01771, + 0.030235, + 0.073866, + 0.081451, + 0.081918, + 0.06951, + 0.087479, + 0.112543, + 0.144408, + 0.132194, + 0.126737 + ], + "USPTO_Backgrounds|s2|cyc8": [ + 0.020393, + 0.023525, + 0.008893, + 0.034894, + 0.080007, + 0.084207, + 0.087204, + 0.07322, + 0.101235, + 0.138472, + 0.148984, + 0.130282, + 0.119517 + ], + "USPTO_Backgrounds|s2|cyc9": [ + 0.019755, + 0.011677, + 0.008178, + 0.042014, + 0.070422, + 0.088109, + 0.091219, + 0.096442, + 0.115349, + 0.140778, + 0.148888, + 0.121954, + 0.117006 + ], + "USPTO_Backgrounds|s2|cyc10": [ + 0.008317, + 0.012069, + 0.01007, + 0.063953, + 0.080223, + 0.092583, + 0.10983, + 0.110716, + 0.140892, + 0.147997, + 0.146991, + 0.127408, + 0.113612 + ], + "USPTO_Backgrounds|s2|cyc11": [ + 0.008639, + 0.016929, + 0.046753, + 0.074839, + 0.080849, + 0.106884, + 0.123335, + 0.124225, + 0.154001, + 0.143189, + 0.14714, + 0.127519, + 0.12648 + ], + "USPTO_Backgrounds|s2|cyc12": [ + 0.009872, + 0.049666, + 0.065962, + 0.074193, + 0.097229, + 0.11849, + 0.123644, + 0.11783, + 0.143395, + 0.147371, + 0.144168, + 0.137864, + 0.13498 + ], + "USPTO_Backgrounds|s2|shuf0": [ + 0.042187, + 0.049666, + 0.057443, + 0.067578, + 0.080007, + 0.092583, + 0.087818, + 0.124225, + 0.130571, + 0.143189, + 0.100076, + 0.127519, + 0.099123 + ], + "USPTO_Backgrounds|s2|shuf1": [ + 0.049281, + 0.011677, + 0.065962, + 0.034894, + 0.107982, + 0.0978, + 0.091219, + 0.062417, + 0.101235, + 0.158176, + 0.148888, + 0.121954, + 0.113612 + ], + "USPTO_Backgrounds|s2|shuf2": [ + 0.020393, + 0.012069, + 0.046753, + 0.074839, + 0.107982, + 0.071271, + 0.090921, + 0.11783, + 0.143395, + 0.138472, + 0.161882, + 0.105153, + 0.089925 + ], + "USPTO_Backgrounds|s2|shuf3": [ + 0.058917, + 0.012069, + 0.01771, + 0.080846, + 0.067966, + 0.088109, + 0.096515, + 0.110716, + 0.143395, + 0.061062, + 0.148984, + 0.127408, + 0.099123 + ], + "USPTO_Backgrounds|s2|shuf4": [ + 0.058917, + 0.059073, + 0.066516, + 0.034894, + 0.076883, + 0.105421, + 0.087204, + 0.079571, + 0.140892, + 0.143189, + 0.161882, + 0.048091, + 0.117006 + ], + "USPTO_Backgrounds|s2|bandconst": [ + 0.035164, + 0.044048, + 0.036942, + 0.081749, + 0.100618, + 0.111772, + 0.123621, + 0.11849, + 0.157264, + 0.164064, + 0.160857, + 0.151474, + 0.136311 + ], + "Pile-CC|s0|real": [ + 0.043383, + 0.054278, + 0.054987, + 0.082634, + 0.104972, + 0.109047, + 0.103725, + 0.115066, + 0.158457, + 0.158497, + 0.158238, + 0.154234, + 0.151105 + ], + "Pile-CC|s0|cyc1": [ + 0.047614, + 0.050189, + 0.050407, + 0.073282, + 0.0948, + 0.088522, + 0.102289, + 0.103615, + 0.149009, + 0.150199, + 0.15928, + 0.154365, + 0.057334 + ], + "Pile-CC|s0|cyc2": [ + 0.043873, + 0.049701, + 0.037294, + 0.063563, + 0.080567, + 0.089894, + 0.095474, + 0.094741, + 0.138194, + 0.153741, + 0.161903, + 0.060701, + 0.074673 + ], + "Pile-CC|s0|cyc3": [ + 0.046344, + 0.040175, + 0.035569, + 0.052972, + 0.075866, + 0.089016, + 0.086443, + 0.083422, + 0.14371, + 0.155596, + 0.085106, + 0.070085, + 0.094626 + ], + "Pile-CC|s0|cyc4": [ + 0.038152, + 0.037729, + 0.031198, + 0.053455, + 0.077798, + 0.083449, + 0.077414, + 0.100923, + 0.139334, + 0.054183, + 0.093071, + 0.093894, + 0.10419 + ], + "Pile-CC|s0|cyc5": [ + 0.033187, + 0.034749, + 0.021029, + 0.051562, + 0.072436, + 0.071073, + 0.088917, + 0.103386, + 0.068689, + 0.08399, + 0.12169, + 0.113032, + 0.114013 + ], + "Pile-CC|s0|cyc6": [ + 0.03213, + 0.024089, + 0.016689, + 0.037762, + 0.069291, + 0.075075, + 0.094471, + 0.073454, + 0.090991, + 0.110706, + 0.126664, + 0.128799, + 0.120413 + ], + "Pile-CC|s0|cyc7": [ + 0.022333, + 0.020213, + 0.01432, + 0.027667, + 0.076768, + 0.081894, + 0.092038, + 0.083491, + 0.111941, + 0.121386, + 0.146708, + 0.134865, + 0.123867 + ], + "Pile-CC|s0|cyc8": [ + 0.01481, + 0.01583, + 0.007753, + 0.029499, + 0.07975, + 0.086692, + 0.09494, + 0.096858, + 0.115713, + 0.135128, + 0.154654, + 0.133102, + 0.11432 + ], + "Pile-CC|s0|cyc9": [ + 0.014943, + 0.009761, + 0.008224, + 0.034483, + 0.078478, + 0.086658, + 0.106964, + 0.101791, + 0.124868, + 0.155198, + 0.153976, + 0.124389, + 0.116811 + ], + "Pile-CC|s0|cyc10": [ + 0.007732, + 0.011342, + 0.008081, + 0.073888, + 0.081603, + 0.100796, + 0.098902, + 0.109104, + 0.151489, + 0.157037, + 0.151075, + 0.134461, + 0.113792 + ], + "Pile-CC|s0|cyc11": [ + 0.008168, + 0.012224, + 0.048497, + 0.082874, + 0.090218, + 0.100353, + 0.111688, + 0.117109, + 0.167141, + 0.147205, + 0.155174, + 0.134172, + 0.134743 + ], + "Pile-CC|s0|cyc12": [ + 0.008402, + 0.043213, + 0.062219, + 0.087278, + 0.094697, + 0.110059, + 0.11132, + 0.108805, + 0.15069, + 0.156875, + 0.148642, + 0.14583, + 0.140875 + ], + "Pile-CC|s0|shuf0": [ + 0.03213, + 0.043213, + 0.037294, + 0.051562, + 0.07975, + 0.100796, + 0.088917, + 0.117109, + 0.138194, + 0.147205, + 0.093071, + 0.134172, + 0.10419 + ], + "Pile-CC|s0|shuf1": [ + 0.038152, + 0.009761, + 0.062219, + 0.029499, + 0.0948, + 0.089016, + 0.106964, + 0.073454, + 0.115713, + 0.155596, + 0.153976, + 0.124389, + 0.113792 + ], + "Pile-CC|s0|shuf2": [ + 0.01481, + 0.011342, + 0.048497, + 0.082874, + 0.0948, + 0.071073, + 0.086443, + 0.108805, + 0.15069, + 0.135128, + 0.161903, + 0.113032, + 0.094626 + ], + "Pile-CC|s0|shuf3": [ + 0.043873, + 0.011342, + 0.01432, + 0.052972, + 0.069291, + 0.086658, + 0.094471, + 0.109104, + 0.15069, + 0.054183, + 0.154654, + 0.134461, + 0.10419 + ], + "Pile-CC|s0|shuf4": [ + 0.043873, + 0.040175, + 0.050407, + 0.029499, + 0.072436, + 0.088522, + 0.09494, + 0.083422, + 0.151489, + 0.147205, + 0.161903, + 0.060701, + 0.116811 + ], + "Pile-CC|s0|bandconst": [ + 0.027304, + 0.030839, + 0.026024, + 0.07042, + 0.096593, + 0.104024, + 0.119398, + 0.117442, + 0.165372, + 0.170617, + 0.163119, + 0.155888, + 0.136558 + ], + "Pile-CC|s1|real": [ + 0.043162, + 0.054225, + 0.055653, + 0.087024, + 0.106119, + 0.110478, + 0.106386, + 0.11517, + 0.160888, + 0.158067, + 0.158055, + 0.15396, + 0.151012 + ], + "Pile-CC|s1|cyc1": [ + 0.048831, + 0.052623, + 0.053784, + 0.076957, + 0.096999, + 0.090674, + 0.102181, + 0.104625, + 0.150476, + 0.148353, + 0.159555, + 0.154296, + 0.057913 + ], + "Pile-CC|s1|cyc2": [ + 0.046451, + 0.052032, + 0.041007, + 0.066634, + 0.083585, + 0.092565, + 0.095345, + 0.096043, + 0.138227, + 0.154489, + 0.161758, + 0.058612, + 0.073855 + ], + "Pile-CC|s1|cyc3": [ + 0.051012, + 0.043221, + 0.035959, + 0.05764, + 0.07802, + 0.089473, + 0.085113, + 0.083457, + 0.14325, + 0.155225, + 0.081206, + 0.071538, + 0.094869 + ], + "Pile-CC|s1|cyc4": [ + 0.038709, + 0.038596, + 0.031813, + 0.056094, + 0.079412, + 0.083155, + 0.076763, + 0.100684, + 0.139172, + 0.053962, + 0.092198, + 0.094962, + 0.105247 + ], + "Pile-CC|s1|cyc5": [ + 0.034378, + 0.035379, + 0.022106, + 0.052414, + 0.074032, + 0.070574, + 0.08838, + 0.103736, + 0.067173, + 0.084849, + 0.12151, + 0.114722, + 0.115136 + ], + "Pile-CC|s1|cyc6": [ + 0.032422, + 0.025172, + 0.017136, + 0.039656, + 0.068768, + 0.076242, + 0.094794, + 0.070925, + 0.088875, + 0.108589, + 0.127759, + 0.129816, + 0.120399 + ], + "Pile-CC|s1|cyc7": [ + 0.022035, + 0.019837, + 0.014872, + 0.027774, + 0.076678, + 0.082316, + 0.090432, + 0.080534, + 0.113924, + 0.120086, + 0.147142, + 0.137321, + 0.12543 + ], + "Pile-CC|s1|cyc8": [ + 0.01481, + 0.016979, + 0.007975, + 0.029389, + 0.079937, + 0.086305, + 0.096802, + 0.097147, + 0.11709, + 0.137055, + 0.157522, + 0.135801, + 0.115912 + ], + "Pile-CC|s1|cyc9": [ + 0.01487, + 0.010312, + 0.007947, + 0.034372, + 0.077379, + 0.089109, + 0.108758, + 0.101026, + 0.127165, + 0.155815, + 0.156438, + 0.126806, + 0.117345 + ], + "Pile-CC|s1|cyc10": [ + 0.007434, + 0.011712, + 0.008379, + 0.075559, + 0.082536, + 0.103367, + 0.102395, + 0.114787, + 0.150334, + 0.161248, + 0.152415, + 0.135622, + 0.114762 + ], + "Pile-CC|s1|cyc11": [ + 0.007871, + 0.012152, + 0.048337, + 0.083243, + 0.088253, + 0.103539, + 0.116826, + 0.116724, + 0.169074, + 0.148992, + 0.154914, + 0.135729, + 0.13471 + ], + "Pile-CC|s1|cyc12": [ + 0.008105, + 0.04519, + 0.061795, + 0.08876, + 0.09589, + 0.111808, + 0.114315, + 0.110533, + 0.153004, + 0.158076, + 0.148864, + 0.146725, + 0.140599 + ], + "Pile-CC|s1|shuf0": [ + 0.032422, + 0.04519, + 0.041007, + 0.052414, + 0.079937, + 0.103367, + 0.08838, + 0.116724, + 0.138227, + 0.148992, + 0.092198, + 0.135729, + 0.105247 + ], + "Pile-CC|s1|shuf1": [ + 0.038709, + 0.010312, + 0.061795, + 0.029389, + 0.096999, + 0.089473, + 0.108758, + 0.070925, + 0.11709, + 0.155225, + 0.156438, + 0.126806, + 0.114762 + ], + "Pile-CC|s1|shuf2": [ + 0.01481, + 0.011712, + 0.048337, + 0.083243, + 0.096999, + 0.070574, + 0.085113, + 0.110533, + 0.153004, + 0.137055, + 0.161758, + 0.114722, + 0.094869 + ], + "Pile-CC|s1|shuf3": [ + 0.046451, + 0.011712, + 0.014872, + 0.05764, + 0.068768, + 0.089109, + 0.094794, + 0.114787, + 0.153004, + 0.053962, + 0.157522, + 0.135622, + 0.105247 + ], + "Pile-CC|s1|shuf4": [ + 0.046451, + 0.043221, + 0.053784, + 0.029389, + 0.074032, + 0.090674, + 0.096802, + 0.083457, + 0.150334, + 0.148992, + 0.161758, + 0.058612, + 0.117345 + ], + "Pile-CC|s1|bandconst": [ + 0.028861, + 0.032181, + 0.026885, + 0.073093, + 0.09713, + 0.104886, + 0.119551, + 0.118092, + 0.166484, + 0.171097, + 0.163431, + 0.154991, + 0.139302 + ], + "Pile-CC|s2|real": [ + 0.041971, + 0.054091, + 0.054931, + 0.082894, + 0.104447, + 0.107676, + 0.10527, + 0.115703, + 0.156624, + 0.155328, + 0.157018, + 0.154907, + 0.150662 + ], + "Pile-CC|s2|cyc1": [ + 0.04751, + 0.050469, + 0.049628, + 0.074206, + 0.093863, + 0.088096, + 0.099437, + 0.103497, + 0.147773, + 0.147372, + 0.1583, + 0.15426, + 0.059122 + ], + "Pile-CC|s2|cyc2": [ + 0.04465, + 0.046657, + 0.037198, + 0.066302, + 0.080591, + 0.091132, + 0.092863, + 0.094935, + 0.138414, + 0.152885, + 0.162128, + 0.061619, + 0.07177 + ], + "Pile-CC|s2|cyc3": [ + 0.047446, + 0.040668, + 0.036347, + 0.056027, + 0.07605, + 0.089071, + 0.084331, + 0.08322, + 0.141976, + 0.155173, + 0.083572, + 0.071258, + 0.092884 + ], + "Pile-CC|s2|cyc4": [ + 0.038665, + 0.037108, + 0.030924, + 0.055488, + 0.078426, + 0.080876, + 0.07545, + 0.099756, + 0.140411, + 0.051015, + 0.092388, + 0.094768, + 0.102557 + ], + "Pile-CC|s2|cyc5": [ + 0.033211, + 0.035087, + 0.021607, + 0.050487, + 0.071806, + 0.069591, + 0.088802, + 0.103754, + 0.066176, + 0.081633, + 0.120451, + 0.112033, + 0.114287 + ], + "Pile-CC|s2|cyc6": [ + 0.031541, + 0.023534, + 0.016582, + 0.038434, + 0.068176, + 0.074827, + 0.095126, + 0.070542, + 0.087828, + 0.110173, + 0.128031, + 0.127542, + 0.121645 + ], + "Pile-CC|s2|cyc7": [ + 0.021585, + 0.020545, + 0.014248, + 0.02748, + 0.075487, + 0.081967, + 0.08816, + 0.080598, + 0.112789, + 0.11716, + 0.146467, + 0.134695, + 0.123818 + ], + "Pile-CC|s2|cyc8": [ + 0.014512, + 0.01646, + 0.007528, + 0.029129, + 0.079978, + 0.085491, + 0.09269, + 0.096384, + 0.113181, + 0.135262, + 0.154602, + 0.132369, + 0.115617 + ], + "Pile-CC|s2|cyc9": [ + 0.014645, + 0.010021, + 0.008019, + 0.034909, + 0.075399, + 0.086193, + 0.106635, + 0.098956, + 0.124571, + 0.156619, + 0.154031, + 0.123749, + 0.117062 + ], + "Pile-CC|s2|cyc10": [ + 0.007434, + 0.011415, + 0.008081, + 0.072394, + 0.080622, + 0.099199, + 0.099175, + 0.111264, + 0.151124, + 0.161122, + 0.149854, + 0.133666, + 0.113196 + ], + "Pile-CC|s2|cyc11": [ + 0.008241, + 0.012484, + 0.045252, + 0.08073, + 0.087579, + 0.099518, + 0.112615, + 0.117774, + 0.170265, + 0.147161, + 0.153312, + 0.132327, + 0.134375 + ], + "Pile-CC|s2|cyc12": [ + 0.008402, + 0.042262, + 0.060541, + 0.087205, + 0.093339, + 0.107819, + 0.113475, + 0.108409, + 0.152012, + 0.155744, + 0.147785, + 0.144575, + 0.14108 + ], + "Pile-CC|s2|shuf0": [ + 0.031541, + 0.042262, + 0.037198, + 0.050487, + 0.079978, + 0.099199, + 0.088802, + 0.117774, + 0.138414, + 0.147161, + 0.092388, + 0.132327, + 0.102557 + ], + "Pile-CC|s2|shuf1": [ + 0.038665, + 0.010021, + 0.060541, + 0.029129, + 0.093863, + 0.089071, + 0.106635, + 0.070542, + 0.113181, + 0.155173, + 0.154031, + 0.123749, + 0.113196 + ], + "Pile-CC|s2|shuf2": [ + 0.014512, + 0.011415, + 0.045252, + 0.08073, + 0.093863, + 0.069591, + 0.084331, + 0.108409, + 0.152012, + 0.135262, + 0.162128, + 0.112033, + 0.092884 + ], + "Pile-CC|s2|shuf3": [ + 0.04465, + 0.011415, + 0.014248, + 0.056027, + 0.068176, + 0.086193, + 0.095126, + 0.111264, + 0.152012, + 0.051015, + 0.154602, + 0.133666, + 0.102557 + ], + "Pile-CC|s2|shuf4": [ + 0.04465, + 0.040668, + 0.049628, + 0.029129, + 0.071806, + 0.088096, + 0.09269, + 0.08322, + 0.151124, + 0.147161, + 0.162128, + 0.061619, + 0.117062 + ], + "Pile-CC|s2|bandconst": [ + 0.027315, + 0.031956, + 0.025695, + 0.070864, + 0.095476, + 0.101753, + 0.117674, + 0.115552, + 0.166234, + 0.169268, + 0.161277, + 0.153625, + 0.137448 + ], + "StackExchange|s0|real": [ + 0.050373, + 0.057417, + 0.068171, + 0.092493, + 0.093034, + 0.105191, + 0.110804, + 0.126573, + 0.160572, + 0.168212, + 0.158293, + 0.158145, + 0.151805 + ], + "StackExchange|s0|cyc1": [ + 0.043845, + 0.063843, + 0.069743, + 0.076203, + 0.091272, + 0.097779, + 0.115048, + 0.114458, + 0.154535, + 0.155107, + 0.161404, + 0.158673, + 0.041334 + ], + "StackExchange|s0|cyc2": [ + 0.049617, + 0.067121, + 0.044474, + 0.06709, + 0.090035, + 0.099575, + 0.105682, + 0.1048, + 0.145902, + 0.157074, + 0.164336, + 0.029011, + 0.052304 + ], + "StackExchange|s0|cyc3": [ + 0.057395, + 0.046782, + 0.033459, + 0.07135, + 0.087804, + 0.096645, + 0.101316, + 0.091624, + 0.148863, + 0.15985, + 0.06141, + 0.029358, + 0.085693 + ], + "StackExchange|s0|cyc4": [ + 0.040416, + 0.037078, + 0.03522, + 0.072714, + 0.089086, + 0.092604, + 0.083611, + 0.106334, + 0.143888, + 0.037379, + 0.076233, + 0.074237, + 0.095666 + ], + "StackExchange|s0|cyc5": [ + 0.033662, + 0.03703, + 0.029129, + 0.072396, + 0.085418, + 0.076737, + 0.094242, + 0.105136, + 0.05888, + 0.046857, + 0.109698, + 0.100889, + 0.103393 + ], + "StackExchange|s0|cyc6": [ + 0.033575, + 0.034353, + 0.023144, + 0.051606, + 0.074966, + 0.082936, + 0.095677, + 0.066956, + 0.067736, + 0.090301, + 0.111469, + 0.103628, + 0.105015 + ], + "StackExchange|s0|cyc7": [ + 0.028377, + 0.02682, + 0.016733, + 0.038726, + 0.07936, + 0.08395, + 0.090195, + 0.070593, + 0.104449, + 0.113481, + 0.119832, + 0.109877, + 0.112636 + ], + "StackExchange|s0|cyc8": [ + 0.022012, + 0.021248, + 0.008605, + 0.039966, + 0.078524, + 0.088342, + 0.085395, + 0.089665, + 0.11416, + 0.112457, + 0.133199, + 0.122242, + 0.112685 + ], + "StackExchange|s0|cyc9": [ + 0.019937, + 0.013354, + 0.008963, + 0.040034, + 0.08327, + 0.084191, + 0.105985, + 0.099822, + 0.106643, + 0.134419, + 0.143331, + 0.135706, + 0.116457 + ], + "StackExchange|s0|cyc10": [ + 0.011183, + 0.013525, + 0.009413, + 0.079169, + 0.079726, + 0.098149, + 0.103961, + 0.091981, + 0.135641, + 0.146441, + 0.154798, + 0.135915, + 0.117879 + ], + "StackExchange|s0|cyc11": [ + 0.010103, + 0.01508, + 0.06773, + 0.078859, + 0.091011, + 0.099637, + 0.100531, + 0.110808, + 0.156143, + 0.14598, + 0.158715, + 0.140263, + 0.135642 + ], + "StackExchange|s0|cyc12": [ + 0.010929, + 0.055822, + 0.061598, + 0.091081, + 0.092208, + 0.096041, + 0.108301, + 0.111196, + 0.151699, + 0.159944, + 0.157256, + 0.152852, + 0.145348 + ], + "StackExchange|s0|shuf0": [ + 0.033575, + 0.055822, + 0.044474, + 0.072396, + 0.078524, + 0.098149, + 0.094242, + 0.110808, + 0.145902, + 0.14598, + 0.076233, + 0.140263, + 0.095666 + ], + "StackExchange|s0|shuf1": [ + 0.040416, + 0.013354, + 0.061598, + 0.039966, + 0.091272, + 0.096645, + 0.105985, + 0.066956, + 0.11416, + 0.15985, + 0.143331, + 0.135706, + 0.117879 + ], + "StackExchange|s0|shuf2": [ + 0.022012, + 0.013525, + 0.06773, + 0.078859, + 0.091272, + 0.076737, + 0.101316, + 0.111196, + 0.151699, + 0.112457, + 0.164336, + 0.100889, + 0.085693 + ], + "StackExchange|s0|shuf3": [ + 0.049617, + 0.013525, + 0.016733, + 0.07135, + 0.074966, + 0.084191, + 0.095677, + 0.091981, + 0.151699, + 0.037379, + 0.133199, + 0.135915, + 0.095666 + ], + "StackExchange|s0|shuf4": [ + 0.049617, + 0.046782, + 0.069743, + 0.039966, + 0.085418, + 0.097779, + 0.085395, + 0.091624, + 0.135641, + 0.14598, + 0.164336, + 0.029011, + 0.116457 + ], + "StackExchange|s0|bandconst": [ + 0.033543, + 0.038868, + 0.036682, + 0.08402, + 0.097718, + 0.106808, + 0.119124, + 0.120905, + 0.15974, + 0.165404, + 0.161088, + 0.149908, + 0.134506 + ], + "StackExchange|s1|real": [ + 0.049403, + 0.056255, + 0.065253, + 0.09198, + 0.091076, + 0.104369, + 0.11078, + 0.126027, + 0.159668, + 0.167597, + 0.158761, + 0.156205, + 0.151917 + ], + "StackExchange|s1|cyc1": [ + 0.041315, + 0.063208, + 0.071512, + 0.073169, + 0.092641, + 0.096542, + 0.114618, + 0.116738, + 0.154703, + 0.154387, + 0.160457, + 0.157084, + 0.036144 + ], + "StackExchange|s1|cyc2": [ + 0.048867, + 0.068951, + 0.044816, + 0.068295, + 0.089564, + 0.098362, + 0.10692, + 0.104101, + 0.144275, + 0.156391, + 0.164708, + 0.026785, + 0.049475 + ], + "StackExchange|s1|cyc3": [ + 0.061268, + 0.046199, + 0.032954, + 0.072121, + 0.087075, + 0.095483, + 0.102723, + 0.091572, + 0.146844, + 0.158861, + 0.058622, + 0.02872, + 0.083981 + ], + "StackExchange|s1|cyc4": [ + 0.040691, + 0.037743, + 0.034132, + 0.072453, + 0.088328, + 0.092748, + 0.08492, + 0.104684, + 0.143369, + 0.034895, + 0.072445, + 0.069547, + 0.09528 + ], + "StackExchange|s1|cyc5": [ + 0.032954, + 0.036508, + 0.028344, + 0.074388, + 0.086453, + 0.077201, + 0.09288, + 0.105396, + 0.058498, + 0.04283, + 0.104512, + 0.099997, + 0.100787 + ], + "StackExchange|s1|cyc6": [ + 0.032069, + 0.034133, + 0.022729, + 0.052458, + 0.075561, + 0.081696, + 0.095715, + 0.063808, + 0.064097, + 0.082901, + 0.111448, + 0.102073, + 0.104953 + ], + "StackExchange|s1|cyc7": [ + 0.027668, + 0.026073, + 0.01658, + 0.039139, + 0.07933, + 0.082771, + 0.086939, + 0.065809, + 0.098634, + 0.112055, + 0.118599, + 0.111554, + 0.112307 + ], + "StackExchange|s1|cyc8": [ + 0.021491, + 0.020763, + 0.008975, + 0.039183, + 0.079078, + 0.085107, + 0.084051, + 0.085129, + 0.114036, + 0.1094, + 0.130746, + 0.121219, + 0.112224 + ], + "StackExchange|s1|cyc9": [ + 0.017926, + 0.013095, + 0.009036, + 0.040256, + 0.08067, + 0.082286, + 0.103811, + 0.099234, + 0.106436, + 0.133602, + 0.141128, + 0.133954, + 0.116338 + ], + "StackExchange|s1|cyc10": [ + 0.011443, + 0.013338, + 0.009268, + 0.077274, + 0.077961, + 0.096621, + 0.105113, + 0.08737, + 0.134384, + 0.146439, + 0.153116, + 0.134267, + 0.115376 + ], + "StackExchange|s1|cyc11": [ + 0.010103, + 0.015267, + 0.065436, + 0.074425, + 0.08692, + 0.09849, + 0.09802, + 0.110722, + 0.153716, + 0.147575, + 0.15555, + 0.139738, + 0.132995 + ], + "StackExchange|s1|cyc12": [ + 0.01071, + 0.055111, + 0.060517, + 0.087555, + 0.09253, + 0.09501, + 0.109136, + 0.108634, + 0.150684, + 0.157163, + 0.157874, + 0.151258, + 0.143219 + ], + "StackExchange|s1|shuf0": [ + 0.032069, + 0.055111, + 0.044816, + 0.074388, + 0.079078, + 0.096621, + 0.09288, + 0.110722, + 0.144275, + 0.147575, + 0.072445, + 0.139738, + 0.09528 + ], + "StackExchange|s1|shuf1": [ + 0.040691, + 0.013095, + 0.060517, + 0.039183, + 0.092641, + 0.095483, + 0.103811, + 0.063808, + 0.114036, + 0.158861, + 0.141128, + 0.133954, + 0.115376 + ], + "StackExchange|s1|shuf2": [ + 0.021491, + 0.013338, + 0.065436, + 0.074425, + 0.092641, + 0.077201, + 0.102723, + 0.108634, + 0.150684, + 0.1094, + 0.164708, + 0.099997, + 0.083981 + ], + "StackExchange|s1|shuf3": [ + 0.048867, + 0.013338, + 0.01658, + 0.072121, + 0.075561, + 0.082286, + 0.095715, + 0.08737, + 0.150684, + 0.034895, + 0.130746, + 0.134267, + 0.09528 + ], + "StackExchange|s1|shuf4": [ + 0.048867, + 0.046199, + 0.071512, + 0.039183, + 0.086453, + 0.096542, + 0.084051, + 0.091572, + 0.134384, + 0.147575, + 0.164708, + 0.026785, + 0.116338 + ], + "StackExchange|s1|bandconst": [ + 0.032163, + 0.038789, + 0.034893, + 0.084163, + 0.096391, + 0.104875, + 0.118207, + 0.119152, + 0.159118, + 0.16299, + 0.159132, + 0.149766, + 0.132813 + ], + "StackExchange|s2|real": [ + 0.047182, + 0.056247, + 0.065664, + 0.091764, + 0.092421, + 0.102849, + 0.11237, + 0.125618, + 0.15941, + 0.167394, + 0.157778, + 0.157295, + 0.152706 + ], + "StackExchange|s2|cyc1": [ + 0.042172, + 0.061912, + 0.07061, + 0.074535, + 0.092086, + 0.096764, + 0.113991, + 0.114459, + 0.154748, + 0.155839, + 0.159717, + 0.157596, + 0.036749 + ], + "StackExchange|s2|cyc2": [ + 0.048467, + 0.069047, + 0.042928, + 0.067234, + 0.090142, + 0.098753, + 0.10621, + 0.104658, + 0.144592, + 0.156261, + 0.163986, + 0.026362, + 0.05017 + ], + "StackExchange|s2|cyc3": [ + 0.060938, + 0.046611, + 0.031843, + 0.072369, + 0.086857, + 0.09562, + 0.102283, + 0.091587, + 0.146695, + 0.159705, + 0.056987, + 0.028998, + 0.084487 + ], + "StackExchange|s2|cyc4": [ + 0.03943, + 0.036666, + 0.033283, + 0.073419, + 0.088671, + 0.092201, + 0.084668, + 0.105312, + 0.143975, + 0.034434, + 0.07143, + 0.073316, + 0.095557 + ], + "StackExchange|s2|cyc5": [ + 0.032362, + 0.035801, + 0.028905, + 0.072948, + 0.086121, + 0.076626, + 0.093882, + 0.105511, + 0.057582, + 0.043802, + 0.107605, + 0.100333, + 0.101853 + ], + "StackExchange|s2|cyc6": [ + 0.031891, + 0.034095, + 0.023214, + 0.052165, + 0.075634, + 0.082161, + 0.095882, + 0.064705, + 0.06474, + 0.083473, + 0.111619, + 0.102845, + 0.102781 + ], + "StackExchange|s2|cyc7": [ + 0.027408, + 0.026259, + 0.016508, + 0.039731, + 0.079807, + 0.083216, + 0.087785, + 0.067978, + 0.097372, + 0.110373, + 0.119033, + 0.10978, + 0.111744 + ], + "StackExchange|s2|cyc8": [ + 0.021529, + 0.021173, + 0.008975, + 0.04015, + 0.079116, + 0.082855, + 0.084525, + 0.088025, + 0.112452, + 0.110512, + 0.130791, + 0.121568, + 0.1116 + ], + "StackExchange|s2|cyc9": [ + 0.01874, + 0.01332, + 0.008963, + 0.040515, + 0.079333, + 0.08268, + 0.102667, + 0.099347, + 0.106324, + 0.131843, + 0.142526, + 0.133491, + 0.115718 + ], + "StackExchange|s2|cyc10": [ + 0.011475, + 0.013598, + 0.009341, + 0.073603, + 0.077891, + 0.097758, + 0.105376, + 0.088358, + 0.132738, + 0.143809, + 0.153538, + 0.135233, + 0.116201 + ], + "StackExchange|s2|cyc11": [ + 0.009812, + 0.015267, + 0.060191, + 0.073132, + 0.089526, + 0.099074, + 0.099508, + 0.108095, + 0.155466, + 0.145017, + 0.156455, + 0.139368, + 0.133124 + ], + "StackExchange|s2|cyc12": [ + 0.010638, + 0.051629, + 0.057868, + 0.08728, + 0.092679, + 0.094746, + 0.104927, + 0.111089, + 0.150222, + 0.157012, + 0.157911, + 0.15283, + 0.144293 + ], + "StackExchange|s2|shuf0": [ + 0.031891, + 0.051629, + 0.042928, + 0.072948, + 0.079116, + 0.097758, + 0.093882, + 0.108095, + 0.144592, + 0.145017, + 0.07143, + 0.139368, + 0.095557 + ], + "StackExchange|s2|shuf1": [ + 0.03943, + 0.01332, + 0.057868, + 0.04015, + 0.092086, + 0.09562, + 0.102667, + 0.064705, + 0.112452, + 0.159705, + 0.142526, + 0.133491, + 0.116201 + ], + "StackExchange|s2|shuf2": [ + 0.021529, + 0.013598, + 0.060191, + 0.073132, + 0.092086, + 0.076626, + 0.102283, + 0.111089, + 0.150222, + 0.110512, + 0.163986, + 0.100333, + 0.084487 + ], + "StackExchange|s2|shuf3": [ + 0.048467, + 0.013598, + 0.016508, + 0.072369, + 0.075634, + 0.08268, + 0.095882, + 0.088358, + 0.150222, + 0.034434, + 0.130791, + 0.135233, + 0.095557 + ], + "StackExchange|s2|shuf4": [ + 0.048467, + 0.046611, + 0.07061, + 0.04015, + 0.086121, + 0.096764, + 0.084525, + 0.091587, + 0.132738, + 0.145017, + 0.163986, + 0.026362, + 0.115718 + ], + "StackExchange|s2|bandconst": [ + 0.032017, + 0.038567, + 0.035526, + 0.083796, + 0.0961, + 0.104608, + 0.117713, + 0.11977, + 0.160143, + 0.164877, + 0.160176, + 0.148759, + 0.133249 + ], + "Github|s0|real": [ + 0.061661, + 0.063693, + 0.063466, + 0.084312, + 0.085188, + 0.094214, + 0.104046, + 0.098932, + 0.109859, + 0.12134, + 0.135163, + 0.13676, + 0.141728 + ], + "Github|s0|cyc1": [ + 0.060177, + 0.069636, + 0.07183, + 0.06997, + 0.088528, + 0.093862, + 0.097853, + 0.08574, + 0.1241, + 0.118249, + 0.148013, + 0.144774, + 0.053036 + ], + "Github|s0|cyc2": [ + 0.055617, + 0.0783, + 0.050226, + 0.083174, + 0.089481, + 0.088854, + 0.085369, + 0.083175, + 0.119189, + 0.135741, + 0.154774, + 0.04315, + 0.056829 + ], + "Github|s0|cyc3": [ + 0.061477, + 0.062165, + 0.047744, + 0.082292, + 0.085153, + 0.081245, + 0.084847, + 0.079537, + 0.135043, + 0.145741, + 0.061383, + 0.046253, + 0.08079 + ], + "Github|s0|cyc4": [ + 0.050671, + 0.062551, + 0.050076, + 0.065163, + 0.080884, + 0.081572, + 0.080213, + 0.098849, + 0.139725, + 0.03466, + 0.082234, + 0.074614, + 0.091091 + ], + "Github|s0|cyc5": [ + 0.044277, + 0.067496, + 0.036574, + 0.055086, + 0.085087, + 0.076504, + 0.0929, + 0.099619, + 0.050535, + 0.058061, + 0.098944, + 0.08962, + 0.100245 + ], + "Github|s0|cyc6": [ + 0.04482, + 0.044426, + 0.022792, + 0.043475, + 0.077932, + 0.080727, + 0.090706, + 0.055484, + 0.070138, + 0.078174, + 0.110354, + 0.091403, + 0.096327 + ], + "Github|s0|cyc7": [ + 0.025992, + 0.027892, + 0.014703, + 0.04242, + 0.08385, + 0.080507, + 0.079553, + 0.071334, + 0.085204, + 0.102456, + 0.11512, + 0.103204, + 0.104079 + ], + "Github|s0|cyc8": [ + 0.016784, + 0.02379, + 0.013974, + 0.041076, + 0.080755, + 0.076087, + 0.087268, + 0.075361, + 0.103334, + 0.095528, + 0.116944, + 0.111506, + 0.095323 + ], + "Github|s0|cyc9": [ + 0.017146, + 0.018221, + 0.011748, + 0.037281, + 0.079971, + 0.08533, + 0.08714, + 0.08883, + 0.100945, + 0.104316, + 0.122761, + 0.093056, + 0.08677 + ], + "Github|s0|cyc10": [ + 0.012166, + 0.020665, + 0.009338, + 0.069722, + 0.085928, + 0.086998, + 0.090629, + 0.084142, + 0.110486, + 0.115468, + 0.120968, + 0.08395, + 0.090437 + ], + "Github|s0|cyc11": [ + 0.012349, + 0.018691, + 0.058876, + 0.075999, + 0.085947, + 0.092623, + 0.091617, + 0.095738, + 0.123175, + 0.107652, + 0.118654, + 0.095317, + 0.100818 + ], + "Github|s0|cyc12": [ + 0.010346, + 0.05834, + 0.060771, + 0.076097, + 0.089828, + 0.08986, + 0.094848, + 0.103658, + 0.119238, + 0.094698, + 0.130227, + 0.10946, + 0.122392 + ], + "Github|s0|shuf0": [ + 0.04482, + 0.05834, + 0.050226, + 0.055086, + 0.080755, + 0.086998, + 0.0929, + 0.095738, + 0.119189, + 0.107652, + 0.082234, + 0.095317, + 0.091091 + ], + "Github|s0|shuf1": [ + 0.050671, + 0.018221, + 0.060771, + 0.041076, + 0.088528, + 0.081245, + 0.08714, + 0.055484, + 0.103334, + 0.145741, + 0.122761, + 0.093056, + 0.090437 + ], + "Github|s0|shuf2": [ + 0.016784, + 0.020665, + 0.058876, + 0.075999, + 0.088528, + 0.076504, + 0.084847, + 0.103658, + 0.119238, + 0.095528, + 0.154774, + 0.08962, + 0.08079 + ], + "Github|s0|shuf3": [ + 0.055617, + 0.020665, + 0.014703, + 0.082292, + 0.077932, + 0.08533, + 0.090706, + 0.084142, + 0.119238, + 0.03466, + 0.116944, + 0.08395, + 0.091091 + ], + "Github|s0|shuf4": [ + 0.055617, + 0.062165, + 0.07183, + 0.041076, + 0.085087, + 0.093862, + 0.087268, + 0.079537, + 0.110486, + 0.107652, + 0.154774, + 0.04315, + 0.08677 + ], + "Github|s0|bandconst": [ + 0.039763, + 0.053889, + 0.044466, + 0.078159, + 0.091546, + 0.094876, + 0.106034, + 0.108664, + 0.139519, + 0.140653, + 0.146455, + 0.134479, + 0.12165 + ], + "Github|s1|real": [ + 0.059157, + 0.062318, + 0.06397, + 0.085101, + 0.085395, + 0.096753, + 0.106207, + 0.104574, + 0.120782, + 0.134149, + 0.139493, + 0.140874, + 0.143885 + ], + "Github|s1|cyc1": [ + 0.058507, + 0.066873, + 0.074476, + 0.070641, + 0.088044, + 0.0938, + 0.102226, + 0.09365, + 0.128406, + 0.125643, + 0.14981, + 0.148029, + 0.053521 + ], + "Github|s1|cyc2": [ + 0.054361, + 0.078819, + 0.047134, + 0.082836, + 0.092586, + 0.091218, + 0.094474, + 0.08872, + 0.122846, + 0.140678, + 0.157614, + 0.04428, + 0.05961 + ], + "Github|s1|cyc3": [ + 0.06059, + 0.06232, + 0.044896, + 0.078858, + 0.086448, + 0.085921, + 0.08894, + 0.082487, + 0.138346, + 0.147393, + 0.061123, + 0.051587, + 0.081458 + ], + "Github|s1|cyc4": [ + 0.045927, + 0.057065, + 0.048402, + 0.068792, + 0.08467, + 0.085666, + 0.082015, + 0.100601, + 0.14046, + 0.035791, + 0.084385, + 0.078547, + 0.092542 + ], + "Github|s1|cyc5": [ + 0.038624, + 0.064932, + 0.035679, + 0.062573, + 0.08635, + 0.077649, + 0.093391, + 0.102117, + 0.050389, + 0.059824, + 0.101162, + 0.093827, + 0.101373 + ], + "Github|s1|cyc6": [ + 0.041706, + 0.045613, + 0.026158, + 0.048889, + 0.079379, + 0.082232, + 0.091759, + 0.053585, + 0.071616, + 0.081069, + 0.110289, + 0.094688, + 0.09689 + ], + "Github|s1|cyc7": [ + 0.025653, + 0.030454, + 0.016855, + 0.043459, + 0.084663, + 0.082165, + 0.079732, + 0.070848, + 0.090595, + 0.105418, + 0.114463, + 0.105883, + 0.103071 + ], + "Github|s1|cyc8": [ + 0.018301, + 0.02461, + 0.013489, + 0.042485, + 0.081641, + 0.077634, + 0.087803, + 0.078031, + 0.107409, + 0.099417, + 0.117481, + 0.110903, + 0.097657 + ], + "Github|s1|cyc9": [ + 0.017921, + 0.018293, + 0.011561, + 0.038704, + 0.080473, + 0.086721, + 0.090991, + 0.088583, + 0.104158, + 0.108394, + 0.123878, + 0.102525, + 0.088935 + ], + "Github|s1|cyc10": [ + 0.011906, + 0.019733, + 0.009933, + 0.067515, + 0.085636, + 0.088686, + 0.095228, + 0.087104, + 0.111162, + 0.118063, + 0.128339, + 0.09349, + 0.096033 + ], + "Github|s1|cyc11": [ + 0.013011, + 0.018431, + 0.056996, + 0.074478, + 0.087061, + 0.093928, + 0.094011, + 0.094767, + 0.127857, + 0.115343, + 0.12754, + 0.105353, + 0.104571 + ], + "Github|s1|cyc12": [ + 0.010049, + 0.05665, + 0.057562, + 0.076484, + 0.09126, + 0.092658, + 0.097541, + 0.10277, + 0.12754, + 0.108872, + 0.135867, + 0.114504, + 0.125131 + ], + "Github|s1|shuf0": [ + 0.041706, + 0.05665, + 0.047134, + 0.062573, + 0.081641, + 0.088686, + 0.093391, + 0.094767, + 0.122846, + 0.115343, + 0.084385, + 0.105353, + 0.092542 + ], + "Github|s1|shuf1": [ + 0.045927, + 0.018293, + 0.057562, + 0.042485, + 0.088044, + 0.085921, + 0.090991, + 0.053585, + 0.107409, + 0.147393, + 0.123878, + 0.102525, + 0.096033 + ], + "Github|s1|shuf2": [ + 0.018301, + 0.019733, + 0.056996, + 0.074478, + 0.088044, + 0.077649, + 0.08894, + 0.10277, + 0.12754, + 0.099417, + 0.157614, + 0.093827, + 0.081458 + ], + "Github|s1|shuf3": [ + 0.054361, + 0.019733, + 0.016855, + 0.078858, + 0.079379, + 0.086721, + 0.091759, + 0.087104, + 0.12754, + 0.035791, + 0.117481, + 0.09349, + 0.092542 + ], + "Github|s1|shuf4": [ + 0.054361, + 0.06232, + 0.074476, + 0.042485, + 0.08635, + 0.0938, + 0.087803, + 0.082487, + 0.111162, + 0.115343, + 0.157614, + 0.04428, + 0.088935 + ], + "Github|s1|bandconst": [ + 0.03861, + 0.052669, + 0.042758, + 0.080457, + 0.093588, + 0.098323, + 0.108328, + 0.108031, + 0.143925, + 0.145056, + 0.147815, + 0.138118, + 0.118728 + ], + "Github|s2|real": [ + 0.063115, + 0.063986, + 0.067414, + 0.085201, + 0.089056, + 0.094917, + 0.105258, + 0.102559, + 0.111473, + 0.125025, + 0.13669, + 0.137444, + 0.142722 + ], + "Github|s2|cyc1": [ + 0.060657, + 0.070558, + 0.076261, + 0.072435, + 0.088338, + 0.095341, + 0.101209, + 0.089433, + 0.123743, + 0.119487, + 0.149497, + 0.14537, + 0.054819 + ], + "Github|s2|cyc2": [ + 0.057415, + 0.080261, + 0.050705, + 0.081391, + 0.091234, + 0.090178, + 0.088866, + 0.082879, + 0.1205, + 0.136349, + 0.157371, + 0.045963, + 0.057349 + ], + "Github|s2|cyc3": [ + 0.062869, + 0.063093, + 0.048077, + 0.079484, + 0.085703, + 0.082616, + 0.084476, + 0.079922, + 0.13657, + 0.146105, + 0.065501, + 0.049457, + 0.080766 + ], + "Github|s2|cyc4": [ + 0.051111, + 0.061734, + 0.049082, + 0.066522, + 0.080804, + 0.082432, + 0.079794, + 0.098219, + 0.141894, + 0.040112, + 0.082922, + 0.074328, + 0.092054 + ], + "Github|s2|cyc5": [ + 0.042974, + 0.06565, + 0.039159, + 0.056602, + 0.083861, + 0.075908, + 0.093876, + 0.10034, + 0.05463, + 0.063025, + 0.09957, + 0.090533, + 0.101415 + ], + "Github|s2|cyc6": [ + 0.043462, + 0.045316, + 0.023865, + 0.045182, + 0.077115, + 0.08195, + 0.091358, + 0.056218, + 0.073703, + 0.081227, + 0.112478, + 0.092864, + 0.098567 + ], + "Github|s2|cyc7": [ + 0.027474, + 0.028225, + 0.016409, + 0.042905, + 0.083888, + 0.080614, + 0.079979, + 0.072782, + 0.088979, + 0.107528, + 0.114126, + 0.105916, + 0.107624 + ], + "Github|s2|cyc8": [ + 0.016856, + 0.023233, + 0.014494, + 0.042115, + 0.080866, + 0.078805, + 0.088145, + 0.077439, + 0.107277, + 0.097117, + 0.119443, + 0.110976, + 0.09806 + ], + "Github|s2|cyc9": [ + 0.018073, + 0.018183, + 0.01145, + 0.039244, + 0.080601, + 0.085794, + 0.089171, + 0.091198, + 0.102841, + 0.10635, + 0.122504, + 0.09775, + 0.087758 + ], + "Github|s2|cyc10": [ + 0.012128, + 0.021367, + 0.009635, + 0.07137, + 0.086635, + 0.089076, + 0.092934, + 0.087621, + 0.112607, + 0.116973, + 0.125414, + 0.087406, + 0.090844 + ], + "Github|s2|cyc11": [ + 0.012681, + 0.01895, + 0.059803, + 0.078375, + 0.087416, + 0.092735, + 0.093177, + 0.096209, + 0.125814, + 0.109553, + 0.119971, + 0.095494, + 0.102614 + ], + "Github|s2|cyc12": [ + 0.010049, + 0.059973, + 0.060473, + 0.077655, + 0.089214, + 0.091554, + 0.095095, + 0.105731, + 0.122719, + 0.098951, + 0.131142, + 0.108027, + 0.122746 + ], + "Github|s2|shuf0": [ + 0.043462, + 0.059973, + 0.050705, + 0.056602, + 0.080866, + 0.089076, + 0.093876, + 0.096209, + 0.1205, + 0.109553, + 0.082922, + 0.095494, + 0.092054 + ], + "Github|s2|shuf1": [ + 0.051111, + 0.018183, + 0.060473, + 0.042115, + 0.088338, + 0.082616, + 0.089171, + 0.056218, + 0.107277, + 0.146105, + 0.122504, + 0.09775, + 0.090844 + ], + "Github|s2|shuf2": [ + 0.016856, + 0.021367, + 0.059803, + 0.078375, + 0.088338, + 0.075908, + 0.084476, + 0.105731, + 0.122719, + 0.097117, + 0.157371, + 0.090533, + 0.080766 + ], + "Github|s2|shuf3": [ + 0.057415, + 0.021367, + 0.016409, + 0.079484, + 0.077115, + 0.085794, + 0.091358, + 0.087621, + 0.122719, + 0.040112, + 0.119443, + 0.087406, + 0.092054 + ], + "Github|s2|shuf4": [ + 0.057415, + 0.063093, + 0.076261, + 0.042115, + 0.083861, + 0.095341, + 0.088145, + 0.079922, + 0.112607, + 0.109553, + 0.157371, + 0.045963, + 0.087758 + ], + "Github|s2|bandconst": [ + 0.040203, + 0.053923, + 0.044019, + 0.077602, + 0.091653, + 0.096652, + 0.107086, + 0.109214, + 0.141312, + 0.14275, + 0.146014, + 0.133775, + 0.121132 + ], + "OOD_News_2024|s0|real": [ + 0.04269, + 0.045047, + 0.048554, + 0.071652, + 0.089733, + 0.098823, + 0.083745, + 0.10083, + 0.139179, + 0.136232, + 0.14484, + 0.149485, + 0.147025 + ], + "OOD_News_2024|s0|cyc1": [ + 0.046194, + 0.051224, + 0.035946, + 0.061138, + 0.08432, + 0.073046, + 0.083061, + 0.090557, + 0.128939, + 0.137358, + 0.153335, + 0.150235, + 0.044091 + ], + "OOD_News_2024|s0|cyc2": [ + 0.049738, + 0.03898, + 0.028781, + 0.060106, + 0.064573, + 0.077818, + 0.081031, + 0.086274, + 0.131186, + 0.147617, + 0.15791, + 0.05298, + 0.06353 + ], + "OOD_News_2024|s0|cyc3": [ + 0.038594, + 0.033138, + 0.027932, + 0.042946, + 0.063775, + 0.075016, + 0.079803, + 0.077561, + 0.138611, + 0.151505, + 0.071628, + 0.065207, + 0.084226 + ], + "OOD_News_2024|s0|cyc4": [ + 0.033149, + 0.031546, + 0.022308, + 0.044333, + 0.065077, + 0.072754, + 0.069376, + 0.097927, + 0.13428, + 0.051419, + 0.085165, + 0.089856, + 0.09349 + ], + "OOD_News_2024|s0|cyc5": [ + 0.02862, + 0.027015, + 0.015067, + 0.037537, + 0.065247, + 0.064902, + 0.083223, + 0.100035, + 0.064909, + 0.07402, + 0.110725, + 0.100728, + 0.113704 + ], + "OOD_News_2024|s0|cyc6": [ + 0.023971, + 0.019, + 0.012456, + 0.032582, + 0.058779, + 0.069789, + 0.089805, + 0.064024, + 0.074572, + 0.099633, + 0.114093, + 0.127469, + 0.118633 + ], + "OOD_News_2024|s0|cyc7": [ + 0.013826, + 0.015536, + 0.01247, + 0.02406, + 0.066554, + 0.075197, + 0.081575, + 0.072794, + 0.09237, + 0.096297, + 0.137424, + 0.125925, + 0.1093 + ], + "OOD_News_2024|s0|cyc8": [ + 0.00949, + 0.01513, + 0.00712, + 0.028461, + 0.074257, + 0.07601, + 0.082066, + 0.087298, + 0.090193, + 0.128688, + 0.153722, + 0.10639, + 0.100352 + ], + "OOD_News_2024|s0|cyc9": [ + 0.010788, + 0.010128, + 0.006306, + 0.029908, + 0.069287, + 0.077158, + 0.097462, + 0.083974, + 0.112245, + 0.147821, + 0.132853, + 0.100203, + 0.105329 + ], + "OOD_News_2024|s0|cyc10": [ + 0.006845, + 0.011245, + 0.008026, + 0.064194, + 0.07191, + 0.094384, + 0.08495, + 0.103607, + 0.138234, + 0.138305, + 0.128927, + 0.112721, + 0.101372 + ], + "OOD_News_2024|s0|cyc11": [ + 0.007724, + 0.012449, + 0.048677, + 0.068977, + 0.083367, + 0.090946, + 0.101868, + 0.10863, + 0.144072, + 0.12522, + 0.136663, + 0.112991, + 0.12439 + ], + "OOD_News_2024|s0|cyc12": [ + 0.007481, + 0.042927, + 0.044871, + 0.075949, + 0.07992, + 0.098731, + 0.101973, + 0.090652, + 0.13145, + 0.133838, + 0.130066, + 0.132169, + 0.135278 + ], + "OOD_News_2024|s0|shuf0": [ + 0.023971, + 0.042927, + 0.028781, + 0.037537, + 0.074257, + 0.094384, + 0.083223, + 0.10863, + 0.131186, + 0.12522, + 0.085165, + 0.112991, + 0.09349 + ], + "OOD_News_2024|s0|shuf1": [ + 0.033149, + 0.010128, + 0.044871, + 0.028461, + 0.08432, + 0.075016, + 0.097462, + 0.064024, + 0.090193, + 0.151505, + 0.132853, + 0.100203, + 0.101372 + ], + "OOD_News_2024|s0|shuf2": [ + 0.00949, + 0.011245, + 0.048677, + 0.068977, + 0.08432, + 0.064902, + 0.079803, + 0.090652, + 0.13145, + 0.128688, + 0.15791, + 0.100728, + 0.084226 + ], + "OOD_News_2024|s0|shuf3": [ + 0.049738, + 0.011245, + 0.01247, + 0.042946, + 0.058779, + 0.077158, + 0.089805, + 0.103607, + 0.13145, + 0.051419, + 0.153722, + 0.112721, + 0.09349 + ], + "OOD_News_2024|s0|shuf4": [ + 0.049738, + 0.033138, + 0.035946, + 0.028461, + 0.065247, + 0.073046, + 0.082066, + 0.077561, + 0.138234, + 0.12522, + 0.15791, + 0.05298, + 0.105329 + ], + "OOD_News_2024|s0|bandconst": [ + 0.021868, + 0.026651, + 0.02105, + 0.059221, + 0.083034, + 0.088114, + 0.106718, + 0.108887, + 0.153424, + 0.157535, + 0.151059, + 0.140191, + 0.123674 + ], + "OOD_News_2024|s1|real": [ + 0.04424, + 0.044563, + 0.052438, + 0.070651, + 0.094554, + 0.099257, + 0.084654, + 0.101201, + 0.139242, + 0.138771, + 0.144956, + 0.14928, + 0.14637 + ], + "OOD_News_2024|s1|cyc1": [ + 0.046384, + 0.051186, + 0.035447, + 0.062787, + 0.087046, + 0.075202, + 0.085108, + 0.091531, + 0.130685, + 0.138198, + 0.154409, + 0.149801, + 0.042795 + ], + "OOD_News_2024|s1|cyc2": [ + 0.047824, + 0.038215, + 0.029762, + 0.061816, + 0.066612, + 0.076747, + 0.081303, + 0.085716, + 0.13198, + 0.148837, + 0.158728, + 0.054552, + 0.064641 + ], + "OOD_News_2024|s1|cyc3": [ + 0.03663, + 0.033613, + 0.029678, + 0.044062, + 0.065084, + 0.075437, + 0.079401, + 0.077488, + 0.138877, + 0.151687, + 0.071156, + 0.066065, + 0.083815 + ], + "OOD_News_2024|s1|cyc4": [ + 0.033233, + 0.032252, + 0.023015, + 0.044252, + 0.066042, + 0.073469, + 0.070834, + 0.097701, + 0.13391, + 0.050493, + 0.086059, + 0.089436, + 0.092868 + ], + "OOD_News_2024|s1|cyc5": [ + 0.027698, + 0.027275, + 0.014977, + 0.038946, + 0.065354, + 0.065051, + 0.083285, + 0.100757, + 0.063561, + 0.077013, + 0.109421, + 0.097986, + 0.113177 + ], + "OOD_News_2024|s1|cyc6": [ + 0.024265, + 0.019073, + 0.013013, + 0.032394, + 0.059406, + 0.070198, + 0.090906, + 0.066149, + 0.078122, + 0.100902, + 0.113587, + 0.126074, + 0.117598 + ], + "OOD_News_2024|s1|cyc7": [ + 0.016352, + 0.015311, + 0.012764, + 0.024472, + 0.068112, + 0.076219, + 0.082989, + 0.075316, + 0.095276, + 0.094217, + 0.136453, + 0.124631, + 0.110585 + ], + "OOD_News_2024|s1|cyc8": [ + 0.010273, + 0.01524, + 0.00738, + 0.028461, + 0.074112, + 0.077852, + 0.083455, + 0.088839, + 0.090453, + 0.126186, + 0.150328, + 0.10514, + 0.101828 + ], + "OOD_News_2024|s1|cyc9": [ + 0.011608, + 0.010017, + 0.006451, + 0.030355, + 0.072334, + 0.079316, + 0.099204, + 0.086257, + 0.112866, + 0.147018, + 0.133023, + 0.099949, + 0.106631 + ], + "OOD_News_2024|s1|cyc10": [ + 0.006917, + 0.011173, + 0.008026, + 0.063051, + 0.074983, + 0.094586, + 0.08527, + 0.101841, + 0.138465, + 0.13693, + 0.13084, + 0.114925, + 0.102157 + ], + "OOD_News_2024|s1|cyc11": [ + 0.007877, + 0.012449, + 0.049288, + 0.068923, + 0.08424, + 0.091095, + 0.101041, + 0.108288, + 0.14388, + 0.124078, + 0.137603, + 0.113598, + 0.124532 + ], + "OOD_News_2024|s1|cyc12": [ + 0.007443, + 0.042071, + 0.045911, + 0.076131, + 0.082054, + 0.096736, + 0.104667, + 0.089405, + 0.130401, + 0.137207, + 0.133016, + 0.133614, + 0.137136 + ], + "OOD_News_2024|s1|shuf0": [ + 0.024265, + 0.042071, + 0.029762, + 0.038946, + 0.074112, + 0.094586, + 0.083285, + 0.108288, + 0.13198, + 0.124078, + 0.086059, + 0.113598, + 0.092868 + ], + "OOD_News_2024|s1|shuf1": [ + 0.033233, + 0.010017, + 0.045911, + 0.028461, + 0.087046, + 0.075437, + 0.099204, + 0.066149, + 0.090453, + 0.151687, + 0.133023, + 0.099949, + 0.102157 + ], + "OOD_News_2024|s1|shuf2": [ + 0.010273, + 0.011173, + 0.049288, + 0.068923, + 0.087046, + 0.065051, + 0.079401, + 0.089405, + 0.130401, + 0.126186, + 0.158728, + 0.097986, + 0.083815 + ], + "OOD_News_2024|s1|shuf3": [ + 0.047824, + 0.011173, + 0.012764, + 0.044062, + 0.059406, + 0.079316, + 0.090906, + 0.101841, + 0.130401, + 0.050493, + 0.150328, + 0.114925, + 0.092868 + ], + "OOD_News_2024|s1|shuf4": [ + 0.047824, + 0.033613, + 0.035447, + 0.028461, + 0.065354, + 0.075202, + 0.083455, + 0.077488, + 0.138465, + 0.124078, + 0.158728, + 0.054552, + 0.106631 + ], + "OOD_News_2024|s1|bandconst": [ + 0.022685, + 0.026596, + 0.021383, + 0.059675, + 0.084125, + 0.090836, + 0.106234, + 0.109756, + 0.153493, + 0.156891, + 0.151261, + 0.142476, + 0.124987 + ], + "OOD_News_2024|s2|real": [ + 0.042608, + 0.044323, + 0.048239, + 0.068314, + 0.094824, + 0.098085, + 0.08472, + 0.101447, + 0.138757, + 0.137076, + 0.144479, + 0.148687, + 0.145857 + ], + "OOD_News_2024|s2|cyc1": [ + 0.045563, + 0.049091, + 0.03411, + 0.064194, + 0.086597, + 0.07378, + 0.083082, + 0.089802, + 0.130295, + 0.136475, + 0.152562, + 0.149531, + 0.039585 + ], + "OOD_News_2024|s2|cyc2": [ + 0.0471, + 0.037997, + 0.029502, + 0.061493, + 0.065833, + 0.075814, + 0.080584, + 0.085937, + 0.131529, + 0.147394, + 0.159008, + 0.049284, + 0.059979 + ], + "OOD_News_2024|s2|cyc3": [ + 0.036332, + 0.033891, + 0.028604, + 0.042981, + 0.064034, + 0.074322, + 0.078746, + 0.077641, + 0.14003, + 0.151743, + 0.069333, + 0.061033, + 0.080744 + ], + "OOD_News_2024|s2|cyc4": [ + 0.034054, + 0.032183, + 0.021508, + 0.042123, + 0.064896, + 0.072376, + 0.070021, + 0.099357, + 0.134115, + 0.045732, + 0.084112, + 0.083307, + 0.091235 + ], + "OOD_News_2024|s2|cyc5": [ + 0.028608, + 0.026714, + 0.015306, + 0.036798, + 0.065319, + 0.06452, + 0.082915, + 0.101352, + 0.062011, + 0.073483, + 0.106016, + 0.094716, + 0.112678 + ], + "OOD_News_2024|s2|cyc6": [ + 0.023763, + 0.018553, + 0.012165, + 0.031043, + 0.058592, + 0.069942, + 0.090137, + 0.059451, + 0.075777, + 0.097691, + 0.111579, + 0.125285, + 0.116302 + ], + "OOD_News_2024|s2|cyc7": [ + 0.014902, + 0.014462, + 0.012057, + 0.024063, + 0.068095, + 0.076074, + 0.078632, + 0.07332, + 0.090775, + 0.088797, + 0.136496, + 0.124728, + 0.108505 + ], + "OOD_News_2024|s2|cyc8": [ + 0.008895, + 0.015803, + 0.00738, + 0.028721, + 0.073887, + 0.076478, + 0.082201, + 0.086203, + 0.087028, + 0.127611, + 0.149411, + 0.103576, + 0.100699 + ], + "OOD_News_2024|s2|cyc9": [ + 0.011681, + 0.010353, + 0.006748, + 0.030355, + 0.06681, + 0.076255, + 0.098023, + 0.082163, + 0.111377, + 0.148065, + 0.133678, + 0.099948, + 0.103753 + ], + "OOD_News_2024|s2|cyc10": [ + 0.006917, + 0.010947, + 0.008026, + 0.0616, + 0.072307, + 0.0926, + 0.083098, + 0.10317, + 0.135924, + 0.137116, + 0.129968, + 0.113715, + 0.102195 + ], + "OOD_News_2024|s2|cyc11": [ + 0.007651, + 0.012449, + 0.048326, + 0.066769, + 0.082404, + 0.089007, + 0.104504, + 0.107798, + 0.1447, + 0.124698, + 0.137763, + 0.113129, + 0.124951 + ], + "OOD_News_2024|s2|cyc12": [ + 0.007487, + 0.04188, + 0.045908, + 0.074062, + 0.079614, + 0.100073, + 0.103042, + 0.088455, + 0.131021, + 0.133075, + 0.131748, + 0.132697, + 0.136989 + ], + "OOD_News_2024|s2|shuf0": [ + 0.023763, + 0.04188, + 0.029502, + 0.036798, + 0.073887, + 0.0926, + 0.082915, + 0.107798, + 0.131529, + 0.124698, + 0.084112, + 0.113129, + 0.091235 + ], + "OOD_News_2024|s2|shuf1": [ + 0.034054, + 0.010353, + 0.045908, + 0.028721, + 0.086597, + 0.074322, + 0.098023, + 0.059451, + 0.087028, + 0.151743, + 0.133678, + 0.099948, + 0.102195 + ], + "OOD_News_2024|s2|shuf2": [ + 0.008895, + 0.010947, + 0.048326, + 0.066769, + 0.086597, + 0.06452, + 0.078746, + 0.088455, + 0.131021, + 0.127611, + 0.159008, + 0.094716, + 0.080744 + ], + "OOD_News_2024|s2|shuf3": [ + 0.0471, + 0.010947, + 0.012057, + 0.042981, + 0.058592, + 0.076255, + 0.090137, + 0.10317, + 0.131021, + 0.045732, + 0.149411, + 0.113715, + 0.091235 + ], + "OOD_News_2024|s2|shuf4": [ + 0.0471, + 0.033891, + 0.03411, + 0.028721, + 0.065319, + 0.07378, + 0.082201, + 0.077641, + 0.135924, + 0.124698, + 0.159008, + 0.049284, + 0.103753 + ], + "OOD_News_2024|s2|bandconst": [ + 0.021868, + 0.026651, + 0.021056, + 0.058636, + 0.084067, + 0.090919, + 0.105375, + 0.107935, + 0.154667, + 0.157157, + 0.151789, + 0.142511, + 0.124464 + ], + "OOD_arXiv_2023|s0|real": [ + 0.052916, + 0.052158, + 0.047972, + 0.09187, + 0.109422, + 0.104295, + 0.092789, + 0.111932, + 0.145577, + 0.148881, + 0.146061, + 0.149361, + 0.150065 + ], + "OOD_arXiv_2023|s0|cyc1": [ + 0.051897, + 0.03957, + 0.052903, + 0.080997, + 0.085892, + 0.083465, + 0.100052, + 0.099917, + 0.139954, + 0.141151, + 0.151227, + 0.154052, + 0.045248 + ], + "OOD_arXiv_2023|s0|cyc2": [ + 0.04622, + 0.04567, + 0.036084, + 0.063265, + 0.075732, + 0.092975, + 0.093642, + 0.092509, + 0.128605, + 0.15053, + 0.160682, + 0.048514, + 0.054849 + ], + "OOD_arXiv_2023|s0|cyc3": [ + 0.043837, + 0.038745, + 0.0289, + 0.053038, + 0.080708, + 0.088679, + 0.089857, + 0.080384, + 0.142065, + 0.157016, + 0.084337, + 0.044476, + 0.078978 + ], + "OOD_arXiv_2023|s0|cyc4": [ + 0.036525, + 0.033795, + 0.027642, + 0.065131, + 0.079899, + 0.081675, + 0.078105, + 0.09598, + 0.142691, + 0.051723, + 0.088271, + 0.07455, + 0.09778 + ], + "OOD_arXiv_2023|s0|cyc5": [ + 0.029117, + 0.031837, + 0.024841, + 0.058404, + 0.07551, + 0.070929, + 0.086334, + 0.101684, + 0.070486, + 0.056805, + 0.107758, + 0.116613, + 0.107345 + ], + "OOD_arXiv_2023|s0|cyc6": [ + 0.03044, + 0.026417, + 0.018296, + 0.044093, + 0.066753, + 0.072855, + 0.091489, + 0.066635, + 0.073152, + 0.069831, + 0.11866, + 0.128565, + 0.110448 + ], + "OOD_arXiv_2023|s0|cyc7": [ + 0.023613, + 0.021618, + 0.014893, + 0.033155, + 0.071993, + 0.07791, + 0.093584, + 0.062421, + 0.089645, + 0.117343, + 0.139866, + 0.126332, + 0.107031 + ], + "OOD_arXiv_2023|s0|cyc8": [ + 0.016235, + 0.019911, + 0.00878, + 0.033205, + 0.075789, + 0.09097, + 0.088014, + 0.073298, + 0.113738, + 0.133107, + 0.141352, + 0.109726, + 0.102317 + ], + "OOD_arXiv_2023|s0|cyc9": [ + 0.017, + 0.011893, + 0.008362, + 0.035397, + 0.078537, + 0.08762, + 0.092958, + 0.101932, + 0.121839, + 0.139378, + 0.135647, + 0.113977, + 0.106702 + ], + "OOD_arXiv_2023|s0|cyc10": [ + 0.009207, + 0.012664, + 0.009151, + 0.085966, + 0.081684, + 0.090303, + 0.110484, + 0.103814, + 0.138543, + 0.134463, + 0.136564, + 0.122028, + 0.108169 + ], + "OOD_arXiv_2023|s0|cyc11": [ + 0.008877, + 0.01389, + 0.063809, + 0.089476, + 0.081446, + 0.107151, + 0.11538, + 0.107259, + 0.138379, + 0.137471, + 0.143715, + 0.124563, + 0.123611 + ], + "OOD_arXiv_2023|s0|cyc12": [ + 0.008585, + 0.046204, + 0.068914, + 0.080682, + 0.101354, + 0.111408, + 0.107738, + 0.088895, + 0.133775, + 0.144517, + 0.14272, + 0.140497, + 0.136592 + ], + "OOD_arXiv_2023|s0|shuf0": [ + 0.03044, + 0.046204, + 0.036084, + 0.058404, + 0.075789, + 0.090303, + 0.086334, + 0.107259, + 0.128605, + 0.137471, + 0.088271, + 0.124563, + 0.09778 + ], + "OOD_arXiv_2023|s0|shuf1": [ + 0.036525, + 0.011893, + 0.068914, + 0.033205, + 0.085892, + 0.088679, + 0.092958, + 0.066635, + 0.113738, + 0.157016, + 0.135647, + 0.113977, + 0.108169 + ], + "OOD_arXiv_2023|s0|shuf2": [ + 0.016235, + 0.012664, + 0.063809, + 0.089476, + 0.085892, + 0.070929, + 0.089857, + 0.088895, + 0.133775, + 0.133107, + 0.160682, + 0.116613, + 0.078978 + ], + "OOD_arXiv_2023|s0|shuf3": [ + 0.04622, + 0.012664, + 0.014893, + 0.053038, + 0.066753, + 0.08762, + 0.091489, + 0.103814, + 0.133775, + 0.051723, + 0.141352, + 0.122028, + 0.09778 + ], + "OOD_arXiv_2023|s0|shuf4": [ + 0.04622, + 0.038745, + 0.052903, + 0.033205, + 0.07551, + 0.083465, + 0.088014, + 0.080384, + 0.138543, + 0.137471, + 0.160682, + 0.048514, + 0.106702 + ], + "OOD_arXiv_2023|s0|bandconst": [ + 0.028766, + 0.032526, + 0.02782, + 0.07312, + 0.094325, + 0.102653, + 0.113775, + 0.109023, + 0.151934, + 0.164991, + 0.157482, + 0.148871, + 0.129624 + ], + "OOD_arXiv_2023|s1|real": [ + 0.053468, + 0.051937, + 0.046449, + 0.093052, + 0.110053, + 0.104676, + 0.091449, + 0.112986, + 0.146754, + 0.148605, + 0.146026, + 0.149162, + 0.1494 + ], + "OOD_arXiv_2023|s1|cyc1": [ + 0.050023, + 0.038455, + 0.053162, + 0.081246, + 0.086636, + 0.084423, + 0.100943, + 0.09953, + 0.140433, + 0.140762, + 0.152207, + 0.15303, + 0.046108 + ], + "OOD_arXiv_2023|s1|cyc2": [ + 0.043807, + 0.046823, + 0.036047, + 0.062431, + 0.07571, + 0.093047, + 0.094536, + 0.092738, + 0.130072, + 0.151198, + 0.161034, + 0.048009, + 0.055178 + ], + "OOD_arXiv_2023|s1|cyc3": [ + 0.044692, + 0.038409, + 0.028641, + 0.052145, + 0.079636, + 0.087759, + 0.088703, + 0.080529, + 0.141953, + 0.156666, + 0.083935, + 0.043573, + 0.081122 + ], + "OOD_arXiv_2023|s1|cyc4": [ + 0.036747, + 0.033313, + 0.026787, + 0.065353, + 0.079958, + 0.081858, + 0.07823, + 0.0975, + 0.142989, + 0.051267, + 0.086998, + 0.077486, + 0.098051 + ], + "OOD_arXiv_2023|s1|cyc5": [ + 0.028019, + 0.031837, + 0.023948, + 0.058145, + 0.075769, + 0.071424, + 0.087502, + 0.102404, + 0.067559, + 0.057962, + 0.110625, + 0.116124, + 0.108222 + ], + "OOD_arXiv_2023|s1|cyc6": [ + 0.030769, + 0.026047, + 0.017878, + 0.043352, + 0.066455, + 0.073974, + 0.091489, + 0.067394, + 0.071159, + 0.071222, + 0.123553, + 0.128419, + 0.110009 + ], + "OOD_arXiv_2023|s1|cyc7": [ + 0.023137, + 0.02095, + 0.013927, + 0.033602, + 0.07192, + 0.078375, + 0.093613, + 0.062795, + 0.089629, + 0.119054, + 0.139413, + 0.125508, + 0.108345 + ], + "OOD_arXiv_2023|s1|cyc8": [ + 0.017188, + 0.019166, + 0.008489, + 0.032582, + 0.076236, + 0.09157, + 0.087222, + 0.073474, + 0.114893, + 0.133394, + 0.142994, + 0.112059, + 0.10294 + ], + "OOD_arXiv_2023|s1|cyc9": [ + 0.016737, + 0.011893, + 0.008064, + 0.035695, + 0.075439, + 0.087789, + 0.094488, + 0.102182, + 0.120047, + 0.139489, + 0.135305, + 0.113733, + 0.10841 + ], + "OOD_arXiv_2023|s1|cyc10": [ + 0.009207, + 0.0123, + 0.009078, + 0.088487, + 0.07965, + 0.092899, + 0.108999, + 0.104512, + 0.137926, + 0.134482, + 0.137792, + 0.124898, + 0.109196 + ], + "OOD_arXiv_2023|s1|cyc11": [ + 0.008506, + 0.014222, + 0.06246, + 0.087527, + 0.082001, + 0.107098, + 0.11553, + 0.107173, + 0.13986, + 0.136883, + 0.144056, + 0.124648, + 0.123651 + ], + "OOD_arXiv_2023|s1|cyc12": [ + 0.008883, + 0.044051, + 0.066882, + 0.080942, + 0.101871, + 0.113162, + 0.106361, + 0.090062, + 0.132734, + 0.144659, + 0.141975, + 0.141369, + 0.135078 + ], + "OOD_arXiv_2023|s1|shuf0": [ + 0.030769, + 0.044051, + 0.036047, + 0.058145, + 0.076236, + 0.092899, + 0.087502, + 0.107173, + 0.130072, + 0.136883, + 0.086998, + 0.124648, + 0.098051 + ], + "OOD_arXiv_2023|s1|shuf1": [ + 0.036747, + 0.011893, + 0.066882, + 0.032582, + 0.086636, + 0.087759, + 0.094488, + 0.067394, + 0.114893, + 0.156666, + 0.135305, + 0.113733, + 0.109196 + ], + "OOD_arXiv_2023|s1|shuf2": [ + 0.017188, + 0.0123, + 0.06246, + 0.087527, + 0.086636, + 0.071424, + 0.088703, + 0.090062, + 0.132734, + 0.133394, + 0.161034, + 0.116124, + 0.081122 + ], + "OOD_arXiv_2023|s1|shuf3": [ + 0.043807, + 0.0123, + 0.013927, + 0.052145, + 0.066455, + 0.087789, + 0.091489, + 0.104512, + 0.132734, + 0.051267, + 0.142994, + 0.124898, + 0.098051 + ], + "OOD_arXiv_2023|s1|shuf4": [ + 0.043807, + 0.038409, + 0.053162, + 0.032582, + 0.075769, + 0.084423, + 0.087222, + 0.080529, + 0.137926, + 0.136883, + 0.161034, + 0.048009, + 0.10841 + ], + "OOD_arXiv_2023|s1|bandconst": [ + 0.028728, + 0.032633, + 0.0273, + 0.072259, + 0.094062, + 0.103023, + 0.113828, + 0.110742, + 0.151302, + 0.163894, + 0.156927, + 0.148524, + 0.129446 + ], + "OOD_arXiv_2023|s2|real": [ + 0.052581, + 0.055031, + 0.048282, + 0.092936, + 0.107914, + 0.10363, + 0.091054, + 0.111842, + 0.145084, + 0.14786, + 0.146358, + 0.149422, + 0.150307 + ], + "OOD_arXiv_2023|s2|cyc1": [ + 0.050061, + 0.038271, + 0.054093, + 0.082472, + 0.084964, + 0.081325, + 0.099816, + 0.099403, + 0.138255, + 0.142027, + 0.151445, + 0.152954, + 0.043137 + ], + "OOD_arXiv_2023|s2|cyc2": [ + 0.045181, + 0.045454, + 0.036084, + 0.064226, + 0.074118, + 0.09257, + 0.093144, + 0.092801, + 0.129737, + 0.150881, + 0.160797, + 0.046503, + 0.05343 + ], + "OOD_arXiv_2023|s2|cyc3": [ + 0.041126, + 0.038652, + 0.027768, + 0.052018, + 0.078926, + 0.085647, + 0.089573, + 0.080826, + 0.141112, + 0.157166, + 0.082893, + 0.040577, + 0.075675 + ], + "OOD_arXiv_2023|s2|cyc4": [ + 0.036002, + 0.033015, + 0.026229, + 0.067136, + 0.080086, + 0.082404, + 0.078407, + 0.095121, + 0.142478, + 0.04961, + 0.086579, + 0.075054, + 0.096746 + ], + "OOD_arXiv_2023|s2|cyc5": [ + 0.028892, + 0.030982, + 0.02402, + 0.058222, + 0.07514, + 0.071501, + 0.088025, + 0.101732, + 0.06825, + 0.056705, + 0.110071, + 0.115378, + 0.103373 + ], + "OOD_arXiv_2023|s2|cyc6": [ + 0.030585, + 0.026947, + 0.017781, + 0.043425, + 0.066123, + 0.073302, + 0.09097, + 0.064632, + 0.074575, + 0.068899, + 0.119639, + 0.126258, + 0.108764 + ], + "OOD_arXiv_2023|s2|cyc7": [ + 0.024211, + 0.021283, + 0.014152, + 0.032228, + 0.070843, + 0.077783, + 0.092694, + 0.061733, + 0.08723, + 0.117715, + 0.139014, + 0.124443, + 0.105932 + ], + "OOD_arXiv_2023|s2|cyc8": [ + 0.015004, + 0.019686, + 0.009046, + 0.032278, + 0.076087, + 0.090094, + 0.089131, + 0.071162, + 0.114838, + 0.13072, + 0.142339, + 0.110256, + 0.103008 + ], + "OOD_arXiv_2023|s2|cyc9": [ + 0.017297, + 0.012191, + 0.008064, + 0.035099, + 0.07695, + 0.088192, + 0.093606, + 0.102948, + 0.119919, + 0.13466, + 0.133363, + 0.112665, + 0.106398 + ], + "OOD_arXiv_2023|s2|cyc10": [ + 0.009103, + 0.012626, + 0.009151, + 0.090077, + 0.084491, + 0.091468, + 0.11053, + 0.104065, + 0.137849, + 0.132476, + 0.136254, + 0.123452, + 0.108724 + ], + "OOD_arXiv_2023|s2|cyc11": [ + 0.008617, + 0.013962, + 0.065637, + 0.089231, + 0.081619, + 0.107835, + 0.114418, + 0.106772, + 0.136994, + 0.136088, + 0.14322, + 0.125937, + 0.124081 + ], + "OOD_arXiv_2023|s2|cyc12": [ + 0.008068, + 0.045148, + 0.067326, + 0.081269, + 0.101868, + 0.11197, + 0.106025, + 0.088683, + 0.131591, + 0.142213, + 0.14066, + 0.140793, + 0.136907 + ], + "OOD_arXiv_2023|s2|shuf0": [ + 0.030585, + 0.045148, + 0.036084, + 0.058222, + 0.076087, + 0.091468, + 0.088025, + 0.106772, + 0.129737, + 0.136088, + 0.086579, + 0.125937, + 0.096746 + ], + "OOD_arXiv_2023|s2|shuf1": [ + 0.036002, + 0.012191, + 0.067326, + 0.032278, + 0.084964, + 0.085647, + 0.093606, + 0.064632, + 0.114838, + 0.157166, + 0.133363, + 0.112665, + 0.108724 + ], + "OOD_arXiv_2023|s2|shuf2": [ + 0.015004, + 0.012626, + 0.065637, + 0.089231, + 0.084964, + 0.071501, + 0.089573, + 0.088683, + 0.131591, + 0.13072, + 0.160797, + 0.115378, + 0.075675 + ], + "OOD_arXiv_2023|s2|shuf3": [ + 0.045181, + 0.012626, + 0.014152, + 0.052018, + 0.066123, + 0.088192, + 0.09097, + 0.104065, + 0.131591, + 0.04961, + 0.142339, + 0.123452, + 0.096746 + ], + "OOD_arXiv_2023|s2|shuf4": [ + 0.045181, + 0.038652, + 0.054093, + 0.032278, + 0.07514, + 0.081325, + 0.089131, + 0.080826, + 0.137849, + 0.136088, + 0.160797, + 0.046503, + 0.106398 + ], + "OOD_arXiv_2023|s2|bandconst": [ + 0.029063, + 0.032301, + 0.028231, + 0.07417, + 0.095126, + 0.103083, + 0.114814, + 0.108501, + 0.152038, + 0.161831, + 0.156907, + 0.1478, + 0.128344 + ], + "OOD_CommonPile|s0|real": [ + 0.055579, + 0.062111, + 0.054786, + 0.094346, + 0.112595, + 0.098532, + 0.093028, + 0.107859, + 0.141961, + 0.148417, + 0.146895, + 0.149309, + 0.150662 + ], + "OOD_CommonPile|s0|cyc1": [ + 0.052787, + 0.050315, + 0.054654, + 0.09108, + 0.088903, + 0.083517, + 0.097521, + 0.098435, + 0.140767, + 0.142831, + 0.152468, + 0.154197, + 0.058903 + ], + "OOD_CommonPile|s0|cyc2": [ + 0.046057, + 0.050921, + 0.044653, + 0.068657, + 0.07788, + 0.090863, + 0.092443, + 0.091479, + 0.131335, + 0.149985, + 0.161104, + 0.060964, + 0.059513 + ], + "OOD_CommonPile|s0|cyc3": [ + 0.045901, + 0.048058, + 0.02696, + 0.058861, + 0.081737, + 0.08463, + 0.091911, + 0.083958, + 0.143856, + 0.157735, + 0.088467, + 0.051339, + 0.089208 + ], + "OOD_CommonPile|s0|cyc4": [ + 0.039601, + 0.030393, + 0.025436, + 0.069521, + 0.079363, + 0.082896, + 0.081513, + 0.095426, + 0.144797, + 0.067394, + 0.083277, + 0.088015, + 0.104459 + ], + "OOD_CommonPile|s0|cyc5": [ + 0.027862, + 0.031041, + 0.024031, + 0.059956, + 0.07606, + 0.072172, + 0.088382, + 0.101301, + 0.07232, + 0.070894, + 0.117155, + 0.122537, + 0.109911 + ], + "OOD_CommonPile|s0|cyc6": [ + 0.028706, + 0.027763, + 0.015506, + 0.045671, + 0.068342, + 0.074707, + 0.092308, + 0.074343, + 0.069456, + 0.089478, + 0.124729, + 0.126216, + 0.113008 + ], + "OOD_CommonPile|s0|cyc7": [ + 0.022834, + 0.019278, + 0.013031, + 0.033228, + 0.074358, + 0.079097, + 0.097067, + 0.063476, + 0.089866, + 0.121926, + 0.13899, + 0.121141, + 0.109473 + ], + "OOD_CommonPile|s0|cyc8": [ + 0.014259, + 0.018462, + 0.007816, + 0.032599, + 0.077786, + 0.09949, + 0.08781, + 0.075301, + 0.11773, + 0.132799, + 0.1363, + 0.113184, + 0.106328 + ], + "OOD_CommonPile|s0|cyc9": [ + 0.015438, + 0.011553, + 0.008805, + 0.036716, + 0.090665, + 0.089859, + 0.098434, + 0.099064, + 0.125428, + 0.129604, + 0.132278, + 0.113476, + 0.107149 + ], + "OOD_CommonPile|s0|cyc10": [ + 0.007059, + 0.012809, + 0.008746, + 0.0904, + 0.087085, + 0.091204, + 0.108248, + 0.103087, + 0.128848, + 0.13067, + 0.136244, + 0.121296, + 0.109507 + ], + "OOD_CommonPile|s0|cyc11": [ + 0.008173, + 0.014302, + 0.069785, + 0.089627, + 0.087224, + 0.10647, + 0.112549, + 0.098745, + 0.129373, + 0.130809, + 0.142852, + 0.122267, + 0.125614 + ], + "OOD_CommonPile|s0|cyc12": [ + 0.009038, + 0.055006, + 0.066253, + 0.080719, + 0.100546, + 0.112388, + 0.0987, + 0.08521, + 0.126493, + 0.143127, + 0.14463, + 0.141834, + 0.134321 + ], + "OOD_CommonPile|s0|shuf0": [ + 0.028706, + 0.055006, + 0.044653, + 0.059956, + 0.077786, + 0.091204, + 0.088382, + 0.098745, + 0.131335, + 0.130809, + 0.083277, + 0.122267, + 0.104459 + ], + "OOD_CommonPile|s0|shuf1": [ + 0.039601, + 0.011553, + 0.066253, + 0.032599, + 0.088903, + 0.08463, + 0.098434, + 0.074343, + 0.11773, + 0.157735, + 0.132278, + 0.113476, + 0.109507 + ], + "OOD_CommonPile|s0|shuf2": [ + 0.014259, + 0.012809, + 0.069785, + 0.089627, + 0.088903, + 0.072172, + 0.091911, + 0.08521, + 0.126493, + 0.132799, + 0.161104, + 0.122537, + 0.089208 + ], + "OOD_CommonPile|s0|shuf3": [ + 0.046057, + 0.012809, + 0.013031, + 0.058861, + 0.068342, + 0.089859, + 0.092308, + 0.103087, + 0.126493, + 0.067394, + 0.1363, + 0.121296, + 0.104459 + ], + "OOD_CommonPile|s0|shuf4": [ + 0.046057, + 0.048058, + 0.054654, + 0.032599, + 0.07606, + 0.083517, + 0.08781, + 0.083958, + 0.128848, + 0.130809, + 0.161104, + 0.060964, + 0.107149 + ], + "OOD_CommonPile|s0|bandconst": [ + 0.026959, + 0.033421, + 0.028853, + 0.077015, + 0.095682, + 0.100277, + 0.113133, + 0.110208, + 0.14654, + 0.159661, + 0.158418, + 0.147464, + 0.130901 + ], + "OOD_CommonPile|s1|real": [ + 0.051536, + 0.056712, + 0.049886, + 0.091625, + 0.114972, + 0.09921, + 0.088174, + 0.1071, + 0.141305, + 0.148172, + 0.147383, + 0.149321, + 0.150827 + ], + "OOD_CommonPile|s1|cyc1": [ + 0.046127, + 0.042528, + 0.052798, + 0.08824, + 0.086031, + 0.080993, + 0.098398, + 0.098738, + 0.141717, + 0.142218, + 0.152468, + 0.153972, + 0.054958 + ], + "OOD_CommonPile|s1|cyc2": [ + 0.038189, + 0.047232, + 0.040515, + 0.059919, + 0.072798, + 0.090275, + 0.092767, + 0.091631, + 0.130906, + 0.150198, + 0.160692, + 0.056167, + 0.057279 + ], + "OOD_CommonPile|s1|cyc3": [ + 0.041188, + 0.046295, + 0.02654, + 0.051945, + 0.078474, + 0.084728, + 0.091507, + 0.081732, + 0.144302, + 0.156925, + 0.083674, + 0.047159, + 0.088357 + ], + "OOD_CommonPile|s1|cyc4": [ + 0.036356, + 0.029561, + 0.026142, + 0.066041, + 0.076621, + 0.083497, + 0.081379, + 0.095368, + 0.142697, + 0.063333, + 0.080577, + 0.082616, + 0.103648 + ], + "OOD_CommonPile|s1|cyc5": [ + 0.026342, + 0.028758, + 0.021396, + 0.053952, + 0.073386, + 0.071245, + 0.088419, + 0.100782, + 0.067853, + 0.065619, + 0.112969, + 0.12044, + 0.108467 + ], + "OOD_CommonPile|s1|cyc6": [ + 0.02849, + 0.024718, + 0.01525, + 0.04441, + 0.066715, + 0.074828, + 0.091878, + 0.072114, + 0.067549, + 0.084961, + 0.123624, + 0.125678, + 0.110078 + ], + "OOD_CommonPile|s1|cyc7": [ + 0.020649, + 0.018537, + 0.012997, + 0.031299, + 0.073804, + 0.077723, + 0.095176, + 0.061274, + 0.089196, + 0.117199, + 0.140081, + 0.120787, + 0.109557 + ], + "OOD_CommonPile|s1|cyc8": [ + 0.015338, + 0.018344, + 0.008036, + 0.032117, + 0.076934, + 0.097193, + 0.08658, + 0.073513, + 0.113445, + 0.132428, + 0.134752, + 0.109996, + 0.104195 + ], + "OOD_CommonPile|s1|cyc9": [ + 0.016362, + 0.011259, + 0.008434, + 0.036176, + 0.086305, + 0.08706, + 0.094899, + 0.09792, + 0.125484, + 0.129833, + 0.132075, + 0.112072, + 0.107574 + ], + "OOD_CommonPile|s1|cyc10": [ + 0.007135, + 0.012072, + 0.009005, + 0.089175, + 0.084056, + 0.091198, + 0.107638, + 0.102531, + 0.12716, + 0.128676, + 0.136945, + 0.119917, + 0.109875 + ], + "OOD_CommonPile|s1|cyc11": [ + 0.008392, + 0.014042, + 0.067097, + 0.089099, + 0.083941, + 0.106417, + 0.114245, + 0.096444, + 0.128174, + 0.130312, + 0.143114, + 0.123378, + 0.12512 + ], + "OOD_CommonPile|s1|cyc12": [ + 0.008559, + 0.050964, + 0.06762, + 0.080541, + 0.099248, + 0.114287, + 0.095256, + 0.082348, + 0.126484, + 0.139905, + 0.142913, + 0.140724, + 0.135093 + ], + "OOD_CommonPile|s1|shuf0": [ + 0.02849, + 0.050964, + 0.040515, + 0.053952, + 0.076934, + 0.091198, + 0.088419, + 0.096444, + 0.130906, + 0.130312, + 0.080577, + 0.123378, + 0.103648 + ], + "OOD_CommonPile|s1|shuf1": [ + 0.036356, + 0.011259, + 0.06762, + 0.032117, + 0.086031, + 0.084728, + 0.094899, + 0.072114, + 0.113445, + 0.156925, + 0.132075, + 0.112072, + 0.109875 + ], + "OOD_CommonPile|s1|shuf2": [ + 0.015338, + 0.012072, + 0.067097, + 0.089099, + 0.086031, + 0.071245, + 0.091507, + 0.082348, + 0.126484, + 0.132428, + 0.160692, + 0.12044, + 0.088357 + ], + "OOD_CommonPile|s1|shuf3": [ + 0.038189, + 0.012072, + 0.012997, + 0.051945, + 0.066715, + 0.08706, + 0.091878, + 0.102531, + 0.126484, + 0.063333, + 0.134752, + 0.119917, + 0.103648 + ], + "OOD_CommonPile|s1|shuf4": [ + 0.038189, + 0.046295, + 0.052798, + 0.032117, + 0.073386, + 0.080993, + 0.08658, + 0.081732, + 0.12716, + 0.130312, + 0.160692, + 0.056167, + 0.107574 + ], + "OOD_CommonPile|s1|bandconst": [ + 0.026796, + 0.031626, + 0.027966, + 0.075051, + 0.094165, + 0.09999, + 0.111837, + 0.108728, + 0.148011, + 0.160386, + 0.157067, + 0.14536, + 0.131258 + ], + "OOD_CommonPile|s2|real": [ + 0.050898, + 0.05769, + 0.050204, + 0.088208, + 0.111145, + 0.095939, + 0.088411, + 0.106589, + 0.14055, + 0.147698, + 0.145063, + 0.150079, + 0.148547 + ], + "OOD_CommonPile|s2|cyc1": [ + 0.048333, + 0.046007, + 0.049124, + 0.089407, + 0.085786, + 0.080584, + 0.096461, + 0.097042, + 0.138795, + 0.140457, + 0.152526, + 0.152524, + 0.061307 + ], + "OOD_CommonPile|s2|cyc2": [ + 0.041895, + 0.044851, + 0.040865, + 0.062143, + 0.073226, + 0.088384, + 0.089752, + 0.088488, + 0.130802, + 0.149866, + 0.158801, + 0.061999, + 0.061023 + ], + "OOD_CommonPile|s2|cyc3": [ + 0.041036, + 0.044985, + 0.025883, + 0.052534, + 0.078263, + 0.083993, + 0.090432, + 0.081385, + 0.142377, + 0.155801, + 0.089952, + 0.052002, + 0.089782 + ], + "OOD_CommonPile|s2|cyc4": [ + 0.038206, + 0.028479, + 0.02358, + 0.066661, + 0.076652, + 0.081777, + 0.081302, + 0.094231, + 0.144513, + 0.06909, + 0.086268, + 0.085359, + 0.105187 + ], + "OOD_CommonPile|s2|cyc5": [ + 0.025782, + 0.028627, + 0.021984, + 0.055835, + 0.074463, + 0.071349, + 0.08727, + 0.101079, + 0.070293, + 0.070996, + 0.118823, + 0.12138, + 0.11202 + ], + "OOD_CommonPile|s2|cyc6": [ + 0.027002, + 0.025602, + 0.015063, + 0.042963, + 0.066822, + 0.073898, + 0.0929, + 0.072749, + 0.069391, + 0.088701, + 0.127553, + 0.125517, + 0.109602 + ], + "OOD_CommonPile|s2|cyc7": [ + 0.02157, + 0.018126, + 0.012619, + 0.030852, + 0.073396, + 0.077723, + 0.095476, + 0.063289, + 0.08995, + 0.11942, + 0.140652, + 0.121682, + 0.108444 + ], + "OOD_CommonPile|s2|cyc8": [ + 0.013624, + 0.018306, + 0.007557, + 0.03193, + 0.076602, + 0.095309, + 0.087338, + 0.073252, + 0.114962, + 0.132182, + 0.135075, + 0.112646, + 0.10578 + ], + "OOD_CommonPile|s2|cyc9": [ + 0.015213, + 0.011183, + 0.008434, + 0.036456, + 0.083044, + 0.084287, + 0.092865, + 0.097165, + 0.123935, + 0.1278, + 0.132391, + 0.112461, + 0.106209 + ], + "OOD_CommonPile|s2|cyc10": [ + 0.006799, + 0.012332, + 0.008746, + 0.089061, + 0.081461, + 0.088879, + 0.105339, + 0.101892, + 0.125349, + 0.127611, + 0.136697, + 0.119572, + 0.107711 + ], + "OOD_CommonPile|s2|cyc11": [ + 0.008173, + 0.014302, + 0.065833, + 0.087561, + 0.083139, + 0.102296, + 0.113009, + 0.096502, + 0.126159, + 0.128881, + 0.141009, + 0.121413, + 0.123003 + ], + "OOD_CommonPile|s2|cyc12": [ + 0.009038, + 0.049362, + 0.064392, + 0.077818, + 0.096373, + 0.112425, + 0.094977, + 0.08341, + 0.125548, + 0.140527, + 0.142825, + 0.139409, + 0.134175 + ], + "OOD_CommonPile|s2|shuf0": [ + 0.027002, + 0.049362, + 0.040865, + 0.055835, + 0.076602, + 0.088879, + 0.08727, + 0.096502, + 0.130802, + 0.128881, + 0.086268, + 0.121413, + 0.105187 + ], + "OOD_CommonPile|s2|shuf1": [ + 0.038206, + 0.011183, + 0.064392, + 0.03193, + 0.085786, + 0.083993, + 0.092865, + 0.072749, + 0.114962, + 0.155801, + 0.132391, + 0.112461, + 0.107711 + ], + "OOD_CommonPile|s2|shuf2": [ + 0.013624, + 0.012332, + 0.065833, + 0.087561, + 0.085786, + 0.071349, + 0.090432, + 0.08341, + 0.125548, + 0.132182, + 0.158801, + 0.12138, + 0.089782 + ], + "OOD_CommonPile|s2|shuf3": [ + 0.041895, + 0.012332, + 0.012619, + 0.052534, + 0.066822, + 0.084287, + 0.0929, + 0.101892, + 0.125548, + 0.06909, + 0.135075, + 0.119572, + 0.105187 + ], + "OOD_CommonPile|s2|shuf4": [ + 0.041895, + 0.044985, + 0.049124, + 0.03193, + 0.074463, + 0.080584, + 0.087338, + 0.081385, + 0.125349, + 0.128881, + 0.158801, + 0.061999, + 0.106209 + ], + "OOD_CommonPile|s2|bandconst": [ + 0.024657, + 0.030456, + 0.026737, + 0.072742, + 0.091541, + 0.098662, + 0.111368, + 0.107207, + 0.147078, + 0.158857, + 0.157659, + 0.14806, + 0.13152 + ] + }, + "band_constant_exploratory": { + "status": "EXPLORATORY \u2014 NOT a matched control. J_bar is a different object with a different norm and spectrum, not a permutation of the existing ones, so it changes more than the cyclic and deranged arms do. Reported only to ask whether layer-specific variation carries anything beyond one average transport over the band. Carries no verdict.", + "by_operator": { + "Wikipedia_en|s0": { + "min": 0.23080356568098068, + "persist": 0.1070764422416687, + "best1L": 0.16058741668239235, + "best1L_layer": 18, + "mean": 0.10206455665174871 + }, + "Wikipedia_en|s1": { + "min": 0.2299209788441658, + "persist": 0.10716487169265747, + "best1L": 0.16031771609559656, + "best1L_layer": 18, + "mean": 0.1020728428221236 + }, + "Wikipedia_en|s2": { + "min": 0.22899930626153947, + "persist": 0.10681172311306, + "best1L": 0.16095257112756373, + "best1L_layer": 18, + "mean": 0.10175060416356875 + }, + "USPTO_Backgrounds|s0": { + "min": 0.23426722176373005, + "persist": 0.11483514632564037, + "best1L": 0.16527951117604972, + "best1L_layer": 18, + "mean": 0.10867785495132781 + }, + "USPTO_Backgrounds|s1": { + "min": 0.2342967141419649, + "persist": 0.11485129769425839, + "best1L": 0.16437398698180913, + "best1L_layer": 18, + "mean": 0.10867276147294502 + }, + "USPTO_Backgrounds|s2": { + "min": 0.23589950986206532, + "persist": 0.1150258719921112, + "best1L": 0.16406371546909213, + "best1L_layer": 18, + "mean": 0.10941340843072304 + }, + "Pile-CC|s0": { + "min": 0.24449960179626942, + "persist": 0.11225685998797416, + "best1L": 0.17061694664880633, + "best1L_layer": 18, + "mean": 0.10643054950588311 + }, + "Pile-CC|s1": { + "min": 0.24591544978320598, + "persist": 0.11155272945761681, + "best1L": 0.1710970656014979, + "best1L_layer": 18, + "mean": 0.1073834315300561 + }, + "Pile-CC|s2": { + "min": 0.24536319486796856, + "persist": 0.10984451100230216, + "best1L": 0.1692681179381907, + "best1L_layer": 18, + "mean": 0.10570269180103564 + }, + "StackExchange|s0": { + "min": 0.24481524713337421, + "persist": 0.11209553233347833, + "best1L": 0.16540382704697548, + "best1L_layer": 18, + "mean": 0.10833185257237905 + }, + "StackExchange|s1": { + "min": 0.24460886009037494, + "persist": 0.11167673370800912, + "best1L": 0.16298971185460687, + "best1L_layer": 18, + "mean": 0.10711174795735412 + }, + "StackExchange|s2": { + "min": 0.2436747681349516, + "persist": 0.11179751630406827, + "best1L": 0.1648765112273395, + "best1L_layer": 18, + "mean": 0.10733086892197696 + }, + "Github|s0": { + "min": 0.22249324433505535, + "persist": 0.1049712629057467, + "best1L": 0.146455295663327, + "best1L_layer": 19, + "mean": 0.10001176173368899 + }, + "Github|s1": { + "min": 0.22421207651495934, + "persist": 0.10424974793568254, + "best1L": 0.14781491560861468, + "best1L_layer": 19, + "mean": 0.10126203092603156 + }, + "Github|s2": { + "min": 0.22234967462718486, + "persist": 0.10431253546848893, + "best1L": 0.14601412219926715, + "best1L_layer": 19, + "mean": 0.10041043559590784 + }, + "OOD_News_2024|s0": { + "min": 0.22683808468282224, + "persist": 0.10250141583383084, + "best1L": 0.15753491427749394, + "best1L_layer": 18, + "mean": 0.09549426190519275 + }, + "OOD_News_2024|s1": { + "min": 0.22812642492353916, + "persist": 0.10240800827741622, + "best1L": 0.15689077079296113, + "best1L_layer": 18, + "mean": 0.09618429566841 + }, + "OOD_News_2024|s2": { + "min": 0.22892262898385524, + "persist": 0.10177812576293946, + "best1L": 0.15715738590806722, + "best1L_layer": 18, + "mean": 0.09593027555741943 + }, + "OOD_arXiv_2023|s0": { + "min": 0.23787386305630207, + "persist": 0.10584816969931125, + "best1L": 0.16499051703140138, + "best1L_layer": 18, + "mean": 0.10268529859693865 + }, + "OOD_arXiv_2023|s1": { + "min": 0.2359683346003294, + "persist": 0.10609778203070164, + "best1L": 0.16389426989480854, + "best1L_layer": 18, + "mean": 0.1025130225936524 + }, + "OOD_arXiv_2023|s2": { + "min": 0.23785456232726573, + "persist": 0.10529082268476486, + "best1L": 0.1618311653845012, + "best1L_layer": 18, + "mean": 0.1024776491193244 + }, + "OOD_CommonPile|s0": { + "min": 0.23026493601500989, + "persist": 0.10630818158388138, + "best1L": 0.1596614788286388, + "best1L_layer": 18, + "mean": 0.1021949142557927 + }, + "OOD_CommonPile|s1": { + "min": 0.2300564032047987, + "persist": 0.10716964155435563, + "best1L": 0.16038610143586993, + "best1L_layer": 18, + "mean": 0.1014031785253722 + }, + "OOD_CommonPile|s2": { + "min": 0.22871493734419346, + "persist": 0.10555092766880989, + "best1L": 0.15885662641376258, + "best1L_layer": 18, + "mean": 0.10050331637299119 + } + } + }, + "arms_per_operator": { + "Wikipedia_en|s0|real": { + "min": 0.24112006239593028, + "persist": 0.10448124296963215, + "best1L": 0.149345647636801, + "best1L_layer": 21, + "mean": 0.10186347586255019 + }, + "Wikipedia_en|s0|cyc1": { + "min": 0.24465489909052848, + "persist": 0.08934329934418202, + "best1L": 0.15265186629258096, + "best1L_layer": 20, + "mean": 0.09317943890077564 + }, + "Wikipedia_en|s0|cyc2": { + "min": 0.24774179682135583, + "persist": 0.08122235767077654, + "best1L": 0.15898456126451493, + "best1L_layer": 19, + "mean": 0.08500005606463394 + }, + "Wikipedia_en|s0|cyc3": { + "min": 0.2410713393241167, + "persist": 0.07539298878982663, + "best1L": 0.1547305072657764, + "best1L_layer": 18, + "mean": 0.07952465178767362 + }, + "Wikipedia_en|s0|cyc4": { + "min": 0.2295370601117611, + "persist": 0.07084742113947869, + "best1L": 0.14085483751259745, + "best1L_layer": 17, + "mean": 0.0748938307519823 + }, + "Wikipedia_en|s0|cyc5": { + "min": 0.2196815822273493, + "persist": 0.06846536304801702, + "best1L": 0.11119886208325624, + "best1L_layer": 20, + "mean": 0.0729712767410092 + }, + "Wikipedia_en|s0|cyc6": { + "min": 0.21996808499097825, + "persist": 0.07129673603922129, + "best1L": 0.12630914775654672, + "best1L_layer": 19, + "mean": 0.07274960733484477 + }, + "Wikipedia_en|s0|cyc7": { + "min": 0.2130806405097246, + "persist": 0.07354356050491333, + "best1L": 0.1327294260263443, + "best1L_layer": 19, + "mean": 0.0737249891130397 + }, + "Wikipedia_en|s0|cyc8": { + "min": 0.22506533190608025, + "persist": 0.07460274435579776, + "best1L": 0.14550709184259175, + "best1L_layer": 19, + "mean": 0.07635188705216234 + }, + "Wikipedia_en|s0|cyc9": { + "min": 0.22194473072886467, + "persist": 0.07699530236423016, + "best1L": 0.14107747003436089, + "best1L_layer": 19, + "mean": 0.07783953984303829 + }, + "Wikipedia_en|s0|cyc10": { + "min": 0.23527912124991418, + "persist": 0.08221811093389989, + "best1L": 0.14535923502407969, + "best1L_layer": 18, + "mean": 0.08305542542276761 + }, + "Wikipedia_en|s0|cyc11": { + "min": 0.24940552487969397, + "persist": 0.08873683214187622, + "best1L": 0.1508416782133281, + "best1L_layer": 17, + "mean": 0.08990662201809195 + }, + "Wikipedia_en|s0|cyc12": { + "min": 0.2481809973716736, + "persist": 0.09482559959869832, + "best1L": 0.14169441666454077, + "best1L_layer": 18, + "mean": 0.09476932000476294 + }, + "Wikipedia_en|s0|shuf0": { + "min": 0.24828002080321313, + "persist": 0.0813209842890501, + "best1L": 0.14124351693317294, + "best1L_layer": 18, + "mean": 0.0848243370347728 + }, + "Wikipedia_en|s0|shuf1": { + "min": 0.25063798800110815, + "persist": 0.07815886037424206, + "best1L": 0.1547305072657764, + "best1L_layer": 18, + "mean": 0.08327821177439405 + }, + "Wikipedia_en|s0|shuf2": { + "min": 0.27249130234122276, + "persist": 0.07532001119107008, + "best1L": 0.15898456126451493, + "best1L_layer": 19, + "mean": 0.08407428325512088 + }, + "Wikipedia_en|s0|shuf3": { + "min": 0.2572912372648716, + "persist": 0.07884126789867878, + "best1L": 0.14550709184259175, + "best1L_layer": 19, + "mean": 0.08067107106460474 + }, + "Wikipedia_en|s0|shuf4": { + "min": 0.27122137174010275, + "persist": 0.07891678363084793, + "best1L": 0.15898456126451493, + "best1L_layer": 19, + "mean": 0.08414260460099635 + }, + "Wikipedia_en|s1|real": { + "min": 0.24326577521860598, + "persist": 0.10463058389723301, + "best1L": 0.14915861464105545, + "best1L_layer": 20, + "mean": 0.10257191960789407 + }, + "Wikipedia_en|s1|cyc1": { + "min": 0.2464993428438902, + "persist": 0.08979243226349354, + "best1L": 0.15380188724957405, + "best1L_layer": 20, + "mean": 0.0931138812080742 + }, + "Wikipedia_en|s1|cyc2": { + "min": 0.2451518040150404, + "persist": 0.08038937114179134, + "best1L": 0.15894237011671067, + "best1L_layer": 19, + "mean": 0.0847948356195747 + }, + "Wikipedia_en|s1|cyc3": { + "min": 0.24064997844398023, + "persist": 0.0749247282743454, + "best1L": 0.15522975875064732, + "best1L_layer": 18, + "mean": 0.07989380616211118 + }, + "Wikipedia_en|s1|cyc4": { + "min": 0.23126568533480168, + "persist": 0.07191995345056057, + "best1L": 0.1410394778009504, + "best1L_layer": 17, + "mean": 0.0750126819117353 + }, + "Wikipedia_en|s1|cyc5": { + "min": 0.21937125362455845, + "persist": 0.06973953358829021, + "best1L": 0.11109491419047117, + "best1L_layer": 20, + "mean": 0.07334538957802579 + }, + "Wikipedia_en|s1|cyc6": { + "min": 0.22078891471028328, + "persist": 0.07189883068203926, + "best1L": 0.12500804690644146, + "best1L_layer": 19, + "mean": 0.07292733843832348 + }, + "Wikipedia_en|s1|cyc7": { + "min": 0.21491295993328094, + "persist": 0.07543061077594757, + "best1L": 0.13377069178968667, + "best1L_layer": 19, + "mean": 0.07423216304204498 + }, + "Wikipedia_en|s1|cyc8": { + "min": 0.22404550090432168, + "persist": 0.07623262777924537, + "best1L": 0.14468113137409092, + "best1L_layer": 19, + "mean": 0.07597249398151269 + }, + "Wikipedia_en|s1|cyc9": { + "min": 0.2201889619231224, + "persist": 0.07600862942636014, + "best1L": 0.14083444650750607, + "best1L_layer": 19, + "mean": 0.07761462475531376 + }, + "Wikipedia_en|s1|cyc10": { + "min": 0.23417429402470588, + "persist": 0.08185376562178134, + "best1L": 0.14817312858067452, + "best1L_layer": 18, + "mean": 0.08332084288569884 + }, + "Wikipedia_en|s1|cyc11": { + "min": 0.25029473602771757, + "persist": 0.08967118039727211, + "best1L": 0.15152264293283224, + "best1L_layer": 17, + "mean": 0.09031645406682332 + }, + "Wikipedia_en|s1|cyc12": { + "min": 0.24850083589553834, + "persist": 0.09680024485569447, + "best1L": 0.1444552014581859, + "best1L_layer": 18, + "mean": 0.09551119891872917 + }, + "Wikipedia_en|s1|shuf0": { + "min": 0.24736156240105628, + "persist": 0.08280326202511787, + "best1L": 0.1434199701063335, + "best1L_layer": 18, + "mean": 0.08492322588034977 + }, + "Wikipedia_en|s1|shuf1": { + "min": 0.2511155292391777, + "persist": 0.07869168482720852, + "best1L": 0.15522975875064732, + "best1L_layer": 18, + "mean": 0.08352150246680069 + }, + "Wikipedia_en|s1|shuf2": { + "min": 0.2717252686619759, + "persist": 0.07610932774841786, + "best1L": 0.15894237011671067, + "best1L_layer": 19, + "mean": 0.08378845315713149 + }, + "Wikipedia_en|s1|shuf3": { + "min": 0.2539973951876163, + "persist": 0.07892063967883586, + "best1L": 0.14468113137409092, + "best1L_layer": 19, + "mean": 0.08045883837666434 + }, + "Wikipedia_en|s1|shuf4": { + "min": 0.26881202794611453, + "persist": 0.07768305167555808, + "best1L": 0.15894237011671067, + "best1L_layer": 19, + "mean": 0.08426683409802185 + }, + "Wikipedia_en|s2|real": { + "min": 0.2404395245015621, + "persist": 0.1035530712455511, + "best1L": 0.14927313113585114, + "best1L_layer": 21, + "mean": 0.10122094393322746 + }, + "Wikipedia_en|s2|cyc1": { + "min": 0.24745286628603935, + "persist": 0.08989957869052886, + "best1L": 0.1540093337651342, + "best1L_layer": 20, + "mean": 0.09260734868629907 + }, + "Wikipedia_en|s2|cyc2": { + "min": 0.2476446010172367, + "persist": 0.0821324611082673, + "best1L": 0.15996434539556503, + "best1L_layer": 19, + "mean": 0.08437192303244956 + }, + "Wikipedia_en|s2|cyc3": { + "min": 0.24047864861786367, + "persist": 0.07544096242636442, + "best1L": 0.1558563913218677, + "best1L_layer": 18, + "mean": 0.07963592791168664 + }, + "Wikipedia_en|s2|cyc4": { + "min": 0.23137674368917943, + "persist": 0.0726619554683566, + "best1L": 0.14113222709856926, + "best1L_layer": 17, + "mean": 0.07518701685297016 + }, + "Wikipedia_en|s2|cyc5": { + "min": 0.22085850983858107, + "persist": 0.06853464171290398, + "best1L": 0.10852986900135875, + "best1L_layer": 20, + "mean": 0.073510498198 + }, + "Wikipedia_en|s2|cyc6": { + "min": 0.2215069506317377, + "persist": 0.07089707776904106, + "best1L": 0.1254404212348163, + "best1L_layer": 19, + "mean": 0.07274680074053602 + }, + "Wikipedia_en|s2|cyc7": { + "min": 0.21479458957910538, + "persist": 0.0727221466600895, + "best1L": 0.13317099828273057, + "best1L_layer": 19, + "mean": 0.07377628867980093 + }, + "Wikipedia_en|s2|cyc8": { + "min": 0.22394790276885032, + "persist": 0.07652443125844002, + "best1L": 0.14521459843963386, + "best1L_layer": 19, + "mean": 0.0764743829665419 + }, + "Wikipedia_en|s2|cyc9": { + "min": 0.22065867334604264, + "persist": 0.07727269493043423, + "best1L": 0.14111508678179235, + "best1L_layer": 19, + "mean": 0.0779379863686 + }, + "Wikipedia_en|s2|cyc10": { + "min": 0.23209379762411117, + "persist": 0.08171238414943219, + "best1L": 0.14783612648025155, + "best1L_layer": 18, + "mean": 0.08293861371930689 + }, + "Wikipedia_en|s2|cyc11": { + "min": 0.24836737662553787, + "persist": 0.08940134309232235, + "best1L": 0.15008457554504276, + "best1L_layer": 17, + "mean": 0.08937052386239744 + }, + "Wikipedia_en|s2|cyc12": { + "min": 0.24544059112668037, + "persist": 0.09578449341934174, + "best1L": 0.1431495850905776, + "best1L_layer": 18, + "mean": 0.09450498728099495 + }, + "Wikipedia_en|s2|shuf0": { + "min": 0.24522592797875403, + "persist": 0.08325023017823696, + "best1L": 0.13993550688028336, + "best1L_layer": 18, + "mean": 0.08433288286457985 + }, + "Wikipedia_en|s2|shuf1": { + "min": 0.24893060624599456, + "persist": 0.07850877046585084, + "best1L": 0.1558563913218677, + "best1L_layer": 18, + "mean": 0.0835768440080126 + }, + "Wikipedia_en|s2|shuf2": { + "min": 0.2693440858274698, + "persist": 0.0764667907729745, + "best1L": 0.15996434539556503, + "best1L_layer": 19, + "mean": 0.0836080540705902 + }, + "Wikipedia_en|s2|shuf3": { + "min": 0.2519726946949959, + "persist": 0.07845090553164483, + "best1L": 0.14521459843963386, + "best1L_layer": 19, + "mean": 0.08017132774649116 + }, + "Wikipedia_en|s2|shuf4": { + "min": 0.26741342917084693, + "persist": 0.07803469803184271, + "best1L": 0.15996434539556503, + "best1L_layer": 19, + "mean": 0.08335260870830657 + }, + "USPTO_Backgrounds|s0|real": { + "min": 0.259354492649436, + "persist": 0.12008049613796175, + "best1L": 0.15072583509609103, + "best1L_layer": 21, + "mean": 0.11354535975977062 + }, + "USPTO_Backgrounds|s0|cyc1": { + "min": 0.2660733174532652, + "persist": 0.10286976075731218, + "best1L": 0.15527732679620385, + "best1L_layer": 20, + "mean": 0.10442221711541956 + }, + "USPTO_Backgrounds|s0|cyc2": { + "min": 0.26701751463115214, + "persist": 0.09106933686416596, + "best1L": 0.15978973160963506, + "best1L_layer": 19, + "mean": 0.092897947784961 + }, + "USPTO_Backgrounds|s0|cyc3": { + "min": 0.25231800191104414, + "persist": 0.08340665891300887, + "best1L": 0.1578873535618186, + "best1L_layer": 18, + "mean": 0.08587530906392762 + }, + "USPTO_Backgrounds|s0|cyc4": { + "min": 0.24658580273389816, + "persist": 0.07722584009170533, + "best1L": 0.14428129056468605, + "best1L_layer": 17, + "mean": 0.08008580781304492 + }, + "USPTO_Backgrounds|s0|cyc5": { + "min": 0.22676849067211152, + "persist": 0.07362625505775214, + "best1L": 0.12000870862975717, + "best1L_layer": 19, + "mean": 0.07747449026568434 + }, + "USPTO_Backgrounds|s0|cyc6": { + "min": 0.2272262915968895, + "persist": 0.07392723641823977, + "best1L": 0.12301107589155436, + "best1L_layer": 20, + "mean": 0.07580054189266565 + }, + "USPTO_Backgrounds|s0|cyc7": { + "min": 0.23238108009099961, + "persist": 0.07773103311192245, + "best1L": 0.14315947089344264, + "best1L_layer": 19, + "mean": 0.07806017687544227 + }, + "USPTO_Backgrounds|s0|cyc8": { + "min": 0.24720819145441056, + "persist": 0.07524540713056922, + "best1L": 0.14876709184609355, + "best1L_layer": 19, + "mean": 0.08062946553377864 + }, + "USPTO_Backgrounds|s0|cyc9": { + "min": 0.23515085279941558, + "persist": 0.0812265558866784, + "best1L": 0.14804828269407153, + "best1L_layer": 19, + "mean": 0.08227562350220978 + }, + "USPTO_Backgrounds|s0|cyc10": { + "min": 0.2463291682302952, + "persist": 0.09212690480053425, + "best1L": 0.14985272530466318, + "best1L_layer": 18, + "mean": 0.0892354204856719 + }, + "USPTO_Backgrounds|s0|cyc11": { + "min": 0.2507125727832317, + "persist": 0.1008718129247427, + "best1L": 0.15315032359212638, + "best1L_layer": 17, + "mean": 0.09754605619281602 + }, + "USPTO_Backgrounds|s0|cyc12": { + "min": 0.2639938496053219, + "persist": 0.10622173002921045, + "best1L": 0.1456082358956337, + "best1L_layer": 18, + "mean": 0.10459881655406207 + }, + "USPTO_Backgrounds|s0|shuf0": { + "min": 0.25584742724895476, + "persist": 0.0909794308245182, + "best1L": 0.14089913871139287, + "best1L_layer": 18, + "mean": 0.09151404553110926 + }, + "USPTO_Backgrounds|s0|shuf1": { + "min": 0.26351498067379, + "persist": 0.08494102142285556, + "best1L": 0.1578873535618186, + "best1L_layer": 18, + "mean": 0.08914851853444886 + }, + "USPTO_Backgrounds|s0|shuf2": { + "min": 0.26983229219913485, + "persist": 0.08603019118309022, + "best1L": 0.15978973160963506, + "best1L_layer": 19, + "mean": 0.08996220240772415 + }, + "USPTO_Backgrounds|s0|shuf3": { + "min": 0.2610111027956009, + "persist": 0.08238022662699222, + "best1L": 0.14876709184609355, + "best1L_layer": 19, + "mean": 0.08525083182701984 + }, + "USPTO_Backgrounds|s0|shuf4": { + "min": 0.2718806996941566, + "persist": 0.08888753782957792, + "best1L": 0.15978973160963506, + "best1L_layer": 19, + "mean": 0.09029005719751765 + }, + "USPTO_Backgrounds|s1|real": { + "min": 0.26035019345581534, + "persist": 0.11979121798649431, + "best1L": 0.15048738671466708, + "best1L_layer": 21, + "mean": 0.11336708609820702 + }, + "USPTO_Backgrounds|s1|cyc1": { + "min": 0.2664876949042082, + "persist": 0.10320484701078385, + "best1L": 0.15772240916267039, + "best1L_layer": 20, + "mean": 0.10432179550317905 + }, + "USPTO_Backgrounds|s1|cyc2": { + "min": 0.2635386260226369, + "persist": 0.09056942078750581, + "best1L": 0.16132688473444431, + "best1L_layer": 19, + "mean": 0.09276327165380979 + }, + "USPTO_Backgrounds|s1|cyc3": { + "min": 0.253836914896965, + "persist": 0.0834438684163615, + "best1L": 0.15824909862130881, + "best1L_layer": 18, + "mean": 0.08616400295241665 + }, + "USPTO_Backgrounds|s1|cyc4": { + "min": 0.2477096289396286, + "persist": 0.07724606692790985, + "best1L": 0.14514176901429893, + "best1L_layer": 17, + "mean": 0.08030854575156879 + }, + "USPTO_Backgrounds|s1|cyc5": { + "min": 0.22877835482358932, + "persist": 0.0736608987674117, + "best1L": 0.12022542068734765, + "best1L_layer": 19, + "mean": 0.07776771868095518 + }, + "USPTO_Backgrounds|s1|cyc6": { + "min": 0.22654253169894217, + "persist": 0.07244238487910479, + "best1L": 0.12259903494268656, + "best1L_layer": 20, + "mean": 0.07542417480681951 + }, + "USPTO_Backgrounds|s1|cyc7": { + "min": 0.2308446027338505, + "persist": 0.07685582942795008, + "best1L": 0.1413387143984437, + "best1L_layer": 19, + "mean": 0.07770328933838755 + }, + "USPTO_Backgrounds|s1|cyc8": { + "min": 0.24601733461022376, + "persist": 0.07351846862584352, + "best1L": 0.14999909452162682, + "best1L_layer": 19, + "mean": 0.07992122141250338 + }, + "USPTO_Backgrounds|s1|cyc9": { + "min": 0.2360354594886303, + "persist": 0.08106237396132202, + "best1L": 0.1491251359693706, + "best1L_layer": 19, + "mean": 0.0821598298364104 + }, + "USPTO_Backgrounds|s1|cyc10": { + "min": 0.24493970572948456, + "persist": 0.0907769862562418, + "best1L": 0.1484671885147691, + "best1L_layer": 18, + "mean": 0.08858364602335944 + }, + "USPTO_Backgrounds|s1|cyc11": { + "min": 0.25200355648994444, + "persist": 0.10203451290726662, + "best1L": 0.15572992507368327, + "best1L_layer": 17, + "mean": 0.09719980190902089 + }, + "USPTO_Backgrounds|s1|cyc12": { + "min": 0.26441720277071, + "persist": 0.10686601563356816, + "best1L": 0.14777875300496818, + "best1L_layer": 18, + "mean": 0.10421839669266214 + }, + "USPTO_Backgrounds|s1|shuf0": { + "min": 0.25631540194153785, + "persist": 0.08966481909155846, + "best1L": 0.14284658748656512, + "best1L_layer": 18, + "mean": 0.09108227839102395 + }, + "USPTO_Backgrounds|s1|shuf1": { + "min": 0.26299020983278754, + "persist": 0.08335184752941131, + "best1L": 0.15824909862130881, + "best1L_layer": 18, + "mean": 0.08902300013488947 + }, + "USPTO_Backgrounds|s1|shuf2": { + "min": 0.26882662773132326, + "persist": 0.08593744933605194, + "best1L": 0.16132688473444431, + "best1L_layer": 19, + "mean": 0.08994547622457433 + }, + "USPTO_Backgrounds|s1|shuf3": { + "min": 0.26178502663969994, + "persist": 0.08251192029565572, + "best1L": 0.14999909452162682, + "best1L_layer": 19, + "mean": 0.08509996933325266 + }, + "USPTO_Backgrounds|s1|shuf4": { + "min": 0.2712178058922291, + "persist": 0.08784533105790615, + "best1L": 0.16132688473444431, + "best1L_layer": 19, + "mean": 0.09006686771575075 + }, + "USPTO_Backgrounds|s2|real": { + "min": 0.2636730432510376, + "persist": 0.12028511771932245, + "best1L": 0.1508549521677196, + "best1L_layer": 21, + "mean": 0.11420551870788376 + }, + "USPTO_Backgrounds|s2|cyc1": { + "min": 0.2721596110612154, + "persist": 0.10350246492307633, + "best1L": 0.15694167772307993, + "best1L_layer": 20, + "mean": 0.10517446865112735 + }, + "USPTO_Backgrounds|s2|cyc2": { + "min": 0.2663179863244295, + "persist": 0.09205169032793492, + "best1L": 0.16188167135696857, + "best1L_layer": 19, + "mean": 0.09353948302002839 + }, + "USPTO_Backgrounds|s2|cyc3": { + "min": 0.25458351261913775, + "persist": 0.08337027903180569, + "best1L": 0.15817591305822135, + "best1L_layer": 18, + "mean": 0.08640075325652455 + }, + "USPTO_Backgrounds|s2|cyc4": { + "min": 0.2493179753422737, + "persist": 0.07659747153520584, + "best1L": 0.14452740056440233, + "best1L_layer": 17, + "mean": 0.08064560225914018 + }, + "USPTO_Backgrounds|s2|cyc5": { + "min": 0.23044797629117966, + "persist": 0.07321974840015173, + "best1L": 0.11941050002351403, + "best1L_layer": 19, + "mean": 0.07799167365928251 + }, + "USPTO_Backgrounds|s2|cyc6": { + "min": 0.22808412313461304, + "persist": 0.07146476625930517, + "best1L": 0.1258844243362546, + "best1L_layer": 20, + "mean": 0.07597456493173153 + }, + "USPTO_Backgrounds|s2|cyc7": { + "min": 0.23484571725130082, + "persist": 0.07656294738408179, + "best1L": 0.14440803807228803, + "best1L_layer": 19, + "mean": 0.07852794426636626 + }, + "USPTO_Backgrounds|s2|cyc8": { + "min": 0.2502021752297878, + "persist": 0.07379736891016364, + "best1L": 0.14898359826765956, + "best1L_layer": 19, + "mean": 0.08083325527572574 + }, + "USPTO_Backgrounds|s2|cyc9": { + "min": 0.2386375717818737, + "persist": 0.08108652718365192, + "best1L": 0.14888764671050012, + "best1L_layer": 19, + "mean": 0.08244533874452688 + }, + "USPTO_Backgrounds|s2|cyc10": { + "min": 0.24681735411286354, + "persist": 0.0920442946255207, + "best1L": 0.14799681212753057, + "best1L_layer": 18, + "mean": 0.08958936037603192 + }, + "USPTO_Backgrounds|s2|cyc11": { + "min": 0.25675220787525177, + "persist": 0.1035269170999527, + "best1L": 0.15400127712637185, + "best1L_layer": 17, + "mean": 0.09852168752202907 + }, + "USPTO_Backgrounds|s2|cyc12": { + "min": 0.26823230013251304, + "persist": 0.10654133583884687, + "best1L": 0.14737074133008718, + "best1L_layer": 18, + "mean": 0.10497405434815356 + }, + "USPTO_Backgrounds|s2|shuf0": { + "min": 0.25852382332086565, + "persist": 0.09051615260541439, + "best1L": 0.14318923261016608, + "best1L_layer": 18, + "mean": 0.09246030792038745 + }, + "USPTO_Backgrounds|s2|shuf1": { + "min": 0.2640672240406275, + "persist": 0.08496731789782644, + "best1L": 0.15817591305822135, + "best1L_layer": 18, + "mean": 0.08962283564301637 + }, + "USPTO_Backgrounds|s2|shuf2": { + "min": 0.27396334931254385, + "persist": 0.08608829639852048, + "best1L": 0.16188167135696857, + "best1L_layer": 19, + "mean": 0.09083722877101257 + }, + "USPTO_Backgrounds|s2|shuf3": { + "min": 0.265429225564003, + "persist": 0.0844848508015275, + "best1L": 0.14898359826765956, + "best1L_layer": 19, + "mean": 0.08560147087018077 + }, + "USPTO_Backgrounds|s2|shuf4": { + "min": 0.27192303650081157, + "persist": 0.08831145130097866, + "best1L": 0.16188167135696857, + "best1L_layer": 19, + "mean": 0.09073381491829283 + }, + "Pile-CC|s0|real": { + "min": 0.26216798909008504, + "persist": 0.11054239969234914, + "best1L": 0.1584967777132988, + "best1L_layer": 18, + "mean": 0.11143265816006953 + }, + "Pile-CC|s0|cyc1": { + "min": 0.258773285523057, + "persist": 0.09200247563421726, + "best1L": 0.1592800036072731, + "best1L_layer": 19, + "mean": 0.09853129417662365 + }, + "Pile-CC|s0|cyc2": { + "min": 0.2580602984875441, + "persist": 0.08368945233523846, + "best1L": 0.16190255880355836, + "best1L_layer": 19, + "mean": 0.08802455526669152 + }, + "Pile-CC|s0|cyc3": { + "min": 0.24546110183000563, + "persist": 0.07930920347571373, + "best1L": 0.1555955003015697, + "best1L_layer": 18, + "mean": 0.08145600411718568 + }, + "Pile-CC|s0|cyc4": { + "min": 0.24290884509682656, + "persist": 0.07255633361637592, + "best1L": 0.1393344914074987, + "best1L_layer": 17, + "mean": 0.07575321051990613 + }, + "Pile-CC|s0|cyc5": { + "min": 0.2318360410630703, + "persist": 0.07307335759978742, + "best1L": 0.1216898700920865, + "best1L_layer": 19, + "mean": 0.07521187127209626 + }, + "Pile-CC|s0|cyc6": { + "min": 0.2349184848368168, + "persist": 0.07597332696896046, + "best1L": 0.12879882878623902, + "best1L_layer": 20, + "mean": 0.0769642194141992 + }, + "Pile-CC|s0|cyc7": { + "min": 0.24081215113401414, + "persist": 0.08178141999524087, + "best1L": 0.14670771937817334, + "best1L_layer": 19, + "mean": 0.08134540224161285 + }, + "Pile-CC|s0|cyc8": { + "min": 0.24220702350139617, + "persist": 0.08436877615749835, + "best1L": 0.15465431462507695, + "best1L_layer": 19, + "mean": 0.08300377049196798 + }, + "Pile-CC|s0|cyc9": { + "min": 0.2494048297405243, + "persist": 0.0846515854820609, + "best1L": 0.15519776437431573, + "best1L_layer": 18, + "mean": 0.08588795474228951 + }, + "Pile-CC|s0|cyc10": { + "min": 0.25588576048612593, + "persist": 0.09110317777376623, + "best1L": 0.157037487719208, + "best1L_layer": 18, + "mean": 0.09225385306725421 + }, + "Pile-CC|s0|cyc11": { + "min": 0.26856088638305664, + "persist": 0.10063425626140088, + "best1L": 0.16714127464219927, + "best1L_layer": 17, + "mean": 0.1007358947589707 + }, + "Pile-CC|s0|cyc12": { + "min": 0.2707030847668648, + "persist": 0.10266181107144803, + "best1L": 0.15687472838908434, + "best1L_layer": 18, + "mean": 0.10530030547480028 + }, + "Pile-CC|s0|shuf0": { + "min": 0.2740033358335495, + "persist": 0.08584446720778942, + "best1L": 0.1472047993913293, + "best1L_layer": 18, + "mean": 0.08981567861763044 + }, + "Pile-CC|s0|shuf1": { + "min": 0.2677078492939472, + "persist": 0.08377785235643387, + "best1L": 0.1555955003015697, + "best1L_layer": 18, + "mean": 0.08979466315537978 + }, + "Pile-CC|s0|shuf2": { + "min": 0.2843741521239281, + "persist": 0.08186221458017826, + "best1L": 0.16190255880355836, + "best1L_layer": 19, + "mean": 0.09030934698127496 + }, + "Pile-CC|s0|shuf3": { + "min": 0.26529838889837265, + "persist": 0.08023232966661453, + "best1L": 0.15465431462507695, + "best1L_layer": 19, + "mean": 0.08309319255013879 + }, + "Pile-CC|s0|shuf4": { + "min": 0.27600362077355384, + "persist": 0.08126350324600935, + "best1L": 0.16190255880355836, + "best1L_layer": 19, + "mean": 0.08779865524982318 + }, + "Pile-CC|s1|real": { + "min": 0.2650879543274641, + "persist": 0.1118273551343009, + "best1L": 0.16088795913383364, + "best1L_layer": 17, + "mean": 0.11232283977755847 + }, + "Pile-CC|s1|cyc1": { + "min": 0.26205273531377316, + "persist": 0.09270810969173908, + "best1L": 0.15955481678247452, + "best1L_layer": 19, + "mean": 0.0997897786692752 + }, + "Pile-CC|s1|cyc2": { + "min": 0.2592153411358595, + "persist": 0.08589368350803853, + "best1L": 0.1617575243115425, + "best1L_layer": 19, + "mean": 0.08927686329966841 + }, + "Pile-CC|s1|cyc3": { + "min": 0.2467532079666853, + "persist": 0.07935975659638643, + "best1L": 0.15522535247728228, + "best1L_layer": 18, + "mean": 0.0823064294057146 + }, + "Pile-CC|s1|cyc4": { + "min": 0.2454092249274254, + "persist": 0.07274248618632555, + "best1L": 0.13917180649004876, + "best1L_layer": 17, + "mean": 0.07621288322037659 + }, + "Pile-CC|s1|cyc5": { + "min": 0.23234946504235268, + "persist": 0.07503292493056506, + "best1L": 0.1215099724708125, + "best1L_layer": 19, + "mean": 0.07572221392049239 + }, + "Pile-CC|s1|cyc6": { + "min": 0.2356440953910351, + "persist": 0.07468877932988108, + "best1L": 0.1298155628144741, + "best1L_layer": 20, + "mean": 0.07696555396900154 + }, + "Pile-CC|s1|cyc7": { + "min": 0.24180326163768767, + "persist": 0.08094410325866193, + "best1L": 0.1471419846639037, + "best1L_layer": 19, + "mean": 0.08141400154787475 + }, + "Pile-CC|s1|cyc8": { + "min": 0.2431012399494648, + "persist": 0.08301130570471286, + "best1L": 0.15752222638111563, + "best1L_layer": 19, + "mean": 0.0840556892602203 + }, + "Pile-CC|s1|cyc9": { + "min": 0.25209871530532835, + "persist": 0.08470968976616859, + "best1L": 0.15643848637118934, + "best1L_layer": 19, + "mean": 0.08671879971113343 + }, + "Pile-CC|s1|cyc10": { + "min": 0.2604607939720154, + "persist": 0.093659982457757, + "best1L": 0.1612477601505816, + "best1L_layer": 18, + "mean": 0.09388853535951616 + }, + "Pile-CC|s1|cyc11": { + "min": 0.2715131685137749, + "persist": 0.10136037096381187, + "best1L": 0.16907403273507954, + "best1L_layer": 17, + "mean": 0.10156652816177275 + }, + "Pile-CC|s1|cyc12": { + "min": 0.27170284315943716, + "persist": 0.1036932236282155, + "best1L": 0.1580763639882207, + "best1L_layer": 18, + "mean": 0.10643569146318792 + }, + "Pile-CC|s1|shuf0": { + "min": 0.2786212250590324, + "persist": 0.08509123313706368, + "best1L": 0.14899237360805273, + "best1L_layer": 18, + "mean": 0.09075644738936367 + }, + "Pile-CC|s1|shuf1": { + "min": 0.2705193042755127, + "persist": 0.08387383986264467, + "best1L": 0.15643848637118934, + "best1L_layer": 19, + "mean": 0.09051406377066787 + }, + "Pile-CC|s1|shuf2": { + "min": 0.2866306558251381, + "persist": 0.08191019371151924, + "best1L": 0.1617575243115425, + "best1L_layer": 19, + "mean": 0.09097907653508278 + }, + "Pile-CC|s1|shuf3": { + "min": 0.2700627721846104, + "persist": 0.08150973357260227, + "best1L": 0.15752222638111563, + "best1L_layer": 19, + "mean": 0.0848838487890764 + }, + "Pile-CC|s1|shuf4": { + "min": 0.27726985663175585, + "persist": 0.08268851265311242, + "best1L": 0.1617575243115425, + "best1L_layer": 19, + "mean": 0.08883470476855738 + }, + "Pile-CC|s2|real": { + "min": 0.25994115322828293, + "persist": 0.11091751407366246, + "best1L": 0.15701769748702646, + "best1L_layer": 19, + "mean": 0.11088641090855862 + }, + "Pile-CC|s2|cyc1": { + "min": 0.25636298544704916, + "persist": 0.09105798788368702, + "best1L": 0.15830021649599074, + "best1L_layer": 19, + "mean": 0.09796410900128719 + }, + "Pile-CC|s2|cyc2": { + "min": 0.25696867145597935, + "persist": 0.08409138682764024, + "best1L": 0.16212766021490096, + "best1L_layer": 19, + "mean": 0.08778035886084232 + }, + "Pile-CC|s2|cyc3": { + "min": 0.24683286286890507, + "persist": 0.07874860353767872, + "best1L": 0.15517307473346592, + "best1L_layer": 18, + "mean": 0.0813863001145924 + }, + "Pile-CC|s2|cyc4": { + "min": 0.2433391124010086, + "persist": 0.07199897654354573, + "best1L": 0.1404113297816366, + "best1L_layer": 17, + "mean": 0.07521801450415157 + }, + "Pile-CC|s2|cyc5": { + "min": 0.23074265345931053, + "persist": 0.0722140698460862, + "best1L": 0.12045118983369321, + "best1L_layer": 19, + "mean": 0.07453282273625239 + }, + "Pile-CC|s2|cyc6": { + "min": 0.2331268884241581, + "persist": 0.07365265290718526, + "best1L": 0.12803147742524743, + "best1L_layer": 19, + "mean": 0.07646005755254569 + }, + "Pile-CC|s2|cyc7": { + "min": 0.23985243290662767, + "persist": 0.0785519766388461, + "best1L": 0.14646733682602645, + "best1L_layer": 19, + "mean": 0.08038441904533941 + }, + "Pile-CC|s2|cyc8": { + "min": 0.24046728387475014, + "persist": 0.08197234906256198, + "best1L": 0.15460197678767146, + "best1L_layer": 19, + "mean": 0.08255411631141145 + }, + "Pile-CC|s2|cyc9": { + "min": 0.24881202280521392, + "persist": 0.08302169982343913, + "best1L": 0.15661919508129357, + "best1L_layer": 18, + "mean": 0.08544680853732503 + }, + "Pile-CC|s2|cyc10": { + "min": 0.26123038977384566, + "persist": 0.08996095582842827, + "best1L": 0.16112186117097735, + "best1L_layer": 18, + "mean": 0.09219580762124119 + }, + "Pile-CC|s2|cyc11": { + "min": 0.2678173676133156, + "persist": 0.09852816946804524, + "best1L": 0.17026458689942955, + "best1L_layer": 17, + "mean": 0.10012558604089113 + }, + "Pile-CC|s2|cyc12": { + "min": 0.26895916312932966, + "persist": 0.10345436993520707, + "best1L": 0.15574411321431397, + "best1L_layer": 18, + "mean": 0.10481903528024514 + }, + "Pile-CC|s2|shuf0": { + "min": 0.2740327961742878, + "persist": 0.0852287447778508, + "best1L": 0.14716136623173953, + "best1L_layer": 18, + "mean": 0.08923754596975274 + }, + "Pile-CC|s2|shuf1": { + "min": 0.2686062276363373, + "persist": 0.0828045567497611, + "best1L": 0.15517307473346592, + "best1L_layer": 18, + "mean": 0.08906121647451073 + }, + "Pile-CC|s2|shuf2": { + "min": 0.2827077329158783, + "persist": 0.08076190613210202, + "best1L": 0.16212766021490096, + "best1L_layer": 19, + "mean": 0.08941692448077866 + }, + "Pile-CC|s2|shuf3": { + "min": 0.2685593143105507, + "persist": 0.07991705778986216, + "best1L": 0.15460197678767146, + "best1L_layer": 19, + "mean": 0.083149988277672 + }, + "Pile-CC|s2|shuf4": { + "min": 0.27765234634280206, + "persist": 0.0815402265638113, + "best1L": 0.16212766021490096, + "best1L_layer": 19, + "mean": 0.0876139697421772 + }, + "StackExchange|s0|real": { + "min": 0.2665700364857912, + "persist": 0.11692521655932069, + "best1L": 0.16821235921233893, + "best1L_layer": 18, + "mean": 0.11546809655488827 + }, + "StackExchange|s0|cyc1": { + "min": 0.2613341700285673, + "persist": 0.1012859835056588, + "best1L": 0.16140388152562082, + "best1L_layer": 19, + "mean": 0.10332647514475796 + }, + "StackExchange|s0|cyc2": { + "min": 0.2552523840218782, + "persist": 0.08860837917309254, + "best1L": 0.1643361995695159, + "best1L_layer": 19, + "mean": 0.09054001804593449 + }, + "StackExchange|s0|cyc3": { + "min": 0.2473218720406294, + "persist": 0.08135761003941297, + "best1L": 0.15984972920268775, + "best1L_layer": 18, + "mean": 0.08242686621423882 + }, + "StackExchange|s0|cyc4": { + "min": 0.23495023064315318, + "persist": 0.0764957282692194, + "best1L": 0.14388806475326418, + "best1L_layer": 17, + "mean": 0.07572815640462348 + }, + "StackExchange|s0|cyc5": { + "min": 0.2179786454886198, + "persist": 0.07426857380196453, + "best1L": 0.10969796988647432, + "best1L_layer": 19, + "mean": 0.07334350702406552 + }, + "StackExchange|s0|cyc6": { + "min": 0.21610805802047253, + "persist": 0.07185408053919673, + "best1L": 0.11146931275725365, + "best1L_layer": 19, + "mean": 0.07241244158044888 + }, + "StackExchange|s0|cyc7": { + "min": 0.22547364346683024, + "persist": 0.07468668799847364, + "best1L": 0.11983248244505376, + "best1L_layer": 19, + "mean": 0.07654083334715464 + }, + "StackExchange|s0|cyc8": { + "min": 0.23464636504650116, + "persist": 0.0770730247721076, + "best1L": 0.13319911709986626, + "best1L_layer": 19, + "mean": 0.07911538233837256 + }, + "StackExchange|s0|cyc9": { + "min": 0.2458835296332836, + "persist": 0.08183340206742287, + "best1L": 0.1433313896879554, + "best1L_layer": 19, + "mean": 0.08400868300635081 + }, + "StackExchange|s0|cyc10": { + "min": 0.2574793234467506, + "persist": 0.08737516540568321, + "best1L": 0.1547978099435568, + "best1L_layer": 19, + "mean": 0.09059865800126528 + }, + "StackExchange|s0|cyc11": { + "min": 0.269672379642725, + "persist": 0.0971110253361985, + "best1L": 0.15871521178632975, + "best1L_layer": 19, + "mean": 0.10080789913280079 + }, + "StackExchange|s0|cyc12": { + "min": 0.2723156429827213, + "persist": 0.09960463433526456, + "best1L": 0.15994415376335383, + "best1L_layer": 18, + "mean": 0.10725189768530143 + }, + "StackExchange|s0|shuf0": { + "min": 0.26500423550605773, + "persist": 0.08735615264158696, + "best1L": 0.14597954116761686, + "best1L_layer": 18, + "mean": 0.09169479947304353 + }, + "StackExchange|s0|shuf1": { + "min": 0.27110425978899, + "persist": 0.08621515452396125, + "best1L": 0.15984972920268775, + "best1L_layer": 18, + "mean": 0.09131687145295123 + }, + "StackExchange|s0|shuf2": { + "min": 0.2754390150308609, + "persist": 0.08668109960854053, + "best1L": 0.1643361995695159, + "best1L_layer": 19, + "mean": 0.09059390218641895 + }, + "StackExchange|s0|shuf3": { + "min": 0.25646587610244753, + "persist": 0.07744382806122303, + "best1L": 0.1516985133290291, + "best1L_layer": 17, + "mean": 0.08091517091155626 + }, + "StackExchange|s0|shuf4": { + "min": 0.26446273662149905, + "persist": 0.08563328310847282, + "best1L": 0.1643361995695159, + "best1L_layer": 19, + "mean": 0.08905768384798789 + }, + "StackExchange|s1|real": { + "min": 0.2635776612907648, + "persist": 0.11572976196184755, + "best1L": 0.16759706046432257, + "best1L_layer": 18, + "mean": 0.11456084991805256 + }, + "StackExchange|s1|cyc1": { + "min": 0.2605585116893053, + "persist": 0.09921706530731171, + "best1L": 0.16045724998693914, + "best1L_layer": 19, + "mean": 0.10250129072049347 + }, + "StackExchange|s1|cyc2": { + "min": 0.2533643342554569, + "persist": 0.08753219544887543, + "best1L": 0.16470781466923654, + "best1L_layer": 19, + "mean": 0.09011608662997157 + }, + "StackExchange|s1|cyc3": { + "min": 0.2447812009602785, + "persist": 0.08105674516409636, + "best1L": 0.15886051561683417, + "best1L_layer": 18, + "mean": 0.08203265672916761 + }, + "StackExchange|s1|cyc4": { + "min": 0.23115344271063804, + "persist": 0.0750043960288167, + "best1L": 0.14336858047172427, + "best1L_layer": 17, + "mean": 0.07471037016097958 + }, + "StackExchange|s1|cyc5": { + "min": 0.21276201866567135, + "persist": 0.074287299066782, + "best1L": 0.1053957961499691, + "best1L_layer": 16, + "mean": 0.07236526516767648 + }, + "StackExchange|s1|cyc6": { + "min": 0.21204972602427005, + "persist": 0.07032811222597957, + "best1L": 0.11144841387867928, + "best1L_layer": 19, + "mean": 0.07104933996272918 + }, + "StackExchange|s1|cyc7": { + "min": 0.2203646067529917, + "persist": 0.07297673318535089, + "best1L": 0.11859915878158063, + "best1L_layer": 19, + "mean": 0.07518919960935959 + }, + "StackExchange|s1|cyc8": { + "min": 0.229991415143013, + "persist": 0.07496839771047234, + "best1L": 0.1307459036586806, + "best1L_layer": 19, + "mean": 0.07780013679025265 + }, + "StackExchange|s1|cyc9": { + "min": 0.24363282471895217, + "persist": 0.08008556142449379, + "best1L": 0.14112758142873644, + "best1L_layer": 19, + "mean": 0.08290532047394664 + }, + "StackExchange|s1|cyc10": { + "min": 0.25483701899647715, + "persist": 0.08757720764260739, + "best1L": 0.15311564020812513, + "best1L_layer": 19, + "mean": 0.08938233276936584 + }, + "StackExchange|s1|cyc11": { + "min": 0.26434001624584197, + "persist": 0.09480583060067146, + "best1L": 0.15555008295923473, + "best1L_layer": 19, + "mean": 0.09915058250144984 + }, + "StackExchange|s1|cyc12": { + "min": 0.2687639605253935, + "persist": 0.09995818524621428, + "best1L": 0.15787384565919638, + "best1L_layer": 19, + "mean": 0.10610774766671686 + }, + "StackExchange|s1|shuf0": { + "min": 0.26482513546943665, + "persist": 0.08486844457220286, + "best1L": 0.14757475964725017, + "best1L_layer": 18, + "mean": 0.0911536838241423 + }, + "StackExchange|s1|shuf1": { + "min": 0.2690747335553169, + "persist": 0.08523496259003878, + "best1L": 0.15886051561683417, + "best1L_layer": 18, + "mean": 0.09019860788293255 + }, + "StackExchange|s1|shuf2": { + "min": 0.2727171786129475, + "persist": 0.08461906053125859, + "best1L": 0.16470781466923654, + "best1L_layer": 19, + "mean": 0.08958903274928721 + }, + "StackExchange|s1|shuf3": { + "min": 0.2517006419599056, + "persist": 0.0757306307554245, + "best1L": 0.15068354364484549, + "best1L_layer": 17, + "mean": 0.07982377837089678 + }, + "StackExchange|s1|shuf4": { + "min": 0.26243913993239404, + "persist": 0.08559647388756275, + "best1L": 0.16470781466923654, + "best1L_layer": 19, + "mean": 0.08878237669212886 + }, + "StackExchange|s2|real": { + "min": 0.26277029849588873, + "persist": 0.11578959980979561, + "best1L": 0.1673937773332, + "best1L_layer": 18, + "mean": 0.11451531928122187 + }, + "StackExchange|s2|cyc1": { + "min": 0.25876003615558146, + "persist": 0.09979466262739152, + "best1L": 0.15971698116045446, + "best1L_layer": 19, + "mean": 0.10239830035626746 + }, + "StackExchange|s2|cyc2": { + "min": 0.2523853961378336, + "persist": 0.08782981932163239, + "best1L": 0.1639862934825942, + "best1L_layer": 19, + "mean": 0.08990849661148297 + }, + "StackExchange|s2|cyc3": { + "min": 0.24522266425192357, + "persist": 0.08020176719874143, + "best1L": 0.15970469620078803, + "best1L_layer": 18, + "mean": 0.08192148140515201 + }, + "StackExchange|s2|cyc4": { + "min": 0.23324109837412835, + "persist": 0.07594887902960182, + "best1L": 0.1439747354015708, + "best1L_layer": 17, + "mean": 0.07479698956863454 + }, + "StackExchange|s2|cyc5": { + "min": 0.21450462341308593, + "persist": 0.07493740683421493, + "best1L": 0.10760536591988057, + "best1L_layer": 19, + "mean": 0.07256387811055622 + }, + "StackExchange|s2|cyc6": { + "min": 0.21421590484678746, + "persist": 0.06988869858905673, + "best1L": 0.1116186536848545, + "best1L_layer": 19, + "mean": 0.07116953996457875 + }, + "StackExchange|s2|cyc7": { + "min": 0.22130074612796308, + "persist": 0.07392877032980323, + "best1L": 0.1190327825723216, + "best1L_layer": 19, + "mean": 0.07515325878585617 + }, + "StackExchange|s2|cyc8": { + "min": 0.23164934143424035, + "persist": 0.07534541552886367, + "best1L": 0.13079134176950902, + "best1L_layer": 19, + "mean": 0.07794378561218485 + }, + "StackExchange|s2|cyc9": { + "min": 0.24120622724294663, + "persist": 0.08033088967204094, + "best1L": 0.142526294849813, + "best1L_layer": 19, + "mean": 0.08272828275445276 + }, + "StackExchange|s2|cyc10": { + "min": 0.25613289326429367, + "persist": 0.0875804441748187, + "best1L": 0.15353806279599666, + "best1L_layer": 19, + "mean": 0.08914758749306202 + }, + "StackExchange|s2|cyc11": { + "min": 0.2670810602605343, + "persist": 0.09447033393662423, + "best1L": 0.15645478311926125, + "best1L_layer": 19, + "mean": 0.09877197273397961 + }, + "StackExchange|s2|cyc12": { + "min": 0.2682420652359724, + "persist": 0.09851444452069699, + "best1L": 0.1579108839854598, + "best1L_layer": 19, + "mean": 0.10562481058295817 + }, + "StackExchange|s2|shuf0": { + "min": 0.26502151116728784, + "persist": 0.08450331743806601, + "best1L": 0.14501739460974933, + "best1L_layer": 18, + "mean": 0.09032381329273519 + }, + "StackExchange|s2|shuf1": { + "min": 0.26919378712773323, + "persist": 0.08529306538403034, + "best1L": 0.15970469620078803, + "best1L_layer": 18, + "mean": 0.09001687288042516 + }, + "StackExchange|s2|shuf2": { + "min": 0.27218492031097413, + "persist": 0.0856543831527233, + "best1L": 0.1639862934825942, + "best1L_layer": 19, + "mean": 0.08923640738850316 + }, + "StackExchange|s2|shuf3": { + "min": 0.25335856787860395, + "persist": 0.07592109628021718, + "best1L": 0.1502217585220933, + "best1L_layer": 17, + "mean": 0.07997933049471333 + }, + "StackExchange|s2|shuf4": { + "min": 0.26091468334198, + "persist": 0.08539159186184406, + "best1L": 0.1639862934825942, + "best1L_layer": 19, + "mean": 0.0883581805598134 + }, + "Github|s0|real": { + "min": 0.23628932386636733, + "persist": 0.09901561820879579, + "best1L": 0.14172825450077653, + "best1L_layer": 21, + "mean": 0.10002779970527627 + }, + "Github|s0|cyc1": { + "min": 0.24918047972023488, + "persist": 0.08971556711476296, + "best1L": 0.14801262295804918, + "best1L_layer": 19, + "mean": 0.09428976904690409 + }, + "Github|s0|cyc2": { + "min": 0.24668156318366527, + "persist": 0.08578659044578671, + "best1L": 0.15477379213552922, + "best1L_layer": 19, + "mean": 0.08645217665471136 + }, + "Github|s0|cyc3": { + "min": 0.22907505556941032, + "persist": 0.08211773307994008, + "best1L": 0.14574102172628045, + "best1L_layer": 18, + "mean": 0.08105144954931277 + }, + "Github|s0|cyc4": { + "min": 0.21767748408019544, + "persist": 0.07695391960442066, + "best1L": 0.1397254491923377, + "best1L_layer": 17, + "mean": 0.07633109574850935 + }, + "Github|s0|cyc5": { + "min": 0.19537636451423168, + "persist": 0.07678973907604814, + "best1L": 0.1002447223290801, + "best1L_layer": 21, + "mean": 0.07345756523740979 + }, + "Github|s0|cyc6": { + "min": 0.18537968620657921, + "persist": 0.07428613863885403, + "best1L": 0.11035448964685202, + "best1L_layer": 19, + "mean": 0.06975080790266824 + }, + "Github|s0|cyc7": { + "min": 0.1888928882777691, + "persist": 0.0767789407633245, + "best1L": 0.11511950250715017, + "best1L_layer": 19, + "mean": 0.07202409162329367 + }, + "Github|s0|cyc8": { + "min": 0.19047731012105942, + "persist": 0.0756934721954167, + "best1L": 0.11694383872672916, + "best1L_layer": 19, + "mean": 0.07213314212410926 + }, + "Github|s0|cyc9": { + "min": 0.19170695617794992, + "persist": 0.07800743475090713, + "best1L": 0.12276060823351145, + "best1L_layer": 19, + "mean": 0.07180869370125807 + }, + "Github|s0|cyc10": { + "min": 0.20684023946523666, + "persist": 0.07926999262999743, + "best1L": 0.12096772957593202, + "best1L_layer": 19, + "mean": 0.07545352222469563 + }, + "Github|s0|cyc11": { + "min": 0.21978304758667946, + "persist": 0.08710520691238344, + "best1L": 0.12317505283281208, + "best1L_layer": 17, + "mean": 0.08288115266567239 + }, + "Github|s0|cyc12": { + "min": 0.23171783983707428, + "persist": 0.0902141378261149, + "best1L": 0.13022656198590993, + "best1L_layer": 19, + "mean": 0.08921251023689715 + }, + "Github|s0|shuf0": { + "min": 0.22602029740810395, + "persist": 0.08236480634659529, + "best1L": 0.11918865544721484, + "best1L_layer": 17, + "mean": 0.08156515554292128 + }, + "Github|s0|shuf1": { + "min": 0.23050989732146263, + "persist": 0.07923254787456244, + "best1L": 0.14574102172628045, + "best1L_layer": 18, + "mean": 0.07988187497875725 + }, + "Github|s0|shuf2": { + "min": 0.24628960341215134, + "persist": 0.08386189788579941, + "best1L": 0.15477379213552922, + "best1L_layer": 19, + "mean": 0.08198528926049431 + }, + "Github|s0|shuf3": { + "min": 0.20614932514727116, + "persist": 0.0770264295861125, + "best1L": 0.11923791421577334, + "best1L_layer": 17, + "mean": 0.07363603530594935 + }, + "Github|s0|shuf4": { + "min": 0.23921805135905744, + "persist": 0.08045144844800234, + "best1L": 0.15477379213552922, + "best1L_layer": 19, + "mean": 0.08302106933865266 + }, + "Github|s1|real": { + "min": 0.2431370198726654, + "persist": 0.10285943346098066, + "best1L": 0.14388514393940569, + "best1L_layer": 21, + "mean": 0.10328132233877953 + }, + "Github|s1|cyc1": { + "min": 0.2507873121649027, + "persist": 0.09414449795149267, + "best1L": 0.1498102034209296, + "best1L_layer": 19, + "mean": 0.0964327606851629 + }, + "Github|s1|cyc2": { + "min": 0.24921251721680165, + "persist": 0.08901641878765076, + "best1L": 0.15761379897594452, + "best1L_layer": 19, + "mean": 0.08885961532494269 + }, + "Github|s1|cyc3": { + "min": 0.23064695931971074, + "persist": 0.08470613565295934, + "best1L": 0.14739258168265224, + "best1L_layer": 18, + "mean": 0.0823358444407439 + }, + "Github|s1|cyc4": { + "min": 0.2164762318134308, + "persist": 0.07870175316929817, + "best1L": 0.14046045443974436, + "best1L_layer": 17, + "mean": 0.0772972326801839 + }, + "Github|s1|cyc5": { + "min": 0.19749009497463704, + "persist": 0.07966927289962769, + "best1L": 0.10211706683039665, + "best1L_layer": 16, + "mean": 0.0744530377828946 + }, + "Github|s1|cyc6": { + "min": 0.18942782767117022, + "persist": 0.07571506043896079, + "best1L": 0.11028909143060446, + "best1L_layer": 19, + "mean": 0.07106705168441224 + }, + "Github|s1|cyc7": { + "min": 0.1920121766626835, + "persist": 0.0779442111030221, + "best1L": 0.11446320358663797, + "best1L_layer": 19, + "mean": 0.07332770438112605 + }, + "Github|s1|cyc8": { + "min": 0.19534170180559157, + "persist": 0.07733739698305726, + "best1L": 0.11748056812211871, + "best1L_layer": 19, + "mean": 0.07360464462544769 + }, + "Github|s1|cyc9": { + "min": 0.19909432530403137, + "persist": 0.0802134029334411, + "best1L": 0.12387841995805501, + "best1L_layer": 19, + "mean": 0.07393361570564314 + }, + "Github|s1|cyc10": { + "min": 0.21164898872375487, + "persist": 0.08012283786665648, + "best1L": 0.12833928037434816, + "best1L_layer": 19, + "mean": 0.07790984824443092 + }, + "Github|s1|cyc11": { + "min": 0.2260049231350422, + "persist": 0.08770988443866372, + "best1L": 0.12785694357007743, + "best1L_layer": 17, + "mean": 0.08564214797941251 + }, + "Github|s1|cyc12": { + "min": 0.23558300361037254, + "persist": 0.09133961694315076, + "best1L": 0.13586734775453807, + "best1L_layer": 19, + "mean": 0.09206826824408311 + }, + "Github|s1|shuf0": { + "min": 0.23540843054652213, + "persist": 0.08247843962162733, + "best1L": 0.12284629298374057, + "best1L_layer": 17, + "mean": 0.08361669440368692 + }, + "Github|s1|shuf1": { + "min": 0.2322789803147316, + "persist": 0.08251860772725195, + "best1L": 0.14739258168265224, + "best1L_layer": 18, + "mean": 0.0815420002269093 + }, + "Github|s1|shuf2": { + "min": 0.25017106980085374, + "persist": 0.08669814679306001, + "best1L": 0.15761379897594452, + "best1L_layer": 19, + "mean": 0.08359735563504868 + }, + "Github|s1|shuf3": { + "min": 0.21141563914716244, + "persist": 0.07803292572498322, + "best1L": 0.12753971768543124, + "best1L_layer": 17, + "mean": 0.07550863375385794 + }, + "Github|s1|shuf4": { + "min": 0.2424768466502428, + "persist": 0.08245038855820894, + "best1L": 0.15761379897594452, + "best1L_layer": 19, + "mean": 0.08472432143663844 + }, + "Github|s2|real": { + "min": 0.2420657031238079, + "persist": 0.10037821242585779, + "best1L": 0.14272244507446885, + "best1L_layer": 21, + "mean": 0.10191230223281308 + }, + "Github|s2|cyc1": { + "min": 0.25226836614310744, + "persist": 0.0902076771017164, + "best1L": 0.1494966673431918, + "best1L_layer": 19, + "mean": 0.09593452669662209 + }, + "Github|s2|cyc2": { + "min": 0.24756524451076983, + "persist": 0.08842656414490194, + "best1L": 0.15737078960519285, + "best1L_layer": 19, + "mean": 0.08772775764048744 + }, + "Github|s2|cyc3": { + "min": 0.22927598394453524, + "persist": 0.08200733670964837, + "best1L": 0.14610508335754274, + "best1L_layer": 18, + "mean": 0.08189533086887632 + }, + "Github|s2|cyc4": { + "min": 0.21718583218753337, + "persist": 0.07769525684416294, + "best1L": 0.14189365962520242, + "best1L_layer": 17, + "mean": 0.07700072689555013 + }, + "Github|s2|cyc5": { + "min": 0.19567096345126628, + "persist": 0.07964687580242753, + "best1L": 0.10141456443816424, + "best1L_layer": 21, + "mean": 0.07442652953681178 + }, + "Github|s2|cyc6": { + "min": 0.18716027066111565, + "persist": 0.0765820202184841, + "best1L": 0.11247781533747911, + "best1L_layer": 19, + "mean": 0.07102344845457433 + }, + "Github|s2|cyc7": { + "min": 0.1917248085141182, + "persist": 0.07959713144227862, + "best1L": 0.1141260800883174, + "best1L_layer": 19, + "mean": 0.07357308230935954 + }, + "Github|s2|cyc8": { + "min": 0.1942688047885895, + "persist": 0.07647162312641739, + "best1L": 0.11944311214610934, + "best1L_layer": 19, + "mean": 0.0734481180122552 + }, + "Github|s2|cyc9": { + "min": 0.19864364787936212, + "persist": 0.08017552343662829, + "best1L": 0.12250371035188437, + "best1L_layer": 19, + "mean": 0.07314747899340895 + }, + "Github|s2|cyc10": { + "min": 0.20913663655519485, + "persist": 0.0833418711554259, + "best1L": 0.12541419230401515, + "best1L_layer": 19, + "mean": 0.07723162951700104 + }, + "Github|s2|cyc11": { + "min": 0.22196120619773865, + "persist": 0.08844244717620313, + "best1L": 0.1258139593526721, + "best1L_layer": 17, + "mean": 0.08406100377727013 + }, + "Github|s2|cyc12": { + "min": 0.234267058968544, + "persist": 0.09042844409123063, + "best1L": 0.13114164266735315, + "best1L_layer": 19, + "mean": 0.09025606581081565 + }, + "Github|s2|shuf0": { + "min": 0.22873966172337531, + "persist": 0.08199466541409492, + "best1L": 0.12049962682649493, + "best1L_layer": 17, + "mean": 0.0824072357141771 + }, + "Github|s2|shuf1": { + "min": 0.23314449191093445, + "persist": 0.0824049681192264, + "best1L": 0.14610508335754274, + "best1L_layer": 18, + "mean": 0.08097723820784057 + }, + "Github|s2|shuf2": { + "min": 0.250272324681282, + "persist": 0.08569017052650452, + "best1L": 0.15737078960519285, + "best1L_layer": 19, + "mean": 0.08302783919677426 + }, + "Github|s2|shuf3": { + "min": 0.2102797579020262, + "persist": 0.07784030102193355, + "best1L": 0.12271936638280749, + "best1L_layer": 17, + "mean": 0.07525365441822662 + }, + "Github|s2|shuf4": { + "min": 0.24373132549226284, + "persist": 0.0819853462278843, + "best1L": 0.15737078960519285, + "best1L_layer": 19, + "mean": 0.08456955167303722 + }, + "OOD_News_2024|s0|real": { + "min": 0.24892510399222373, + "persist": 0.0986472788034007, + "best1L": 0.14948488723021, + "best1L_layer": 20, + "mean": 0.0998334468808025 + }, + "OOD_News_2024|s0|cyc1": { + "min": 0.24727923050522804, + "persist": 0.07609396949410438, + "best1L": 0.1533345952630043, + "best1L_layer": 19, + "mean": 0.08764945882139728 + }, + "OOD_News_2024|s0|cyc2": { + "min": 0.24841095358133317, + "persist": 0.06927309352904558, + "best1L": 0.15790979962330312, + "best1L_layer": 19, + "mean": 0.08004022030016551 + }, + "OOD_News_2024|s0|cyc3": { + "min": 0.2394300740212202, + "persist": 0.063960344158113, + "best1L": 0.15150519469752907, + "best1L_layer": 18, + "mean": 0.07307249533164745 + }, + "OOD_News_2024|s0|cyc4": { + "min": 0.23021967709064484, + "persist": 0.062089438736438754, + "best1L": 0.13428029916249215, + "best1L_layer": 17, + "mean": 0.06851385573390871 + }, + "OOD_News_2024|s0|cyc5": { + "min": 0.22367033809423448, + "persist": 0.06111552237998694, + "best1L": 0.11370432777330279, + "best1L_layer": 21, + "mean": 0.06813320208591624 + }, + "OOD_News_2024|s0|cyc6": { + "min": 0.2168418888002634, + "persist": 0.06659831765573472, + "best1L": 0.1274686561897397, + "best1L_layer": 20, + "mean": 0.06960036545597877 + }, + "OOD_News_2024|s0|cyc7": { + "min": 0.21730304509401321, + "persist": 0.07115896109025925, + "best1L": 0.1374243201687932, + "best1L_layer": 19, + "mean": 0.07102509807580365 + }, + "OOD_News_2024|s0|cyc8": { + "min": 0.22523545250296592, + "persist": 0.07165419813245535, + "best1L": 0.15372183173894882, + "best1L_layer": 19, + "mean": 0.07378290279123646 + }, + "OOD_News_2024|s0|cyc9": { + "min": 0.23528325334191322, + "persist": 0.07334950044751168, + "best1L": 0.14782054927200078, + "best1L_layer": 18, + "mean": 0.07565091468859464 + }, + "OOD_News_2024|s0|cyc10": { + "min": 0.23740548864006997, + "persist": 0.08093129198532552, + "best1L": 0.1383049980737269, + "best1L_layer": 18, + "mean": 0.08190157136951501 + }, + "OOD_News_2024|s0|cyc11": { + "min": 0.24768006950616836, + "persist": 0.09100507088005542, + "best1L": 0.1440723062492907, + "best1L_layer": 17, + "mean": 0.08969033680175646 + }, + "OOD_News_2024|s0|cyc12": { + "min": 0.24640265479683876, + "persist": 0.09219894497655332, + "best1L": 0.13527830145321787, + "best1L_layer": 21, + "mean": 0.09271569182344068 + }, + "OOD_News_2024|s0|shuf0": { + "min": 0.2541537590324879, + "persist": 0.07370346884708852, + "best1L": 0.131186162494123, + "best1L_layer": 17, + "mean": 0.08013544477379093 + }, + "OOD_News_2024|s0|shuf1": { + "min": 0.24952763319015503, + "persist": 0.072022794932127, + "best1L": 0.15150519469752907, + "best1L_layer": 18, + "mean": 0.0779659233826141 + }, + "OOD_News_2024|s0|shuf2": { + "min": 0.27084239572286606, + "persist": 0.07044114768505097, + "best1L": 0.15790979962330312, + "best1L_layer": 19, + "mean": 0.08162064346341559 + }, + "OOD_News_2024|s0|shuf3": { + "min": 0.2475816510617733, + "persist": 0.07144842091947794, + "best1L": 0.15372183173894882, + "best1L_layer": 19, + "mean": 0.0760423588548572 + }, + "OOD_News_2024|s0|shuf4": { + "min": 0.26790230087935923, + "persist": 0.06905782185494899, + "best1L": 0.15790979962330312, + "best1L_layer": 19, + "mean": 0.07883657239637194 + }, + "OOD_News_2024|s1|real": { + "min": 0.2498748254030943, + "persist": 0.0974247383652255, + "best1L": 0.14928000222425908, + "best1L_layer": 20, + "mean": 0.10078284283473085 + }, + "OOD_News_2024|s1|cyc1": { + "min": 0.24921858683228493, + "persist": 0.07759257033467293, + "best1L": 0.15440886169672013, + "best1L_layer": 19, + "mean": 0.08850601836770343 + }, + "OOD_News_2024|s1|cyc2": { + "min": 0.2484479110687971, + "persist": 0.07042569387704134, + "best1L": 0.15872800648212432, + "best1L_layer": 19, + "mean": 0.08051793128072929 + }, + "OOD_News_2024|s1|cyc3": { + "min": 0.2370097879320383, + "persist": 0.06501695420593023, + "best1L": 0.15168743589892983, + "best1L_layer": 18, + "mean": 0.07330712947967605 + }, + "OOD_News_2024|s1|cyc4": { + "min": 0.23094834983348847, + "persist": 0.062324636802077296, + "best1L": 0.13391016176901757, + "best1L_layer": 17, + "mean": 0.06873570074004909 + }, + "OOD_News_2024|s1|cyc5": { + "min": 0.22319307066500188, + "persist": 0.061417451570741834, + "best1L": 0.11317731933668256, + "best1L_layer": 21, + "mean": 0.06803849124743674 + }, + "OOD_News_2024|s1|cyc6": { + "min": 0.21828910894691944, + "persist": 0.06693381431978196, + "best1L": 0.1260743947699666, + "best1L_layer": 20, + "mean": 0.07012988192280038 + }, + "OOD_News_2024|s1|cyc7": { + "min": 0.22046925500035286, + "persist": 0.07017162467818708, + "best1L": 0.13645295575261115, + "best1L_layer": 19, + "mean": 0.07174602013726074 + }, + "OOD_News_2024|s1|cyc8": { + "min": 0.22531190887093544, + "persist": 0.07028057370334864, + "best1L": 0.15032769590616227, + "best1L_layer": 19, + "mean": 0.07381129727627223 + }, + "OOD_News_2024|s1|cyc9": { + "min": 0.23558367565274238, + "persist": 0.07511108517646789, + "best1L": 0.1470176162198186, + "best1L_layer": 18, + "mean": 0.0765407134922078 + }, + "OOD_News_2024|s1|cyc10": { + "min": 0.2382078655064106, + "persist": 0.08211878836154937, + "best1L": 0.13846481535583735, + "best1L_layer": 17, + "mean": 0.08224315939005464 + }, + "OOD_News_2024|s1|cyc11": { + "min": 0.24780959635972977, + "persist": 0.09035323709249496, + "best1L": 0.14387996355071664, + "best1L_layer": 17, + "mean": 0.0897608347523671 + }, + "OOD_News_2024|s1|cyc12": { + "min": 0.24939174056053162, + "persist": 0.09138143684249371, + "best1L": 0.13720696028321983, + "best1L_layer": 18, + "mean": 0.0935222933624083 + }, + "OOD_News_2024|s1|shuf0": { + "min": 0.2539294570684433, + "persist": 0.07335275076329709, + "best1L": 0.13197980877012014, + "best1L_layer": 17, + "mean": 0.0802997493561214 + }, + "OOD_News_2024|s1|shuf1": { + "min": 0.2506876736879349, + "persist": 0.07231609672307968, + "best1L": 0.15168743589892983, + "best1L_layer": 18, + "mean": 0.07867134449502024 + }, + "OOD_News_2024|s1|shuf2": { + "min": 0.26873930990695954, + "persist": 0.07153821419924497, + "best1L": 0.15872800648212432, + "best1L_layer": 19, + "mean": 0.08135952041484415 + }, + "OOD_News_2024|s1|shuf3": { + "min": 0.24649405032396315, + "persist": 0.070631312020123, + "best1L": 0.15032769590616227, + "best1L_layer": 19, + "mean": 0.07586957910098135 + }, + "OOD_News_2024|s1|shuf4": { + "min": 0.2676916144788265, + "persist": 0.06975763961672783, + "best1L": 0.15872800648212432, + "best1L_layer": 19, + "mean": 0.07917677774497021 + }, + "OOD_News_2024|s2|real": { + "min": 0.2513899695128202, + "persist": 0.0976877189008519, + "best1L": 0.14868734579067677, + "best1L_layer": 20, + "mean": 0.09980119719134214 + }, + "OOD_News_2024|s2|cyc1": { + "min": 0.248661507666111, + "persist": 0.07841357663273811, + "best1L": 0.1525622583925724, + "best1L_layer": 19, + "mean": 0.08728210587376872 + }, + "OOD_News_2024|s2|cyc2": { + "min": 0.248103953525424, + "persist": 0.06806455701589584, + "best1L": 0.1590079665184021, + "best1L_layer": 19, + "mean": 0.07934255206133597 + }, + "OOD_News_2024|s2|cyc3": { + "min": 0.23622534088790417, + "persist": 0.063995642028749, + "best1L": 0.15174297550693155, + "best1L_layer": 18, + "mean": 0.07226418680735291 + }, + "OOD_News_2024|s2|cyc4": { + "min": 0.22769222781062126, + "persist": 0.06100828610360622, + "best1L": 0.13411503783427178, + "best1L_layer": 17, + "mean": 0.06730900587728963 + }, + "OOD_News_2024|s2|cyc5": { + "min": 0.22063464894890786, + "persist": 0.06014908261131495, + "best1L": 0.11267777448520064, + "best1L_layer": 21, + "mean": 0.06695673129915332 + }, + "OOD_News_2024|s2|cyc6": { + "min": 0.21537539325654506, + "persist": 0.06585373270791024, + "best1L": 0.12528522554785013, + "best1L_layer": 20, + "mean": 0.0684831412240433 + }, + "OOD_News_2024|s2|cyc7": { + "min": 0.21594582200050355, + "persist": 0.06878018516581505, + "best1L": 0.13649575784802437, + "best1L_layer": 19, + "mean": 0.0700697965042379 + }, + "OOD_News_2024|s2|cyc8": { + "min": 0.2237995445728302, + "persist": 0.07037505228072405, + "best1L": 0.1494111016392708, + "best1L_layer": 19, + "mean": 0.07291484132695657 + }, + "OOD_News_2024|s2|cyc9": { + "min": 0.2355883814394474, + "persist": 0.07181922048330307, + "best1L": 0.14806489516049623, + "best1L_layer": 18, + "mean": 0.0753237189939962 + }, + "OOD_News_2024|s2|cyc10": { + "min": 0.2368263028562069, + "persist": 0.08157219290733338, + "best1L": 0.13711604634299873, + "best1L_layer": 18, + "mean": 0.08135251819979973 + }, + "OOD_News_2024|s2|cyc11": { + "min": 0.24707725048065185, + "persist": 0.09037345796823501, + "best1L": 0.14469974348321557, + "best1L_layer": 17, + "mean": 0.08955000150375642 + }, + "OOD_News_2024|s2|cyc12": { + "min": 0.24698940962553023, + "persist": 0.09000955547671766, + "best1L": 0.13698905492201446, + "best1L_layer": 21, + "mean": 0.09277318297670438 + }, + "OOD_News_2024|s2|shuf0": { + "min": 0.25065327659249304, + "persist": 0.07303814720362425, + "best1L": 0.13152852412313223, + "best1L_layer": 17, + "mean": 0.07952657614189845 + }, + "OOD_News_2024|s2|shuf1": { + "min": 0.24929407984018326, + "persist": 0.07085146754980087, + "best1L": 0.15174297550693155, + "best1L_layer": 18, + "mean": 0.07784770133081251 + }, + "OOD_News_2024|s2|shuf2": { + "min": 0.2683473452925682, + "persist": 0.06865867599844933, + "best1L": 0.1590079665184021, + "best1L_layer": 19, + "mean": 0.08048894156773503 + }, + "OOD_News_2024|s2|shuf3": { + "min": 0.2447837095707655, + "persist": 0.07009160798043013, + "best1L": 0.1494111016392708, + "best1L_layer": 19, + "mean": 0.07479643790015521 + }, + "OOD_News_2024|s2|shuf4": { + "min": 0.2669975996017456, + "persist": 0.06849358156323433, + "best1L": 0.1590079665184021, + "best1L_layer": 19, + "mean": 0.07810996786127082 + }, + "OOD_arXiv_2023|s0|real": { + "min": 0.2689389446750283, + "persist": 0.10620390176773072, + "best1L": 0.1500653130467981, + "best1L_layer": 21, + "mean": 0.10794616465707524 + }, + "OOD_arXiv_2023|s0|cyc1": { + "min": 0.2599913142621517, + "persist": 0.08979958370327949, + "best1L": 0.15405166200362147, + "best1L_layer": 20, + "mean": 0.09433254165789828 + }, + "OOD_arXiv_2023|s0|cyc2": { + "min": 0.26034976318478587, + "persist": 0.0767205398529768, + "best1L": 0.160682013630867, + "best1L_layer": 19, + "mean": 0.08379061142958773 + }, + "OOD_arXiv_2023|s0|cyc3": { + "min": 0.2450237762182951, + "persist": 0.07334241345524788, + "best1L": 0.15701562371104955, + "best1L_layer": 18, + "mean": 0.07777066982906455 + }, + "OOD_arXiv_2023|s0|cyc4": { + "min": 0.24308281689882277, + "persist": 0.06942177405580878, + "best1L": 0.14269127598963677, + "best1L_layer": 17, + "mean": 0.07336663441786256 + }, + "OOD_arXiv_2023|s0|cyc5": { + "min": 0.22944177091121673, + "persist": 0.0677750394679606, + "best1L": 0.11661325683817267, + "best1L_layer": 20, + "mean": 0.07212795837012191 + }, + "OOD_arXiv_2023|s0|cyc6": { + "min": 0.22876061946153642, + "persist": 0.06380815720185637, + "best1L": 0.12856529960408808, + "best1L_layer": 20, + "mean": 0.07058732495643198 + }, + "OOD_arXiv_2023|s0|cyc7": { + "min": 0.23469268158078194, + "persist": 0.0689063424244523, + "best1L": 0.13986600465141236, + "best1L_layer": 19, + "mean": 0.0753387153148651 + }, + "OOD_arXiv_2023|s0|cyc8": { + "min": 0.24054780304431916, + "persist": 0.07275609858334064, + "best1L": 0.1413517483510077, + "best1L_layer": 19, + "mean": 0.07741867736913263 + }, + "OOD_arXiv_2023|s0|cyc9": { + "min": 0.2428005039691925, + "persist": 0.08011427267920226, + "best1L": 0.13937763832509517, + "best1L_layer": 18, + "mean": 0.08086473594571894 + }, + "OOD_arXiv_2023|s0|cyc10": { + "min": 0.26230942606925967, + "persist": 0.08652528189122677, + "best1L": 0.13854260202497243, + "best1L_layer": 17, + "mean": 0.08792617739482711 + }, + "OOD_arXiv_2023|s0|cyc11": { + "min": 0.2701844315975904, + "persist": 0.09329471662640572, + "best1L": 0.14371524760499596, + "best1L_layer": 19, + "mean": 0.09654048324535744 + }, + "OOD_arXiv_2023|s0|cyc12": { + "min": 0.2762877721339464, + "persist": 0.0955369658768177, + "best1L": 0.14451740141957997, + "best1L_layer": 18, + "mean": 0.10091401587795609 + }, + "OOD_arXiv_2023|s0|shuf0": { + "min": 0.2580361794680357, + "persist": 0.08164319396018982, + "best1L": 0.137471385858953, + "best1L_layer": 18, + "mean": 0.0851929060922255 + }, + "OOD_arXiv_2023|s0|shuf1": { + "min": 0.2720124736428261, + "persist": 0.08086159788072109, + "best1L": 0.15701562371104955, + "best1L_layer": 18, + "mean": 0.08563452432326113 + }, + "OOD_arXiv_2023|s0|shuf2": { + "min": 0.29188896268606185, + "persist": 0.07931843660771846, + "best1L": 0.160682013630867, + "best1L_layer": 19, + "mean": 0.08776248212760458 + }, + "OOD_arXiv_2023|s0|shuf3": { + "min": 0.2627900928258896, + "persist": 0.07451336067169904, + "best1L": 0.1413517483510077, + "best1L_layer": 19, + "mean": 0.07870380660304083 + }, + "OOD_arXiv_2023|s0|shuf4": { + "min": 0.27306791618466375, + "persist": 0.08041031993925571, + "best1L": 0.160682013630867, + "best1L_layer": 19, + "mean": 0.0838736379756413 + }, + "OOD_arXiv_2023|s1|real": { + "min": 0.26879445035010574, + "persist": 0.10661298111081123, + "best1L": 0.14940013266168534, + "best1L_layer": 21, + "mean": 0.10800126081180329 + }, + "OOD_arXiv_2023|s1|cyc1": { + "min": 0.25937286838889123, + "persist": 0.09000121727585793, + "best1L": 0.15302968672476708, + "best1L_layer": 20, + "mean": 0.09438137409922022 + }, + "OOD_arXiv_2023|s1|cyc2": { + "min": 0.25903744660317896, + "persist": 0.0771851535886526, + "best1L": 0.161034494638443, + "best1L_layer": 19, + "mean": 0.08389458549791015 + }, + "OOD_arXiv_2023|s1|cyc3": { + "min": 0.242216120287776, + "persist": 0.07436438780277968, + "best1L": 0.1566657116636634, + "best1L_layer": 18, + "mean": 0.07752020744553123 + }, + "OOD_arXiv_2023|s1|cyc4": { + "min": 0.24364814534783363, + "persist": 0.0695155831053853, + "best1L": 0.14298889390192926, + "best1L_layer": 17, + "mean": 0.07357969795837283 + }, + "OOD_arXiv_2023|s1|cyc5": { + "min": 0.22968224808573723, + "persist": 0.06829195013269782, + "best1L": 0.11612438904121518, + "best1L_layer": 20, + "mean": 0.07227240919914597 + }, + "OOD_arXiv_2023|s1|cyc6": { + "min": 0.2247220791876316, + "persist": 0.06432763850316406, + "best1L": 0.12841879329644143, + "best1L_layer": 20, + "mean": 0.07090154881947315 + }, + "OOD_arXiv_2023|s1|cyc7": { + "min": 0.23189474046230316, + "persist": 0.06981468172743917, + "best1L": 0.1394132254179567, + "best1L_layer": 19, + "mean": 0.0754053802945866 + }, + "OOD_arXiv_2023|s1|cyc8": { + "min": 0.24115060940384864, + "persist": 0.07390220630913973, + "best1L": 0.14299439466558397, + "best1L_layer": 19, + "mean": 0.07786205183141506 + }, + "OOD_arXiv_2023|s1|cyc9": { + "min": 0.24196035265922547, + "persist": 0.07942629072349519, + "best1L": 0.13948948830366134, + "best1L_layer": 18, + "mean": 0.08071313558253818 + }, + "OOD_arXiv_2023|s1|cyc10": { + "min": 0.2605405643582344, + "persist": 0.08763419520109891, + "best1L": 0.13792606946080924, + "best1L_layer": 17, + "mean": 0.08841729964031121 + }, + "OOD_arXiv_2023|s1|cyc11": { + "min": 0.26781916432082653, + "persist": 0.09510925151407719, + "best1L": 0.14405585089698433, + "best1L_layer": 19, + "mean": 0.09643188364973937 + }, + "OOD_arXiv_2023|s1|cyc12": { + "min": 0.27214032523334025, + "persist": 0.096482764929533, + "best1L": 0.14465879555791616, + "best1L_layer": 18, + "mean": 0.10061749830179346 + }, + "OOD_arXiv_2023|s1|shuf0": { + "min": 0.25658972561359406, + "persist": 0.08155045211315155, + "best1L": 0.13688263203948736, + "best1L_layer": 18, + "mean": 0.08534403256042145 + }, + "OOD_arXiv_2023|s1|shuf1": { + "min": 0.26868619695305823, + "persist": 0.08128509055823088, + "best1L": 0.1566657116636634, + "best1L_layer": 18, + "mean": 0.08570556889932889 + }, + "OOD_arXiv_2023|s1|shuf2": { + "min": 0.29177426397800443, + "persist": 0.07937072664499283, + "best1L": 0.161034494638443, + "best1L_layer": 19, + "mean": 0.08774685529687513 + }, + "OOD_arXiv_2023|s1|shuf3": { + "min": 0.2612253025174141, + "persist": 0.0759934613481164, + "best1L": 0.14299439466558397, + "best1L_layer": 19, + "mean": 0.07864374442097659 + }, + "OOD_arXiv_2023|s1|shuf4": { + "min": 0.2692541994154453, + "persist": 0.07981508262455464, + "best1L": 0.161034494638443, + "best1L_layer": 19, + "mean": 0.0837050593964075 + }, + "OOD_arXiv_2023|s2|real": { + "min": 0.27008571829646827, + "persist": 0.10486989989876747, + "best1L": 0.15030740327201783, + "best1L_layer": 21, + "mean": 0.10786939886971735 + }, + "OOD_arXiv_2023|s2|cyc1": { + "min": 0.26244997307658197, + "persist": 0.0891899362206459, + "best1L": 0.15295392922125756, + "best1L_layer": 20, + "mean": 0.09370945760657867 + }, + "OOD_arXiv_2023|s2|cyc2": { + "min": 0.26016926281154157, + "persist": 0.07588321231305599, + "best1L": 0.160796719789505, + "best1L_layer": 19, + "mean": 0.08345595765724563 + }, + "OOD_arXiv_2023|s2|cyc3": { + "min": 0.2412263412028551, + "persist": 0.07223417703062296, + "best1L": 0.15716563668102027, + "best1L_layer": 18, + "mean": 0.07630439413120398 + }, + "OOD_arXiv_2023|s2|cyc4": { + "min": 0.2436048425734043, + "persist": 0.07131247045472264, + "best1L": 0.14247803441248835, + "best1L_layer": 17, + "mean": 0.07298982571586608 + }, + "OOD_arXiv_2023|s2|cyc5": { + "min": 0.2284705802798271, + "persist": 0.06894528865814209, + "best1L": 0.11537822773680091, + "best1L_layer": 20, + "mean": 0.07171462068755108 + }, + "OOD_arXiv_2023|s2|cyc6": { + "min": 0.22836322635412215, + "persist": 0.06273132031783461, + "best1L": 0.1262575877364725, + "best1L_layer": 20, + "mean": 0.07014613625808404 + }, + "OOD_arXiv_2023|s2|cyc7": { + "min": 0.2321656472980976, + "persist": 0.06871654633432626, + "best1L": 0.13901360160671175, + "best1L_layer": 19, + "mean": 0.0745584827919419 + }, + "OOD_arXiv_2023|s2|cyc8": { + "min": 0.24170712307095527, + "persist": 0.0717713269405067, + "best1L": 0.14233868471346794, + "best1L_layer": 19, + "mean": 0.07720353956728314 + }, + "OOD_arXiv_2023|s2|cyc9": { + "min": 0.2397597461938858, + "persist": 0.08066813084296882, + "best1L": 0.13465987034142018, + "best1L_layer": 18, + "mean": 0.0801041355338664 + }, + "OOD_arXiv_2023|s2|cyc10": { + "min": 0.26445486694574355, + "persist": 0.0872489795088768, + "best1L": 0.13784926142543555, + "best1L_layer": 17, + "mean": 0.08848206349558985 + }, + "OOD_arXiv_2023|s2|cyc11": { + "min": 0.2680948875844479, + "persist": 0.09265537299215794, + "best1L": 0.1432203055359423, + "best1L_layer": 19, + "mean": 0.09649318460097464 + }, + "OOD_arXiv_2023|s2|cyc12": { + "min": 0.27268675081431865, + "persist": 0.09357050023972988, + "best1L": 0.14221262335777282, + "best1L_layer": 18, + "mean": 0.10019393882517201 + }, + "OOD_arXiv_2023|s2|shuf0": { + "min": 0.2571215949952602, + "persist": 0.08120271135121584, + "best1L": 0.1360876625403762, + "best1L_layer": 18, + "mean": 0.08519055319529098 + }, + "OOD_arXiv_2023|s2|shuf1": { + "min": 0.27173766642808916, + "persist": 0.07898571649566292, + "best1L": 0.15716563668102027, + "best1L_layer": 18, + "mean": 0.08487706368547292 + }, + "OOD_arXiv_2023|s2|shuf2": { + "min": 0.29140356555581093, + "persist": 0.07789169233292341, + "best1L": 0.160796719789505, + "best1L_layer": 19, + "mean": 0.08702920557787786 + }, + "OOD_arXiv_2023|s2|shuf3": { + "min": 0.25893412753939626, + "persist": 0.0748466819524765, + "best1L": 0.14233868471346794, + "best1L_layer": 19, + "mean": 0.07823581470248217 + }, + "OOD_arXiv_2023|s2|shuf4": { + "min": 0.2704723007977009, + "persist": 0.07978044673800469, + "best1L": 0.160796719789505, + "best1L_layer": 19, + "mean": 0.08340466896725747 + }, + "OOD_CommonPile|s0|real": { + "min": 0.2531457645818591, + "persist": 0.10742316469550132, + "best1L": 0.15066220397129654, + "best1L_layer": 21, + "mean": 0.10892914802086755 + }, + "OOD_CommonPile|s0|cyc1": { + "min": 0.2627125080674887, + "persist": 0.09244429729878903, + "best1L": 0.15419739685021341, + "best1L_layer": 20, + "mean": 0.09741357403959577 + }, + "OOD_CommonPile|s0|cyc2": { + "min": 0.26012763045728204, + "persist": 0.08084794022142887, + "best1L": 0.16110447645187378, + "best1L_layer": 19, + "mean": 0.08660409054718912 + }, + "OOD_CommonPile|s0|cyc3": { + "min": 0.24968226067721844, + "persist": 0.07524253893643618, + "best1L": 0.15773502048105, + "best1L_layer": 18, + "mean": 0.08097087280968061 + }, + "OOD_CommonPile|s0|cyc4": { + "min": 0.24207183569669724, + "persist": 0.07128295265138149, + "best1L": 0.1447966059204191, + "best1L_layer": 17, + "mean": 0.07631471455186749 + }, + "OOD_CommonPile|s0|cyc5": { + "min": 0.23257821202278137, + "persist": 0.06925702821463346, + "best1L": 0.12253700504079461, + "best1L_layer": 20, + "mean": 0.0748938831675332 + }, + "OOD_CommonPile|s0|cyc6": { + "min": 0.21952449679374694, + "persist": 0.06901387060061097, + "best1L": 0.12621573833748698, + "best1L_layer": 20, + "mean": 0.07309467419981956 + }, + "OOD_CommonPile|s0|cyc7": { + "min": 0.2220759093761444, + "persist": 0.07254374772310257, + "best1L": 0.13899049693718551, + "best1L_layer": 19, + "mean": 0.07567421749162559 + }, + "OOD_CommonPile|s0|cyc8": { + "min": 0.23089288026094437, + "persist": 0.07848773952573537, + "best1L": 0.13630021042190493, + "best1L_layer": 19, + "mean": 0.07845097130451065 + }, + "OOD_CommonPile|s0|cyc9": { + "min": 0.23008935898542404, + "persist": 0.08131065811030566, + "best1L": 0.1322776926215738, + "best1L_layer": 19, + "mean": 0.08142051991308108 + }, + "OOD_CommonPile|s0|cyc10": { + "min": 0.2434085987508297, + "persist": 0.08847136597614735, + "best1L": 0.1362439488992095, + "best1L_layer": 19, + "mean": 0.08732332531452322 + }, + "OOD_CommonPile|s0|cyc11": { + "min": 0.2524479102343321, + "persist": 0.09383858114015312, + "best1L": 0.14285152908414603, + "best1L_layer": 19, + "mean": 0.09521454903386677 + }, + "OOD_CommonPile|s0|cyc12": { + "min": 0.25662991553545, + "persist": 0.09491675160825253, + "best1L": 0.14463012916967272, + "best1L_layer": 19, + "mean": 0.09986660404650208 + }, + "OOD_CommonPile|s0|shuf0": { + "min": 0.2543313570320606, + "persist": 0.07998228296637536, + "best1L": 0.13133459638338535, + "best1L_layer": 17, + "mean": 0.08589117893134243 + }, + "OOD_CommonPile|s0|shuf1": { + "min": 0.26050583124160764, + "persist": 0.08328658873215318, + "best1L": 0.15773502048105, + "best1L_layer": 18, + "mean": 0.08669545783195644 + }, + "OOD_CommonPile|s0|shuf2": { + "min": 0.2809336222708225, + "persist": 0.07980556171387435, + "best1L": 0.16110447645187378, + "best1L_layer": 19, + "mean": 0.08898586003073992 + }, + "OOD_CommonPile|s0|shuf3": { + "min": 0.24981799349188805, + "persist": 0.07883550319820642, + "best1L": 0.13630021042190493, + "best1L_layer": 19, + "mean": 0.08002273270573754 + }, + "OOD_CommonPile|s0|shuf4": { + "min": 0.2634584128856659, + "persist": 0.08045977549627423, + "best1L": 0.16110447645187378, + "best1L_layer": 19, + "mean": 0.08473745891466164 + }, + "OOD_CommonPile|s1|real": { + "min": 0.25511594451963904, + "persist": 0.10653829500079155, + "best1L": 0.1508266285993159, + "best1L_layer": 21, + "mean": 0.10740168782873438 + }, + "OOD_CommonPile|s1|cyc1": { + "min": 0.262430252879858, + "persist": 0.09121811166405677, + "best1L": 0.15397229841910304, + "best1L_layer": 20, + "mean": 0.09532200081112722 + }, + "OOD_CommonPile|s1|cyc2": { + "min": 0.25738223306834696, + "persist": 0.07685153149068355, + "best1L": 0.16069214791059494, + "best1L_layer": 19, + "mean": 0.08373599015982248 + }, + "OOD_CommonPile|s1|cyc3": { + "min": 0.24732892848551274, + "persist": 0.07387349773198366, + "best1L": 0.15692478474229574, + "best1L_layer": 18, + "mean": 0.0786788050049486 + }, + "OOD_CommonPile|s1|cyc4": { + "min": 0.2429734468460083, + "persist": 0.0686999642290175, + "best1L": 0.14269711724482476, + "best1L_layer": 17, + "mean": 0.07444880712965432 + }, + "OOD_CommonPile|s1|cyc5": { + "min": 0.22865016311407088, + "persist": 0.0674042111262679, + "best1L": 0.12044034404680133, + "best1L_layer": 20, + "mean": 0.07227921931368585 + }, + "OOD_CommonPile|s1|cyc6": { + "min": 0.21865845173597337, + "persist": 0.06744857300072908, + "best1L": 0.1256784507073462, + "best1L_layer": 20, + "mean": 0.07156100457964035 + }, + "OOD_CommonPile|s1|cyc7": { + "min": 0.22386411875486373, + "persist": 0.07004166077822446, + "best1L": 0.14008135432377458, + "best1L_layer": 19, + "mean": 0.07448294431400987 + }, + "OOD_CommonPile|s1|cyc8": { + "min": 0.23431778103113174, + "persist": 0.07617354784160853, + "best1L": 0.13475188738666474, + "best1L_layer": 19, + "mean": 0.07714395286479535 + }, + "OOD_CommonPile|s1|cyc9": { + "min": 0.23173676431179047, + "persist": 0.08209528918378055, + "best1L": 0.1320748451165855, + "best1L_layer": 19, + "mean": 0.0804194741962979 + }, + "OOD_CommonPile|s1|cyc10": { + "min": 0.24649499878287315, + "persist": 0.08706701498012989, + "best1L": 0.13694460298866035, + "best1L_layer": 19, + "mean": 0.0865678532855012 + }, + "OOD_CommonPile|s1|cyc11": { + "min": 0.2528634499758482, + "persist": 0.09123027175664902, + "best1L": 0.14311446901410818, + "best1L_layer": 19, + "mean": 0.0945980385850327 + }, + "OOD_CommonPile|s1|cyc12": { + "min": 0.2573898326605558, + "persist": 0.09367291294038296, + "best1L": 0.14291334906592965, + "best1L_layer": 19, + "mean": 0.09876475866364602 + }, + "OOD_CommonPile|s1|shuf0": { + "min": 0.2539062112569809, + "persist": 0.07729471735656261, + "best1L": 0.13090635638218373, + "best1L_layer": 17, + "mean": 0.08428751054766158 + }, + "OOD_CommonPile|s1|shuf1": { + "min": 0.26217192634940145, + "persist": 0.07961883107200265, + "best1L": 0.15692478474229574, + "best1L_layer": 18, + "mean": 0.08534733428118321 + }, + "OOD_CommonPile|s1|shuf2": { + "min": 0.2828520558774471, + "persist": 0.0780245989561081, + "best1L": 0.16069214791059494, + "best1L_layer": 19, + "mean": 0.08793366246211987 + }, + "OOD_CommonPile|s1|shuf3": { + "min": 0.24786126241087914, + "persist": 0.07607217859476804, + "best1L": 0.13475188738666474, + "best1L_layer": 19, + "mean": 0.07780913293648224 + }, + "OOD_CommonPile|s1|shuf4": { + "min": 0.26146751791238787, + "persist": 0.07973866332322359, + "best1L": 0.16069214791059494, + "best1L_layer": 19, + "mean": 0.08261492469598754 + }, + "OOD_CommonPile|s2|real": { + "min": 0.2503817303106189, + "persist": 0.1059548944234848, + "best1L": 0.15007893643341957, + "best1L_layer": 20, + "mean": 0.10623238773729939 + }, + "OOD_CommonPile|s2|cyc1": { + "min": 0.2586239531636238, + "persist": 0.09090781398117542, + "best1L": 0.15252565890550612, + "best1L_layer": 19, + "mean": 0.09525797984007603 + }, + "OOD_CommonPile|s2|cyc2": { + "min": 0.25594792813062667, + "persist": 0.07763439416885376, + "best1L": 0.15880076885223388, + "best1L_layer": 19, + "mean": 0.08400726578055093 + }, + "OOD_CommonPile|s2|cyc3": { + "min": 0.2481913235038519, + "persist": 0.0740542409941554, + "best1L": 0.15580145921558142, + "best1L_layer": 18, + "mean": 0.07910962627233507 + }, + "OOD_CommonPile|s2|cyc4": { + "min": 0.24340879842638968, + "persist": 0.06959064593538641, + "best1L": 0.1445134202716872, + "best1L_layer": 17, + "mean": 0.07548500023728523 + }, + "OOD_CommonPile|s2|cyc5": { + "min": 0.23236385583877564, + "persist": 0.06958994772285224, + "best1L": 0.12137967376038432, + "best1L_layer": 20, + "mean": 0.07383852944118328 + }, + "OOD_CommonPile|s2|cyc6": { + "min": 0.21841877400875093, + "persist": 0.06768160546198487, + "best1L": 0.12755314093083142, + "best1L_layer": 19, + "mean": 0.07213564105690098 + }, + "OOD_CommonPile|s2|cyc7": { + "min": 0.22373739778995513, + "persist": 0.07244668863713741, + "best1L": 0.14065227964892985, + "best1L_layer": 19, + "mean": 0.07486138190501011 + }, + "OOD_CommonPile|s2|cyc8": { + "min": 0.2302233688533306, + "persist": 0.07742505099158734, + "best1L": 0.13507484206929804, + "best1L_layer": 19, + "mean": 0.07727401566405136 + }, + "OOD_CommonPile|s2|cyc9": { + "min": 0.22676139771938325, + "persist": 0.08173806383274496, + "best1L": 0.13239063150249422, + "best1L_layer": 19, + "mean": 0.07934174434586357 + }, + "OOD_CommonPile|s2|cyc10": { + "min": 0.24038296043872834, + "persist": 0.08646663043182343, + "best1L": 0.1366969565860927, + "best1L_layer": 19, + "mean": 0.08549604031006591 + }, + "OOD_CommonPile|s2|cyc11": { + "min": 0.24679399654269218, + "persist": 0.09086780219804495, + "best1L": 0.14100920166820288, + "best1L_layer": 19, + "mean": 0.0931753066001245 + }, + "OOD_CommonPile|s2|cyc12": { + "min": 0.2537793729454279, + "persist": 0.09413428418338299, + "best1L": 0.14282543417066335, + "best1L_layer": 19, + "mean": 0.09771374927681441 + }, + "OOD_CommonPile|s2|shuf0": { + "min": 0.2520745679736137, + "persist": 0.07946023121476173, + "best1L": 0.13080244611483066, + "best1L_layer": 17, + "mean": 0.08422058678026167 + }, + "OOD_CommonPile|s2|shuf1": { + "min": 0.2588454566895962, + "persist": 0.08081897310912609, + "best1L": 0.15580145921558142, + "best1L_layer": 18, + "mean": 0.08495625513605773 + }, + "OOD_CommonPile|s2|shuf2": { + "min": 0.277788944542408, + "persist": 0.07980298958718776, + "best1L": 0.15880076885223388, + "best1L_layer": 19, + "mean": 0.08753995095881131 + }, + "OOD_CommonPile|s2|shuf3": { + "min": 0.24596437588334083, + "persist": 0.07772078756242991, + "best1L": 0.13507484206929804, + "best1L_layer": 19, + "mean": 0.07844246996638293 + }, + "OOD_CommonPile|s2|shuf4": { + "min": 0.2595244117081165, + "persist": 0.07968054311349988, + "best1L": 0.15880076885223388, + "best1L_layer": 19, + "mean": 0.0825342004885897 + } + }, + "provenance": { + "schema_version": 1, + "experiment": "DA1", + "utc": "2026-09-02T00:58:31+00:00", + "script": { + "path": "experiments/da1_derangement_adjudication.py", + "sha256": "054d1f0baf14f5f9767cedd4bd1d89c102aef6f07d6bf0f3d3a11b979c3dbc4d", + "n_lines": 554 + }, + "git": { + "commit": "271454ebce2fcff5c6e8efc26ed26536367d5936", + "commit_short": "271454ebce2f", + "git_repository_present": true, + "branch": "main", + "dirty": true, + "dirty_files": [ + "docs/experiments/preregs/DA1_derangement_adjudication.md", + "experiments/da1_derangement_adjudication.py", + "experiments/t59_read_dose.py", + "ocs/reproducibility/SCRIPT_PROVENANCE.md", + "repro/exp/da1_derangement.sh", + "repro/exp/r10_t59_readout.sh", + "repro/scripts_at_run/76473cf24c5b__t59_read_dose.py", + "tools/da1b_clustered_t.py", + "tools/emit_exp_modules.py", + "tools/r10_t59_readout_reconciliation.py" + ], + "dirty_diff_sha256": "4b9a825f5c9d7c0df2ccb7fe9a4474b14aabbc47236f6ed60d8740097d85eb3b", + "describe": "271454e-dirty" + }, + "argv": [ + "experiments/da1_derangement_adjudication.py", + "--device", + "cpu" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s0.pt", + "sha256": "efcf8b669825016233690a19162ff645dc58ed2e2628a2e113f26713e27cf998", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s1.pt", + "sha256": "bfa5aee55a338b05ccb9fc7d66a631da83b97c6e48cfa387105689bd30805033", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s2.pt", + "sha256": "33d97ade0445171e2329dc86da6ef93a218f178e3390f182e511c237ca2f1ed6", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "6f8f06ad3ce7ef6c3a0dc44afa63838df505ac8cd1617b25fda0aa9747a31524", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "2a2a3f4524ac4db804501e412d4ebc188935058687f1a468e2b5bfe0cdfbbfcd", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "a1e5dfd9f19145467500b542c74c1103bd90a44d430e4e3528aca22cdb589922", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s0.pt", + "sha256": "666e119f8cdcc6b218c020862e3de834c6920dcbed894121a108defa90877325", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s1.pt", + "sha256": "6c8e7e8900cdc01e54107cd6d4d786d12d199a7c50110f29606ea3ddbe64e0bc", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s2.pt", + "sha256": "fb4fc80ba2990f27e9343260491cfafbdb1d3bcaf9abc79cfc66b51a5fd38037", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s0.pt", + "sha256": "fba4f9a51cbd39b523043bea1286f8efc61ec72df542dc224a419c0e2e0a3a9f", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s1.pt", + "sha256": "7552cfc76417a4b50971284df5944f731e579c196137b8c32a3ae1c2d949d582", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s2.pt", + "sha256": "20e1867f9f0c7a5528f7b115ea67479c8875ff496ca35ee2323576e2aae81a63", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s0.pt", + "sha256": "3895668dce09dae91c6300f6adbba2de8359c63bf45fd5676d554f6ed90e9dc8", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s1.pt", + "sha256": "ca08e40ea0973004e8c21790e59334ea3fa6839be08e4753570c8ccfa3784283", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s2.pt", + "sha256": "fabb7cc8e515078c224375f23949ba336c33a4f86f35bf3e96df6cfbdba97bfc", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48_crossover_410m_rstrip.json", + "sha256": "eef75b627898125b090cb0b8b351a398885b1162f27c334bcef895d980064cc6", + "bytes": 111498, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "fc6fcb31101aa9238b96fb98711e83e46b5a43e59d3429e0a9d5dee5030a3522" + } +} \ No newline at end of file diff --git a/results/da1b_clustered_t.json b/results/da1b_clustered_t.json new file mode 100644 index 0000000000000000000000000000000000000000..5d6649966df7ec51cb25bc6292359e86b0b4e2ff --- /dev/null +++ b/results/da1b_clustered_t.json @@ -0,0 +1,366 @@ +{ + "experiment": "DA1b \u2014 corpus-clustered paired t on DA1's arms", + "status": "RECOMPUTATION \u2014 no model, no scoring, no new bar", + "recomputes_not_remeasures": true, + "statistic": "one-sample paired t over the 8 per-corpus values of (real - null), df = 7, two-sided p from Student's t. Transcribed from results/paper_clustered_derangement_t.json.", + "sets_no_threshold": "DA1's pre-registration deliberately created no pass/fail bar. The t is reported because the claim being adjudicated was reported with one.", + "source": "results/da1_derangement_adjudication_410m.json", + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "by_null": { + "cyclic": { + "min": { + "n_corpora": 8, + "mean_gap": 0.016158586757956082, + "sd_gap": 0.005935189272842151, + "t": 7.700409032091719, + "df": 7, + "p_two_sided": 0.0001162082895884551, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": 0.00866188103747037, + "USPTO_Backgrounds": 0.013367152612449418, + "Pile-CC": 0.011950484456287547, + "StackExchange": 0.021557182704822875, + "Github": 0.025896132261388816, + "OOD_News_2024": 0.015564333906190247, + "OOD_arXiv_2023": 0.02063941022174226, + "OOD_CommonPile": 0.011632116863297126 + }, + "n_corpora_real_higher": 8 + }, + "persist": { + "n_corpora": 8, + "mean_gap": 0.026932741740408044, + "sd_gap": 0.004813248791752445, + "t": 15.826586278458048, + "df": 7, + "p_two_sided": 9.745694470193062e-07, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": 0.024912288402103716, + "USPTO_Backgrounds": 0.03391349902829259, + "Pile-CC": 0.02618290100298408, + "StackExchange": 0.032589205108686445, + "Github": 0.01846325099144855, + "OOD_News_2024": 0.024781826972806204, + "OOD_arXiv_2023": 0.02759048957895073, + "OOD_CommonPile": 0.02702847283799202 + }, + "n_corpora_real_higher": 8 + }, + "best1L": { + "n_corpora": 8, + "mean_gap": 0.010585618611043575, + "sd_gap": 0.006528119316292479, + "t": 4.58641292553733, + "df": 7, + "p_two_sided": 0.0025244836066057683, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": 0.007148121246912831, + "USPTO_Backgrounds": 0.0044389816977652985, + "Pile-CC": 0.007965687827284768, + "StackExchange": 0.025273932869054577, + "Github": 0.014144847043401881, + "OOD_News_2024": 0.008197156071481804, + "OOD_arXiv_2023": 0.008032226041218032, + "OOD_CommonPile": 0.009483996091229407 + }, + "n_corpora_real_higher": 8 + }, + "mean": { + "n_corpora": 8, + "mean_gap": 0.024403997495239742, + "sd_gap": 0.0026990833649512412, + "t": 25.57347037297646, + "df": 7, + "p_two_sided": 3.5715863869041014e-08, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": 0.02068865052615909, + "USPTO_Backgrounds": 0.026204439467747004, + "Pile-CC": 0.024389282417118963, + "StackExchange": 0.029249768276760083, + "Github": 0.021976490119874126, + "OOD_News_2024": 0.022576457652938448, + "OOD_arXiv_2023": 0.02548529269619254, + "OOD_CommonPile": 0.024661598805127694 + }, + "n_corpora_real_higher": 8 + } + }, + "shuf": { + "min": { + "n_corpora": 8, + "mean_gap": -0.005223511185807477, + "sd_gap": 0.007681076142808633, + "t": -1.923470155700173, + "df": 7, + "p_two_sided": 0.09583906347439665, + "clears_05": false, + "per_corpus_gap": { + "Wikipedia_en": -0.01677957579493522, + "USPTO_Backgrounds": -0.00401597244044144, + "Pile-CC": -0.012404273003339739, + "StackExchange": -0.0006210960696140799, + "Github": 0.008090302099784202, + "OOD_News_2024": -0.007111757447322231, + "OOD_arXiv_2023": -0.0010599334662159121, + "OOD_CommonPile": -0.007885783364375398 + }, + "n_corpora_real_higher": 1 + }, + "persist": { + "n_corpora": 8, + "mean_gap": 0.027495992561453023, + "sd_gap": 0.004445116490390382, + "t": 17.49569698584057, + "df": 7, + "p_two_sided": 4.905678869008229e-07, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": 0.025456481482833623, + "USPTO_Backgrounds": 0.033592421001133826, + "Pile-CC": 0.02854199816628049, + "StackExchange": 0.03240535648384442, + "Github": 0.0190823487068216, + "OOD_News_2024": 0.026893035499379034, + "OOD_arXiv_2023": 0.02673099617784222, + "OOD_CommonPile": 0.027265302973488975 + }, + "n_corpora_real_higher": 8 + }, + "best1L": { + "n_corpora": 8, + "mean_gap": 0.0009394872553336112, + "sd_gap": 0.004549157575910848, + "t": 0.5841238057810929, + "df": 7, + "p_two_sided": 0.5774675390776101, + "clears_05": false, + "per_corpus_gap": { + "Wikipedia_en": -0.0028476039599627245, + "USPTO_Backgrounds": -0.003643520946303991, + "Pile-CC": 0.0022061711053053545, + "StackExchange": 0.010690994511047991, + "Github": 0.0020594797221322736, + "OOD_News_2024": -0.0011414066531384992, + "OOD_arXiv_2023": -0.0016090460959821973, + "OOD_CommonPile": 0.001800830359570682 + }, + "n_corpora_real_higher": 4 + }, + "mean": { + "n_corpora": 8, + "mean_gap": 0.022790310339493237, + "sd_gap": 0.002493348995303512, + "t": 25.853072340464518, + "df": 7, + "p_two_sided": 3.312162360490782e-08, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": 0.018619374527434757, + "USPTO_Backgrounds": 0.02433006116060707, + "Pile-CC": 0.023196681431936428, + "StackExchange": 0.02677872111755185, + "Github": 0.02071954481942484, + "OOD_News_2024": 0.021422659716634554, + "OOD_arXiv_2023": 0.02386894652458768, + "OOD_CommonPile": 0.023386493417768717 + }, + "n_corpora_real_higher": 8 + } + }, + "bandconst_EXPLORATORY": { + "min": { + "n_corpora": 8, + "mean_gap": 0.021238462246644003, + "sd_gap": 0.006216932701958535, + "t": 9.66255315702249, + "df": 7, + "p_two_sided": 2.681493607990114e-05, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": 0.011700503776470805, + "USPTO_Backgrounds": 0.026304761196176224, + "Pile-CC": 0.01713961673279607, + "StackExchange": 0.019939706971247972, + "Github": 0.01747901712854702, + "OOD_News_2024": 0.022100920105973865, + "OOD_arXiv_2023": 0.03204078444590172, + "OOD_CommonPile": 0.02320238761603835 + }, + "n_corpora_real_higher": 8, + "caveat": "J_bar is not a permutation of the band's Jacobians; it is a different object with a different norm and spectrum. Exploratory only." + }, + "persist": { + "n_corpora": 8, + "mean_gap": -0.0001377559587126577, + "sd_gap": 0.0034932725417077966, + "t": -0.11153801644912968, + "df": 7, + "p_two_sided": 0.9143206055226831, + "clears_05": false, + "per_corpus_gap": { + "Wikipedia_en": -0.0027960463116566414, + "USPTO_Backgrounds": 0.00514817194392285, + "Pile-CC": -0.00012227718252687259, + "StackExchange": 0.0042915986618027036, + "Github": -0.00376009407142798, + "OOD_News_2024": -0.004309271268236145, + "OOD_arXiv_2023": 0.00015000278751055307, + "OOD_CommonPile": 0.0002958677709102714 + }, + "n_corpora_real_higher": 4, + "caveat": "J_bar is not a permutation of the band's Jacobians; it is a different object with a different norm and spectrum. Exploratory only." + }, + "best1L": { + "n_corpora": 8, + "mean_gap": -0.008530615242004085, + "sd_gap": 0.0057730551983264855, + "t": -4.179454848838448, + "df": 7, + "p_two_sided": 0.004140061276152293, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": -0.01136010349728167, + "USPTO_Backgrounds": -0.013883013216157758, + "Pile-CC": -0.011526565284778661, + "StackExchange": 0.003311048960313201, + "Github": -0.003982829985519243, + "OOD_News_2024": -0.008043611911125487, + "OOD_arXiv_2023": -0.013647701110069965, + "OOD_CommonPile": -0.009112145891413098 + }, + "n_corpora_real_higher": 1, + "caveat": "J_bar is not a permutation of the band's Jacobians; it is a different object with a different norm and spectrum. Exploratory only." + }, + "mean": { + "n_corpora": 8, + "mean_gap": 0.004248576085303331, + "sd_gap": 0.0024787876802077552, + "t": 4.847848783971993, + "df": 7, + "p_two_sided": 0.0018609087759251224, + "clears_05": true, + "per_corpus_gap": { + "Wikipedia_en": -7.722141125644788e-05, + "USPTO_Backgrounds": 0.0047846465702885005, + "Pile-CC": 0.005041745336403919, + "StackExchange": 0.00725659876748419, + "Github": 0.001179065340413496, + "OOD_News_2024": 0.004269551258617776, + "OOD_arXiv_2023": 0.00538028467622681, + "OOD_CommonPile": 0.006153938144248403 + }, + "n_corpora_real_higher": 7, + "caveat": "J_bar is not a permutation of the band's Jacobians; it is a different object with a different norm and spectrum. Exploratory only." + } + } + }, + "control_C1_reproduces_paper_clustered_t": { + "requirement": "DA1's shuf gaps reproduce paper_clustered_derangement_t.json (corrected readout) to < 1e-9 on mean_gap and < 1e-6 on t", + "fails_if": "any larger difference, which would mean DA1 is not scoring the same gaps", + "per_agg": { + "min": { + "da1_mean_gap": -0.005223511185807477, + "stored_mean_gap": -0.005223511185807477, + "abs_diff_mean_gap": 0.0, + "da1_t": -1.923470155700173, + "stored_t": -1.923470155700173, + "abs_diff_t": 0.0, + "da1_p": 0.09583906347439665, + "stored_p": 0.0958390634744094 + }, + "persist": { + "da1_mean_gap": 0.027495992561453023, + "stored_mean_gap": 0.027495992561453023, + "abs_diff_mean_gap": 0.0, + "da1_t": 17.49569698584057, + "stored_t": 17.49569698584057, + "abs_diff_t": 0.0, + "da1_p": 4.905678869008229e-07, + "stored_p": 4.905678869008232e-07 + } + }, + "max_abs_diff_mean_gap": 0.0, + "max_abs_diff_t": 0.0, + "fires": true + }, + "provenance": { + "schema_version": 1, + "experiment": "DA1b", + "utc": "2026-09-02T00:47:22+00:00", + "script": { + "path": "tools/da1b_clustered_t.py", + "sha256": "e164a21674f9d62d84a687e4b46fcf97e6bffd4a1ca65a184f1c8c72cb56ece0", + "n_lines": 166 + }, + "git": { + "commit": "271454ebce2fcff5c6e8efc26ed26536367d5936", + "commit_short": "271454ebce2f", + "git_repository_present": true, + "branch": "main", + "dirty": true, + "dirty_files": [ + "docs/experiments/preregs/DA1_derangement_adjudication.md", + "experiments/da1_derangement_adjudication.py", + "repro/exp/da1_derangement.sh", + "repro/exp/r10_t59_readout.sh", + "tools/da1b_clustered_t.py", + "tools/emit_exp_modules.py", + "tools/r10_t59_readout_reconciliation.py", + "xperiments/t59_read_dose.py" + ], + "dirty_diff_sha256": "a46271449f3ded51ab02128ce1febc52dc813bec5b8f6c96b35adfe782721bf5", + "describe": "271454e-dirty" + }, + "argv": [ + "tools/da1b_clustered_t.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/da1_derangement_adjudication_410m.json", + "sha256": "537a2b38683e180ee48b226631c5ab7d7b1bb3b2f2a7f1f8e86c5b5e11a15cc5", + "bytes": 1112404, + "exists": true + }, + { + "path": "results/paper_clustered_derangement_t.json", + "sha256": "017f2439a6249ba4bc2d3cb5c0ee84c55a2bf5d6f073b91f506e675878b32fe5", + "bytes": 5897, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "d3edcff106a9ddd6265b88f437138c5daf3a81a13a05135fc0931b5ea6f9397a" + } +} \ No newline at end of file diff --git a/results/dn1_design_nulls.json b/results/dn1_design_nulls.json new file mode 100644 index 0000000000000000000000000000000000000000..f8e40ac56f62c7c4f27dfe9f3147c6037672ac7a --- /dev/null +++ b/results/dn1_design_nulls.json @@ -0,0 +1,405 @@ +{ + "experiment": "DN1 \u2014 null distributions of three already-registered bars", + "prereg": "docs/experiments/preregs/DN1_design_nulls.md", + "status": "PRE-REGISTERED", + "measures_nothing_new": true, + "scope": "Calibrates thresholds, not measurements. Cannot overturn CV6, CV7 or R9; prices the bars they were judged against.", + "n_sim": 1000000, + "n_perm": 100000, + "A1_design_null": { + "definition": "R = range(group means) / sqrt(mean(per-group sample var)), G=5 groups of n=3", + "distribution": "iid normal, scale-free in sigma and location-free in mu", + "mean": 1.4558214009529349, + "sd": 0.6683661758236041, + "percentiles": { + "p50": 1.3504426642962697, + "p90": 2.3197482513624257, + "p95": 2.685370632595311, + "p99": 3.539988075003452, + "p99.9": 4.87893156220862 + }, + "P_R_ge_10": 2e-06, + "P_R_ge_10_exceedances_observed": 2, + "note": "A one-sided exceedance of 0 in n_sim draws is reported as < 1/n_sim, not as 0. The bar is calibrated against this column." + }, + "C1_scale_freedom": { + "required": "|p99(sigma=1) - p99(sigma=100)| <= 0.02", + "p99_sigma_1": 3.539988075003452, + "p99_sigma_100": 3.5461058294242056, + "abs_diff": 0.006117754420753574, + "fires": true, + "falsification_witness": { + "what": "same code with the denominator replaced by the constant 1.0", + "p99_sigma_1": 2.6567914459131163, + "p99_sigma_100": 265.1280524472683, + "abs_diff": 262.4712610013552, + "control_would_fire": false, + "note": "this must be False, or C1 cannot distinguish a scale-free statistic from any other" + } + }, + "A2_permutation_null_410m": { + "source": "results/d3_corpus_by_family_410m.json -> per_arm.|s..z", + "unit": "15 z values per family, permuted across 5 corpus labels, 3 per label", + "by_family": { + "multihop": { + "observed_R": 25.361005428912275, + "null_mean": 1.4354219513761668, + "null_sd": 0.8281876972206043, + "null_p50": 1.277465107028326, + "null_p95": 2.515198551887543, + "null_p99": 5.61098563909792, + "null_max": 25.43264217979271, + "n_null_ge_bar": 23, + "P_null_ge_bar": 0.00023, + "n_null_ge_observed": 2, + "p_observed_one_sided": 2.999970000299997e-05 + }, + "multilingual": { + "observed_R": 19.426787847831193, + "null_mean": 1.4472181163211184, + "null_sd": 0.7043406387033073, + "null_p50": 1.3324022652684364, + "null_p95": 2.660570482190145, + "null_p99": 3.8541903041621643, + "null_max": 15.066763195787532, + "n_null_ge_bar": 10, + "P_null_ge_bar": 0.0001, + "n_null_ge_observed": 0, + "p_observed_one_sided": 9.99990000099999e-06 + }, + "order-ops": { + "observed_R": 24.632511354365725, + "null_mean": 1.4600481090175972, + "null_sd": 0.724071411525985, + "null_p50": 1.341316486899336, + "null_p95": 2.7854524342316607, + "null_p99": 3.4035854960502476, + "null_max": 24.632511354365725, + "n_null_ge_bar": 10, + "P_null_ge_bar": 0.0001, + "n_null_ge_observed": 1, + "p_observed_one_sided": 1.999980000199998e-05 + }, + "poetry": { + "observed_R": 42.686224720210525, + "null_mean": 1.4207336216576643, + "null_sd": 1.1226321014484546, + "null_p50": 1.0953311671022932, + "null_p95": 2.289972367428842, + "null_p99": 8.799193358965976, + "null_max": 42.686224720210525, + "n_null_ge_bar": 571, + "P_null_ge_bar": 0.00571, + "n_null_ge_observed": 1, + "p_observed_one_sided": 1.999980000199998e-05 + }, + "typo": { + "observed_R": 27.014210896207846, + "null_mean": 1.4510950885052967, + "null_sd": 0.7238398592296635, + "null_p50": 1.3394268870958137, + "null_p95": 2.644590252011299, + "null_p99": 4.196756766414129, + "null_max": 11.924406661562124, + "n_null_ge_bar": 11, + "P_null_ge_bar": 0.00011, + "n_null_ge_observed": 0, + "p_observed_one_sided": 9.99990000099999e-06 + } + } + }, + "C2_pooled_sd_recomputes": { + "required": "<= 1e-12 on all 5 families", + "by_family": { + "multihop": { + "recomputed": 0.017453481001782855, + "stored": 0.017453481001782855, + "abs_diff": 0.0 + }, + "multilingual": { + "recomputed": 0.019695881058294912, + "stored": 0.019695881058294912, + "abs_diff": 0.0 + }, + "order-ops": { + "recomputed": 0.012061420042855183, + "stored": 0.012061420042855183, + "abs_diff": 0.0 + }, + "poetry": { + "recomputed": 0.005578110253433368, + "stored": 0.005578110253433368, + "abs_diff": 0.0 + }, + "typo": { + "recomputed": 0.019451275697770446, + "stored": 0.019451275697770446, + "abs_diff": 0.0 + } + }, + "max_abs_diff": 0.0, + "fires": true + }, + "C3_R_recomputes": { + "required": "<= 1e-12 on all 5 families", + "by_family": { + "multihop": { + "recomputed": 25.361005428912275, + "stored": 25.36100542891225, + "abs_diff": 2.4868995751603507e-14 + }, + "multilingual": { + "recomputed": 19.426787847831193, + "stored": 19.426787847831193, + "abs_diff": 0.0 + }, + "order-ops": { + "recomputed": 24.632511354365725, + "stored": 24.632511354365725, + "abs_diff": 0.0 + }, + "poetry": { + "recomputed": 42.686224720210525, + "stored": 42.686224720210525, + "abs_diff": 0.0 + }, + "typo": { + "recomputed": 27.014210896207846, + "stored": 27.014210896207846, + "abs_diff": 0.0 + } + }, + "max_abs_diff": 2.4868995751603507e-14, + "fires": true + }, + "C4_identity_in_support": { + "required": "the unpermuted assignment reproduces the observed R exactly", + "by_family": { + "multihop": { + "identity_R": 25.361005428912275, + "observed_R": 25.361005428912275, + "abs_diff": 0.0 + }, + "multilingual": { + "identity_R": 19.426787847831193, + "observed_R": 19.426787847831193, + "abs_diff": 0.0 + }, + "order-ops": { + "identity_R": 24.632511354365725, + "observed_R": 24.632511354365725, + "abs_diff": 0.0 + }, + "poetry": { + "identity_R": 42.686224720210525, + "observed_R": 42.686224720210525, + "abs_diff": 0.0 + }, + "typo": { + "identity_R": 27.014210896207846, + "observed_R": 27.014210896207846, + "abs_diff": 0.0 + } + }, + "max_abs_diff": 0.0, + "fires": true + }, + "C5_design_vs_permutation": { + "required": "the two 99th percentiles agree to within a factor of 1.25", + "p99_design": 3.539988075003452, + "p99_permutation_worst_family": 8.799193358965976, + "ratio": 2.485656214804451, + "fires": false + }, + "B_tau_null_exact": { + "n": 5, + "n_orderings": 120, + "bar": 0.6, + "n_orderings_clearing_bar": 14, + "P_single_family_tau_ge_bar": 0.11666666666666667, + "distinct_tau_values": [ + -1.0, + -0.8, + -0.6, + -0.4, + -0.2, + 0.0, + 0.2, + 0.4, + 0.6, + 0.8, + 1.0 + ], + "P_k_of_5_families_clear_under_independence": { + "0": 0.5378028459362139, + "1": 0.3551528227880658, + "2": 0.09381395318930041, + "3": 0.012390522119341565, + "4": 0.0008182420267489713, + "5": 2.1613940329218108e-05 + }, + "P_at_least_3_of_5_under_independence": 0.013230378086419754, + "DEPENDENCE_CAVEAT": "the five families share the same five operators and one activation cache, so they are not independent; the independence figure is an UPPER bound on the evidence, not an estimate of it" + }, + "C_r9_count_null_exact": { + "design": "8 corpora, 15 own-derangement draws each", + "P_one_corpus_clears_all_draws_under_H0": 0.0625, + "expected_count_under_H0": 0.5, + "P_count_equals_zero_under_H0": 0.5967194738332182, + "observed_count": 0, + "reading": "a count of 0 is the modal outcome under the exact null, so the count alone carries almost no information; the direction does", + "sign_test_on_direction": { + "statistic": "number of corpora whose real z_vs_null is negative, min aggregation", + "n_negative": 7, + "n_total": 8, + "z_by_corpus": { + "Wikipedia_en": -1.6709216685460513, + "USPTO_Backgrounds": -0.7050665067014444, + "Pile-CC": -1.9874216306081265, + "StackExchange": -0.09090121377251528, + "Github": 0.593144297659444, + "OOD_News_2024": -0.7490796022292463, + "OOD_arXiv_2023": -0.0881460749226426, + "OOD_CommonPile": -0.7064942600047919 + }, + "p_one_sided_exact_binomial": 0.03515625, + "DEPENDENCE_CAVEAT": "the eight corpora share one activation cache and one 551-item battery; the sign test assumes independence and therefore understates the p-value" + }, + "positive_control": { + "what": "persist on the same draws returns 8 of 8", + "n_beating": 8, + "why_it_matters": "the design CAN return the positive branch, so min's 0 of 8 is a failure of the statistic and not a lack of power in the null" + } + }, + "controls_fired": { + "C1_scale_freedom": true, + "C2_pooled_sd_recomputes": true, + "C3_R_recomputes": true, + "C4_identity_in_support": true, + "C5_design_vs_permutation": false + }, + "VERDICT": "UNCLEAR \u2014 a control did not fire: C5_design_vs_permutation", + "provenance": { + "schema_version": 1, + "experiment": "DN1", + "utc": "2026-08-31T23:19:44+00:00", + "script": { + "path": "experiments/dn1_design_nulls.py", + "sha256": "06e93d9dbd43feadaf2975d5854fd7252ba55264109a66909b2f4df18ea7f6e6", + "n_lines": 305 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/dn1_design_nulls.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/d3_corpus_by_family_410m.json", + "sha256": "795676289554af8111a4b4464617203271848f210fe1ce29a9946d44ab5dad5a", + "bytes": 23102, + "exists": true + }, + { + "path": "results/r9_permutation_calibrated_min.json", + "sha256": "773e2a6423db3022d96f71373c859fd0ae313f529abf569c396fc7de56461b89", + "bytes": 22358, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "201a8d6db3dfd8bcb12d81b23aa253b0ee6e7795a039ddb802b39ef668518dfe" + } +} \ No newline at end of file diff --git a/results/e13_rank_target_layer.json b/results/e13_rank_target_layer.json new file mode 100644 index 0000000000000000000000000000000000000000..bfeac6b300cf87384996abd0d92dba32204c5be6 --- /dev/null +++ b/results/e13_rank_target_layer.json @@ -0,0 +1,134 @@ +{ + "experiment": "E13 \u2014 is T4's stable-rank ladder a target-layer artifact?", + "status": "EXPLORATORY", + "models": { + "70m": { + "n_layers": 6, + "d_model": 512, + "pen_max": 50.798, + "fin_max": 8.516, + "max_ratio": 5.965, + "pen_mean": 40.063, + "fin_mean": 5.453, + "mean_ratio": 7.346, + "pen_per_layer": [ + 44.589, + 37.779, + 50.798, + 27.086 + ], + "fin_per_layer": [ + 8.516, + 5.107, + 5.442, + 4.864, + 3.338 + ] + }, + "160m": { + "n_layers": 12, + "d_model": 768, + "pen_max": 78.005, + "fin_max": 7.58, + "max_ratio": 10.291, + "pen_mean": 50.634, + "fin_mean": 6.495, + "mean_ratio": 7.796, + "pen_per_layer": [ + 39.265, + 40.895, + 40.69, + 46.243, + 48.052, + 51.682, + 59.528, + 50.263, + 51.714, + 78.005 + ], + "fin_per_layer": [ + 7.53, + 7.58, + 6.506, + 6.089, + 5.585, + 5.269, + 7.241, + 6.329, + 7.049, + 7.264, + 5.0 + ] + }, + "410m": { + "n_layers": 24, + "d_model": 1024, + "pen_max": 38.222, + "fin_max": 66.849, + "max_ratio": 0.572, + "pen_mean": 20.869, + "fin_mean": 27.602, + "mean_ratio": 0.756, + "pen_per_layer": [ + 20.386, + 20.32, + 19.552, + 19.707, + 20.209, + 18.724, + 19.699, + 18.88, + 17.592, + 17.29, + 17.48, + 18.007, + 20.805, + 21.058, + 21.658, + 20.136, + 21.896, + 21.622, + 20.532, + 20.543, + 24.794, + 38.222 + ], + "fin_per_layer": [ + 16.157, + 15.743, + 15.567, + 14.286, + 14.057, + 12.036, + 12.618, + 12.277, + 10.85, + 12.803, + 12.425, + 13.622, + 23.337, + 25.413, + 29.53, + 29.332, + 39.031, + 44.068, + 51.464, + 62.108, + 61.321, + 66.849, + 39.963 + ] + }, + "1b": { + "n_layers": 16, + "d_model": 2048, + "pen_max": 151.886, + "fin_max": 75.233, + "max_ratio": 2.019, + "pen_mean": 45.772, + "fin_mean": 30.935, + "mean_ratio": 1.48 + } + }, + "note": "same model, corpus, n=200, dim_batch=128, skip_first=16; only target_layer differs" +} \ No newline at end of file diff --git a/results/e28_Github_410m_n100_s0_reads.json b/results/e28_Github_410m_n100_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..f4836db11b18e917a9afc4db881fb3391709b971 --- /dev/null +++ b/results/e28_Github_410m_n100_s0_reads.json @@ -0,0 +1,634 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/e28_Github_410m_n100_s0.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 6, + "best1L": 3, + "mean": 5, + "persist": 4 + }, + "n_sets_stable": 4, + "n_sets_aggregation_dependent": 2, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "results": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "best1L_layer": 19, + "mean": 0.033, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0476 + }, + "jlens": { + "min": 0.1973, + "best1L": 0.1413, + "best1L_layer": 21, + "mean": 0.0293, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0255 + }, + "shuffled": { + "min": 0.1081, + "best1L": 0.0814, + "best1L_layer": 21, + "mean": 0.0099, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0029 + }, + "per_layer_jlens": { + "0": 0.0209, + "1": 0.0175, + "2": 0.0102, + "3": 0.0073, + "4": 0.0044, + "5": 0.0015, + "6": 0.0029, + "7": 0.0015, + "8": 0.0, + "9": 0.0015, + "10": 0.0, + "11": 0.0029, + "12": 0.0088, + "13": 0.0044, + "14": 0.0102, + "15": 0.0345, + "16": 0.0321, + "17": 0.0411, + "18": 0.0695, + "19": 0.1152, + "20": 0.1178, + "21": 0.1413 + }, + "per_layer_shuffled": { + "0": 0.0015, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0044, + "6": 0.0, + "7": 0.0068, + "8": 0.0015, + "9": 0.0, + "10": 0.0015, + "11": 0.0, + "12": 0.0015, + "13": 0.0, + "14": 0.0029, + "15": 0.0248, + "16": 0.0595, + "17": 0.0044, + "18": 0.0, + "19": 0.0044, + "20": 0.0236, + "21": 0.0814 + }, + "layer_specific": { + "min": 0.0891, + "best1L": 0.0599, + "mean": 0.0194, + "persist": 0.0226 + }, + "generic": { + "min": -0.0786, + "best1L": -0.0689, + "mean": -0.0231, + "persist": -0.0447 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "best1L_layer": 17, + "mean": 0.0165, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0212 + }, + "jlens": { + "min": 0.1253, + "best1L": 0.0844, + "best1L_layer": 17, + "mean": 0.0199, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0284 + }, + "shuffled": { + "min": 0.0831, + "best1L": 0.0435, + "best1L_layer": 16, + "mean": 0.0086, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0059 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0023, + "3": 0.0018, + "4": 0.0018, + "5": 0.0011, + "6": 0.0008, + "7": 0.0008, + "8": 0.0004, + "9": 0.0011, + "10": 0.0023, + "11": 0.0004, + "12": 0.0018, + "13": 0.0048, + "14": 0.0376, + "15": 0.0375, + "16": 0.0519, + "17": 0.0844, + "18": 0.0589, + "19": 0.0562, + "20": 0.0495, + "21": 0.0431 + }, + "per_layer_shuffled": { + "0": 0.0004, + "1": 0.0, + "2": 0.0004, + "3": 0.0004, + "4": 0.0014, + "5": 0.0, + "6": 0.0011, + "7": 0.0008, + "8": 0.001, + "9": 0.0004, + "10": 0.0004, + "11": 0.0051, + "12": 0.0025, + "13": 0.0125, + "14": 0.0199, + "15": 0.0345, + "16": 0.0435, + "17": 0.0048, + "18": 0.0062, + "19": 0.0033, + "20": 0.0227, + "21": 0.029 + }, + "layer_specific": { + "min": 0.0422, + "best1L": 0.0409, + "mean": 0.0113, + "persist": 0.0225 + }, + "generic": { + "min": -0.0284, + "best1L": -0.0411, + "mean": -0.0079, + "persist": -0.0154 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "best1L_layer": 19, + "mean": 0.1449, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2915 + }, + "jlens": { + "min": 0.7162, + "best1L": 0.3652, + "best1L_layer": 21, + "mean": 0.2817, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.3555 + }, + "shuffled": { + "min": 0.6915, + "best1L": 0.3643, + "best1L_layer": 13, + "mean": 0.1811, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.241 + }, + "per_layer_jlens": { + "0": 0.0374, + "1": 0.1467, + "2": 0.2001, + "3": 0.2856, + "4": 0.2113, + "5": 0.2865, + "6": 0.3081, + "7": 0.3101, + "8": 0.2638, + "9": 0.2036, + "10": 0.242, + "11": 0.2357, + "12": 0.3351, + "13": 0.3554, + "14": 0.3439, + "15": 0.3577, + "16": 0.3306, + "17": 0.333, + "18": 0.3502, + "19": 0.3434, + "20": 0.3525, + "21": 0.3652 + }, + "per_layer_shuffled": { + "0": 0.0704, + "1": 0.0437, + "2": 0.035, + "3": 0.0205, + "4": 0.028, + "5": 0.2228, + "6": 0.0485, + "7": 0.2013, + "8": 0.239, + "9": 0.2194, + "10": 0.2258, + "11": 0.0339, + "12": 0.2516, + "13": 0.3643, + "14": 0.3426, + "15": 0.2943, + "16": 0.3183, + "17": 0.1447, + "18": 0.0292, + "19": 0.1583, + "20": 0.3432, + "21": 0.3486 + }, + "layer_specific": { + "min": 0.0247, + "best1L": 0.0009, + "mean": 0.1007, + "persist": 0.1145 + }, + "generic": { + "min": 0.1765, + "best1L": -0.0268, + "mean": 0.0361, + "persist": -0.0505 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "best1L_layer": 18, + "mean": 0.0021, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0312, + "best1L": 0.0123, + "best1L_layer": 18, + "mean": 0.0037, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0031 + }, + "shuffled": { + "min": 0.0335, + "best1L": 0.0087, + "best1L_layer": 20, + "mean": 0.0024, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0015, + "2": 0.0072, + "3": 0.0015, + "4": 0.0046, + "5": 0.0015, + "6": 0.0015, + "7": 0.0046, + "8": 0.0031, + "9": 0.0, + "10": 0.0, + "11": 0.0, + "12": 0.0015, + "13": 0.0, + "14": 0.0015, + "15": 0.0056, + "16": 0.0056, + "17": 0.0074, + "18": 0.0123, + "19": 0.0072, + "20": 0.0046, + "21": 0.0072 + }, + "per_layer_shuffled": { + "0": 0.0056, + "1": 0.0, + "2": 0.0015, + "3": 0.0, + "4": 0.0046, + "5": 0.0, + "6": 0.0, + "7": 0.0074, + "8": 0.0015, + "9": 0.0031, + "10": 0.0015, + "11": 0.0, + "12": 0.0, + "13": 0.0015, + "14": 0.0015, + "15": 0.0056, + "16": 0.0, + "17": 0.0031, + "18": 0.0, + "19": 0.0, + "20": 0.0087, + "21": 0.0061 + }, + "layer_specific": { + "min": -0.0023, + "best1L": 0.0036, + "mean": 0.0013, + "persist": 0.0031 + }, + "generic": { + "min": 0.0079, + "best1L": 0.001, + "mean": 0.0003, + "persist": -0.0015 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "best1L_layer": 21, + "mean": 0.0345, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0475 + }, + "jlens": { + "min": 0.2871, + "best1L": 0.1582, + "best1L_layer": 19, + "mean": 0.0747, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.085 + }, + "shuffled": { + "min": 0.2616, + "best1L": 0.1293, + "best1L_layer": 21, + "mean": 0.0465, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0521 + }, + "per_layer_jlens": { + "0": 0.0057, + "1": 0.0073, + "2": 0.0125, + "3": 0.0183, + "4": 0.0456, + "5": 0.0542, + "6": 0.0652, + "7": 0.0709, + "8": 0.0844, + "9": 0.0936, + "10": 0.0698, + "11": 0.0526, + "12": 0.0594, + "13": 0.0605, + "14": 0.0745, + "15": 0.0795, + "16": 0.0873, + "17": 0.0951, + "18": 0.1449, + "19": 0.1582, + "20": 0.1527, + "21": 0.1512 + }, + "per_layer_shuffled": { + "0": 0.0016, + "1": 0.0076, + "2": 0.0, + "3": 0.0016, + "4": 0.0466, + "5": 0.0378, + "6": 0.0545, + "7": 0.0394, + "8": 0.0667, + "9": 0.0743, + "10": 0.0667, + "11": 0.0232, + "12": 0.0701, + "13": 0.0602, + "14": 0.049, + "15": 0.0831, + "16": 0.059, + "17": 0.0363, + "18": 0.0357, + "19": 0.0464, + "20": 0.0336, + "21": 0.1293 + }, + "layer_specific": { + "min": 0.0255, + "best1L": 0.0289, + "mean": 0.0282, + "persist": 0.0328 + }, + "generic": { + "min": 0.0859, + "best1L": 0.0247, + "mean": 0.0119, + "persist": 0.0046 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "best1L_layer": 13, + "mean": 0.0005, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "jlens": { + "min": 0.0221, + "best1L": 0.0079, + "best1L_layer": 8, + "mean": 0.0018, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0082, + "best1L": 0.0033, + "best1L_layer": 5, + "mean": 0.0006, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0033, + "1": 0.0033, + "2": 0.0016, + "3": 0.0, + "4": 0.0016, + "5": 0.0044, + "6": 0.0044, + "7": 0.0016, + "8": 0.0079, + "9": 0.0016, + "10": 0.0, + "11": 0.0033, + "12": 0.0016, + "13": 0.0, + "14": 0.0016, + "15": 0.0, + "16": 0.0016, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0016 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0033, + "6": 0.0, + "7": 0.0016, + "8": 0.0016, + "9": 0.0, + "10": 0.0016, + "11": 0.0, + "12": 0.0033, + "13": 0.0, + "14": 0.0, + "15": 0.0016, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0 + }, + "layer_specific": { + "min": 0.0139, + "best1L": 0.0046, + "mean": 0.0012, + "persist": 0.0 + }, + "generic": { + "min": 0.0006, + "best1L": -0.0011, + "mean": 0.0001, + "persist": 0.0 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 143.0 +} \ No newline at end of file diff --git a/results/e28_Github_410m_n100_s1_reads.json b/results/e28_Github_410m_n100_s1_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..e248c66d168a8b91e101654ee73e39fea7013744 --- /dev/null +++ b/results/e28_Github_410m_n100_s1_reads.json @@ -0,0 +1,634 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/e28_Github_410m_n100_s1.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 6, + "best1L": 4, + "mean": 6, + "persist": 4 + }, + "n_sets_stable": 3, + "n_sets_aggregation_dependent": 3, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "results": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "best1L_layer": 19, + "mean": 0.033, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0476 + }, + "jlens": { + "min": 0.2106, + "best1L": 0.1445, + "best1L_layer": 21, + "mean": 0.0333, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0294 + }, + "shuffled": { + "min": 0.1213, + "best1L": 0.0889, + "best1L_layer": 21, + "mean": 0.0116, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0029 + }, + "per_layer_jlens": { + "0": 0.0209, + "1": 0.0175, + "2": 0.0253, + "3": 0.0088, + "4": 0.0088, + "5": 0.0015, + "6": 0.0015, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0, + "11": 0.0073, + "12": 0.02, + "13": 0.0102, + "14": 0.0156, + "15": 0.0376, + "16": 0.0374, + "17": 0.0472, + "18": 0.0848, + "19": 0.12, + "20": 0.1246, + "21": 0.1445 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0058, + "6": 0.0, + "7": 0.0068, + "8": 0.0015, + "9": 0.0, + "10": 0.0039, + "11": 0.0, + "12": 0.0015, + "13": 0.0058, + "14": 0.0058, + "15": 0.0272, + "16": 0.0651, + "17": 0.0044, + "18": 0.0, + "19": 0.0097, + "20": 0.0292, + "21": 0.0889 + }, + "layer_specific": { + "min": 0.0894, + "best1L": 0.0556, + "mean": 0.0217, + "persist": 0.0265 + }, + "generic": { + "min": -0.0655, + "best1L": -0.0614, + "mean": -0.0214, + "persist": -0.0447 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "best1L_layer": 17, + "mean": 0.0165, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0212 + }, + "jlens": { + "min": 0.1243, + "best1L": 0.0858, + "best1L_layer": 17, + "mean": 0.0198, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0274 + }, + "shuffled": { + "min": 0.0781, + "best1L": 0.0442, + "best1L_layer": 16, + "mean": 0.008, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0057 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0029, + "3": 0.0008, + "4": 0.0018, + "5": 0.0011, + "6": 0.0008, + "7": 0.0008, + "8": 0.0004, + "9": 0.0011, + "10": 0.0023, + "11": 0.0004, + "12": 0.0014, + "13": 0.0071, + "14": 0.0326, + "15": 0.0331, + "16": 0.0523, + "17": 0.0858, + "18": 0.0598, + "19": 0.0567, + "20": 0.0498, + "21": 0.0438 + }, + "per_layer_shuffled": { + "0": 0.0008, + "1": 0.0, + "2": 0.0004, + "3": 0.0004, + "4": 0.0004, + "5": 0.0, + "6": 0.0015, + "7": 0.0008, + "8": 0.0004, + "9": 0.0008, + "10": 0.0004, + "11": 0.0048, + "12": 0.0022, + "13": 0.0144, + "14": 0.0156, + "15": 0.0317, + "16": 0.0442, + "17": 0.0042, + "18": 0.0019, + "19": 0.0019, + "20": 0.0187, + "21": 0.03 + }, + "layer_specific": { + "min": 0.0462, + "best1L": 0.0416, + "mean": 0.0118, + "persist": 0.0217 + }, + "generic": { + "min": -0.0334, + "best1L": -0.0404, + "mean": -0.0086, + "persist": -0.0155 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "best1L_layer": 19, + "mean": 0.1449, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2915 + }, + "jlens": { + "min": 0.7033, + "best1L": 0.3691, + "best1L_layer": 21, + "mean": 0.2875, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.3634 + }, + "shuffled": { + "min": 0.6878, + "best1L": 0.3804, + "best1L_layer": 20, + "mean": 0.187, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2616 + }, + "per_layer_jlens": { + "0": 0.0627, + "1": 0.1901, + "2": 0.2301, + "3": 0.2897, + "4": 0.2178, + "5": 0.2958, + "6": 0.2931, + "7": 0.2931, + "8": 0.2476, + "9": 0.1976, + "10": 0.2411, + "11": 0.2682, + "12": 0.3271, + "13": 0.3616, + "14": 0.3561, + "15": 0.3675, + "16": 0.3279, + "17": 0.342, + "18": 0.3518, + "19": 0.3431, + "20": 0.3529, + "21": 0.3691 + }, + "per_layer_shuffled": { + "0": 0.0529, + "1": 0.0498, + "2": 0.0357, + "3": 0.0227, + "4": 0.0384, + "5": 0.2386, + "6": 0.053, + "7": 0.2253, + "8": 0.2428, + "9": 0.214, + "10": 0.2401, + "11": 0.0373, + "12": 0.2423, + "13": 0.367, + "14": 0.3505, + "15": 0.2804, + "16": 0.3252, + "17": 0.1492, + "18": 0.0521, + "19": 0.1683, + "20": 0.3804, + "21": 0.3486 + }, + "layer_specific": { + "min": 0.0154, + "best1L": -0.0114, + "mean": 0.1005, + "persist": 0.1018 + }, + "generic": { + "min": 0.1729, + "best1L": -0.0107, + "mean": 0.0421, + "persist": -0.0299 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "best1L_layer": 18, + "mean": 0.0021, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0296, + "best1L": 0.0123, + "best1L_layer": 18, + "mean": 0.0037, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0031 + }, + "shuffled": { + "min": 0.0322, + "best1L": 0.0077, + "best1L_layer": 20, + "mean": 0.0024, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0015, + "1": 0.0046, + "2": 0.0056, + "3": 0.0031, + "4": 0.0056, + "5": 0.0, + "6": 0.0, + "7": 0.0015, + "8": 0.0031, + "9": 0.0015, + "10": 0.0015, + "11": 0.0, + "12": 0.0015, + "13": 0.0, + "14": 0.0015, + "15": 0.0056, + "16": 0.0074, + "17": 0.0074, + "18": 0.0123, + "19": 0.0089, + "20": 0.0046, + "21": 0.0031 + }, + "per_layer_shuffled": { + "0": 0.0056, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0046, + "5": 0.0031, + "6": 0.0, + "7": 0.0041, + "8": 0.0, + "9": 0.0031, + "10": 0.0046, + "11": 0.0, + "12": 0.0031, + "13": 0.0015, + "14": 0.0031, + "15": 0.0056, + "16": 0.0, + "17": 0.0015, + "18": 0.0, + "19": 0.0, + "20": 0.0077, + "21": 0.0061 + }, + "layer_specific": { + "min": -0.0026, + "best1L": 0.0046, + "mean": 0.0012, + "persist": 0.0031 + }, + "generic": { + "min": 0.0066, + "best1L": 0.0, + "mean": 0.0004, + "persist": -0.0015 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "best1L_layer": 21, + "mean": 0.0345, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0475 + }, + "jlens": { + "min": 0.2819, + "best1L": 0.1613, + "best1L_layer": 20, + "mean": 0.0689, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0821 + }, + "shuffled": { + "min": 0.2564, + "best1L": 0.1353, + "best1L_layer": 21, + "mean": 0.0397, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0394 + }, + "per_layer_jlens": { + "0": 0.0042, + "1": 0.0073, + "2": 0.0063, + "3": 0.011, + "4": 0.0388, + "5": 0.0354, + "6": 0.0498, + "7": 0.0558, + "8": 0.0693, + "9": 0.0761, + "10": 0.0537, + "11": 0.0365, + "12": 0.0464, + "13": 0.0555, + "14": 0.0667, + "15": 0.0813, + "16": 0.086, + "17": 0.1032, + "18": 0.1509, + "19": 0.1603, + "20": 0.1613, + "21": 0.1606 + }, + "per_layer_shuffled": { + "0": 0.0016, + "1": 0.0076, + "2": 0.0, + "3": 0.0016, + "4": 0.0295, + "5": 0.0216, + "6": 0.0514, + "7": 0.024, + "8": 0.05, + "9": 0.0631, + "10": 0.048, + "11": 0.0232, + "12": 0.0579, + "13": 0.0552, + "14": 0.0399, + "15": 0.0824, + "16": 0.0605, + "17": 0.0261, + "18": 0.0271, + "19": 0.0394, + "20": 0.0279, + "21": 0.1353 + }, + "layer_specific": { + "min": 0.0255, + "best1L": 0.026, + "mean": 0.0292, + "persist": 0.0428 + }, + "generic": { + "min": 0.0807, + "best1L": 0.0307, + "mean": 0.0051, + "persist": -0.0081 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "best1L_layer": 13, + "mean": 0.0005, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "jlens": { + "min": 0.0128, + "best1L": 0.0063, + "best1L_layer": 5, + "mean": 0.0008, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0109, + "best1L": 0.0044, + "best1L_layer": 15, + "mean": 0.0006, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0016, + "1": 0.0016, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0063, + "6": 0.0016, + "7": 0.0, + "8": 0.0016, + "9": 0.0016, + "10": 0.0, + "11": 0.0, + "12": 0.0016, + "13": 0.0016, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0033, + "6": 0.0, + "7": 0.0, + "8": 0.0016, + "9": 0.0, + "10": 0.0016, + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0016, + "15": 0.0044, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0 + }, + "layer_specific": { + "min": 0.0019, + "best1L": 0.0019, + "mean": 0.0002, + "persist": 0.0 + }, + "generic": { + "min": 0.0033, + "best1L": 0.0, + "mean": 0.0001, + "persist": 0.0 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 157.7 +} \ No newline at end of file diff --git a/results/e28_Github_410m_n200_s0_reads.json b/results/e28_Github_410m_n200_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..b9f73c425e209b4e47330daee1c8da549896cd76 --- /dev/null +++ b/results/e28_Github_410m_n200_s0_reads.json @@ -0,0 +1,634 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/e28_Github_410m_n200_s0.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 6, + "best1L": 3, + "mean": 5, + "persist": 4 + }, + "n_sets_stable": 4, + "n_sets_aggregation_dependent": 2, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "results": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "best1L_layer": 19, + "mean": 0.033, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0476 + }, + "jlens": { + "min": 0.1897, + "best1L": 0.1397, + "best1L_layer": 21, + "mean": 0.0277, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0255 + }, + "shuffled": { + "min": 0.112, + "best1L": 0.08, + "best1L_layer": 21, + "mean": 0.0105, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0029 + }, + "per_layer_jlens": { + "0": 0.0088, + "1": 0.0132, + "2": 0.0117, + "3": 0.0073, + "4": 0.0044, + "5": 0.0015, + "6": 0.0029, + "7": 0.0015, + "8": 0.0, + "9": 0.0, + "10": 0.0, + "11": 0.0044, + "12": 0.0088, + "13": 0.0058, + "14": 0.0127, + "15": 0.0369, + "16": 0.0292, + "17": 0.037, + "18": 0.0622, + "19": 0.1074, + "20": 0.1137, + "21": 0.1397 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0068, + "6": 0.0, + "7": 0.0068, + "8": 0.0015, + "9": 0.0, + "10": 0.0039, + "11": 0.0, + "12": 0.0015, + "13": 0.0, + "14": 0.0044, + "15": 0.0287, + "16": 0.062, + "17": 0.0044, + "18": 0.0, + "19": 0.0044, + "20": 0.0258, + "21": 0.08 + }, + "layer_specific": { + "min": 0.0777, + "best1L": 0.0597, + "mean": 0.0172, + "persist": 0.0226 + }, + "generic": { + "min": -0.0747, + "best1L": -0.0704, + "mean": -0.0225, + "persist": -0.0447 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "best1L_layer": 17, + "mean": 0.0165, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0212 + }, + "jlens": { + "min": 0.1151, + "best1L": 0.075, + "best1L_layer": 17, + "mean": 0.0184, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0236 + }, + "shuffled": { + "min": 0.0798, + "best1L": 0.0439, + "best1L_layer": 16, + "mean": 0.0081, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0048 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0023, + "3": 0.0008, + "4": 0.0018, + "5": 0.0008, + "6": 0.0004, + "7": 0.0008, + "8": 0.0, + "9": 0.0011, + "10": 0.0023, + "11": 0.0004, + "12": 0.0022, + "13": 0.006, + "14": 0.0346, + "15": 0.0353, + "16": 0.0484, + "17": 0.075, + "18": 0.0558, + "19": 0.0523, + "20": 0.0455, + "21": 0.0398 + }, + "per_layer_shuffled": { + "0": 0.0004, + "1": 0.0, + "2": 0.0004, + "3": 0.0004, + "4": 0.0008, + "5": 0.0, + "6": 0.0015, + "7": 0.0008, + "8": 0.001, + "9": 0.0008, + "10": 0.0004, + "11": 0.0045, + "12": 0.0018, + "13": 0.013, + "14": 0.0156, + "15": 0.0317, + "16": 0.0439, + "17": 0.0052, + "18": 0.0056, + "19": 0.0037, + "20": 0.0192, + "21": 0.0268 + }, + "layer_specific": { + "min": 0.0353, + "best1L": 0.0311, + "mean": 0.0104, + "persist": 0.0187 + }, + "generic": { + "min": -0.0317, + "best1L": -0.0407, + "mean": -0.0085, + "persist": -0.0164 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "best1L_layer": 19, + "mean": 0.1449, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2915 + }, + "jlens": { + "min": 0.7244, + "best1L": 0.3716, + "best1L_layer": 15, + "mean": 0.2835, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.3569 + }, + "shuffled": { + "min": 0.6963, + "best1L": 0.3711, + "best1L_layer": 13, + "mean": 0.1873, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2521 + }, + "per_layer_jlens": { + "0": 0.0369, + "1": 0.1228, + "2": 0.2135, + "3": 0.2896, + "4": 0.2106, + "5": 0.2992, + "6": 0.3028, + "7": 0.3055, + "8": 0.2574, + "9": 0.2045, + "10": 0.259, + "11": 0.2611, + "12": 0.3498, + "13": 0.3591, + "14": 0.3495, + "15": 0.3716, + "16": 0.3206, + "17": 0.3223, + "18": 0.3336, + "19": 0.3481, + "20": 0.3538, + "21": 0.3654 + }, + "per_layer_shuffled": { + "0": 0.0631, + "1": 0.0512, + "2": 0.0357, + "3": 0.0205, + "4": 0.0348, + "5": 0.244, + "6": 0.0523, + "7": 0.2126, + "8": 0.2481, + "9": 0.2188, + "10": 0.2374, + "11": 0.036, + "12": 0.2622, + "13": 0.3711, + "14": 0.3535, + "15": 0.2749, + "16": 0.3238, + "17": 0.1563, + "18": 0.0569, + "19": 0.1646, + "20": 0.35, + "21": 0.3518 + }, + "layer_specific": { + "min": 0.0282, + "best1L": 0.0005, + "mean": 0.0962, + "persist": 0.1048 + }, + "generic": { + "min": 0.1813, + "best1L": -0.02, + "mean": 0.0423, + "persist": -0.0394 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "best1L_layer": 18, + "mean": 0.0021, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0286, + "best1L": 0.0123, + "best1L_layer": 18, + "mean": 0.0037, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0031 + }, + "shuffled": { + "min": 0.0371, + "best1L": 0.0074, + "best1L_layer": 7, + "mean": 0.0027, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0046, + "1": 0.0031, + "2": 0.0056, + "3": 0.0031, + "4": 0.0046, + "5": 0.0015, + "6": 0.0, + "7": 0.0015, + "8": 0.0015, + "9": 0.0031, + "10": 0.0, + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0015, + "15": 0.0056, + "16": 0.0056, + "17": 0.0092, + "18": 0.0123, + "19": 0.0107, + "20": 0.0046, + "21": 0.0031 + }, + "per_layer_shuffled": { + "0": 0.0056, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0046, + "5": 0.0061, + "6": 0.0, + "7": 0.0074, + "8": 0.0015, + "9": 0.0015, + "10": 0.0046, + "11": 0.0, + "12": 0.0, + "13": 0.0015, + "14": 0.0031, + "15": 0.0056, + "16": 0.0, + "17": 0.0031, + "18": 0.0, + "19": 0.0015, + "20": 0.0061, + "21": 0.0061 + }, + "layer_specific": { + "min": -0.0084, + "best1L": 0.0049, + "mean": 0.001, + "persist": 0.0031 + }, + "generic": { + "min": 0.0115, + "best1L": -0.0003, + "mean": 0.0006, + "persist": -0.0015 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "best1L_layer": 21, + "mean": 0.0345, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0475 + }, + "jlens": { + "min": 0.2731, + "best1L": 0.1572, + "best1L_layer": 21, + "mean": 0.0664, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0727 + }, + "shuffled": { + "min": 0.2554, + "best1L": 0.1319, + "best1L_layer": 21, + "mean": 0.0415, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0435 + }, + "per_layer_jlens": { + "0": 0.0016, + "1": 0.0063, + "2": 0.0094, + "3": 0.0151, + "4": 0.0414, + "5": 0.0409, + "6": 0.0542, + "7": 0.0652, + "8": 0.0758, + "9": 0.086, + "10": 0.0526, + "11": 0.0357, + "12": 0.0472, + "13": 0.0526, + "14": 0.0605, + "15": 0.0706, + "16": 0.0767, + "17": 0.0847, + "18": 0.1282, + "19": 0.1504, + "20": 0.1493, + "21": 0.1572 + }, + "per_layer_shuffled": { + "0": 0.0031, + "1": 0.0091, + "2": 0.0, + "3": 0.0016, + "4": 0.0378, + "5": 0.025, + "6": 0.0508, + "7": 0.0342, + "8": 0.0552, + "9": 0.0683, + "10": 0.0571, + "11": 0.0232, + "12": 0.0631, + "13": 0.0516, + "14": 0.0449, + "15": 0.0771, + "16": 0.059, + "17": 0.0261, + "18": 0.0289, + "19": 0.0368, + "20": 0.0289, + "21": 0.1319 + }, + "layer_specific": { + "min": 0.0177, + "best1L": 0.0253, + "mean": 0.0249, + "persist": 0.0292 + }, + "generic": { + "min": 0.0796, + "best1L": 0.0273, + "mean": 0.007, + "persist": -0.0039 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "best1L_layer": 13, + "mean": 0.0005, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "jlens": { + "min": 0.0234, + "best1L": 0.0082, + "best1L_layer": 0, + "mean": 0.0015, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0125, + "best1L": 0.0044, + "best1L_layer": 15, + "mean": 0.0008, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0082, + "1": 0.0049, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0044, + "6": 0.0044, + "7": 0.0, + "8": 0.006, + "9": 0.0016, + "10": 0.0, + "11": 0.0, + "12": 0.0016, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0016 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0033, + "6": 0.0, + "7": 0.0016, + "8": 0.0016, + "9": 0.0, + "10": 0.0016, + "11": 0.0, + "12": 0.0033, + "13": 0.0, + "14": 0.0016, + "15": 0.0044, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0 + }, + "layer_specific": { + "min": 0.0109, + "best1L": 0.0038, + "mean": 0.0007, + "persist": 0.0 + }, + "generic": { + "min": 0.0049, + "best1L": 0.0, + "mean": 0.0003, + "persist": 0.0 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 154.4 +} \ No newline at end of file diff --git a/results/e28_Github_410m_n200_s1_reads.json b/results/e28_Github_410m_n200_s1_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..8fdbcebe18fa16e9c5285b926c4320bb316761f4 --- /dev/null +++ b/results/e28_Github_410m_n200_s1_reads.json @@ -0,0 +1,634 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/e28_Github_410m_n200_s1.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 6, + "best1L": 4, + "mean": 5, + "persist": 4 + }, + "n_sets_stable": 4, + "n_sets_aggregation_dependent": 2, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "results": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "best1L_layer": 19, + "mean": 0.033, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0476 + }, + "jlens": { + "min": 0.2116, + "best1L": 0.143, + "best1L_layer": 21, + "mean": 0.0324, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.028 + }, + "shuffled": { + "min": 0.1188, + "best1L": 0.0848, + "best1L_layer": 21, + "mean": 0.011, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0029 + }, + "per_layer_jlens": { + "0": 0.0226, + "1": 0.02, + "2": 0.0253, + "3": 0.0088, + "4": 0.0058, + "5": 0.0015, + "6": 0.0015, + "7": 0.0, + "8": 0.0, + "9": 0.0015, + "10": 0.0, + "11": 0.0058, + "12": 0.0195, + "13": 0.0058, + "14": 0.0127, + "15": 0.0352, + "16": 0.0369, + "17": 0.0457, + "18": 0.0802, + "19": 0.1183, + "20": 0.1229, + "21": 0.143 + }, + "per_layer_shuffled": { + "0": 0.0015, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0058, + "6": 0.0, + "7": 0.0068, + "8": 0.0015, + "9": 0.0, + "10": 0.0039, + "11": 0.0, + "12": 0.0015, + "13": 0.0, + "14": 0.0044, + "15": 0.0272, + "16": 0.0627, + "17": 0.0044, + "18": 0.0, + "19": 0.0073, + "20": 0.0306, + "21": 0.0848 + }, + "layer_specific": { + "min": 0.0928, + "best1L": 0.0582, + "mean": 0.0214, + "persist": 0.025 + }, + "generic": { + "min": -0.0679, + "best1L": -0.0655, + "mean": -0.022, + "persist": -0.0447 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "best1L_layer": 17, + "mean": 0.0165, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0212 + }, + "jlens": { + "min": 0.128, + "best1L": 0.0874, + "best1L_layer": 17, + "mean": 0.02, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0283 + }, + "shuffled": { + "min": 0.0789, + "best1L": 0.0442, + "best1L_layer": 16, + "mean": 0.0081, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0055 + }, + "per_layer_jlens": { + "0": 0.0004, + "1": 0.0, + "2": 0.0029, + "3": 0.0008, + "4": 0.0018, + "5": 0.0008, + "6": 0.0008, + "7": 0.0008, + "8": 0.0008, + "9": 0.0011, + "10": 0.0019, + "11": 0.0004, + "12": 0.0022, + "13": 0.006, + "14": 0.0357, + "15": 0.0352, + "16": 0.0514, + "17": 0.0874, + "18": 0.0619, + "19": 0.0567, + "20": 0.0493, + "21": 0.0426 + }, + "per_layer_shuffled": { + "0": 0.0008, + "1": 0.0, + "2": 0.0004, + "3": 0.0004, + "4": 0.0004, + "5": 0.0, + "6": 0.0015, + "7": 0.0004, + "8": 0.001, + "9": 0.0004, + "10": 0.001, + "11": 0.0048, + "12": 0.0011, + "13": 0.0143, + "14": 0.017, + "15": 0.0336, + "16": 0.0442, + "17": 0.0048, + "18": 0.0015, + "19": 0.0023, + "20": 0.0193, + "21": 0.0294 + }, + "layer_specific": { + "min": 0.0491, + "best1L": 0.0432, + "mean": 0.0119, + "persist": 0.0229 + }, + "generic": { + "min": -0.0326, + "best1L": -0.0404, + "mean": -0.0084, + "persist": -0.0158 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "best1L_layer": 19, + "mean": 0.1449, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2915 + }, + "jlens": { + "min": 0.7121, + "best1L": 0.3682, + "best1L_layer": 15, + "mean": 0.2848, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.3584 + }, + "shuffled": { + "min": 0.6976, + "best1L": 0.3666, + "best1L_layer": 20, + "mean": 0.1842, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2555 + }, + "per_layer_jlens": { + "0": 0.0419, + "1": 0.161, + "2": 0.231, + "3": 0.2815, + "4": 0.2328, + "5": 0.2924, + "6": 0.3019, + "7": 0.3035, + "8": 0.249, + "9": 0.1931, + "10": 0.2411, + "11": 0.2566, + "12": 0.3323, + "13": 0.3557, + "14": 0.3518, + "15": 0.3682, + "16": 0.3257, + "17": 0.3345, + "18": 0.3557, + "19": 0.34, + "20": 0.3484, + "21": 0.3672 + }, + "per_layer_shuffled": { + "0": 0.0683, + "1": 0.0489, + "2": 0.035, + "3": 0.022, + "4": 0.0348, + "5": 0.2163, + "6": 0.0498, + "7": 0.2069, + "8": 0.2512, + "9": 0.204, + "10": 0.2542, + "11": 0.0346, + "12": 0.2409, + "13": 0.3607, + "14": 0.3489, + "15": 0.2802, + "16": 0.3236, + "17": 0.1561, + "18": 0.0401, + "19": 0.1612, + "20": 0.3666, + "21": 0.3488 + }, + "layer_specific": { + "min": 0.0145, + "best1L": 0.0016, + "mean": 0.1005, + "persist": 0.1029 + }, + "generic": { + "min": 0.1827, + "best1L": -0.0246, + "mean": 0.0393, + "persist": -0.036 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "best1L_layer": 18, + "mean": 0.0021, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0273, + "best1L": 0.0123, + "best1L_layer": 18, + "mean": 0.0038, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0056 + }, + "shuffled": { + "min": 0.0312, + "best1L": 0.0061, + "best1L_layer": 20, + "mean": 0.0023, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0031, + "2": 0.0074, + "3": 0.0031, + "4": 0.0031, + "5": 0.0, + "6": 0.0, + "7": 0.0031, + "8": 0.0031, + "9": 0.0031, + "10": 0.0015, + "11": 0.0, + "12": 0.0015, + "13": 0.0, + "14": 0.0041, + "15": 0.0056, + "16": 0.0074, + "17": 0.0074, + "18": 0.0123, + "19": 0.0072, + "20": 0.0046, + "21": 0.0031 + }, + "per_layer_shuffled": { + "0": 0.0056, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0031, + "5": 0.0031, + "6": 0.0, + "7": 0.0056, + "8": 0.0, + "9": 0.0031, + "10": 0.0046, + "11": 0.0, + "12": 0.0031, + "13": 0.0015, + "14": 0.0031, + "15": 0.0031, + "16": 0.0, + "17": 0.0015, + "18": 0.0, + "19": 0.0015, + "20": 0.0061, + "21": 0.0061 + }, + "layer_specific": { + "min": -0.0039, + "best1L": 0.0061, + "mean": 0.0015, + "persist": 0.0056 + }, + "generic": { + "min": 0.0056, + "best1L": -0.0015, + "mean": 0.0003, + "persist": -0.0015 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "best1L_layer": 21, + "mean": 0.0345, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0475 + }, + "jlens": { + "min": 0.2957, + "best1L": 0.1634, + "best1L_layer": 19, + "mean": 0.0754, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0892 + }, + "shuffled": { + "min": 0.2666, + "best1L": 0.1327, + "best1L_layer": 21, + "mean": 0.0453, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0521 + }, + "per_layer_jlens": { + "0": 0.0042, + "1": 0.0063, + "2": 0.0141, + "3": 0.0141, + "4": 0.05, + "5": 0.0464, + "6": 0.061, + "7": 0.0667, + "8": 0.0784, + "9": 0.0881, + "10": 0.0641, + "11": 0.05, + "12": 0.0534, + "13": 0.0607, + "14": 0.0737, + "15": 0.0863, + "16": 0.0967, + "17": 0.1082, + "18": 0.1553, + "19": 0.1634, + "20": 0.1618, + "21": 0.1569 + }, + "per_layer_shuffled": { + "0": 0.0016, + "1": 0.0076, + "2": 0.0, + "3": 0.0016, + "4": 0.0414, + "5": 0.0308, + "6": 0.0527, + "7": 0.0355, + "8": 0.0597, + "9": 0.0761, + "10": 0.0633, + "11": 0.0232, + "12": 0.0623, + "13": 0.0678, + "14": 0.0506, + "15": 0.0865, + "16": 0.0616, + "17": 0.0303, + "18": 0.0305, + "19": 0.0496, + "20": 0.0321, + "21": 0.1327 + }, + "layer_specific": { + "min": 0.0292, + "best1L": 0.0307, + "mean": 0.0301, + "persist": 0.037 + }, + "generic": { + "min": 0.0908, + "best1L": 0.0281, + "mean": 0.0108, + "persist": 0.0046 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "best1L_layer": 13, + "mean": 0.0005, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "jlens": { + "min": 0.0163, + "best1L": 0.0082, + "best1L_layer": 5, + "mean": 0.0013, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0093, + "best1L": 0.0044, + "best1L_layer": 15, + "mean": 0.0006, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0033, + "1": 0.0033, + "2": 0.0016, + "3": 0.0, + "4": 0.0, + "5": 0.0082, + "6": 0.0044, + "7": 0.0, + "8": 0.0016, + "9": 0.0016, + "10": 0.0, + "11": 0.0, + "12": 0.0016, + "13": 0.0016, + "14": 0.0016, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0016, + "6": 0.0, + "7": 0.0016, + "8": 0.0016, + "9": 0.0016, + "10": 0.0016, + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0016, + "15": 0.0044, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0 + }, + "layer_specific": { + "min": 0.0071, + "best1L": 0.0038, + "mean": 0.0007, + "persist": 0.0 + }, + "generic": { + "min": 0.0016, + "best1L": 0.0, + "mean": 0.0001, + "persist": 0.0 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 143.1 +} \ No newline at end of file diff --git a/results/e28_Github_70m_n100_s0_reads.json b/results/e28_Github_70m_n100_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..c078d56ee00b5f1c54bd5f576e6dcee80bc3d47c --- /dev/null +++ b/results/e28_Github_70m_n100_s0_reads.json @@ -0,0 +1,371 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/e28_Github_70m_n100_s0.pt", + "layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 4, + "best1L": 4, + "mean": 3, + "persist": 4 + }, + "n_sets_stable": 6, + "n_sets_aggregation_dependent": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "results": { + "multihop": { + "logit": { + "min": 0.0083, + "best1L": 0.0058, + "best1L_layer": 2, + "mean": 0.0035, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0083 + }, + "jlens": { + "min": 0.0085, + "best1L": 0.007, + "best1L_layer": 3, + "mean": 0.0025, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0085 + }, + "shuffled": { + "min": 0.0015, + "best1L": 0.0015, + "best1L_layer": 2, + "mean": 0.0004, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0015 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0029, + "3": 0.007 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0 + }, + "layer_specific": { + "min": 0.007, + "best1L": 0.0056, + "mean": 0.0021, + "persist": 0.007 + }, + "generic": { + "min": -0.0068, + "best1L": -0.0044, + "mean": -0.0032, + "persist": -0.0068 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.0395, + "best1L": 0.0187, + "best1L_layer": 0, + "mean": 0.0125, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0161 + }, + "jlens": { + "min": 0.0149, + "best1L": 0.0107, + "best1L_layer": 3, + "mean": 0.0047, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0141 + }, + "shuffled": { + "min": 0.0094, + "best1L": 0.0072, + "best1L_layer": 2, + "mean": 0.0025, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0072 + }, + "per_layer_jlens": { + "0": 0.0008, + "1": 0.0027, + "2": 0.0046, + "3": 0.0107 + }, + "per_layer_shuffled": { + "0": 0.0018, + "1": 0.0004, + "2": 0.0072, + "3": 0.0008 + }, + "layer_specific": { + "min": 0.0055, + "best1L": 0.0034, + "mean": 0.0021, + "persist": 0.0069 + }, + "generic": { + "min": -0.0301, + "best1L": -0.0114, + "mean": -0.0099, + "persist": -0.0088 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.0606, + "best1L": 0.0565, + "best1L_layer": 3, + "mean": 0.0162, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0592 + }, + "jlens": { + "min": 0.3008, + "best1L": 0.2437, + "best1L_layer": 2, + "mean": 0.1649, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.2704 + }, + "shuffled": { + "min": 0.2812, + "best1L": 0.2094, + "best1L_layer": 1, + "mean": 0.1106, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.1867 + }, + "per_layer_jlens": { + "0": 0.0442, + "1": 0.186, + "2": 0.2437, + "3": 0.1858 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.2094, + "2": 0.1083, + "3": 0.1249 + }, + "layer_specific": { + "min": 0.0196, + "best1L": 0.0344, + "mean": 0.0543, + "persist": 0.0837 + }, + "generic": { + "min": 0.2206, + "best1L": 0.1529, + "mean": 0.0945, + "persist": 0.1275 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0319, + "best1L": 0.0225, + "best1L_layer": 0, + "mean": 0.0155, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0181 + }, + "jlens": { + "min": 0.0143, + "best1L": 0.0097, + "best1L_layer": 3, + "mean": 0.0054, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0112 + }, + "shuffled": { + "min": 0.0105, + "best1L": 0.0059, + "best1L_layer": 1, + "mean": 0.0026, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0031 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0015, + "2": 0.0072, + "3": 0.0097 + }, + "per_layer_shuffled": { + "0": 0.0015, + "1": 0.0059, + "2": 0.0031, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0038, + "best1L": 0.0038, + "mean": 0.0027, + "persist": 0.0082 + }, + "generic": { + "min": -0.0214, + "best1L": -0.0166, + "mean": -0.0129, + "persist": -0.0151 + }, + "verdict": "STABLE: layer-specific" + }, + "typo": { + "logit": { + "min": 0.062, + "best1L": 0.0503, + "best1L_layer": 3, + "mean": 0.0216, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0579 + }, + "jlens": { + "min": 0.0803, + "best1L": 0.0508, + "best1L_layer": 3, + "mean": 0.0376, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0649 + }, + "shuffled": { + "min": 0.0574, + "best1L": 0.03, + "best1L_layer": 3, + "mean": 0.0229, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.043 + }, + "per_layer_jlens": { + "0": 0.03, + "1": 0.0344, + "2": 0.0352, + "3": 0.0508 + }, + "per_layer_shuffled": { + "0": 0.0073, + "1": 0.0287, + "2": 0.0258, + "3": 0.03 + }, + "layer_specific": { + "min": 0.0229, + "best1L": 0.0208, + "mean": 0.0146, + "persist": 0.0218 + }, + "generic": { + "min": -0.0047, + "best1L": -0.0203, + "mean": 0.0013, + "persist": -0.0148 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 3, + "mean": 0.0023, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0044 + }, + "jlens": { + "min": 0.0109, + "best1L": 0.0093, + "best1L_layer": 3, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0093 + }, + "shuffled": { + "min": 0.0044, + "best1L": 0.0044, + "best1L_layer": 1, + "mean": 0.0011, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0016, + "2": 0.0016, + "3": 0.0093 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0044, + "2": 0.0, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0065, + "best1L": 0.0049, + "mean": 0.002, + "persist": 0.0093 + }, + "generic": { + "min": -0.0016, + "best1L": 0.0, + "mean": -0.0012, + "persist": -0.0044 + }, + "verdict": "STABLE: layer-specific" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 23.8 +} \ No newline at end of file diff --git a/results/e28_Github_70m_n200_s0_reads.json b/results/e28_Github_70m_n200_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..6dcfd456cb7e4b08fcda3b0d1899833ec1469fe0 --- /dev/null +++ b/results/e28_Github_70m_n200_s0_reads.json @@ -0,0 +1,371 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/e28_Github_70m_n200_s0.pt", + "layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 3, + "best1L": 2, + "mean": 3, + "persist": 3 + }, + "n_sets_stable": 6, + "n_sets_aggregation_dependent": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "results": { + "multihop": { + "logit": { + "min": 0.0083, + "best1L": 0.0058, + "best1L_layer": 2, + "mean": 0.0035, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0083 + }, + "jlens": { + "min": 0.0053, + "best1L": 0.0039, + "best1L_layer": 3, + "mean": 0.0013, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0053 + }, + "shuffled": { + "min": 0.0029, + "best1L": 0.0015, + "best1L_layer": 2, + "mean": 0.0007, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0029 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0039 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0015 + }, + "layer_specific": { + "min": 0.0024, + "best1L": 0.0024, + "mean": 0.0006, + "persist": 0.0024 + }, + "generic": { + "min": -0.0053, + "best1L": -0.0044, + "mean": -0.0028, + "persist": -0.0053 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.0395, + "best1L": 0.0187, + "best1L_layer": 0, + "mean": 0.0125, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0161 + }, + "jlens": { + "min": 0.0148, + "best1L": 0.0106, + "best1L_layer": 3, + "mean": 0.0045, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0141 + }, + "shuffled": { + "min": 0.0081, + "best1L": 0.0055, + "best1L_layer": 2, + "mean": 0.0022, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0055 + }, + "per_layer_jlens": { + "0": 0.0008, + "1": 0.0027, + "2": 0.0038, + "3": 0.0106 + }, + "per_layer_shuffled": { + "0": 0.0018, + "1": 0.0008, + "2": 0.0055, + "3": 0.0008 + }, + "layer_specific": { + "min": 0.0067, + "best1L": 0.0051, + "mean": 0.0023, + "persist": 0.0085 + }, + "generic": { + "min": -0.0315, + "best1L": -0.0132, + "mean": -0.0102, + "persist": -0.0105 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.0606, + "best1L": 0.0565, + "best1L_layer": 3, + "mean": 0.0162, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0592 + }, + "jlens": { + "min": 0.316, + "best1L": 0.2394, + "best1L_layer": 2, + "mean": 0.1684, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.2704 + }, + "shuffled": { + "min": 0.2703, + "best1L": 0.2064, + "best1L_layer": 1, + "mean": 0.1071, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.1806 + }, + "per_layer_jlens": { + "0": 0.0442, + "1": 0.203, + "2": 0.2394, + "3": 0.1869 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.2064, + "2": 0.1019, + "3": 0.1201 + }, + "layer_specific": { + "min": 0.0457, + "best1L": 0.033, + "mean": 0.0613, + "persist": 0.0898 + }, + "generic": { + "min": 0.2097, + "best1L": 0.15, + "mean": 0.0909, + "persist": 0.1214 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0319, + "best1L": 0.0225, + "best1L_layer": 0, + "mean": 0.0155, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0181 + }, + "jlens": { + "min": 0.0143, + "best1L": 0.0097, + "best1L_layer": 3, + "mean": 0.005, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0112 + }, + "shuffled": { + "min": 0.013, + "best1L": 0.0059, + "best1L_layer": 1, + "mean": 0.0033, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0056 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0, + "2": 0.0072, + "3": 0.0097 + }, + "per_layer_shuffled": { + "0": 0.0015, + "1": 0.0059, + "2": 0.0056, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0013, + "best1L": 0.0038, + "mean": 0.0017, + "persist": 0.0056 + }, + "generic": { + "min": -0.0189, + "best1L": -0.0166, + "mean": -0.0123, + "persist": -0.0125 + }, + "verdict": "STABLE: layer-specific" + }, + "typo": { + "logit": { + "min": 0.062, + "best1L": 0.0503, + "best1L_layer": 3, + "mean": 0.0216, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0579 + }, + "jlens": { + "min": 0.0758, + "best1L": 0.0482, + "best1L_layer": 3, + "mean": 0.0365, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0605 + }, + "shuffled": { + "min": 0.0618, + "best1L": 0.0318, + "best1L_layer": 3, + "mean": 0.0237, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0449 + }, + "per_layer_jlens": { + "0": 0.03, + "1": 0.0344, + "2": 0.0333, + "3": 0.0482 + }, + "per_layer_shuffled": { + "0": 0.0073, + "1": 0.0297, + "2": 0.0258, + "3": 0.0318 + }, + "layer_specific": { + "min": 0.014, + "best1L": 0.0164, + "mean": 0.0128, + "persist": 0.0156 + }, + "generic": { + "min": -0.0003, + "best1L": -0.0185, + "mean": 0.002, + "persist": -0.013 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 3, + "mean": 0.0023, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0044 + }, + "jlens": { + "min": 0.0109, + "best1L": 0.0093, + "best1L_layer": 3, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0093 + }, + "shuffled": { + "min": 0.0033, + "best1L": 0.0016, + "best1L_layer": 1, + "mean": 0.0008, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0016 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0016, + "2": 0.0016, + "3": 0.0093 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0016, + "2": 0.0, + "3": 0.0016 + }, + "layer_specific": { + "min": 0.0076, + "best1L": 0.0076, + "mean": 0.0023, + "persist": 0.0076 + }, + "generic": { + "min": -0.0027, + "best1L": -0.0027, + "mean": -0.0015, + "persist": -0.0027 + }, + "verdict": "STABLE: layer-specific" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 24.0 +} \ No newline at end of file diff --git a/results/e28_Github_70m_n25_s0_reads.json b/results/e28_Github_70m_n25_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..4d04fd3eea2f0da72d4615e8d58676b5c9114790 --- /dev/null +++ b/results/e28_Github_70m_n25_s0_reads.json @@ -0,0 +1,371 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/e28_Github_70m_n25_s0.pt", + "layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 4, + "best1L": 3, + "mean": 3, + "persist": 4 + }, + "n_sets_stable": 6, + "n_sets_aggregation_dependent": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "results": { + "multihop": { + "logit": { + "min": 0.0083, + "best1L": 0.0058, + "best1L_layer": 2, + "mean": 0.0035, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0083 + }, + "jlens": { + "min": 0.0085, + "best1L": 0.0056, + "best1L_layer": 3, + "mean": 0.0021, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0085 + }, + "shuffled": { + "min": 0.0015, + "best1L": 0.0015, + "best1L_layer": 2, + "mean": 0.0004, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0015 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0029, + "3": 0.0056 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0 + }, + "layer_specific": { + "min": 0.007, + "best1L": 0.0041, + "mean": 0.0018, + "persist": 0.007 + }, + "generic": { + "min": -0.0068, + "best1L": -0.0044, + "mean": -0.0032, + "persist": -0.0068 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.0395, + "best1L": 0.0187, + "best1L_layer": 0, + "mean": 0.0125, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0161 + }, + "jlens": { + "min": 0.0169, + "best1L": 0.0119, + "best1L_layer": 3, + "mean": 0.0051, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0161 + }, + "shuffled": { + "min": 0.0095, + "best1L": 0.007, + "best1L_layer": 2, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.007 + }, + "per_layer_jlens": { + "0": 0.0008, + "1": 0.0027, + "2": 0.005, + "3": 0.0119 + }, + "per_layer_shuffled": { + "0": 0.0022, + "1": 0.0011, + "2": 0.007, + "3": 0.0023 + }, + "layer_specific": { + "min": 0.0073, + "best1L": 0.0049, + "mean": 0.0019, + "persist": 0.0091 + }, + "generic": { + "min": -0.03, + "best1L": -0.0117, + "mean": -0.0093, + "persist": -0.0091 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.0606, + "best1L": 0.0565, + "best1L_layer": 3, + "mean": 0.0162, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0592 + }, + "jlens": { + "min": 0.2935, + "best1L": 0.2333, + "best1L_layer": 2, + "mean": 0.1575, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.2615 + }, + "shuffled": { + "min": 0.2655, + "best1L": 0.2032, + "best1L_layer": 1, + "mean": 0.0996, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.1585 + }, + "per_layer_jlens": { + "0": 0.0376, + "1": 0.1828, + "2": 0.2333, + "3": 0.1765 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.2032, + "2": 0.0787, + "3": 0.1163 + }, + "layer_specific": { + "min": 0.028, + "best1L": 0.0301, + "mean": 0.058, + "persist": 0.103 + }, + "generic": { + "min": 0.205, + "best1L": 0.1468, + "mean": 0.0834, + "persist": 0.0993 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0319, + "best1L": 0.0225, + "best1L_layer": 0, + "mean": 0.0155, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0181 + }, + "jlens": { + "min": 0.0143, + "best1L": 0.0097, + "best1L_layer": 3, + "mean": 0.0058, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0128 + }, + "shuffled": { + "min": 0.0123, + "best1L": 0.0077, + "best1L_layer": 1, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0031 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0015, + "2": 0.0087, + "3": 0.0097 + }, + "per_layer_shuffled": { + "0": 0.0015, + "1": 0.0077, + "2": 0.0015, + "3": 0.0015 + }, + "layer_specific": { + "min": 0.0021, + "best1L": 0.0021, + "mean": 0.0027, + "persist": 0.0097 + }, + "generic": { + "min": -0.0197, + "best1L": -0.0148, + "mean": -0.0125, + "persist": -0.0151 + }, + "verdict": "STABLE: layer-specific" + }, + "typo": { + "logit": { + "min": 0.062, + "best1L": 0.0503, + "best1L_layer": 3, + "mean": 0.0216, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0579 + }, + "jlens": { + "min": 0.0763, + "best1L": 0.0526, + "best1L_layer": 3, + "mean": 0.0381, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0623 + }, + "shuffled": { + "min": 0.0532, + "best1L": 0.0287, + "best1L_layer": 1, + "mean": 0.0214, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0389 + }, + "per_layer_jlens": { + "0": 0.0284, + "1": 0.0347, + "2": 0.0367, + "3": 0.0526 + }, + "per_layer_shuffled": { + "0": 0.0073, + "1": 0.0287, + "2": 0.0232, + "3": 0.0266 + }, + "layer_specific": { + "min": 0.0231, + "best1L": 0.024, + "mean": 0.0167, + "persist": 0.0234 + }, + "generic": { + "min": -0.0088, + "best1L": -0.0216, + "mean": -0.0002, + "persist": -0.019 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 3, + "mean": 0.0023, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0044 + }, + "jlens": { + "min": 0.0109, + "best1L": 0.0093, + "best1L_layer": 3, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0093 + }, + "shuffled": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 1, + "mean": 0.0015, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0016 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0016, + "2": 0.0016, + "3": 0.0093 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0044, + "2": 0.0, + "3": 0.0016 + }, + "layer_specific": { + "min": 0.0049, + "best1L": 0.0049, + "mean": 0.0016, + "persist": 0.0076 + }, + "generic": { + "min": 0.0, + "best1L": 0.0, + "mean": -0.0008, + "persist": -0.0027 + }, + "verdict": "STABLE: layer-specific" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 24.0 +} \ No newline at end of file diff --git a/results/e28_Github_70m_n400_s0_reads.json b/results/e28_Github_70m_n400_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..a3800db9661a89ddaa69d30e0c334b7f900e33a6 --- /dev/null +++ b/results/e28_Github_70m_n400_s0_reads.json @@ -0,0 +1,371 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/e28_Github_70m_n400_s0.pt", + "layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 3, + "best1L": 2, + "mean": 3, + "persist": 3 + }, + "n_sets_stable": 6, + "n_sets_aggregation_dependent": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "results": { + "multihop": { + "logit": { + "min": 0.0083, + "best1L": 0.0058, + "best1L_layer": 2, + "mean": 0.0035, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0083 + }, + "jlens": { + "min": 0.0053, + "best1L": 0.0039, + "best1L_layer": 3, + "mean": 0.0013, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0053 + }, + "shuffled": { + "min": 0.0015, + "best1L": 0.0015, + "best1L_layer": 2, + "mean": 0.0004, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0015 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0039 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0039, + "best1L": 0.0024, + "mean": 0.001, + "persist": 0.0039 + }, + "generic": { + "min": -0.0068, + "best1L": -0.0044, + "mean": -0.0032, + "persist": -0.0068 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.0395, + "best1L": 0.0187, + "best1L_layer": 0, + "mean": 0.0125, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0161 + }, + "jlens": { + "min": 0.0148, + "best1L": 0.0106, + "best1L_layer": 3, + "mean": 0.0047, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0141 + }, + "shuffled": { + "min": 0.0087, + "best1L": 0.0062, + "best1L_layer": 2, + "mean": 0.0024, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0062 + }, + "per_layer_jlens": { + "0": 0.0008, + "1": 0.0031, + "2": 0.0042, + "3": 0.0106 + }, + "per_layer_shuffled": { + "0": 0.0018, + "1": 0.0008, + "2": 0.0062, + "3": 0.0008 + }, + "layer_specific": { + "min": 0.0061, + "best1L": 0.0044, + "mean": 0.0023, + "persist": 0.0079 + }, + "generic": { + "min": -0.0308, + "best1L": -0.0125, + "mean": -0.0101, + "persist": -0.0099 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.0606, + "best1L": 0.0565, + "best1L_layer": 3, + "mean": 0.0162, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0592 + }, + "jlens": { + "min": 0.3279, + "best1L": 0.2462, + "best1L_layer": 2, + "mean": 0.1737, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.2849 + }, + "shuffled": { + "min": 0.2762, + "best1L": 0.2101, + "best1L_layer": 1, + "mean": 0.1123, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.1951 + }, + "per_layer_jlens": { + "0": 0.0533, + "1": 0.2069, + "2": 0.2462, + "3": 0.1886 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.2101, + "2": 0.1094, + "3": 0.1299 + }, + "layer_specific": { + "min": 0.0516, + "best1L": 0.0362, + "mean": 0.0614, + "persist": 0.0898 + }, + "generic": { + "min": 0.2156, + "best1L": 0.1536, + "mean": 0.0962, + "persist": 0.1359 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0319, + "best1L": 0.0225, + "best1L_layer": 0, + "mean": 0.0155, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0181 + }, + "jlens": { + "min": 0.0143, + "best1L": 0.0097, + "best1L_layer": 3, + "mean": 0.0043, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0112 + }, + "shuffled": { + "min": 0.013, + "best1L": 0.0059, + "best1L_layer": 1, + "mean": 0.0033, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0056 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0, + "2": 0.0046, + "3": 0.0097 + }, + "per_layer_shuffled": { + "0": 0.0015, + "1": 0.0059, + "2": 0.0056, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0013, + "best1L": 0.0038, + "mean": 0.0011, + "persist": 0.0056 + }, + "generic": { + "min": -0.0189, + "best1L": -0.0166, + "mean": -0.0123, + "persist": -0.0125 + }, + "verdict": "STABLE: layer-specific" + }, + "typo": { + "logit": { + "min": 0.062, + "best1L": 0.0503, + "best1L_layer": 3, + "mean": 0.0216, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0579 + }, + "jlens": { + "min": 0.0758, + "best1L": 0.0482, + "best1L_layer": 3, + "mean": 0.0369, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0586 + }, + "shuffled": { + "min": 0.0618, + "best1L": 0.0318, + "best1L_layer": 3, + "mean": 0.0237, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0449 + }, + "per_layer_jlens": { + "0": 0.0318, + "1": 0.036, + "2": 0.0315, + "3": 0.0482 + }, + "per_layer_shuffled": { + "0": 0.0073, + "1": 0.0297, + "2": 0.0258, + "3": 0.0318 + }, + "layer_specific": { + "min": 0.014, + "best1L": 0.0164, + "mean": 0.0132, + "persist": 0.0138 + }, + "generic": { + "min": -0.0003, + "best1L": -0.0185, + "mean": 0.002, + "persist": -0.013 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 3, + "mean": 0.0023, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0044 + }, + "jlens": { + "min": 0.0109, + "best1L": 0.0093, + "best1L_layer": 3, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0093 + }, + "shuffled": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 1, + "mean": 0.0015, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0016 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0016, + "2": 0.0016, + "3": 0.0093 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0044, + "2": 0.0, + "3": 0.0016 + }, + "layer_specific": { + "min": 0.0049, + "best1L": 0.0049, + "mean": 0.0016, + "persist": 0.0076 + }, + "generic": { + "min": 0.0, + "best1L": 0.0, + "mean": -0.0008, + "persist": -0.0027 + }, + "verdict": "STABLE: layer-specific" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 25.8 +} \ No newline at end of file diff --git a/results/e28_Github_70m_n50_s0_reads.json b/results/e28_Github_70m_n50_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..0c40186231b7446a209626b23e7b1e3b8e4d8528 --- /dev/null +++ b/results/e28_Github_70m_n50_s0_reads.json @@ -0,0 +1,371 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/e28_Github_70m_n50_s0.pt", + "layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 4, + "best1L": 3, + "mean": 3, + "persist": 4 + }, + "n_sets_stable": 6, + "n_sets_aggregation_dependent": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "results": { + "multihop": { + "logit": { + "min": 0.0083, + "best1L": 0.0058, + "best1L_layer": 2, + "mean": 0.0035, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0083 + }, + "jlens": { + "min": 0.0085, + "best1L": 0.0056, + "best1L_layer": 3, + "mean": 0.0021, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0085 + }, + "shuffled": { + "min": 0.0015, + "best1L": 0.0015, + "best1L_layer": 2, + "mean": 0.0004, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0015 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0029, + "3": 0.0056 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0 + }, + "layer_specific": { + "min": 0.007, + "best1L": 0.0041, + "mean": 0.0018, + "persist": 0.007 + }, + "generic": { + "min": -0.0068, + "best1L": -0.0044, + "mean": -0.0032, + "persist": -0.0068 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.0395, + "best1L": 0.0187, + "best1L_layer": 0, + "mean": 0.0125, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0161 + }, + "jlens": { + "min": 0.0157, + "best1L": 0.0115, + "best1L_layer": 3, + "mean": 0.0049, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0149 + }, + "shuffled": { + "min": 0.0102, + "best1L": 0.0076, + "best1L_layer": 2, + "mean": 0.0032, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0076 + }, + "per_layer_jlens": { + "0": 0.0008, + "1": 0.0031, + "2": 0.0042, + "3": 0.0115 + }, + "per_layer_shuffled": { + "0": 0.0022, + "1": 0.0008, + "2": 0.0076, + "3": 0.0023 + }, + "layer_specific": { + "min": 0.0055, + "best1L": 0.0039, + "mean": 0.0017, + "persist": 0.0073 + }, + "generic": { + "min": -0.0294, + "best1L": -0.0111, + "mean": -0.0092, + "persist": -0.0085 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.0606, + "best1L": 0.0565, + "best1L_layer": 3, + "mean": 0.0162, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0592 + }, + "jlens": { + "min": 0.3042, + "best1L": 0.2455, + "best1L_layer": 2, + "mean": 0.1664, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.2738 + }, + "shuffled": { + "min": 0.2889, + "best1L": 0.2194, + "best1L_layer": 1, + "mean": 0.111, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.1822 + }, + "per_layer_jlens": { + "0": 0.041, + "1": 0.1921, + "2": 0.2455, + "3": 0.1869 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.2194, + "2": 0.0974, + "3": 0.1272 + }, + "layer_specific": { + "min": 0.0153, + "best1L": 0.0262, + "mean": 0.0554, + "persist": 0.0916 + }, + "generic": { + "min": 0.2284, + "best1L": 0.1629, + "mean": 0.0948, + "persist": 0.1229 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0319, + "best1L": 0.0225, + "best1L_layer": 0, + "mean": 0.0155, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0181 + }, + "jlens": { + "min": 0.0143, + "best1L": 0.0097, + "best1L_layer": 3, + "mean": 0.0054, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0112 + }, + "shuffled": { + "min": 0.0138, + "best1L": 0.0077, + "best1L_layer": 1, + "mean": 0.0034, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0031 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0015, + "2": 0.0072, + "3": 0.0097 + }, + "per_layer_shuffled": { + "0": 0.0031, + "1": 0.0077, + "2": 0.0031, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0005, + "best1L": 0.0021, + "mean": 0.0019, + "persist": 0.0082 + }, + "generic": { + "min": -0.0181, + "best1L": -0.0148, + "mean": -0.0121, + "persist": -0.0151 + }, + "verdict": "STABLE: layer-specific" + }, + "typo": { + "logit": { + "min": 0.062, + "best1L": 0.0503, + "best1L_layer": 3, + "mean": 0.0216, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0579 + }, + "jlens": { + "min": 0.0782, + "best1L": 0.0526, + "best1L_layer": 3, + "mean": 0.0384, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0641 + }, + "shuffled": { + "min": 0.0532, + "best1L": 0.0287, + "best1L_layer": 1, + "mean": 0.0221, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0415 + }, + "per_layer_jlens": { + "0": 0.0302, + "1": 0.0373, + "2": 0.0336, + "3": 0.0526 + }, + "per_layer_shuffled": { + "0": 0.0073, + "1": 0.0287, + "2": 0.0258, + "3": 0.0266 + }, + "layer_specific": { + "min": 0.025, + "best1L": 0.024, + "mean": 0.0163, + "persist": 0.0226 + }, + "generic": { + "min": -0.0088, + "best1L": -0.0216, + "mean": 0.0005, + "persist": -0.0164 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 3, + "mean": 0.0023, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0044 + }, + "jlens": { + "min": 0.0109, + "best1L": 0.0093, + "best1L_layer": 3, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0093 + }, + "shuffled": { + "min": 0.006, + "best1L": 0.006, + "best1L_layer": 1, + "mean": 0.0015, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0016, + "2": 0.0016, + "3": 0.0093 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.006, + "2": 0.0, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0049, + "best1L": 0.0033, + "mean": 0.0016, + "persist": 0.0093 + }, + "generic": { + "min": 0.0, + "best1L": 0.0016, + "mean": -0.0008, + "persist": -0.0044 + }, + "verdict": "STABLE: layer-specific" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 30.2 +} \ No newline at end of file diff --git a/results/e28_Github_70m_n800_s0_reads.json b/results/e28_Github_70m_n800_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..91f3fab58e7fa82dd81f974f4c9433ff58ad2d8d --- /dev/null +++ b/results/e28_Github_70m_n800_s0_reads.json @@ -0,0 +1,371 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/e28_Github_70m_n800_s0.pt", + "layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 3, + "best1L": 2, + "mean": 3, + "persist": 3 + }, + "n_sets_stable": 6, + "n_sets_aggregation_dependent": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "results": { + "multihop": { + "logit": { + "min": 0.0083, + "best1L": 0.0058, + "best1L_layer": 2, + "mean": 0.0035, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0083 + }, + "jlens": { + "min": 0.0053, + "best1L": 0.0039, + "best1L_layer": 3, + "mean": 0.0013, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0053 + }, + "shuffled": { + "min": 0.0029, + "best1L": 0.0015, + "best1L_layer": 2, + "mean": 0.0007, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0029 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0039 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0015, + "3": 0.0015 + }, + "layer_specific": { + "min": 0.0024, + "best1L": 0.0024, + "mean": 0.0006, + "persist": 0.0024 + }, + "generic": { + "min": -0.0053, + "best1L": -0.0044, + "mean": -0.0028, + "persist": -0.0053 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.0395, + "best1L": 0.0187, + "best1L_layer": 0, + "mean": 0.0125, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0161 + }, + "jlens": { + "min": 0.0148, + "best1L": 0.0106, + "best1L_layer": 3, + "mean": 0.0046, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0141 + }, + "shuffled": { + "min": 0.0069, + "best1L": 0.0047, + "best1L_layer": 2, + "mean": 0.0019, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0047 + }, + "per_layer_jlens": { + "0": 0.0008, + "1": 0.0027, + "2": 0.0042, + "3": 0.0106 + }, + "per_layer_shuffled": { + "0": 0.0018, + "1": 0.0004, + "2": 0.0047, + "3": 0.0008 + }, + "layer_specific": { + "min": 0.0079, + "best1L": 0.0059, + "mean": 0.0027, + "persist": 0.0093 + }, + "generic": { + "min": -0.0327, + "best1L": -0.014, + "mean": -0.0105, + "persist": -0.0114 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.0606, + "best1L": 0.0565, + "best1L_layer": 3, + "mean": 0.0162, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0592 + }, + "jlens": { + "min": 0.3256, + "best1L": 0.2431, + "best1L_layer": 2, + "mean": 0.1723, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.2834 + }, + "shuffled": { + "min": 0.2762, + "best1L": 0.2101, + "best1L_layer": 1, + "mean": 0.1123, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.1951 + }, + "per_layer_jlens": { + "0": 0.0487, + "1": 0.2055, + "2": 0.2431, + "3": 0.1917 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.2101, + "2": 0.1071, + "3": 0.1322 + }, + "layer_specific": { + "min": 0.0494, + "best1L": 0.033, + "mean": 0.0599, + "persist": 0.0882 + }, + "generic": { + "min": 0.2156, + "best1L": 0.1536, + "mean": 0.0962, + "persist": 0.1359 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0319, + "best1L": 0.0225, + "best1L_layer": 0, + "mean": 0.0155, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0181 + }, + "jlens": { + "min": 0.0143, + "best1L": 0.0097, + "best1L_layer": 3, + "mean": 0.0043, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0112 + }, + "shuffled": { + "min": 0.013, + "best1L": 0.0059, + "best1L_layer": 1, + "mean": 0.0033, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0056 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0, + "2": 0.0046, + "3": 0.0097 + }, + "per_layer_shuffled": { + "0": 0.0015, + "1": 0.0059, + "2": 0.0056, + "3": 0.0 + }, + "layer_specific": { + "min": 0.0013, + "best1L": 0.0038, + "mean": 0.0011, + "persist": 0.0056 + }, + "generic": { + "min": -0.0189, + "best1L": -0.0166, + "mean": -0.0123, + "persist": -0.0125 + }, + "verdict": "STABLE: layer-specific" + }, + "typo": { + "logit": { + "min": 0.062, + "best1L": 0.0503, + "best1L_layer": 3, + "mean": 0.0216, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0579 + }, + "jlens": { + "min": 0.0758, + "best1L": 0.0482, + "best1L_layer": 3, + "mean": 0.0369, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0605 + }, + "shuffled": { + "min": 0.0618, + "best1L": 0.0318, + "best1L_layer": 3, + "mean": 0.0237, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0449 + }, + "per_layer_jlens": { + "0": 0.03, + "1": 0.036, + "2": 0.0333, + "3": 0.0482 + }, + "per_layer_shuffled": { + "0": 0.0073, + "1": 0.0297, + "2": 0.0258, + "3": 0.0318 + }, + "layer_specific": { + "min": 0.014, + "best1L": 0.0164, + "mean": 0.0132, + "persist": 0.0156 + }, + "generic": { + "min": -0.0003, + "best1L": -0.0185, + "mean": 0.002, + "persist": -0.013 + }, + "verdict": "STABLE: layer-specific" + }, + "association": { + "logit": { + "min": 0.006, + "best1L": 0.0044, + "best1L_layer": 3, + "mean": 0.0023, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0044 + }, + "jlens": { + "min": 0.0109, + "best1L": 0.0093, + "best1L_layer": 3, + "mean": 0.0031, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0093 + }, + "shuffled": { + "min": 0.0033, + "best1L": 0.0016, + "best1L_layer": 1, + "mean": 0.0008, + "persist_K": 1, + "persist_band_size": 2, + "persist": 0.0016 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0016, + "2": 0.0016, + "3": 0.0093 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0016, + "2": 0.0, + "3": 0.0016 + }, + "layer_specific": { + "min": 0.0076, + "best1L": 0.0076, + "mean": 0.0023, + "persist": 0.0076 + }, + "generic": { + "min": -0.0027, + "best1L": -0.0027, + "mean": -0.0015, + "persist": -0.0027 + }, + "verdict": "STABLE: layer-specific" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 24.7 +} \ No newline at end of file diff --git a/results/e28_Pile-CC_410m_n100_s0_reads.json b/results/e28_Pile-CC_410m_n100_s0_reads.json new file mode 100644 index 0000000000000000000000000000000000000000..b67534bc556284e703426efc05e170a5b93e2e0d --- /dev/null +++ b/results/e28_Pile-CC_410m_n100_s0_reads.json @@ -0,0 +1,634 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/e28_Pile-CC_410m_n100_s0.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 6, + "best1L": 5, + "mean": 6, + "persist": 3 + }, + "n_sets_stable": 2, + "n_sets_aggregation_dependent": 4, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "results": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "best1L_layer": 19, + "mean": 0.033, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0476 + }, + "jlens": { + "min": 0.2519, + "best1L": 0.1664, + "best1L_layer": 21, + "mean": 0.0435, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0455 + }, + "shuffled": { + "min": 0.2087, + "best1L": 0.1358, + "best1L_layer": 21, + "mean": 0.0188, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0073 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0029, + "4": 0.0015, + "5": 0.0, + "6": 0.0, + "7": 0.0015, + "8": 0.0056, + "9": 0.0039, + "10": 0.0161, + "11": 0.0175, + "12": 0.017, + "13": 0.0297, + "14": 0.0331, + "15": 0.0306, + "16": 0.052, + "17": 0.113, + "18": 0.138, + "19": 0.1635, + "20": 0.1649, + "21": 0.1664 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0015, + "6": 0.0, + "7": 0.0053, + "8": 0.0015, + "9": 0.0015, + "10": 0.0097, + "11": 0.0, + "12": 0.0102, + "13": 0.0083, + "14": 0.0355, + "15": 0.0248, + "16": 0.0785, + "17": 0.0243, + "18": 0.0, + "19": 0.027, + "20": 0.0491, + "21": 0.1358 + }, + "layer_specific": { + "min": 0.0432, + "best1L": 0.0306, + "mean": 0.0247, + "persist": 0.0382 + }, + "generic": { + "min": 0.0219, + "best1L": -0.0145, + "mean": -0.0142, + "persist": -0.0403 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "best1L_layer": 17, + "mean": 0.0165, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0212 + }, + "jlens": { + "min": 0.188, + "best1L": 0.1523, + "best1L_layer": 17, + "mean": 0.0276, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0361 + }, + "shuffled": { + "min": 0.1127, + "best1L": 0.049, + "best1L_layer": 16, + "mean": 0.0124, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0083 + }, + "per_layer_jlens": { + "0": 0.0042, + "1": 0.0071, + "2": 0.0073, + "3": 0.0075, + "4": 0.0071, + "5": 0.0032, + "6": 0.0019, + "7": 0.0015, + "8": 0.0033, + "9": 0.0011, + "10": 0.0004, + "11": 0.0, + "12": 0.0018, + "13": 0.0011, + "14": 0.0239, + "15": 0.0384, + "16": 0.0726, + "17": 0.1523, + "18": 0.0789, + "19": 0.0712, + "20": 0.0708, + "21": 0.0527 + }, + "per_layer_shuffled": { + "0": 0.0046, + "1": 0.0, + "2": 0.0008, + "3": 0.0004, + "4": 0.0008, + "5": 0.0051, + "6": 0.0, + "7": 0.0068, + "8": 0.0, + "9": 0.0004, + "10": 0.0008, + "11": 0.0023, + "12": 0.0089, + "13": 0.009, + "14": 0.012, + "15": 0.0365, + "16": 0.049, + "17": 0.0412, + "18": 0.0084, + "19": 0.0078, + "20": 0.0292, + "21": 0.0489 + }, + "layer_specific": { + "min": 0.0753, + "best1L": 0.1033, + "mean": 0.0153, + "persist": 0.0278 + }, + "generic": { + "min": 0.0013, + "best1L": -0.0356, + "mean": -0.0042, + "persist": -0.0129 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "best1L_layer": 19, + "mean": 0.1449, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2915 + }, + "jlens": { + "min": 0.6422, + "best1L": 0.382, + "best1L_layer": 21, + "mean": 0.2146, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.3002 + }, + "shuffled": { + "min": 0.5945, + "best1L": 0.4402, + "best1L_layer": 20, + "mean": 0.1551, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.2568 + }, + "per_layer_jlens": { + "0": 0.0041, + "1": 0.0832, + "2": 0.097, + "3": 0.1252, + "4": 0.1341, + "5": 0.1404, + "6": 0.2375, + "7": 0.2409, + "8": 0.165, + "9": 0.0599, + "10": 0.1202, + "11": 0.1356, + "12": 0.2448, + "13": 0.307, + "14": 0.2811, + "15": 0.2545, + "16": 0.2781, + "17": 0.3486, + "18": 0.3477, + "19": 0.3588, + "20": 0.3766, + "21": 0.382 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0041, + "2": 0.0173, + "3": 0.0257, + "4": 0.0027, + "5": 0.1929, + "6": 0.0, + "7": 0.145, + "8": 0.1324, + "9": 0.1115, + "10": 0.0824, + "11": 0.0184, + "12": 0.1816, + "13": 0.269, + "14": 0.3202, + "15": 0.2435, + "16": 0.3265, + "17": 0.1639, + "18": 0.0, + "19": 0.358, + "20": 0.4402, + "21": 0.3768 + }, + "layer_specific": { + "min": 0.0477, + "best1L": -0.0582, + "mean": 0.0595, + "persist": 0.0434 + }, + "generic": { + "min": 0.0795, + "best1L": 0.0491, + "mean": 0.0102, + "persist": -0.0347 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "best1L_layer": 18, + "mean": 0.0021, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0491, + "best1L": 0.0148, + "best1L_layer": 4, + "mean": 0.0051, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0015 + }, + "shuffled": { + "min": 0.068, + "best1L": 0.0189, + "best1L_layer": 17, + "mean": 0.0057, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0015 + }, + "per_layer_jlens": { + "0": 0.0015, + "1": 0.0072, + "2": 0.0046, + "3": 0.0061, + "4": 0.0148, + "5": 0.0143, + "6": 0.0046, + "7": 0.0112, + "8": 0.0031, + "9": 0.0015, + "10": 0.0031, + "11": 0.0015, + "12": 0.0031, + "13": 0.0015, + "14": 0.0046, + "15": 0.0056, + "16": 0.0087, + "17": 0.0015, + "18": 0.0041, + "19": 0.0031, + "20": 0.0046, + "21": 0.0015 + }, + "per_layer_shuffled": { + "0": 0.0031, + "1": 0.0, + "2": 0.0, + "3": 0.0015, + "4": 0.0123, + "5": 0.0148, + "6": 0.0015, + "7": 0.014, + "8": 0.0072, + "9": 0.0015, + "10": 0.0072, + "11": 0.0, + "12": 0.0102, + "13": 0.0, + "14": 0.0031, + "15": 0.0031, + "16": 0.0, + "17": 0.0189, + "18": 0.0041, + "19": 0.012, + "20": 0.0041, + "21": 0.0061 + }, + "layer_specific": { + "min": -0.0189, + "best1L": -0.0041, + "mean": -0.0006, + "persist": 0.0 + }, + "generic": { + "min": 0.0424, + "best1L": 0.0112, + "mean": 0.0036, + "persist": 0.0 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "best1L_layer": 21, + "mean": 0.0345, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0475 + }, + "jlens": { + "min": 0.3757, + "best1L": 0.2115, + "best1L_layer": 18, + "mean": 0.1224, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.166 + }, + "shuffled": { + "min": 0.3773, + "best1L": 0.1889, + "best1L_layer": 13, + "mean": 0.0798, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0853 + }, + "per_layer_jlens": { + "0": 0.0102, + "1": 0.0185, + "2": 0.0485, + "3": 0.0328, + "4": 0.0941, + "5": 0.0803, + "6": 0.0936, + "7": 0.1111, + "8": 0.1165, + "9": 0.1519, + "10": 0.1082, + "11": 0.0952, + "12": 0.1572, + "13": 0.1798, + "14": 0.1918, + "15": 0.186, + "16": 0.1611, + "17": 0.1751, + "18": 0.2115, + "19": 0.1772, + "20": 0.1499, + "21": 0.1424 + }, + "per_layer_shuffled": { + "0": 0.0109, + "1": 0.0, + "2": 0.0016, + "3": 0.0209, + "4": 0.0923, + "5": 0.0464, + "6": 0.099, + "7": 0.062, + "8": 0.0738, + "9": 0.1186, + "10": 0.0993, + "11": 0.0136, + "12": 0.1397, + "13": 0.1889, + "14": 0.1272, + "15": 0.1157, + "16": 0.0637, + "17": 0.0952, + "18": 0.0284, + "19": 0.1175, + "20": 0.1053, + "21": 0.1361 + }, + "layer_specific": { + "min": -0.0016, + "best1L": 0.0227, + "mean": 0.0426, + "persist": 0.0808 + }, + "generic": { + "min": 0.2015, + "best1L": 0.0843, + "mean": 0.0453, + "persist": 0.0378 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "best1L_layer": 13, + "mean": 0.0005, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "jlens": { + "min": 0.0259, + "best1L": 0.0103, + "best1L_layer": 6, + "mean": 0.0024, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0566, + "best1L": 0.0332, + "best1L_layer": 5, + "mean": 0.0048, + "persist_K": 6, + "persist_band_size": 13, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0044, + "1": 0.0016, + "2": 0.0016, + "3": 0.0016, + "4": 0.0033, + "5": 0.0016, + "6": 0.0103, + "7": 0.0079, + "8": 0.006, + "9": 0.0033, + "10": 0.0033, + "11": 0.0, + "12": 0.0016, + "13": 0.0, + "14": 0.0, + "15": 0.0016, + "16": 0.0, + "17": 0.0016, + "18": 0.0033, + "19": 0.0, + "20": 0.0, + "21": 0.0 + }, + "per_layer_shuffled": { + "0": 0.0016, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0332, + "6": 0.0016, + "7": 0.0128, + "8": 0.0016, + "9": 0.0033, + "10": 0.0033, + "11": 0.0, + "12": 0.0016, + "13": 0.0016, + "14": 0.0033, + "15": 0.0016, + "16": 0.0, + "17": 0.0087, + "18": 0.0076, + "19": 0.0153, + "20": 0.0093, + "21": 0.0 + }, + "layer_specific": { + "min": -0.0307, + "best1L": -0.0228, + "mean": -0.0024, + "persist": 0.0 + }, + "generic": { + "min": 0.049, + "best1L": 0.0288, + "mean": 0.0043, + "persist": 0.0 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + } + }, + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 180.1 +} \ No newline at end of file diff --git a/results/e31_local_bakeoff_410m.json b/results/e31_local_bakeoff_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..4bdb0e77e8d6dd556bf65613e80c27bb232baec5 --- /dev/null +++ b/results/e31_local_bakeoff_410m.json @@ -0,0 +1,917 @@ +{ + "experiment": "E31-L (local reanalysis of E31's primary outcome)", + "written": "2026-08-13", + "model": "EleutherAI/pythia-410m-deduped", + "n_match": 400, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "spectral_layer": 15, + "aggregation": "persist", + "outcome_definition": "mean read AUC over 6 eval sets, N>=75, seeds averaged", + "decision_rule_source": "PLAN.tex section 5, E31 -- quoted in this file's docstring, not reinterpreted", + "control": "|r| must stay >= 0.8 under every leave-one-corpus-out refit", + "declared_deviations_from_E31": [ + "single matched N=400 rather than an 11-point trajectory in N", + "fp16 stored lenses (~3e-3 relative floor, << seed SD)", + "no refit: operators are the E28 artifacts" + ], + "inputs": { + "ladder_cells": 15, + "lenses": 15, + "eval_items": 541, + "concept_token_ids": 1310 + }, + "by_corpus": { + "Github": { + "dispersion_L0": 4.954359, + "dispersion_band_mean": 0.528685358974359, + "dispersion_slope": -0.10484419597069597, + "stable_rank": 15.637147585550943, + "top8_spectral_share": 0.17575907707214355, + "condition_number": 4981.613932291667, + "unembed_alignment": 0.11824092268943787, + "mean_prompt_tokens": 2242.0025, + "type_token_ratio": 0.12989583333333332, + "eval_vocab_overlap": 0.32595419847328244, + "model_cross_entropy": 1.2680157721430685, + "plateau_auc": 0.023403378161274385, + "auc_at_400": 0.023366356253742226, + "max_auc": 0.02472728787226573, + "plateau_auc_seed_sd": 0.0003104179835607803 + }, + "Pile-CC": { + "dispersion_L0": 3.1123753333333335, + "dispersion_band_mean": 0.33618071794871796, + "dispersion_slope": -0.06644831318681318, + "stable_rank": 20.31925392150879, + "top8_spectral_share": 0.1333793600400289, + "condition_number": 20436.781575520832, + "unembed_alignment": 0.19340071578820547, + "mean_prompt_tokens": 1273.4625, + "type_token_ratio": 0.21022135416666668, + "eval_vocab_overlap": 0.5720101781170484, + "model_cross_entropy": 3.1277856354912124, + "plateau_auc": 0.03809645690345838, + "auc_at_400": 0.03819741803916201, + "max_auc": 0.03878024464384431, + "plateau_auc_seed_sd": 0.00015295200779028256 + }, + "StackExchange": { + "dispersion_L0": 1.928899, + "dispersion_band_mean": 0.2518260769230769, + "dispersion_slope": -0.04941665018315018, + "stable_rank": 19.971024831136067, + "top8_spectral_share": 0.14192854364713034, + "condition_number": 3118.718505859375, + "unembed_alignment": 0.1548958569765091, + "mean_prompt_tokens": 677.5925, + "type_token_ratio": 0.14688151041666667, + "eval_vocab_overlap": 0.3801526717557252, + "model_cross_entropy": 2.5566212087869644, + "plateau_auc": 0.03806300258396992, + "auc_at_400": 0.03808815800954425, + "max_auc": 0.03868280886302865, + "plateau_auc_seed_sd": 0.00024832041221574055 + }, + "USPTO_Backgrounds": { + "dispersion_L0": 1.6590663333333333, + "dispersion_band_mean": 0.2561091282051282, + "dispersion_slope": -0.04525278021978022, + "stable_rank": 18.85663350423177, + "top8_spectral_share": 0.14590109388033548, + "condition_number": 1771.3484700520833, + "unembed_alignment": 0.15197869141896567, + "mean_prompt_tokens": 810.37, + "type_token_ratio": 0.14515625, + "eval_vocab_overlap": 0.3040712468193384, + "model_cross_entropy": 2.4697054509694376, + "plateau_auc": 0.04144605403894154, + "auc_at_400": 0.04206535970378253, + "max_auc": 0.042369054201694706, + "plateau_auc_seed_sd": 0.0004189487186783741 + }, + "Wikipedia_en": { + "dispersion_L0": 2.9050226666666665, + "dispersion_band_mean": 0.35629546153846153, + "dispersion_slope": -0.06556962271062271, + "stable_rank": 21.379074732462566, + "top8_spectral_share": 0.13833467662334442, + "condition_number": 25153.8798828125, + "unembed_alignment": 0.1686630348364512, + "mean_prompt_tokens": 1053.2416666666666, + "type_token_ratio": 0.19037109375, + "eval_vocab_overlap": 0.5241730279898219, + "model_cross_entropy": 2.466690806671977, + "plateau_auc": 0.032221472284783115, + "auc_at_400": 0.03250449677811019, + "max_auc": 0.03348244042948834, + "plateau_auc_seed_sd": 0.0003691465734544507 + } + }, + "by_cell": { + "Github|s0": { + "dispersion_L0": 4.829229, + "dispersion_band_mean": 0.5161408461538461, + "dispersion_slope": -0.10139051648351649, + "stable_rank": 15.897682189941406, + "top8_spectral_share": 0.1743803322315216, + "condition_number": 4358.82421875, + "unembed_alignment": 0.11723999679088593, + "mean_prompt_tokens": 2149.215, + "type_token_ratio": 0.13177734375, + "eval_vocab_overlap": 0.3114503816793893, + "model_cross_entropy": 1.1752161437179893, + "plateau_auc": 0.023584197988087426, + "auc_at_400": 0.02327154716476798, + "auc_at_25": 0.021977887777168126, + "max_auc": 0.025150195673285497 + }, + "Github|s1": { + "dispersion_L0": 4.767306, + "dispersion_band_mean": 0.5218856923076923, + "dispersion_slope": -0.1032985879120879, + "stable_rank": 15.55256462097168, + "top8_spectral_share": 0.1764976978302002, + "condition_number": 4482.2685546875, + "unembed_alignment": 0.12082402408123016, + "mean_prompt_tokens": 2445.22, + "type_token_ratio": 0.13056640625, + "eval_vocab_overlap": 0.3450381679389313, + "model_cross_entropy": 1.3683314372319728, + "plateau_auc": 0.023580993372047332, + "auc_at_400": 0.024028856085524672, + "auc_at_25": 0.02384093377207007, + "max_auc": 0.024444008489982003 + }, + "Github|s2": { + "dispersion_L0": 5.266542, + "dispersion_band_mean": 0.5480295384615385, + "dispersion_slope": -0.10984348351648351, + "stable_rank": 15.461195945739746, + "top8_spectral_share": 0.17639920115470886, + "condition_number": 6103.7490234375, + "unembed_alignment": 0.11665874719619751, + "mean_prompt_tokens": 2131.5725, + "type_token_ratio": 0.12734375, + "eval_vocab_overlap": 0.32137404580152673, + "model_cross_entropy": 1.260499735479243, + "plateau_auc": 0.0230449431236884, + "auc_at_400": 0.022798665510934023, + "auc_at_25": 0.01983687270521408, + "max_auc": 0.024587659453529687 + }, + "Pile-CC|s0": { + "dispersion_L0": 3.177652, + "dispersion_band_mean": 0.34225938461538463, + "dispersion_slope": -0.06779391758241758, + "stable_rank": 20.433944702148438, + "top8_spectral_share": 0.13306917250156403, + "condition_number": 10958.201171875, + "unembed_alignment": 0.1926257163286209, + "mean_prompt_tokens": 1172.3375, + "type_token_ratio": 0.212890625, + "eval_vocab_overlap": 0.5740458015267176, + "model_cross_entropy": 3.107743479311466, + "plateau_auc": 0.03792208318337404, + "auc_at_400": 0.037824345309109914, + "auc_at_25": 0.03794395289428177, + "max_auc": 0.038648379412257954 + }, + "Pile-CC|s1": { + "dispersion_L0": 3.036196, + "dispersion_band_mean": 0.32970638461538465, + "dispersion_slope": -0.06503617582417583, + "stable_rank": 19.994997024536133, + "top8_spectral_share": 0.1339729279279709, + "condition_number": 14158.3310546875, + "unembed_alignment": 0.19446802139282227, + "mean_prompt_tokens": 1363.555, + "type_token_ratio": 0.20828125, + "eval_vocab_overlap": 0.5580152671755725, + "model_cross_entropy": 3.164168832823634, + "plateau_auc": 0.0381593606213019, + "auc_at_400": 0.0383203767594837, + "auc_at_25": 0.03895468029770113, + "max_auc": 0.03895468029770113 + }, + "Pile-CC|s2": { + "dispersion_L0": 3.123278, + "dispersion_band_mean": 0.33657638461538464, + "dispersion_slope": -0.06651484615384616, + "stable_rank": 20.528820037841797, + "top8_spectral_share": 0.13309597969055176, + "condition_number": 36193.8125, + "unembed_alignment": 0.19310840964317322, + "mean_prompt_tokens": 1284.495, + "type_token_ratio": 0.2094921875, + "eval_vocab_overlap": 0.583969465648855, + "model_cross_entropy": 3.1114445943385363, + "plateau_auc": 0.0382079269056992, + "auc_at_400": 0.03844753204889241, + "auc_at_25": 0.03643960248501527, + "max_auc": 0.03873767422157384 + }, + "StackExchange|s0": { + "dispersion_L0": 1.899756, + "dispersion_band_mean": 0.24748746153846155, + "dispersion_slope": -0.048744197802197804, + "stable_rank": 20.159818649291992, + "top8_spectral_share": 0.14098617434501648, + "condition_number": 3104.291015625, + "unembed_alignment": 0.15546901524066925, + "mean_prompt_tokens": 634.7775, + "type_token_ratio": 0.147421875, + "eval_vocab_overlap": 0.3801526717557252, + "model_cross_entropy": 2.5761693492531776, + "plateau_auc": 0.03834920283406973, + "auc_at_400": 0.03846123142700111, + "auc_at_25": 0.03774874872483668, + "max_auc": 0.03926570919741477 + }, + "StackExchange|s1": { + "dispersion_L0": 1.993767, + "dispersion_band_mean": 0.2612495384615385, + "dispersion_slope": -0.051016967032967035, + "stable_rank": 20.103240966796875, + "top8_spectral_share": 0.1411186307668686, + "condition_number": 2421.12548828125, + "unembed_alignment": 0.15369564294815063, + "mean_prompt_tokens": 727.9625, + "type_token_ratio": 0.1501171875, + "eval_vocab_overlap": 0.3847328244274809, + "model_cross_entropy": 2.536895774304867, + "plateau_auc": 0.03793507116865747, + "auc_at_400": 0.03771718318707177, + "auc_at_25": 0.0363006595566514, + "max_auc": 0.038454936750765355 + }, + "StackExchange|s2": { + "dispersion_L0": 1.893174, + "dispersion_band_mean": 0.24674123076923077, + "dispersion_slope": -0.04848878571428571, + "stable_rank": 19.650014877319336, + "top8_spectral_share": 0.14368082582950592, + "condition_number": 3830.739013671875, + "unembed_alignment": 0.1555229127407074, + "mean_prompt_tokens": 670.0375, + "type_token_ratio": 0.14310546875, + "eval_vocab_overlap": 0.3755725190839695, + "model_cross_entropy": 2.556798502802849, + "plateau_auc": 0.03790473374918255, + "auc_at_400": 0.038086059414559885, + "auc_at_25": 0.03691776728789722, + "max_auc": 0.03832778064090581 + }, + "USPTO_Backgrounds|s0": { + "dispersion_L0": 1.634971, + "dispersion_band_mean": 0.2537289230769231, + "dispersion_slope": -0.04470524725274725, + "stable_rank": 18.908348083496094, + "top8_spectral_share": 0.14572106301784515, + "condition_number": 1735.6766357421875, + "unembed_alignment": 0.1524249017238617, + "mean_prompt_tokens": 847.9175, + "type_token_ratio": 0.1446875, + "eval_vocab_overlap": 0.29389312977099236, + "model_cross_entropy": 2.451203413307667, + "plateau_auc": 0.041781165612437735, + "auc_at_400": 0.0424143901715676, + "auc_at_25": 0.04001120568829633, + "max_auc": 0.042777767493611285 + }, + "USPTO_Backgrounds|s1": { + "dispersion_L0": 1.688143, + "dispersion_band_mean": 0.2587466923076923, + "dispersion_slope": -0.04591918681318682, + "stable_rank": 18.925546646118164, + "top8_spectral_share": 0.1455058753490448, + "condition_number": 1717.1842041015625, + "unembed_alignment": 0.15189313888549805, + "mean_prompt_tokens": 822.3475, + "type_token_ratio": 0.1486328125, + "eval_vocab_overlap": 0.31068702290076333, + "model_cross_entropy": 2.504025334492326, + "plateau_auc": 0.04097635059977177, + "auc_at_400": 0.04216637435768331, + "auc_at_25": 0.039277933937098296, + "max_auc": 0.04216637435768331 + }, + "USPTO_Backgrounds|s2": { + "dispersion_L0": 1.654085, + "dispersion_band_mean": 0.25585176923076924, + "dispersion_slope": -0.045133906593406596, + "stable_rank": 18.736005783081055, + "top8_spectral_share": 0.14647634327411652, + "condition_number": 1861.1845703125, + "unembed_alignment": 0.15161803364753723, + "mean_prompt_tokens": 760.845, + "type_token_ratio": 0.1421484375, + "eval_vocab_overlap": 0.30763358778625954, + "model_cross_entropy": 2.4538876051083207, + "plateau_auc": 0.041580645904615125, + "auc_at_400": 0.04161531458209668, + "auc_at_25": 0.04056764214432665, + "max_auc": 0.04216302075378952 + }, + "Wikipedia_en|s0": { + "dispersion_L0": 2.915317, + "dispersion_band_mean": 0.3576262307692308, + "dispersion_slope": -0.06568178021978022, + "stable_rank": 21.32246208190918, + "top8_spectral_share": 0.1384531557559967, + "condition_number": 15275.1123046875, + "unembed_alignment": 0.16826492547988892, + "mean_prompt_tokens": 1045.385, + "type_token_ratio": 0.1894921875, + "eval_vocab_overlap": 0.5038167938931297, + "model_cross_entropy": 2.464904047548771, + "plateau_auc": 0.032267923316392465, + "auc_at_400": 0.03278358883800961, + "auc_at_25": 0.03367994510613027, + "max_auc": 0.03367994510613027 + }, + "Wikipedia_en|s1": { + "dispersion_L0": 2.899967, + "dispersion_band_mean": 0.3582076923076923, + "dispersion_slope": -0.06588393406593407, + "stable_rank": 21.265737533569336, + "top8_spectral_share": 0.1386982500553131, + "condition_number": 38194.31640625, + "unembed_alignment": 0.16883540153503418, + "mean_prompt_tokens": 1080.31, + "type_token_ratio": 0.1901953125, + "eval_vocab_overlap": 0.5274809160305344, + "model_cross_entropy": 2.4637784250080585, + "plateau_auc": 0.03256519488375761, + "auc_at_400": 0.03280065975905884, + "auc_at_25": 0.033587056677788496, + "max_auc": 0.033587056677788496 + }, + "Wikipedia_en|s2": { + "dispersion_L0": 2.899784, + "dispersion_band_mean": 0.35305246153846154, + "dispersion_slope": -0.06514315384615385, + "stable_rank": 21.54902458190918, + "top8_spectral_share": 0.13785262405872345, + "condition_number": 21992.2109375, + "unembed_alignment": 0.16888877749443054, + "mean_prompt_tokens": 1034.03, + "type_token_ratio": 0.19142578125, + "eval_vocab_overlap": 0.5412213740458015, + "model_cross_entropy": 2.4713899474591017, + "plateau_auc": 0.03183129865419928, + "auc_at_400": 0.03192924173726212, + "auc_at_25": 0.032335397942612566, + "max_auc": 0.03318031950454627 + } + }, + "bakeoff": { + "plateau_auc": { + "dispersion_L0": { + "r": -0.9242882654186496, + "spearman": -0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": -0.6602694430492779, + "Pile-CC": -0.9883267965267698, + "StackExchange": -0.9287734804839324, + "USPTO_Backgrounds": -0.893623860212412, + "Wikipedia_en": -0.9421475362694959 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.6602694430492779, + "leverage_driven": true, + "survives_control": false + }, + "dispersion_band_mean": { + "r": -0.9531498843872271, + "spearman": -0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": -0.7671626905078005, + "Pile-CC": -0.977915566270783, + "StackExchange": -0.9718692145883884, + "USPTO_Backgrounds": -0.9448820211765581, + "Wikipedia_en": -0.9621510293915778 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.7671626905078005, + "leverage_driven": true, + "survives_control": false + }, + "dispersion_slope": { + "r": 0.9445686578610754, + "spearman": 0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": 0.7187361535787741, + "Pile-CC": 0.9823487349640734, + "StackExchange": 0.9483969907762579, + "USPTO_Backgrounds": 0.9253548227763234, + "Wikipedia_en": 0.9657721233058413 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.7187361535787741, + "leverage_driven": true, + "survives_control": false + }, + "stable_rank": { + "r": 0.6217659208337967, + "spearman": 0.09999999999999998, + "significant_n5": false, + "loo": { + "Github": -0.9695644030203718, + "Pile-CC": 0.5911743338547131, + "StackExchange": 0.6039231697189055, + "USPTO_Backgrounds": 0.7998823035318988, + "Wikipedia_en": 0.8815381967499595 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": 0.5911743338547131, + "leverage_driven": true, + "survives_control": false + }, + "top8_spectral_share": { + "r": -0.7882566086770282, + "spearman": -0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": 0.4765579902877013, + "Pile-CC": -0.7760578580365551, + "StackExchange": -0.7820470740144058, + "USPTO_Backgrounds": -0.9089751111716116, + "Wikipedia_en": -0.8989917096460657 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.4765579902877013, + "leverage_driven": true, + "survives_control": false + }, + "condition_number": { + "r": -0.07692108108636014, + "spearman": -0.4999999999999999, + "significant_n5": false, + "loo": { + "Github": -0.7998858953272293, + "Pile-CC": -0.24557369416617256, + "StackExchange": 0.03784692108272241, + "USPTO_Backgrounds": 0.24187061137722737, + "Wikipedia_en": 0.08985480707819898 + }, + "worst_loo_corpus": "StackExchange", + "worst_loo_r": 0.03784692108272241, + "leverage_driven": true, + "survives_control": false + }, + "unembed_alignment": { + "r": 0.6323756162160421, + "spearman": 0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": -0.27176406633776684, + "Pile-CC": 0.6634352056273711, + "StackExchange": 0.6719152191488584, + "USPTO_Backgrounds": 0.824124147263485, + "Wikipedia_en": 0.7076569845195414 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.27176406633776684, + "leverage_driven": true, + "survives_control": false + }, + "mean_prompt_tokens": { + "r": -0.8817956434996898, + "spearman": -0.4999999999999999, + "significant_n5": true, + "loo": { + "Github": -0.3478077213273393, + "Pile-CC": -0.9334358651066791, + "StackExchange": -0.8912236098075476, + "USPTO_Backgrounds": -0.8738302300846262, + "Wikipedia_en": -0.9355318601749332 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.3478077213273393, + "leverage_driven": true, + "survives_control": false + }, + "type_token_ratio": { + "r": 0.30182668187780554, + "spearman": 0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": -0.5210794478633809, + "Pile-CC": 0.1543887990841397, + "StackExchange": 0.4116288520112553, + "USPTO_Backgrounds": 0.5886139787952779, + "Wikipedia_en": 0.4306383068282148 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": 0.1543887990841397, + "leverage_driven": true, + "survives_control": false + }, + "eval_vocab_overlap": { + "r": 0.11840865018179178, + "spearman": -0.09999999999999998, + "significant_n5": false, + "loo": { + "Github": -0.6530075809837673, + "Pile-CC": -0.10454375717470109, + "StackExchange": 0.1795815343927366, + "USPTO_Backgrounds": 0.5784388361066101, + "Wikipedia_en": 0.2431976396090469 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": -0.10454375717470109, + "leverage_driven": true, + "survives_control": false + }, + "model_cross_entropy": { + "r": 0.8335856422219892, + "spearman": 0.6999999999999998, + "significant_n5": false, + "loo": { + "Github": 0.13389888988376378, + "Pile-CC": 0.8798968056268194, + "StackExchange": 0.833411330352734, + "USPTO_Backgrounds": 0.9412820268835245, + "Wikipedia_en": 0.8657204332640084 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.13389888988376378, + "leverage_driven": true, + "survives_control": false + } + }, + "auc_at_400": { + "dispersion_L0": { + "r": -0.9270366304271419, + "spearman": -0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": -0.6738482369278233, + "Pile-CC": -0.9862352864399775, + "StackExchange": -0.936207536964022, + "USPTO_Backgrounds": -0.8960689633184281, + "Wikipedia_en": -0.942986442663324 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.6738482369278233, + "leverage_driven": true, + "survives_control": false + }, + "dispersion_band_mean": { + "r": -0.9517404376690609, + "spearman": -0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": -0.7615264367304332, + "Pile-CC": -0.973408151770571, + "StackExchange": -0.9755190515036716, + "USPTO_Backgrounds": -0.9458941000434438, + "Wikipedia_en": -0.9593170528932303 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.7615264367304332, + "leverage_driven": true, + "survives_control": false + }, + "dispersion_slope": { + "r": 0.9462090338394493, + "spearman": 0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": 0.7294203735833086, + "Pile-CC": 0.9802727643469268, + "StackExchange": 0.9538742091212139, + "USPTO_Backgrounds": 0.9282394240055525, + "Wikipedia_en": 0.9652794817295908 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.7294203735833086, + "leverage_driven": true, + "survives_control": false + }, + "stable_rank": { + "r": 0.6182611048106116, + "spearman": 0.09999999999999998, + "significant_n5": false, + "loo": { + "Github": -0.9790914051670012, + "Pile-CC": 0.589123777338919, + "StackExchange": 0.6008064124287614, + "USPTO_Backgrounds": 0.8110866450963926, + "Wikipedia_en": 0.8684603099052803 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": 0.589123777338919, + "leverage_driven": true, + "survives_control": false + }, + "top8_spectral_share": { + "r": -0.783089511579745, + "spearman": -0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": 0.5069980652604744, + "Pile-CC": -0.7742216915972161, + "StackExchange": -0.7759311525048763, + "USPTO_Backgrounds": -0.9165968266615825, + "Wikipedia_en": -0.8883582342802788 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.5069980652604744, + "leverage_driven": true, + "survives_control": false + }, + "condition_number": { + "r": -0.0778654548657342, + "spearman": -0.4999999999999999, + "significant_n5": false, + "loo": { + "Github": -0.7980412107259645, + "Pile-CC": -0.23818390204655854, + "StackExchange": 0.027696692686508633, + "USPTO_Backgrounds": 0.2582009711718281, + "Wikipedia_en": 0.07695932053163311 + }, + "worst_loo_corpus": "StackExchange", + "worst_loo_r": 0.027696692686508633, + "leverage_driven": true, + "survives_control": false + }, + "unembed_alignment": { + "r": 0.6247633888567198, + "spearman": 0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": -0.2948932668173631, + "Pile-CC": 0.6642851186696328, + "StackExchange": 0.6594542975216466, + "USPTO_Backgrounds": 0.8311554553921144, + "Wikipedia_en": 0.695710826445672 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.2948932668173631, + "leverage_driven": true, + "survives_control": false + }, + "mean_prompt_tokens": { + "r": -0.8815166643122333, + "spearman": -0.4999999999999999, + "significant_n5": true, + "loo": { + "Github": -0.35021033077281305, + "Pile-CC": -0.9288292633935042, + "StackExchange": -0.8970201208367593, + "USPTO_Backgrounds": -0.8779932915022772, + "Wikipedia_en": -0.9317773301268846 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.35021033077281305, + "leverage_driven": true, + "survives_control": false + }, + "type_token_ratio": { + "r": 0.2961005455315703, + "spearman": 0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": -0.5323069313616673, + "Pile-CC": 0.16052348380657647, + "StackExchange": 0.3973004790944023, + "USPTO_Backgrounds": 0.5993348497855877, + "Wikipedia_en": 0.4182580721669106 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": 0.16052348380657647, + "leverage_driven": true, + "survives_control": false + }, + "eval_vocab_overlap": { + "r": 0.10847984021816694, + "spearman": -0.09999999999999998, + "significant_n5": false, + "loo": { + "Github": -0.6729015220908467, + "Pile-CC": -0.10497226868912966, + "StackExchange": 0.1643353026960593, + "USPTO_Backgrounds": 0.5899088644739042, + "Wikipedia_en": 0.22531159256464336 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": -0.10497226868912966, + "leverage_driven": true, + "survives_control": false + }, + "model_cross_entropy": { + "r": 0.8258773997729012, + "spearman": 0.6999999999999998, + "significant_n5": false, + "loo": { + "Github": 0.09768440998487082, + "Pile-CC": 0.877272294811669, + "StackExchange": 0.823887652639768, + "USPTO_Backgrounds": 0.9454225350312015, + "Wikipedia_en": 0.8553014705547857 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.09768440998487082, + "leverage_driven": true, + "survives_control": false + } + }, + "max_auc": { + "dispersion_L0": { + "r": -0.9280308066890505, + "spearman": -0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": -0.6737930865794565, + "Pile-CC": -0.9874645318340407, + "StackExchange": -0.93710694987842, + "USPTO_Backgrounds": -0.8977292738762761, + "Wikipedia_en": -0.9428146598933335 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.6737930865794565, + "leverage_driven": true, + "survives_control": false + }, + "dispersion_band_mean": { + "r": -0.9529610154491082, + "spearman": -0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": -0.761869376243413, + "Pile-CC": -0.9747920491150962, + "StackExchange": -0.9766575599784396, + "USPTO_Backgrounds": -0.9468208142152382, + "Wikipedia_en": -0.9597350157902526 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.761869376243413, + "leverage_driven": true, + "survives_control": false + }, + "dispersion_slope": { + "r": 0.9475518698729031, + "spearman": 0.6999999999999998, + "significant_n5": true, + "loo": { + "Github": 0.7294170852648598, + "Pile-CC": 0.981811584799035, + "StackExchange": 0.9551893315767087, + "USPTO_Backgrounds": 0.9298854575072797, + "Wikipedia_en": 0.9653348453448322 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.7294170852648598, + "leverage_driven": true, + "survives_control": false + }, + "stable_rank": { + "r": 0.6245615173766049, + "spearman": 0.09999999999999998, + "significant_n5": false, + "loo": { + "Github": -0.9789745441924991, + "Pile-CC": 0.5958366186047656, + "StackExchange": 0.607394893205259, + "USPTO_Backgrounds": 0.8152102551885526, + "Wikipedia_en": 0.8705641696386055 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": 0.5958366186047656, + "leverage_driven": true, + "survives_control": false + }, + "top8_spectral_share": { + "r": -0.7877479899999343, + "spearman": -0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": 0.50657377106353, + "Pile-CC": -0.7794742504409932, + "StackExchange": -0.7808428521229787, + "USPTO_Backgrounds": -0.9190748649014199, + "Wikipedia_en": -0.8901150210671723 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.50657377106353, + "leverage_driven": true, + "survives_control": false + }, + "condition_number": { + "r": -0.07139311755512572, + "spearman": -0.4999999999999999, + "significant_n5": false, + "loo": { + "Github": -0.7982922650842919, + "Pile-CC": -0.23088685911396886, + "StackExchange": 0.03550128711059176, + "USPTO_Backgrounds": 0.26312503738389353, + "Wikipedia_en": 0.07919984626343372 + }, + "worst_loo_corpus": "StackExchange", + "worst_loo_r": 0.03550128711059176, + "leverage_driven": true, + "survives_control": false + }, + "unembed_alignment": { + "r": 0.6290901271480333, + "spearman": 0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": -0.2946663706618384, + "Pile-CC": 0.6703838873726189, + "StackExchange": 0.6641607528599512, + "USPTO_Backgrounds": 0.8327498083502494, + "Wikipedia_en": 0.6977510738605228 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.2946663706618384, + "leverage_driven": true, + "survives_control": false + }, + "mean_prompt_tokens": { + "r": -0.8841233039962052, + "spearman": -0.4999999999999999, + "significant_n5": true, + "loo": { + "Github": -0.3504499272686921, + "Pile-CC": -0.9317024537936969, + "StackExchange": -0.8997620487703538, + "USPTO_Backgrounds": -0.8802781627160362, + "Wikipedia_en": -0.9323163737468549 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": -0.3504499272686921, + "leverage_driven": true, + "survives_control": false + }, + "type_token_ratio": { + "r": 0.3013465206695397, + "spearman": 0.29999999999999993, + "significant_n5": false, + "loo": { + "Github": -0.5323317516698883, + "Pile-CC": 0.16812512037905425, + "StackExchange": 0.40338192416350405, + "USPTO_Backgrounds": 0.6019672136381428, + "Wikipedia_en": 0.42041091269475184 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": 0.16812512037905425, + "leverage_driven": true, + "survives_control": false + }, + "eval_vocab_overlap": { + "r": 0.11452847164620714, + "spearman": -0.09999999999999998, + "significant_n5": false, + "loo": { + "Github": -0.6726992818695743, + "Pile-CC": -0.09676026281357074, + "StackExchange": 0.17094727205330956, + "USPTO_Backgrounds": 0.5928723689380736, + "Wikipedia_en": 0.22829151322598118 + }, + "worst_loo_corpus": "Pile-CC", + "worst_loo_r": -0.09676026281357074, + "leverage_driven": true, + "survives_control": false + }, + "model_cross_entropy": { + "r": 0.8291266538400223, + "spearman": 0.6999999999999998, + "significant_n5": false, + "loo": { + "Github": 0.09823389951828312, + "Pile-CC": 0.8812410517459507, + "StackExchange": 0.8273474126737375, + "USPTO_Backgrounds": 0.9463166277299321, + "Wikipedia_en": 0.8570529507075499 + }, + "worst_loo_corpus": "Github", + "worst_loo_r": 0.09823389951828312, + "leverage_driven": true, + "survives_control": false + } + } + } +} \ No newline at end of file diff --git a/results/e33_logit_baseline_410m.json b/results/e33_logit_baseline_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..efc2ea43aecffb91bb114e123a8ad6b1a0f9a920 --- /dev/null +++ b/results/e33_logit_baseline_410m.json @@ -0,0 +1,228 @@ +{ + "experiment": "E33 \u2014 the missing J=I arm on the E28 read ladder", + "prereg": "PLAN.tex section 6 (E33), fixed before this ran; quoted in this file's docstring", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "n_items": 541, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "excluded_sets": { + "association": "floored \u2014 196/205 ladder cells exactly zero (E28 crit 3)" + }, + "reference_lens": "e28_Github_410m_n400_s0.pt", + "by_transport": { + "logit_I": { + "multihop": { + "min": 0.11789181775280408, + "best1L": 0.09847434024725642, + "mean": 0.03894164053989308, + "persist": 0.037447988986968994 + }, + "multilingual": { + "min": 0.11240029175366674, + "best1L": 0.09100797799016748, + "mean": 0.029034417222387025, + "persist": 0.02320522096540247 + }, + "order-ops": { + "min": 0.19740259860243117, + "best1L": 0.17272727404321944, + "mean": 0.052547452971339226, + "persist": 0.03246753276990993 + }, + "poetry": { + "min": 0.020408162714115212, + "best1L": 0.007288629987410137, + "mean": 0.0028033192003411905, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.17559524040137017, + "best1L": 0.11755952372082643, + "mean": 0.054716117680072784, + "persist": 0.04761904717556068 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "jlens_P": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07650471345654555, + "mean": 0.0322697607002088, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428837520735, + "mean": 0.09775641188025475, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "jshuf": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.07489597770784583, + "mean": 0.016963619040325284, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1287164602960859, + "best1L": 0.09862218797206879, + "mean": 0.03132147016003728, + "persist": 0.022480057685502937 + }, + "order-ops": { + "min": 0.07402597367763519, + "best1L": 0.05584415686982019, + "mean": 0.019780219266457216, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.01895043719559908, + "best1L": 0.011661807474281107, + "mean": 0.0037003813444503714, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.27083333155938555, + "best1L": 0.169642853683659, + "mean": 0.08676739848617997, + "persist": 0.07589285887245621 + }, + "association": { + "min": 0.012422360080693449, + "best1L": 0.006211180239915848, + "mean": 0.0010750119191860513, + "persist": 0.0 + } + }, + "rand_matched": { + "multihop": { + "min": 0.024965326434799602, + "best1L": 0.009708737954497337, + "mean": 0.0019204096752218902, + "persist": 0.0 + }, + "multilingual": { + "min": 0.032632342007543357, + "best1L": 0.012327773629554681, + "mean": 0.002816979981227113, + "persist": 0.0 + }, + "order-ops": { + "min": 0.031168831246239797, + "best1L": 0.02337662343467985, + "mean": 0.0023976024240255356, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.004373177752963134, + "mean": 0.0011213276801364763, + "persist": 0.0 + }, + "typo": { + "min": 0.03571428531514747, + "best1L": 0.011904762259551458, + "mean": 0.0029761905898340046, + "persist": 0.0 + }, + "association": { + "min": 0.031055900268256664, + "best1L": 0.009316769960735525, + "mean": 0.002388915354718587, + "persist": 0.0 + } + } + }, + "persist_admitted_mean": { + "logit_I": 0.02843950316309929, + "jlens_P": 0.027925856597721576, + "jshuf": 0.02496830283531121, + "rand_matched": 0.0 + }, + "e28_asymptotes": { + "Github": { + "mean": 0.028084053793529264, + "seed_sd": 0.0003725015802729375 + }, + "Wikipedia_en": { + "mean": 0.03860496947484109, + "seed_sd": 0.000497512452938681 + }, + "StackExchange": { + "mean": 0.0456756031007639, + "seed_sd": 0.0002979844946588909 + }, + "Pile-CC": { + "mean": 0.04571574828415005, + "seed_sd": 0.0001835424093483373 + }, + "USPTO_Backgrounds": { + "mean": 0.049735264846729854, + "seed_sd": 0.0005027384624140496 + } + }, + "corpora_above_logit": [ + "Wikipedia_en", + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds" + ], + "corpora_at_or_below_logit": [ + "Github" + ], + "control_C_jp_beats_jshuf_n_sets": 4, + "control_C_fires": false, + "VERDICT": "I INSIDE THE CORPUS SPREAD \u2014 1 of 5 corpora fit an operator that is NO BETTER than not fitting at all. The crossover exists on the CORPUS axis." +} \ No newline at end of file diff --git a/results/e33_logit_baseline_410m_v2.json b/results/e33_logit_baseline_410m_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..dd956070eb2830a7c320dc1daa0e0b56ed2de205 --- /dev/null +++ b/results/e33_logit_baseline_410m_v2.json @@ -0,0 +1,246 @@ +{ + "experiment": "E33 \u2014 the missing J=I arm on the E28 read ladder", + "prereg": "PLAN.tex section 6 (E33), fixed before this ran; quoted in this file's docstring", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "n_items": 541, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "excluded_sets": { + "association": "floored \u2014 196/205 ladder cells exactly zero (E28 crit 3)" + }, + "reference_lens": "e28_Github_410m_n400_s0.pt", + "derangement": "random", + "derangement_seed": 0, + "derangement_layer_map": { + "9": 12, + "10": 13, + "11": 9, + "12": 11, + "13": 20, + "14": 17, + "15": 21, + "16": 14, + "17": 19, + "18": 15, + "19": 18, + "20": 10, + "21": 16 + }, + "supersedes": "results/e33_logit_baseline_410m.json \u2014 v1, whose CONTROL DID NOT FIRE (4/6) because it used a cyclic shift by one layer. The decision rule is unchanged; only the mis-specified control was fixed. v1 is retained.", + "by_transport": { + "logit_I": { + "multihop": { + "min": 0.11789181775280408, + "best1L": 0.09847434024725642, + "mean": 0.03894164053989308, + "persist": 0.037447988986968994 + }, + "multilingual": { + "min": 0.11240029175366674, + "best1L": 0.09100797799016748, + "mean": 0.029034417222387025, + "persist": 0.02320522096540247 + }, + "order-ops": { + "min": 0.19740259860243117, + "best1L": 0.17272727404321944, + "mean": 0.052547452971339226, + "persist": 0.03246753276990993 + }, + "poetry": { + "min": 0.020408162714115212, + "best1L": 0.007288629987410137, + "mean": 0.0028033192003411905, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.17559524040137017, + "best1L": 0.11755952372082643, + "mean": 0.054716117680072784, + "persist": 0.04761904717556068 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "jlens_P": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07650471345654555, + "mean": 0.0322697607002088, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428837520735, + "mean": 0.09775641188025475, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "jshuf": { + "multihop": { + "min": 0.055478502064943314, + "best1L": 0.03883495128580502, + "mean": 0.010989010999245303, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1240029005067689, + "best1L": 0.09100797692579883, + "mean": 0.02766776377601283, + "persist": 0.02102973165788821 + }, + "order-ops": { + "min": 0.13506493238466127, + "best1L": 0.07532467639872006, + "mean": 0.02417582429812423, + "persist": 0.010389610592808043 + }, + "poetry": { + "min": 0.02623906744910138, + "best1L": 0.013119533658027649, + "mean": 0.004485310654022864, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.2380952425301075, + "best1L": 0.13541666577969277, + "mean": 0.06948260018335921, + "persist": 0.055059523188642094 + }, + "association": { + "min": 0.013975155406764575, + "best1L": 0.006211180239915848, + "mean": 0.0016722407557868532, + "persist": 0.0 + } + }, + "rand_matched": { + "multihop": { + "min": 0.024965326434799602, + "best1L": 0.009708737954497337, + "mean": 0.0019204096752218902, + "persist": 0.0 + }, + "multilingual": { + "min": 0.032632342007543357, + "best1L": 0.012327773629554681, + "mean": 0.002816979981227113, + "persist": 0.0 + }, + "order-ops": { + "min": 0.031168831246239797, + "best1L": 0.02337662343467985, + "mean": 0.0023976024240255356, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.004373177752963134, + "mean": 0.0011213276801364763, + "persist": 0.0 + }, + "typo": { + "min": 0.03571428531514747, + "best1L": 0.011904762259551458, + "mean": 0.0029761905898340046, + "persist": 0.0 + }, + "association": { + "min": 0.031055900268256664, + "best1L": 0.009316769960735525, + "mean": 0.002388915354718587, + "persist": 0.0 + } + } + }, + "persist_admitted_mean": { + "logit_I": 0.02843950316309929, + "jlens_P": 0.027925856597721576, + "jshuf": 0.01814210329736982, + "rand_matched": 0.0 + }, + "e28_asymptotes": { + "Github": { + "mean": 0.028084053793529264, + "seed_sd": 0.0003725015802729375 + }, + "Wikipedia_en": { + "mean": 0.03860496947484109, + "seed_sd": 0.000497512452938681 + }, + "StackExchange": { + "mean": 0.0456756031007639, + "seed_sd": 0.0002979844946588909 + }, + "Pile-CC": { + "mean": 0.04571574828415005, + "seed_sd": 0.0001835424093483373 + }, + "USPTO_Backgrounds": { + "mean": 0.049735264846729854, + "seed_sd": 0.0005027384624140496 + } + }, + "corpora_above_logit": [ + "Wikipedia_en", + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds" + ], + "corpora_at_or_below_logit": [ + "Github" + ], + "control_C_jp_beats_jshuf_n_sets": 5, + "control_C_fires": true, + "VERDICT": "I INSIDE THE CORPUS SPREAD \u2014 1 of 5 corpora fit an operator that is NO BETTER than not fitting at all. The crossover exists on the CORPUS axis." +} \ No newline at end of file diff --git a/results/e33b_tstats_410m.json b/results/e33b_tstats_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..605f625711bf81568829aa017d690d974be06270 --- /dev/null +++ b/results/e33b_tstats_410m.json @@ -0,0 +1,184 @@ +{ + "experiment": "E33b \u2014 the E33 per-corpus t statistics, stored (they were prose-only)", + "why": "discipline #2: 'Github 0.02808, t = -1.65' is cited in SPINE.tex, HANDOFF.md, CLAUDE.md and PREREG_E48_CROSSOVER.md Amendment 1 \u2014 where clause (d) depends on it \u2014 and lived in no results file. No script computed it. This stores it.", + "source": "e33_logit_baseline_410m_v2.json", + "formula": "t = (Y_inf(P) - L) / (seed_sd(P) / sqrt(3)), df = 2, one-sample vs the logit constant L (unfitted, so it has no seed axis)", + "logit_constant_L": 0.02843950316309929, + "t_crit_two_sided_5pct_df2": 4.302652729911275, + "recomputes_not_remeasures": true, + "limitation": "df = 2. The three seed-block means are NOT persisted by t33_logit_baseline.py:165-168 (only their mean and stdev), so they cannot be recovered from the source file \u2014 only from results/ladder410/*.json. E48's hierarchical bootstrap supersedes this as the instrument; this file exists to give the standing citation a home.", + "by_corpus": { + "Github": { + "Y_inf_mean": 0.028084053793529264, + "seed_sd": 0.0003725015802729375, + "n_seed_blocks": 3, + "standard_error": 0.00021506388764414147, + "t": -1.6527617605340326, + "df": 2, + "abs_t_exceeds_crit_5pct": false, + "delta_vs_logit": -0.0003554493695700246, + "pct_vs_logit": -1.2498438089144464 + }, + "Wikipedia_en": { + "Y_inf_mean": 0.03860496947484109, + "seed_sd": 0.000497512452938681, + "n_seed_blocks": 3, + "standard_error": 0.00028723894862933854, + "t": 35.39027823437558, + "df": 2, + "abs_t_exceeds_crit_5pct": true, + "delta_vs_logit": 0.010165466311741804, + "pct_vs_logit": 35.74417687061305 + }, + "StackExchange": { + "Y_inf_mean": 0.0456756031007639, + "seed_sd": 0.0002979844946588909, + "n_seed_blocks": 3, + "standard_error": 0.00017204142820564526, + "t": 100.18575245179832, + "df": 2, + "abs_t_exceeds_crit_5pct": true, + "delta_vs_logit": 0.01723609993766461, + "pct_vs_logit": 60.606192164526725 + }, + "Pile-CC": { + "Y_inf_mean": 0.04571574828415005, + "seed_sd": 0.0001835424093483373, + "n_seed_blocks": 3, + "standard_error": 0.00010596825944497502, + "t": 163.03226278828905, + "df": 2, + "abs_t_exceeds_crit_5pct": true, + "delta_vs_logit": 0.017276245121050762, + "pct_vs_logit": 60.747352096737636 + }, + "USPTO_Backgrounds": { + "Y_inf_mean": 0.049735264846729854, + "seed_sd": 0.0005027384624140496, + "n_seed_blocks": 3, + "standard_error": 0.00029025618660673014, + "t": 73.368846785286, + "df": 2, + "abs_t_exceeds_crit_5pct": true, + "delta_vs_logit": 0.021295761683630565, + "pct_vs_logit": 74.88092025201817 + } + }, + "provenance": { + "schema_version": 1, + "experiment": "E33b", + "utc": "2026-08-31T23:19:46+00:00", + "script": { + "path": "experiments/t33b_store_tstats.py", + "sha256": "0cd64458ccc718a1ff517fd95f2fe01967ff5e8d23ddf7d6ce4b6f310bd39810", + "n_lines": 96 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/t33b_store_tstats.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "db344d95ee0c8c1b208316bcc44fdbf369ab76a9d089365f7ce6b769ab76397a" + } +} \ No newline at end of file diff --git a/results/e34a_dact_vs_floor_410m.json b/results/e34a_dact_vs_floor_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..c997fa547aa99ec029d7378b214f4c70fff3804e --- /dev/null +++ b/results/e34a_dact_vs_floor_410m.json @@ -0,0 +1,179 @@ +{ + "experiment": "E34-a \u2014 D_act(J^P, I) vs the corpus floor. Reference = identity, NOT J^eval", + "amendment": "A1: the pre-registered J^eval fit is infeasible at skip_first=16 \u2014 239/551 prompts survive, 84% poetry+association, ZERO typo. See docstring.", + "prereg": "PLAN.tex section 6 (E34) decision rule, inherited verbatim", + "status": "PRE-REGISTERED (rule) / AMENDED (reference operator)", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_match": 400, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "control_C1_max_dact_self": 0.0, + "control_C1_fires": true, + "control_C2_seed_floor": { + "Github": 0.002150411426773144, + "Wikipedia_en": 0.0009337757912034222, + "StackExchange": 0.0015461445532427077, + "Pile-CC": 0.0011261613026537457, + "USPTO_Backgrounds": 0.0011178285031015977 + }, + "control_C2_fires": true, + "dact_JP_vs_I": { + "Github": 0.8681767469588465, + "Wikipedia_en": 0.9545294361054768, + "StackExchange": 0.9112883282644948, + "Pile-CC": 0.950080806806028, + "USPTO_Backgrounds": 0.9250939900993277 + }, + "dact_JP_vs_I_per_set": { + "Github": { + "multihop": 0.8345621020328486, + "multilingual": 0.8048132287781249, + "order-ops": 0.772059632028774, + "poetry": 1.0138538856947275, + "typo": 0.9290553706450386 + }, + "Wikipedia_en": { + "multihop": 0.9090560011909843, + "multilingual": 0.9272068866426788, + "order-ops": 0.8625896021998382, + "poetry": 1.0291963303508698, + "typo": 0.9945418623737255 + }, + "StackExchange": { + "multihop": 0.8109860278865147, + "multilingual": 0.8925431403423979, + "order-ops": 0.7524357472942202, + "poetry": 1.042740740605713, + "typo": 0.9947378469551004 + }, + "Pile-CC": { + "multihop": 0.8883133749550584, + "multilingual": 0.9265294183590362, + "order-ops": 0.8492925914726198, + "poetry": 1.032478882946538, + "typo": 1.0105672382171873 + }, + "USPTO_Backgrounds": { + "multihop": 0.8497156167659491, + "multilingual": 0.8828099603280244, + "order-ops": 0.8342091280282439, + "poetry": 1.0014411587523424, + "typo": 1.0113597250850692 + } + }, + "dact_pairwise": { + "Github|Wikipedia_en": 0.3165613492324831, + "Github|StackExchange": 0.2587059847931652, + "Github|Pile-CC": 0.3022581633536697, + "Github|USPTO_Backgrounds": 0.32147192436120675, + "Wikipedia_en|StackExchange": 0.15956201850582197, + "Wikipedia_en|Pile-CC": 0.05346862455250874, + "Wikipedia_en|USPTO_Backgrounds": 0.14988926827239746, + "StackExchange|Pile-CC": 0.12337992583814777, + "StackExchange|USPTO_Backgrounds": 0.19569347399014855, + "Pile-CC|USPTO_Backgrounds": 0.10533780146985136 + }, + "Y_inf": { + "Github": 0.028084053793529264, + "Wikipedia_en": 0.03860496947484109, + "StackExchange": 0.0456756031007639, + "Pile-CC": 0.04571574828415005, + "USPTO_Backgrounds": 0.049735264846729854 + }, + "Y_inf_per_set": { + "Github": { + "multihop": 0.0036655439215959335, + "multilingual": 0.028255464585155855, + "order-ops": 0.01898577620218299, + "poetry": 0.004373177752963134, + "typo": 0.0851403065057484 + }, + "Wikipedia_en": { + "multihop": 0.020507231943917518, + "multilingual": 0.028651618145304876, + "order-ops": 0.017195767282704714, + "poetry": 0.0007057244521977545, + "typo": 0.1259645055500806 + }, + "StackExchange": { + "multihop": 0.013902648567494487, + "multilingual": 0.039020683882491934, + "order-ops": 0.00253555960129617, + "poetry": 0.00430376223307483, + "typo": 0.16861536121946208 + }, + "Pile-CC": { + "multihop": 0.016900395698569437, + "multilingual": 0.03651330764430068, + "order-ops": 0.005387205344738153, + "poetry": 0.001457725917654378, + "typo": 0.16832010681548762 + }, + "USPTO_Backgrounds": { + "multihop": 0.01900652395906272, + "multilingual": 0.039440818416772695, + "order-ops": 0.008706108524055078, + "poetry": 0.003833279264942994, + "typo": 0.1776895940688158 + } + }, + "logit_constant": 0.02843950316309929, + "logit_per_set": { + "multihop": 0.037447988986968994, + "multilingual": 0.02320522096540247, + "order-ops": 0.03246753276990993, + "poetry": 0.001457725917654378, + "typo": 0.04761904717556068 + }, + "primary_a": { + "r": 0.6105218453330662, + "loo": { + "Github": -0.6196451936719668, + "Wikipedia_en": 0.8521069610317906, + "StackExchange": 0.6911583648272132, + "Pile-CC": 0.5680249749342092, + "USPTO_Backgrounds": 0.6906158537575058 + }, + "clears": false + }, + "primary_b": { + "r": 0.3400509473881388, + "n_cells": 25, + "loo_by_corpus": { + "Github": 0.19473864032568577, + "Wikipedia_en": 0.40489590181687984, + "StackExchange": 0.2558700582037715, + "Pile-CC": 0.3916482971590668, + "USPTO_Backgrounds": 0.42247082553493803 + }, + "loo_by_set": { + "multihop": 0.2791039625539971, + "multilingual": 0.34308969643520465, + "order-ops": 0.3821507960172952, + "poetry": 0.49914903688422785, + "typo": 0.19676124504996895 + }, + "clears": false + }, + "VERDICT": "NULL \u2014 D_act joins the other 11 predictors. Neither (a) nor (b) clears. Report and retire the instrument as a floor predictor" +} \ No newline at end of file diff --git a/results/e34b_jeval_dact_410m.json b/results/e34b_jeval_dact_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..bfb23ebd3e202224b8847c2bef1803528f8b66da --- /dev/null +++ b/results/e34b_jeval_dact_410m.json @@ -0,0 +1,170 @@ +{ + "experiment": "E34 \u2014 D_act(J^P, J^eval) vs the corpus floor (the pre-registered version)", + "prereg": "PLAN.tex section 6 (E34), decision rule unchanged", + "declared_bias": "J^eval fitted at skip_first=0 (all 551 prompts); every J^P at skip_first=16. C3 sizes that asymmetry.", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "jeval_n_prompts": 551, + "jeval_per_set_counts": { + "multihop": 93, + "multilingual": 107, + "order-ops": 55, + "poetry": 98, + "typo": 96, + "association": 102 + }, + "jeval_skipped": 0, + "control_C1_max_dact_self": 0.0, + "control_C1_fires": true, + "control_C2_seed_floor": { + "Github": 0.002150411426773144, + "Wikipedia_en": 0.0009337757912034222, + "StackExchange": 0.0015461445532427077, + "Pile-CC": 0.0011261613026537457, + "USPTO_Backgrounds": 0.0011178285031015977 + }, + "control_C2_fires": true, + "control_C3_skipfirst_only_dact": 0.0205631296823638, + "control_C3_n": 50, + "control_C3_confounded": false, + "dact_JP_vs_Jeval": { + "Github": 0.20911074680407882, + "Wikipedia_en": 0.20956244883094644, + "StackExchange": 0.21436647667719982, + "Pile-CC": 0.16051075050956173, + "USPTO_Backgrounds": 0.20788212958557906 + }, + "dact_JP_vs_Jeval_per_set": { + "Github": { + "multihop": 0.35077009581342516, + "multilingual": 0.22977268946872112, + "order-ops": 0.4207369967620715, + "poetry": 0.3319886847137658, + "typo": 0.26645240558842753 + }, + "Wikipedia_en": { + "multihop": 0.3542418713341142, + "multilingual": 0.27894386478698663, + "order-ops": 0.4630498791667032, + "poetry": 0.31957432268574915, + "typo": 0.2304223140790766 + }, + "StackExchange": { + "multihop": 0.32471666330651583, + "multilingual": 0.3220759679745572, + "order-ops": 0.4645100354176049, + "poetry": 0.2937623743812321, + "typo": 0.22077422460901633 + }, + "Pile-CC": { + "multihop": 0.3048241910551651, + "multilingual": 0.2752505961714073, + "order-ops": 0.3997745283266268, + "poetry": 0.24540254679940632, + "typo": 0.17340663538798545 + }, + "USPTO_Backgrounds": { + "multihop": 0.26968906549429944, + "multilingual": 0.3540591592010025, + "order-ops": 0.47133281879081074, + "poetry": 0.31464083321508707, + "typo": 0.2385257023991833 + } + }, + "Y_inf": { + "Github": 0.028084053793529264, + "Wikipedia_en": 0.03860496947484109, + "StackExchange": 0.0456756031007639, + "Pile-CC": 0.04571574828415005, + "USPTO_Backgrounds": 0.049735264846729854 + }, + "Y_inf_per_set": { + "Github": { + "multihop": 0.0036655439215959335, + "multilingual": 0.028255464585155855, + "order-ops": 0.01898577620218299, + "poetry": 0.004373177752963134, + "typo": 0.0851403065057484 + }, + "Wikipedia_en": { + "multihop": 0.020507231943917518, + "multilingual": 0.028651618145304876, + "order-ops": 0.017195767282704714, + "poetry": 0.0007057244521977545, + "typo": 0.1259645055500806 + }, + "StackExchange": { + "multihop": 0.013902648567494487, + "multilingual": 0.039020683882491934, + "order-ops": 0.00253555960129617, + "poetry": 0.00430376223307483, + "typo": 0.16861536121946208 + }, + "Pile-CC": { + "multihop": 0.016900395698569437, + "multilingual": 0.03651330764430068, + "order-ops": 0.005387205344738153, + "poetry": 0.001457725917654378, + "typo": 0.16832010681548762 + }, + "USPTO_Backgrounds": { + "multihop": 0.01900652395906272, + "multilingual": 0.039440818416772695, + "order-ops": 0.008706108524055078, + "poetry": 0.003833279264942994, + "typo": 0.1776895940688158 + } + }, + "primary_a": { + "r": -0.25083888729746506, + "loo": { + "Github": -0.12262141626233612, + "Wikipedia_en": -0.21578391981566916, + "StackExchange": -0.38338181276011574, + "Pile-CC": 0.1837720746552409, + "USPTO_Backgrounds": -0.4249544377345405 + }, + "clears": false + }, + "primary_b": { + "r": 0.014376913235039639, + "n_cells": 25, + "loo_by_corpus": { + "Github": 0.17724600473809574, + "Wikipedia_en": -0.0295356101641114, + "StackExchange": 0.03907324594297776, + "Pile-CC": -0.06896970864341105, + "USPTO_Backgrounds": -0.029570918357427263 + }, + "loo_by_set": { + "multihop": 0.12325920121460274, + "multilingual": -0.1895758786259446, + "order-ops": 0.033230085688934465, + "poetry": -0.07531902515534947, + "typo": 0.18029018305295508 + }, + "clears": false + }, + "VERDICT": "NULL \u2014 D_act does not predict the corpus floor, with J^eval as the reference" +} \ No newline at end of file diff --git a/results/e35_containment.json b/results/e35_containment.json new file mode 100644 index 0000000000000000000000000000000000000000..bfaaf59bc7b796629c1625c9b036a652c2129e3d --- /dev/null +++ b/results/e35_containment.json @@ -0,0 +1,316 @@ +{ + "experiment": "E35 / M1 \u2014 n-gram containment against the Pythia stream (inverted query)", + "prereg": "specs/archive/prereg/PREREG_E36_QLADDER.md, section E35", + "mode": "SMOKE (synthetic stream)", + "gates": { + "G1_tokenizer_identical": true, + "G1_mismatched_probes": [], + "G2_vocab_fits_uint16": true, + "vocab_size": 50277 + }, + "k_sweep": [ + 8, + 13, + 20, + 32 + ], + "window": 128, + "tokens_streamed": 31900, + "kgrams_probed": 127531, + "runtime_s": 0.1, + "containment": { + "8": { + "CONTROL_PubMed_2023": { + "containment": 0.020949681864910672, + "n_distinct_kgrams": 287928, + "n_found": 6032 + }, + "Github": { + "containment": 0.0, + "n_distinct_kgrams": 211635, + "n_found": 0 + }, + "OOD_CommonPile": { + "containment": 0.0, + "n_distinct_kgrams": 289835, + "n_found": 0 + }, + "OOD_News_2024": { + "containment": 0.0, + "n_distinct_kgrams": 188471, + "n_found": 0 + }, + "OOD_News_2024_dedup": { + "containment": 0.0, + "n_distinct_kgrams": 188471, + "n_found": 0 + }, + "OOD_Wikipedia_2023": { + "containment": 0.0, + "n_distinct_kgrams": 288503, + "n_found": 0 + }, + "OOD_arXiv_2023": { + "containment": 0.0, + "n_distinct_kgrams": 284907, + "n_found": 0 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 285666, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 285468, + "n_found": 0 + }, + "TRAP_FineWeb": { + "containment": 0.0, + "n_distinct_kgrams": 289236, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 277452, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 288114, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 24200, + "n_found": 0 + } + }, + "13": { + "CONTROL_PubMed_2023": { + "containment": 0.02086225873517161, + "n_distinct_kgrams": 278014, + "n_found": 5800 + }, + "Github": { + "containment": 0.0, + "n_distinct_kgrams": 222172, + "n_found": 0 + }, + "OOD_CommonPile": { + "containment": 0.0, + "n_distinct_kgrams": 278349, + "n_found": 0 + }, + "OOD_News_2024": { + "containment": 0.0, + "n_distinct_kgrams": 182854, + "n_found": 0 + }, + "OOD_News_2024_dedup": { + "containment": 0.0, + "n_distinct_kgrams": 182854, + "n_found": 0 + }, + "OOD_Wikipedia_2023": { + "containment": 0.0, + "n_distinct_kgrams": 277896, + "n_found": 0 + }, + "OOD_arXiv_2023": { + "containment": 0.0, + "n_distinct_kgrams": 277524, + "n_found": 0 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 275001, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 276816, + "n_found": 0 + }, + "TRAP_FineWeb": { + "containment": 0.0, + "n_distinct_kgrams": 277855, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 273271, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 277928, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 23200, + "n_found": 0 + } + }, + "20": { + "CONTROL_PubMed_2023": { + "containment": 0.02083994539551922, + "n_distinct_kgrams": 261517, + "n_found": 5450 + }, + "Github": { + "containment": 0.0, + "n_distinct_kgrams": 219005, + "n_found": 0 + }, + "OOD_CommonPile": { + "containment": 0.0, + "n_distinct_kgrams": 261594, + "n_found": 0 + }, + "OOD_News_2024": { + "containment": 0.0, + "n_distinct_kgrams": 173436, + "n_found": 0 + }, + "OOD_News_2024_dedup": { + "containment": 0.0, + "n_distinct_kgrams": 173436, + "n_found": 0 + }, + "OOD_Wikipedia_2023": { + "containment": 0.0, + "n_distinct_kgrams": 261399, + "n_found": 0 + }, + "OOD_arXiv_2023": { + "containment": 0.0, + "n_distinct_kgrams": 261185, + "n_found": 0 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 259109, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 261072, + "n_found": 0 + }, + "TRAP_FineWeb": { + "containment": 0.0, + "n_distinct_kgrams": 261307, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 257752, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 261403, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 21800, + "n_found": 0 + } + }, + "32": { + "CONTROL_PubMed_2023": { + "containment": 0.020834228274410414, + "n_distinct_kgrams": 232790, + "n_found": 4850 + }, + "Github": { + "containment": 0.0, + "n_distinct_kgrams": 201168, + "n_found": 0 + }, + "OOD_CommonPile": { + "containment": 0.0, + "n_distinct_kgrams": 232800, + "n_found": 0 + }, + "OOD_News_2024": { + "containment": 0.0, + "n_distinct_kgrams": 154916, + "n_found": 0 + }, + "OOD_News_2024_dedup": { + "containment": 0.0, + "n_distinct_kgrams": 154916, + "n_found": 0 + }, + "OOD_Wikipedia_2023": { + "containment": 0.0, + "n_distinct_kgrams": 232729, + "n_found": 0 + }, + "OOD_arXiv_2023": { + "containment": 0.0, + "n_distinct_kgrams": 232587, + "n_found": 0 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 231152, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 232622, + "n_found": 0 + }, + "TRAP_FineWeb": { + "containment": 0.0, + "n_distinct_kgrams": 232667, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 229552, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 232677, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 19400, + "n_found": 0 + } + } + }, + "k_ranges": { + "8": 0.020949681864910672, + "13": 0.02086225873517161, + "20": 0.02083994539551922, + "32": 0.020834228274410414 + }, + "k_selected": null, + "k_selection_rule": "largest k in the sweep whose containment range across real Q corpora is >= 0.30; fixed before running", + "controls": { + "M1_C2_random_containment": { + "8": 0.0, + "13": 0.0, + "20": 0.0, + "32": 0.0 + }, + "M1_C2_fires": true, + "M1_C1_planted_corpus": "CONTROL_PubMed_2023", + "M1_C1_containment": { + "8": 0.020949681864910672, + "13": 0.02086225873517161, + "20": 0.02083994539551922, + "32": 0.020834228274410414 + }, + "M1_C1_fires": false + } +} \ No newline at end of file diff --git a/results/e35_containment_shard0.json b/results/e35_containment_shard0.json new file mode 100644 index 0000000000000000000000000000000000000000..3732fdfab4540eaf1c21b3b348506a80f1119a1e --- /dev/null +++ b/results/e35_containment_shard0.json @@ -0,0 +1,168 @@ +{ + "experiment": "E35 / M1 \u2014 n-gram containment against the Pythia stream (inverted query)", + "prereg": "pythia/specs/PREREG_E36_QLADDER.md, section E35", + "mode": "STREAM", + "gates": { + "G1_tokenizer_identical": true, + "G1_mismatched_probes": [], + "G2_vocab_fits_uint16": true, + "vocab_size": 50277 + }, + "k_sweep": [ + 8, + 13, + 20, + 32 + ], + "window": 128, + "tokens_streamed": 0, + "kgrams_probed": 0, + "runtime_s": 0.0, + "containment": { + "8": { + "Github": { + "containment": 0.5724903725754247, + "n_distinct_kgrams": 211635, + "n_found": 121159 + }, + "Pile-CC": { + "containment": 0.15484516883353286, + "n_distinct_kgrams": 285666, + "n_found": 44234 + }, + "StackExchange": { + "containment": 0.17365869379405047, + "n_distinct_kgrams": 285468, + "n_found": 49574 + }, + "USPTO_Backgrounds": { + "containment": 0.20597797096434697, + "n_distinct_kgrams": 277452, + "n_found": 57149 + }, + "Wikipedia_en": { + "containment": 0.16650006594611855, + "n_distinct_kgrams": 288114, + "n_found": 47971 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 24200, + "n_found": 0 + } + }, + "13": { + "Github": { + "containment": 0.48856291521883943, + "n_distinct_kgrams": 222172, + "n_found": 108545 + }, + "Pile-CC": { + "containment": 0.11586139686764775, + "n_distinct_kgrams": 275001, + "n_found": 31862 + }, + "StackExchange": { + "containment": 0.1010960349112768, + "n_distinct_kgrams": 276816, + "n_found": 27985 + }, + "USPTO_Backgrounds": { + "containment": 0.15446205415137354, + "n_distinct_kgrams": 273271, + "n_found": 42210 + }, + "Wikipedia_en": { + "containment": 0.099935954635733, + "n_distinct_kgrams": 277928, + "n_found": 27775 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 23200, + "n_found": 0 + } + }, + "20": { + "Github": { + "containment": 0.4280769845437319, + "n_distinct_kgrams": 219005, + "n_found": 93751 + }, + "Pile-CC": { + "containment": 0.10712866013916923, + "n_distinct_kgrams": 259109, + "n_found": 27758 + }, + "StackExchange": { + "containment": 0.0840036465036465, + "n_distinct_kgrams": 261072, + "n_found": 21931 + }, + "USPTO_Backgrounds": { + "containment": 0.13277103572426208, + "n_distinct_kgrams": 257752, + "n_found": 34222 + }, + "Wikipedia_en": { + "containment": 0.08057673400840848, + "n_distinct_kgrams": 261403, + "n_found": 21063 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 21800, + "n_found": 0 + } + }, + "32": { + "Github": { + "containment": 0.3742096158434741, + "n_distinct_kgrams": 201168, + "n_found": 75279 + }, + "Pile-CC": { + "containment": 0.09955354052744514, + "n_distinct_kgrams": 231152, + "n_found": 23012 + }, + "StackExchange": { + "containment": 0.07686719226900293, + "n_distinct_kgrams": 232622, + "n_found": 17881 + }, + "USPTO_Backgrounds": { + "containment": 0.12033439046490556, + "n_distinct_kgrams": 229552, + "n_found": 27623 + }, + "Wikipedia_en": { + "containment": 0.07066448338254318, + "n_distinct_kgrams": 232677, + "n_found": 16442 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 19400, + "n_found": 0 + } + } + }, + "k_ranges": { + "8": 0.41764520374189185, + "13": 0.3886269605831064, + "20": 0.3475002505353234, + "32": 0.30354513246093096 + }, + "k_selected": 32, + "k_selection_rule": "largest k in the sweep whose containment range across real Q corpora is >= 0.30; fixed before running", + "controls": { + "M1_C2_random_containment": { + "8": 0.0, + "13": 0.0, + "20": 0.0, + "32": 0.0 + }, + "M1_C2_fires": true + } +} \ No newline at end of file diff --git a/results/e35_containment_smoke.json b/results/e35_containment_smoke.json new file mode 100644 index 0000000000000000000000000000000000000000..54caab08f46947e7e181db11d2c173acbf542462 --- /dev/null +++ b/results/e35_containment_smoke.json @@ -0,0 +1,176 @@ +{ + "experiment": "E35 / M1 \u2014 n-gram containment against the Pythia stream (inverted query)", + "prereg": "pythia/specs/PREREG_E36_QLADDER.md, section E35", + "mode": "SMOKE (synthetic stream)", + "gates": { + "G1_tokenizer_identical": true, + "G1_mismatched_probes": [], + "G2_vocab_fits_uint16": true, + "vocab_size": 50277 + }, + "k_sweep": [ + 8, + 13, + 20, + 32 + ], + "window": 128, + "tokens_streamed": 31900, + "kgrams_probed": 127531, + "runtime_s": 0.0, + "containment": { + "8": { + "Github": { + "containment": 0.42713685193834655, + "n_distinct_kgrams": 12846, + "n_found": 5487 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 14336, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0024281948106008046, + "n_distinct_kgrams": 14414, + "n_found": 35 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 14240, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 14469, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 24200, + "n_found": 0 + } + }, + "13": { + "Github": { + "containment": 0.42905483349996154, + "n_distinct_kgrams": 13003, + "n_found": 5579 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 13795, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0008638065073423554, + "n_distinct_kgrams": 13892, + "n_found": 12 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 13888, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 13917, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 23200, + "n_found": 0 + } + }, + "20": { + "Github": { + "containment": 0.4319035066698618, + "n_distinct_kgrams": 12519, + "n_found": 5407 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 13003, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 13075, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 13079, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 13080, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 21800, + "n_found": 0 + } + }, + "32": { + "Github": { + "containment": 0.43000265981026686, + "n_distinct_kgrams": 11279, + "n_found": 4850 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 11599, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 11640, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 11640, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 11640, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 19400, + "n_found": 0 + } + } + }, + "k_ranges": { + "8": 0.42713685193834655, + "13": 0.42905483349996154, + "20": 0.4319035066698618, + "32": 0.43000265981026686 + }, + "k_selected": 32, + "k_selection_rule": "largest k in the sweep whose containment range across real Q corpora is >= 0.30; fixed before running", + "controls": { + "M1_C2_random_containment": { + "8": 0.0, + "13": 0.0, + "20": 0.0, + "32": 0.0 + }, + "M1_C2_fires": true, + "M1_C1_planted_corpus": "Github", + "M1_C1_containment": { + "8": 0.42713685193834655, + "13": 0.42905483349996154, + "20": 0.4319035066698618, + "32": 0.43000265981026686 + }, + "M1_C1_fires": true + } +} \ No newline at end of file diff --git a/results/e35_smoke_box.json b/results/e35_smoke_box.json new file mode 100644 index 0000000000000000000000000000000000000000..416b0f81dedf3915df4a00a2e33731be3826610f --- /dev/null +++ b/results/e35_smoke_box.json @@ -0,0 +1,176 @@ +{ + "experiment": "E35 / M1 \u2014 n-gram containment against the Pythia stream (inverted query)", + "prereg": "pythia/specs/PREREG_E36_QLADDER.md, section E35", + "mode": "SMOKE (synthetic stream)", + "gates": { + "G1_tokenizer_identical": true, + "G1_mismatched_probes": [], + "G2_vocab_fits_uint16": true, + "vocab_size": 50277 + }, + "k_sweep": [ + 8, + 13, + 20, + 32 + ], + "window": 128, + "tokens_streamed": 31900, + "kgrams_probed": 127531, + "runtime_s": 0.0, + "containment": { + "8": { + "Github": { + "containment": 0.5093288777499304, + "n_distinct_kgrams": 10773, + "n_found": 5487 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 11941, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0018327224258580473, + "n_distinct_kgrams": 12004, + "n_found": 22 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 11931, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 12056, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 24200, + "n_found": 0 + } + }, + "13": { + "Github": { + "containment": 0.5123048668503214, + "n_distinct_kgrams": 10890, + "n_found": 5579 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 11495, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0009505703422053232, + "n_distinct_kgrams": 11572, + "n_found": 11 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 11584, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 11597, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 23200, + "n_found": 0 + } + }, + "20": { + "Github": { + "containment": 0.5155907313817107, + "n_distinct_kgrams": 10487, + "n_found": 5407 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 10836, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 10895, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 10900, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 10900, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 21800, + "n_found": 0 + } + }, + "32": { + "Github": { + "containment": 0.5140434552199258, + "n_distinct_kgrams": 9435, + "n_found": 4850 + }, + "Pile-CC": { + "containment": 0.0, + "n_distinct_kgrams": 9660, + "n_found": 0 + }, + "StackExchange": { + "containment": 0.0, + "n_distinct_kgrams": 9700, + "n_found": 0 + }, + "USPTO_Backgrounds": { + "containment": 0.0, + "n_distinct_kgrams": 9700, + "n_found": 0 + }, + "Wikipedia_en": { + "containment": 0.0, + "n_distinct_kgrams": 9700, + "n_found": 0 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 19400, + "n_found": 0 + } + } + }, + "k_ranges": { + "8": 0.5093288777499304, + "13": 0.5123048668503214, + "20": 0.5155907313817107, + "32": 0.5140434552199258 + }, + "k_selected": 32, + "k_selection_rule": "largest k in the sweep whose containment range across real Q corpora is >= 0.30; fixed before running", + "controls": { + "M1_C2_random_containment": { + "8": 0.0, + "13": 0.0, + "20": 0.0, + "32": 0.0 + }, + "M1_C2_fires": true, + "M1_C1_planted_corpus": "Github", + "M1_C1_containment": { + "8": 0.5093288777499304, + "13": 0.5123048668503214, + "20": 0.5155907313817107, + "32": 0.5140434552199258 + }, + "M1_C1_fires": true + } +} \ No newline at end of file diff --git a/results/e36_qladder_410m.json b/results/e36_qladder_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..54dd57df3c49a5e03dc98a4a89b3418484d37a65 --- /dev/null +++ b/results/e36_qladder_410m.json @@ -0,0 +1,2952 @@ +{ + "experiment": "E36 \u2014 the Q-ladder: vary the READ distribution, hold J^P fixed", + "prereg": "pythia/specs/PREREG_E36_QLADDER.md (2026-08-13, before any E36 number existed)", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "prefix_tokens": 128, + "n_items": 541, + "n_pairs": 893, + "rungs": [ + "Q0", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "USPTO_Backgrounds", + "Github", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "smoke": false, + "x_axis": { + "source": "e48b_exposure_growth.json", + "k": 32, + "coverage_shards": 20, + "note": "measured containment, not a rung number \u2014 this is what E35/M1 was for" + }, + "unavailable_rung": { + "Q5_non_pile_script": "no non-Latin-script corpus is on disk; the ladder does not extend to script-level shift. Recorded, not silently omitted." + }, + "declared_bias": "we shift the READ distribution, not the FIT distribution. EQ2's bias term is ||J^P - J^Q|| where J^Q is the operator that WOULD have been fitted on Q; we never fit J^Q here. E48 supplies that half.", + "ladder": { + "Q0": { + "containment_k32": null, + "is_shuffled": false, + "capability_top10_rate": 0.020156774916013438, + "capability_mean_rank": 4224.63045912654, + "arms": { + "logit": { + "persist": { + "mean": 0.028439503419212996, + "seed_sd": 0.0, + "per_seed": [ + 0.028439503419212996, + 0.028439503419212996, + 0.028439503419212996 + ] + }, + "min": { + "mean": 0.1247396256774664, + "seed_sd": 0.0, + "per_seed": [ + 0.1247396256774664, + 0.1247396256774664, + 0.1247396256774664 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.02855067129712552, + "seed_sd": 0.0, + "per_seed": [ + 0.02855067129712552, + 0.02855067129712552, + 0.02855067129712552 + ] + }, + "min": { + "mean": 0.15541725233197212, + "seed_sd": 0.0, + "per_seed": [ + 0.15541725233197212, + 0.15541725233197212, + 0.15541725233197212 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.027925857156515122, + "seed_sd": 0.0, + "per_seed": [ + 0.027925857156515122, + 0.027925857156515122, + 0.027925857156515122 + ] + }, + "min": { + "mean": 0.11324172168970108, + "seed_sd": 0.0, + "per_seed": [ + 0.11324172168970108, + 0.11324172168970108, + 0.11324172168970108 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0005136465188115835, + "ci_lo": -0.021382784470915794, + "ci_hi": 0.022756576538085938, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.039340307447128, + "seed_sd": 0.0, + "per_seed": [ + 0.039340307447128, + 0.039340307447128, + 0.039340307447128 + ] + }, + "min": { + "mean": 0.13213929273188113, + "seed_sd": 0.0, + "per_seed": [ + 0.13213929273188113, + 0.13213929273188113, + 0.13213929273188113 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01090080477297306, + "ci_lo": -0.014096681028604507, + "ci_hi": 0.047603681683540344, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04615347860381007, + "seed_sd": 0.0, + "per_seed": [ + 0.04615347860381007, + 0.04615347860381007, + 0.04615347860381007 + ] + }, + "min": { + "mean": 0.14736362285912036, + "seed_sd": 0.0, + "per_seed": [ + 0.14736362285912036, + 0.14736362285912036, + 0.14736362285912036 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01771397516131401, + "ci_lo": -0.021887753158807755, + "ci_hi": 0.074996717274189, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.045389215904287995, + "seed_sd": 0.0, + "per_seed": [ + 0.045389215904287995, + 0.045389215904287995, + 0.045389215904287995 + ] + }, + "min": { + "mean": 0.1445665042847395, + "seed_sd": 0.0, + "per_seed": [ + 0.1445665042847395, + 0.1445665042847395, + 0.1445665042847395 + ] + }, + "ci_vs_logit_persist": { + "point": 0.016949711367487907, + "ci_lo": -0.020029015839099884, + "ci_hi": 0.07102534174919128, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.050897271279245614, + "seed_sd": 0.0, + "per_seed": [ + 0.050897271279245614, + 0.050897271279245614, + 0.050897271279245614 + ] + }, + "min": { + "mean": 0.15696468092501165, + "seed_sd": 0.0, + "per_seed": [ + 0.15696468092501165, + 0.15696468092501165, + 0.15696468092501165 + ] + }, + "ci_vs_logit_persist": { + "point": 0.02245776355266571, + "ci_lo": -0.015598583035171032, + "ci_hi": 0.0812041237950325, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "is_shuffled": false, + "capability_top10_rate": 0.022396416573348264, + "capability_mean_rank": 4309.144083613289, + "arms": { + "logit": { + "persist": { + "mean": 0.03629972389123092, + "seed_sd": 0.0022810950733453132, + "per_seed": [ + 0.03462574060540646, + 0.03952490042429417, + 0.03474853064399212 + ] + }, + "min": { + "mean": 0.1330052327364683, + "seed_sd": 0.0014164883328735025, + "per_seed": [ + 0.1340819675475359, + 0.13392978720366955, + 0.1310039434581995 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.03130553908025225, + "seed_sd": 0.0008823959613629814, + "per_seed": [ + 0.030057793483138084, + 0.031912614777684215, + 0.03194620897993446 + ] + }, + "min": { + "mean": 0.1601704572637876, + "seed_sd": 0.0061335209465566, + "per_seed": [ + 0.16515550389885902, + 0.16382546201348305, + 0.1515304058790207 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.026641206163913012, + "seed_sd": 0.0005490161444991766, + "per_seed": [ + 0.02730230875313282, + 0.02666326742619276, + 0.025958042312413453 + ] + }, + "min": { + "mean": 0.11949207074940205, + "seed_sd": 0.002793334988492231, + "per_seed": [ + 0.12006914354860783, + 0.12258795760571957, + 0.11581911109387874 + ] + }, + "ci_vs_logit_persist": { + "point": -0.00798453576862812, + "ci_lo": -0.042686257511377335, + "ci_hi": 0.022087737917900085, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.042335875960998234, + "seed_sd": 0.001032902757615945, + "per_seed": [ + 0.041822236659936604, + 0.04377695198636502, + 0.04140843923669309 + ] + }, + "min": { + "mean": 0.14075611618657907, + "seed_sd": 0.003580610594648241, + "per_seed": [ + 0.13986529037356377, + 0.14551847018301486, + 0.13688458800315856 + ] + }, + "ci_vs_logit_persist": { + "point": 0.007710135076195002, + "ci_lo": -0.03473234921693802, + "ci_hi": 0.057025715708732605, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04749905125548442, + "seed_sd": 0.0024920584976778066, + "per_seed": [ + 0.047684010956436394, + 0.050454501155763866, + 0.044358641654253 + ] + }, + "min": { + "mean": 0.1542269659539064, + "seed_sd": 0.0020765537534632853, + "per_seed": [ + 0.15365934744477272, + 0.15700606480240822, + 0.1520154856145382 + ] + }, + "ci_vs_logit_persist": { + "point": 0.012873309664428234, + "ci_lo": -0.03922334313392639, + "ci_hi": 0.07404104620218277, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.050609867088496685, + "seed_sd": 0.002171233544999387, + "per_seed": [ + 0.05041661513969302, + 0.05336042842827737, + 0.04805255769751966 + ] + }, + "min": { + "mean": 0.1531935981164376, + "seed_sd": 0.004097844888945463, + "per_seed": [ + 0.15286606140434741, + 0.15836815871298313, + 0.14834657423198222 + ] + }, + "ci_vs_logit_persist": { + "point": 0.015984121710062027, + "ci_lo": -0.036915309727191925, + "ci_hi": 0.07962671667337418, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.050712701166048646, + "seed_sd": 0.0015313698759926331, + "per_seed": [ + 0.050940170791000126, + 0.05246412954293191, + 0.0487338031642139 + ] + }, + "min": { + "mean": 0.17039897752304872, + "seed_sd": 0.005706180472277549, + "per_seed": [ + 0.17443899177014827, + 0.17442871890962125, + 0.16232922188937665 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01608695462346077, + "ci_lo": -0.03724583610892296, + "ci_hi": 0.08263824135065079, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "is_shuffled": false, + "capability_top10_rate": 0.01866368047779022, + "capability_mean_rank": 4674.715192235909, + "arms": { + "logit": { + "persist": { + "mean": 0.028078026658234497, + "seed_sd": 0.0016248270852042151, + "per_seed": [ + 0.02840878739953041, + 0.025943371653556823, + 0.029881920921616255 + ] + }, + "min": { + "mean": 0.12258193418383599, + "seed_sd": 0.004003446205338022, + "per_seed": [ + 0.12605063132941724, + 0.11697234660387039, + 0.12472282461822033 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.02416364283611377, + "seed_sd": 0.001003349309462458, + "per_seed": [ + 0.024971047695726157, + 0.022749426774680614, + 0.02477045403793454 + ] + }, + "min": { + "mean": 0.14806132937471073, + "seed_sd": 0.00276723905758561, + "per_seed": [ + 0.15189499780535698, + 0.14546357095241547, + 0.1468254193663597 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.022569557776053746, + "seed_sd": 0.0012074208994716435, + "per_seed": [ + 0.02368360187392682, + 0.020891831861808895, + 0.023133239592425525 + ] + }, + "min": { + "mean": 0.10596823145945868, + "seed_sd": 0.005845325709225712, + "per_seed": [ + 0.10839929915964604, + 0.09791024327278137, + 0.1115951519459486 + ] + }, + "ci_vs_logit_persist": { + "point": -0.005839229095727205, + "ci_lo": -0.03438413888216019, + "ci_hi": 0.024198899045586586, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.03214704647349814, + "seed_sd": 0.0024661763503202403, + "per_seed": [ + 0.03469139921944588, + 0.028809019061736763, + 0.03294072113931179 + ] + }, + "min": { + "mean": 0.11322029903531075, + "seed_sd": 0.005204709409931878, + "per_seed": [ + 0.1151746816933155, + 0.1060974758118391, + 0.11838873960077763 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0037382624577730894, + "ci_lo": -0.03161531314253807, + "ci_hi": 0.04482903704047203, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04227183082451423, + "seed_sd": 0.0014374277309662056, + "per_seed": [ + 0.044263992365449666, + 0.04092536880634725, + 0.04162613130174577 + ] + }, + "min": { + "mean": 0.13252984347442787, + "seed_sd": 0.004803867667689962, + "per_seed": [ + 0.13294915668666363, + 0.12644789181649685, + 0.1381924819201231 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01386304385960102, + "ci_lo": -0.0319400429725647, + "ci_hi": 0.07346367090940475, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.041621484948943056, + "seed_sd": 0.0025450721298467087, + "per_seed": [ + 0.044255858799442646, + 0.03818035263102502, + 0.04242824341636151 + ] + }, + "min": { + "mean": 0.12788050323724748, + "seed_sd": 0.004018488968736439, + "per_seed": [ + 0.12957906983792783, + 0.1223345685750246, + 0.13172787129879 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013212698511779308, + "ci_lo": -0.030285555869340897, + "ci_hi": 0.06918405741453171, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04389908245454232, + "seed_sd": 0.0018030786304362044, + "per_seed": [ + 0.04644289943389594, + 0.04278009617701173, + 0.04247425175271928 + ] + }, + "min": { + "mean": 0.15009770318865776, + "seed_sd": 0.007101317957691461, + "per_seed": [ + 0.15323154553771018, + 0.14026777110993863, + 0.15679379291832446 + ] + }, + "ci_vs_logit_persist": { + "point": 0.015490291640162468, + "ci_lo": -0.03153032809495926, + "ci_hi": 0.07946377247571945, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "is_shuffled": false, + "capability_top10_rate": 0.026502426278462114, + "capability_mean_rank": 4231.8185890257555, + "arms": { + "logit": { + "persist": { + "mean": 0.03157440964908649, + "seed_sd": 0.002796284928918886, + "per_seed": [ + 0.034713071095757185, + 0.027921712095849215, + 0.03208844575565308 + ] + }, + "min": { + "mean": 0.1278687487045924, + "seed_sd": 0.006448430626530321, + "per_seed": [ + 0.1357261061668396, + 0.11993134990334511, + 0.12794879004359244 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.0318799717972676, + "seed_sd": 0.0008508815518589679, + "per_seed": [ + 0.03191518913954496, + 0.032904029544442895, + 0.030820696707814933 + ] + }, + "min": { + "mean": 0.15736184120178223, + "seed_sd": 0.004065740714376282, + "per_seed": [ + 0.1624331034719944, + 0.15247948989272117, + 0.15717293024063111 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.028430391071985164, + "seed_sd": 0.00040201361994103454, + "per_seed": [ + 0.028182579111307858, + 0.028111166879534723, + 0.028997427225112914 + ] + }, + "min": { + "mean": 0.11221431779364745, + "seed_sd": 0.005428481955499573, + "per_seed": [ + 0.11970514766871929, + 0.10701349042356015, + 0.10992431528866291 + ] + }, + "ci_vs_logit_persist": { + "point": -0.006282678805291653, + "ci_lo": -0.03783251345157623, + "ci_hi": 0.022103214636445045, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04342051934606085, + "seed_sd": 0.002331506175030251, + "per_seed": [ + 0.046479623904451726, + 0.04082546669524163, + 0.0429564674384892 + ] + }, + "min": { + "mean": 0.13656094384690126, + "seed_sd": 0.003168943329982689, + "per_seed": [ + 0.13985242135822773, + 0.13228120915591718, + 0.13754920102655888 + ] + }, + "ci_vs_logit_persist": { + "point": 0.008707449771463871, + "ci_lo": -0.03267471864819527, + "ci_hi": 0.05781780183315277, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04891945105046034, + "seed_sd": 0.002041292329157774, + "per_seed": [ + 0.0517880929633975, + 0.04720498416572809, + 0.04776527602225542 + ] + }, + "min": { + "mean": 0.14881497162083784, + "seed_sd": 0.0009700772687800834, + "per_seed": [ + 0.1501594215631485, + 0.147906281799078, + 0.14837921150028704 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014206379652023315, + "ci_lo": -0.03510032221674919, + "ci_hi": 0.07447437942028046, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.05241671356682976, + "seed_sd": 0.002162919922204846, + "per_seed": [ + 0.05540219666436315, + 0.050347342249006034, + 0.051500601787120105 + ] + }, + "min": { + "mean": 0.14973365664482116, + "seed_sd": 0.004798387607280762, + "per_seed": [ + 0.15460128299891948, + 0.1432051584124565, + 0.1513945285230875 + ] + }, + "ci_vs_logit_persist": { + "point": 0.017703641206026077, + "ci_lo": -0.033347200602293015, + "ci_hi": 0.08149312436580658, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.05023885692159335, + "seed_sd": 0.002192391318925561, + "per_seed": [ + 0.05247340011410415, + 0.04726013909094036, + 0.050983031559735534 + ] + }, + "min": { + "mean": 0.16274620306988558, + "seed_sd": 0.001623798532968072, + "per_seed": [ + 0.1649680022150278, + 0.16213808469474317, + 0.16113252229988576 + ] + }, + "ci_vs_logit_persist": { + "point": 0.015525788068771362, + "ci_lo": -0.03374544531106949, + "ci_hi": 0.07918965816497803, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "is_shuffled": false, + "capability_top10_rate": 0.024262784621127288, + "capability_mean_rank": 4211.015304217992, + "arms": { + "logit": { + "persist": { + "mean": 0.028793133182140686, + "seed_sd": 0.0010285076269501952, + "per_seed": [ + 0.030163079011254014, + 0.027684865775518118, + 0.02853145475964993 + ] + }, + "min": { + "mean": 0.12128662702937922, + "seed_sd": 0.0021928223516823725, + "per_seed": [ + 0.1199255108833313, + 0.11955404877662659, + 0.12438032142817974 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.03226851496535043, + "seed_sd": 0.000694903377751064, + "per_seed": [ + 0.032909794221632184, + 0.03130296701565385, + 0.03259278365876526 + ] + }, + "min": { + "mean": 0.15766812091072402, + "seed_sd": 0.0030098658474654143, + "per_seed": [ + 0.16132253482937814, + 0.1539507068693638, + 0.1577311210334301 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.027415493588584167, + "seed_sd": 0.0005045354860205193, + "per_seed": [ + 0.026976512838155033, + 0.028122124611400068, + 0.027147843316197396 + ] + }, + "min": { + "mean": 0.11763123249014219, + "seed_sd": 0.003588175000510387, + "per_seed": [ + 0.11652602553367615, + 0.11389473490417004, + 0.12247293703258037 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0027475852984935045, + "ci_lo": -0.025122208520770073, + "ci_hi": 0.022981896996498108, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04146475471400966, + "seed_sd": 0.0006228269274547368, + "per_seed": [ + 0.04157824940048158, + 0.042164452536962924, + 0.04065156220458448 + ] + }, + "min": { + "mean": 0.13566014481087527, + "seed_sd": 0.004365688431446649, + "per_seed": [ + 0.13475896269083024, + 0.13082114644348622, + 0.14140032529830932 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01130167581140995, + "ci_lo": -0.019091835245490074, + "ci_hi": 0.05440139397978783, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04922135318629444, + "seed_sd": 0.0012973659708061676, + "per_seed": [ + 0.047551121283322575, + 0.04939883570186794, + 0.0507141025736928 + ] + }, + "min": { + "mean": 0.15273317725708088, + "seed_sd": 0.004483918300540694, + "per_seed": [ + 0.15177303440868856, + 0.14778490830212831, + 0.15864158906042575 + ] + }, + "ci_vs_logit_persist": { + "point": 0.019058268517255783, + "ci_lo": -0.022475916892290115, + "ci_hi": 0.07973159849643707, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.049197832588106394, + "seed_sd": 0.0013624558082363772, + "per_seed": [ + 0.04733125129714608, + 0.0497171975672245, + 0.0505450488999486 + ] + }, + "min": { + "mean": 0.14771586476514736, + "seed_sd": 0.0037996570167676273, + "per_seed": [ + 0.14641902185976505, + 0.1438482327386737, + 0.15288033969700338 + ] + }, + "ci_vs_logit_persist": { + "point": 0.019034752622246742, + "ci_lo": -0.02064024843275547, + "ci_hi": 0.07573997229337692, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.05094143764581531, + "seed_sd": 0.002022776255880077, + "per_seed": [ + 0.04855143851600587, + 0.05077505416702479, + 0.053497820254415275 + ] + }, + "min": { + "mean": 0.16123308278620244, + "seed_sd": 0.0013256078180656381, + "per_seed": [ + 0.15953294970095158, + 0.16139903329312802, + 0.1627672653645277 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020778357982635498, + "ci_lo": -0.02221827767789364, + "ci_hi": 0.08543560653924942, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "Github": { + "containment_k32": 0.9340004374453194, + "is_shuffled": false, + "capability_top10_rate": 0.020530048525569243, + "capability_mean_rank": 4797.154535274356, + "arms": { + "logit": { + "persist": { + "mean": 0.023829053342342376, + "seed_sd": 0.0019133100599684207, + "per_seed": [ + 0.02512250132858753, + 0.025240575522184373, + 0.021124083176255228 + ] + }, + "min": { + "mean": 0.11852239407598972, + "seed_sd": 0.0028139239754922797, + "per_seed": [ + 0.12220506258308887, + 0.11798709072172642, + 0.11537502892315388 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.025754503525483113, + "seed_sd": 0.0006269310714382005, + "per_seed": [ + 0.024879138544201852, + 0.026314112078398465, + 0.02607025995384902 + ] + }, + "min": { + "mean": 0.153716907898585, + "seed_sd": 0.003182965315677801, + "per_seed": [ + 0.15006403177976607, + 0.15326534807682038, + 0.15782134383916854 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.023965128883719444, + "seed_sd": 0.0012537102566416024, + "per_seed": [ + 0.02224739659577608, + 0.02520443024113774, + 0.02444355981424451 + ] + }, + "min": { + "mean": 0.10658646300435067, + "seed_sd": 0.0008441858676514157, + "per_seed": [ + 0.10766854919493199, + 0.10648221708834171, + 0.10560862272977829 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0011573706287890673, + "ci_lo": -0.023954618722200394, + "ci_hi": 0.026057662442326546, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.03312810758749644, + "seed_sd": 0.00043170887726855836, + "per_seed": [ + 0.03328255987726152, + 0.033562415605410935, + 0.03253934727981687 + ] + }, + "min": { + "mean": 0.11906188415984313, + "seed_sd": 0.002329785482299106, + "per_seed": [ + 0.11745987385511399, + 0.11736949346959591, + 0.12235628515481949 + ] + }, + "ci_vs_logit_persist": { + "point": 0.008005605079233646, + "ci_lo": -0.02340543456375599, + "ci_hi": 0.047453757375478745, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.045316207610691585, + "seed_sd": 0.0007238705837581695, + "per_seed": [ + 0.04473803648725152, + 0.04487367062829435, + 0.04633691571652889 + ] + }, + "min": { + "mean": 0.14035971301297348, + "seed_sd": 0.0019086486372587475, + "per_seed": [ + 0.14288209639489652, + 0.13826629780232907, + 0.13993074484169482 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020193712785840034, + "ci_lo": -0.021595049649477005, + "ci_hi": 0.08058124035596848, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04261825208862623, + "seed_sd": 0.0007243694688608191, + "per_seed": [ + 0.043500224407762286, + 0.042628553323447706, + 0.04172597853466868 + ] + }, + "min": { + "mean": 0.13725972324609756, + "seed_sd": 0.001667425277131875, + "per_seed": [ + 0.13809503726661204, + 0.13493231534957886, + 0.13875181712210177 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01749575138092041, + "ci_lo": -0.021068150177598, + "ci_hi": 0.07054220139980316, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.044759471978371344, + "seed_sd": 0.0001851358354110141, + "per_seed": [ + 0.04489713786169887, + 0.04449776862747967, + 0.044883509445935485 + ] + }, + "min": { + "mean": 0.1492717611292998, + "seed_sd": 0.0034139530565961015, + "per_seed": [ + 0.14786469452083112, + 0.1459755774587393, + 0.153975011408329 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01963697373867035, + "ci_lo": -0.02300436422228813, + "ci_hi": 0.08291710913181305, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "CONTROL_PubMed_2023": { + "containment_k32": 0.7927273508312213, + "is_shuffled": false, + "capability_top10_rate": 0.019036954087346025, + "capability_mean_rank": 4409.058230683091, + "arms": { + "logit": { + "persist": { + "mean": 0.030234563273067276, + "seed_sd": 0.0015639236905520023, + "per_seed": [ + 0.028575583430938422, + 0.02979731510858983, + 0.032330791279673574 + ] + }, + "min": { + "mean": 0.12175768092274666, + "seed_sd": 0.0024778191012028417, + "per_seed": [ + 0.12339883185923099, + 0.11825580932199956, + 0.12361840158700943 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.03168034661406031, + "seed_sd": 0.000345372082924927, + "per_seed": [ + 0.03132576560601592, + 0.03214854439720512, + 0.031566729838959875 + ] + }, + "min": { + "mean": 0.15922437235713005, + "seed_sd": 0.0028119111024592805, + "per_seed": [ + 0.1608014225959778, + 0.15527436807751654, + 0.1615973263978958 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.02758992228967448, + "seed_sd": 0.00031880767791986667, + "per_seed": [ + 0.02714625564403832, + 0.02774227587506175, + 0.027881235349923372 + ] + }, + "min": { + "mean": 0.11301399742563566, + "seed_sd": 0.0028545590193116235, + "per_seed": [ + 0.11472423039376736, + 0.10899200737476349, + 0.11532575450837612 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0009856612887233496, + "ci_lo": -0.02284880541265011, + "ci_hi": 0.023544693365693092, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.0420196181240802, + "seed_sd": 0.0005438403250338024, + "per_seed": [ + 0.042205870035104454, + 0.04128025218378752, + 0.04257273215334863 + ] + }, + "min": { + "mean": 0.13920330504576364, + "seed_sd": 0.0030836327908162217, + "per_seed": [ + 0.14034029543399812, + 0.13498876728117465, + 0.1422808524221182 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01344403438270092, + "ci_lo": -0.01404797937721014, + "ci_hi": 0.05441785231232643, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04744758574912945, + "seed_sd": 0.001083414083904069, + "per_seed": [ + 0.04734843308106065, + 0.04882128657773137, + 0.046173037588596345 + ] + }, + "min": { + "mean": 0.15173541915913424, + "seed_sd": 0.0034089159953771065, + "per_seed": [ + 0.1532831747084856, + 0.14700750522315503, + 0.15491557754576207 + ] + }, + "ci_vs_logit_persist": { + "point": 0.018872002139687538, + "ci_lo": -0.01906522363424301, + "ci_hi": 0.07450412958860397, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04918796766238908, + "seed_sd": 0.0003472843140221594, + "per_seed": [ + 0.04967892593704164, + 0.048931105574592945, + 0.04895387147553265 + ] + }, + "min": { + "mean": 0.15158311314880848, + "seed_sd": 0.002551027148333135, + "per_seed": [ + 0.15480516105890274, + 0.14856659471988679, + 0.15137758366763593 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020612385123968124, + "ci_lo": -0.01704348437488079, + "ci_hi": 0.07595079392194748, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04972566197781513, + "seed_sd": 0.00010373794261556514, + "per_seed": [ + 0.04978115267585963, + 0.04981553009711206, + 0.0495803031604737 + ] + }, + "min": { + "mean": 0.16385552746554216, + "seed_sd": 0.0024719234343604985, + "per_seed": [ + 0.16457470841705799, + 0.160533219948411, + 0.1664586540311575 + ] + }, + "ci_vs_logit_persist": { + "point": 0.021150080487132072, + "ci_lo": -0.018501339480280876, + "ci_hi": 0.08187014609575272, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "OOD_Wikipedia_2023": { + "containment_k32": 0.26957534299550123, + "is_shuffled": false, + "capability_top10_rate": 0.03471444568868981, + "capability_mean_rank": 4096.241508025383, + "arms": { + "logit": { + "persist": { + "mean": 0.03236095819156617, + "seed_sd": 0.0006463349208366876, + "per_seed": [ + 0.03165938209276646, + 0.03220433469396085, + 0.0332191577879712 + ] + }, + "min": { + "mean": 0.13364263189335665, + "seed_sd": 0.0014895803356980816, + "per_seed": [ + 0.13501964025199414, + 0.1343347653746605, + 0.1315734900534153 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.031634360264676316, + "seed_sd": 0.0003194579934647776, + "per_seed": [ + 0.03163924559485167, + 0.03124068593606353, + 0.03202314926311374 + ] + }, + "min": { + "mean": 0.16317480032642684, + "seed_sd": 0.005029954083515421, + "per_seed": [ + 0.1668417438864708, + 0.166620135307312, + 0.15606252178549768 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.030502137060587606, + "seed_sd": 0.0005318527627935567, + "per_seed": [ + 0.03125416897237301, + 0.0301378277130425, + 0.03011441449634731 + ] + }, + "min": { + "mean": 0.11896193102002144, + "seed_sd": 0.0035537538119840856, + "per_seed": [ + 0.1194708239287138, + 0.1230375561863184, + 0.11437741294503212 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0011572458315640688, + "ci_lo": -0.025759171694517136, + "ci_hi": 0.02485557645559311, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04532940947295477, + "seed_sd": 0.0008049523985396637, + "per_seed": [ + 0.04500848348252475, + 0.04643574636429548, + 0.04454399857204407 + ] + }, + "min": { + "mean": 0.14300246437390646, + "seed_sd": 0.0018359509600638763, + "per_seed": [ + 0.1453766044229269, + 0.14090508222579956, + 0.1427257064729929 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013670025393366814, + "ci_lo": -0.020116280764341354, + "ci_hi": 0.05685994774103165, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.051152476606269676, + "seed_sd": 0.0005932088673280165, + "per_seed": [ + 0.05078884782269597, + 0.05198902431875467, + 0.05067955767735839 + ] + }, + "min": { + "mean": 0.15769512820988893, + "seed_sd": 0.004761395154268062, + "per_seed": [ + 0.15863485671579838, + 0.16299969106912612, + 0.1514508368447423 + ] + }, + "ci_vs_logit_persist": { + "point": 0.019493091851472855, + "ci_lo": -0.02235272154211998, + "ci_hi": 0.07470886409282684, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.05297815694163243, + "seed_sd": 0.0008157224864377802, + "per_seed": [ + 0.052032429119572045, + 0.05402312567457557, + 0.05287891603074968 + ] + }, + "min": { + "mean": 0.15353299925724664, + "seed_sd": 0.001669077283205774, + "per_seed": [ + 0.15557737164199353, + 0.15353264212608336, + 0.15148898400366306 + ] + }, + "ci_vs_logit_persist": { + "point": 0.021318774670362473, + "ci_lo": -0.0209893099963665, + "ci_hi": 0.07794060558080673, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.05419266736134887, + "seed_sd": 0.001034816868778697, + "per_seed": [ + 0.05373580693267286, + 0.05562514401972294, + 0.0532170511316508 + ] + }, + "min": { + "mean": 0.169710739950339, + "seed_sd": 0.001374380223636731, + "per_seed": [ + 0.16799281686544418, + 0.17135706208646298, + 0.16978234089910985 + ] + }, + "ci_vs_logit_persist": { + "point": 0.022533288225531578, + "ci_lo": -0.021409669890999794, + "ci_hi": 0.085838183760643, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "TRAP_FineWeb": { + "containment_k32": 0.06309446548070848, + "is_shuffled": false, + "capability_top10_rate": 0.021649869354236656, + "capability_mean_rank": 4213.683837252706, + "arms": { + "logit": { + "persist": { + "mean": 0.035998970402094224, + "seed_sd": 0.0023369168548651445, + "per_seed": [ + 0.03898334268014878, + 0.03327718588989228, + 0.03573638263624161 + ] + }, + "min": { + "mean": 0.13256995032231014, + "seed_sd": 0.00239487878653843, + "per_seed": [ + 0.13594319708645344, + 0.13114614114165307, + 0.13062051273882389 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.030792023916728795, + "seed_sd": 0.0013693613215323738, + "per_seed": [ + 0.03268995825201273, + 0.030176387797109783, + 0.02950972570106387 + ] + }, + "min": { + "mean": 0.15908302664756777, + "seed_sd": 0.0017480497707151356, + "per_seed": [ + 0.1604225791990757, + 0.1602126158773899, + 0.15661388486623765 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.028455864824354648, + "seed_sd": 0.0008500278521134297, + "per_seed": [ + 0.027818697644397615, + 0.02789164916612208, + 0.02965724766254425 + ] + }, + "min": { + "mean": 0.11889895846446355, + "seed_sd": 0.0026057068605158753, + "per_seed": [ + 0.12236944548785686, + 0.11609070412814618, + 0.11823672577738761 + ] + }, + "ci_vs_logit_persist": { + "point": -0.010527478531002998, + "ci_lo": -0.046978045254945755, + "ci_hi": 0.023138413205742836, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04054118523684641, + "seed_sd": 0.001242298611190451, + "per_seed": [ + 0.04229722085874528, + 0.0397102092159912, + 0.039616125635802746 + ] + }, + "min": { + "mean": 0.13712344678739707, + "seed_sd": 0.0020742287795043157, + "per_seed": [ + 0.14001811109483242, + 0.1352646216750145, + 0.13608760759234428 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0015578410821035504, + "ci_lo": -0.04127275198698044, + "ci_hi": 0.048159196972846985, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04694484616629779, + "seed_sd": 0.0010435078307278341, + "per_seed": [ + 0.048248788248747586, + 0.04689134331420064, + 0.045694406935945155 + ] + }, + "min": { + "mean": 0.15072562620043756, + "seed_sd": 0.0028031406941793026, + "per_seed": [ + 0.1528418455272913, + 0.14676448218524457, + 0.15257055088877677 + ] + }, + "ci_vs_logit_persist": { + "point": 0.007961506024003029, + "ci_lo": -0.04653144255280495, + "ci_hi": 0.07018264383077621, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04928968380360554, + "seed_sd": 0.0006379626847678697, + "per_seed": [ + 0.05010079345665872, + 0.04922628453932702, + 0.048541973414830866 + ] + }, + "min": { + "mean": 0.15071169324219227, + "seed_sd": 0.0024284392989413757, + "per_seed": [ + 0.15404304228723048, + 0.1483231347054243, + 0.14976890273392202 + ] + }, + "ci_vs_logit_persist": { + "point": 0.010306340642273426, + "ci_lo": -0.042944543063640594, + "ci_hi": 0.07253025472164154, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.049697506008669734, + "seed_sd": 0.0017622222859018243, + "per_seed": [ + 0.052007290977053346, + 0.04773217467591166, + 0.049353052373044194 + ] + }, + "min": { + "mean": 0.16596383278568586, + "seed_sd": 0.003502461972567475, + "per_seed": [ + 0.16932464428246022, + 0.16113230511546134, + 0.167434548959136 + ] + }, + "ci_vs_logit_persist": { + "point": 0.010714161209762096, + "ci_lo": -0.04303861781954765, + "ci_hi": 0.07641012221574783, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "is_shuffled": false, + "capability_top10_rate": 0.017543859649122806, + "capability_mean_rank": 4370.228070175438, + "arms": { + "logit": { + "persist": { + "mean": 0.036840112763457, + "seed_sd": 0.0006125081892508191, + "per_seed": [ + 0.037061250093393026, + 0.03600423510652036, + 0.03745485309045762 + ] + }, + "min": { + "mean": 0.12556015662848952, + "seed_sd": 0.0012670002291384453, + "per_seed": [ + 0.12654305137693883, + 0.12636615782976152, + 0.12377126067876816 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.030533886918177208, + "seed_sd": 0.0007017004080437123, + "per_seed": [ + 0.030557604413479565, + 0.03138118665665388, + 0.029662869684398175 + ] + }, + "min": { + "mean": 0.15122126365701358, + "seed_sd": 0.004345917130561741, + "per_seed": [ + 0.15460145026445388, + 0.1539765328168869, + 0.14508580788969994 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.027587826767315467, + "seed_sd": 0.0015278288845761718, + "per_seed": [ + 0.028825605846941472, + 0.028502664342522622, + 0.025435210112482308 + ] + }, + "min": { + "mean": 0.11221578580637773, + "seed_sd": 0.002388753706827494, + "per_seed": [ + 0.11437908075749874, + 0.1133812103420496, + 0.10888706631958485 + ] + }, + "ci_vs_logit_persist": { + "point": -0.009473425336182117, + "ci_lo": -0.05272747203707695, + "ci_hi": 0.029177239164710045, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04160388757785161, + "seed_sd": 0.001092341834007453, + "per_seed": [ + 0.04066372397355735, + 0.04101242083124816, + 0.043135517928749326 + ] + }, + "min": { + "mean": 0.13462657121320565, + "seed_sd": 0.002486481636740192, + "per_seed": [ + 0.13735937923192978, + 0.13517659790813924, + 0.13134373649954795 + ] + }, + "ci_vs_logit_persist": { + "point": 0.004542634356766939, + "ci_lo": -0.044434886425733566, + "ci_hi": 0.05856844782829285, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04644438338776429, + "seed_sd": 0.0009598411331915246, + "per_seed": [ + 0.045579977240413426, + 0.045970195811241864, + 0.04778297711163759 + ] + }, + "min": { + "mean": 0.14397299848496914, + "seed_sd": 0.0030620801082744294, + "per_seed": [ + 0.14578439369797708, + 0.14647371210157872, + 0.13966088965535164 + ] + }, + "ci_vs_logit_persist": { + "point": 0.009383130818605423, + "ci_lo": -0.04959649592638016, + "ci_hi": 0.0770169347524643, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.049888986706112824, + "seed_sd": 0.0003970836175988191, + "per_seed": [ + 0.04968298217281699, + 0.0495395680423826, + 0.05044440990313888 + ] + }, + "min": { + "mean": 0.14638157797356446, + "seed_sd": 0.0015567421245738348, + "per_seed": [ + 0.14661192260682582, + 0.14816255308687687, + 0.1443702582269907 + ] + }, + "ci_vs_logit_persist": { + "point": 0.012827733531594276, + "ci_lo": -0.04676033928990364, + "ci_hi": 0.08359326422214508, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.048988748838504154, + "seed_sd": 0.0009010528224351549, + "per_seed": [ + 0.0498417885042727, + 0.04938203799538314, + 0.04774242001585662 + ] + }, + "min": { + "mean": 0.15854955079654853, + "seed_sd": 0.0032592837034044037, + "per_seed": [ + 0.1596286602318287, + 0.16189085133373737, + 0.15412914082407952 + ] + }, + "ci_vs_logit_persist": { + "point": 0.011927495710551739, + "ci_lo": -0.04547123983502388, + "ci_hi": 0.08188861608505249, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "is_shuffled": false, + "capability_top10_rate": 0.02314296379245987, + "capability_mean_rank": 4401.499066815976, + "arms": { + "logit": { + "persist": { + "mean": 0.028438278171233834, + "seed_sd": 0.001665133926724567, + "per_seed": [ + 0.026223077508620917, + 0.028854017960838974, + 0.030237739044241606 + ] + }, + "min": { + "mean": 0.12067306488752365, + "seed_sd": 0.001669345484239769, + "per_seed": [ + 0.12042249478399754, + 0.12283132374286651, + 0.1187653761357069 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.029294955373431245, + "seed_sd": 0.0006186343222731774, + "per_seed": [ + 0.030160497408360243, + 0.02897259439341724, + 0.028751774318516256 + ] + }, + "min": { + "mean": 0.1527389742434025, + "seed_sd": 0.002334612894349757, + "per_seed": [ + 0.15006249621510506, + 0.15575142204761505, + 0.15240300446748734 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.026148018380627035, + "seed_sd": 0.0008004233053879947, + "per_seed": [ + 0.02531201713718474, + 0.025905078276991844, + 0.027226959727704524 + ] + }, + "min": { + "mean": 0.11367069917420546, + "seed_sd": 0.0015096672898677378, + "per_seed": [ + 0.11164908520877362, + 0.11527602337300777, + 0.114086988940835 + ] + }, + "ci_vs_logit_persist": { + "point": -7.505864050472155e-05, + "ci_lo": -0.0221811905503273, + "ci_hi": 0.02505931816995144, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.03783603221333275, + "seed_sd": 0.0008523357632408745, + "per_seed": [ + 0.037666665157303214, + 0.03688717789482325, + 0.03895425358787179 + ] + }, + "min": { + "mean": 0.13496138068536917, + "seed_sd": 0.0009107089046214201, + "per_seed": [ + 0.13396995030343534, + 0.13474513702094554, + 0.13616905473172664 + ] + }, + "ci_vs_logit_persist": { + "point": 0.011612953618168831, + "ci_lo": -0.01801110990345478, + "ci_hi": 0.05204121768474579, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04517050000528495, + "seed_sd": 0.0012563456946498783, + "per_seed": [ + 0.044943356141448024, + 0.043757994938641784, + 0.04681014893576503 + ] + }, + "min": { + "mean": 0.15203597210347652, + "seed_sd": 0.00233295535119527, + "per_seed": [ + 0.15067538246512413, + 0.1553192686289549, + 0.15011326521635054 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01894741877913475, + "ci_lo": -0.01935446634888649, + "ci_hi": 0.07368820160627365, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04685056325979531, + "seed_sd": 0.0013995171028735106, + "per_seed": [ + 0.047398817632347344, + 0.04492945852689445, + 0.04822341362014413 + ] + }, + "min": { + "mean": 0.14826099450389543, + "seed_sd": 0.0019149920702472346, + "per_seed": [ + 0.14597206562757492, + 0.15065901204943657, + 0.14815190583467483 + ] + }, + "ci_vs_logit_persist": { + "point": 0.02062748745083809, + "ci_lo": -0.018101608380675316, + "ci_hi": 0.07591471076011658, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04756710189394653, + "seed_sd": 0.0008653622927034748, + "per_seed": [ + 0.048763201525434854, + 0.047193299932405355, + 0.04674480422399938 + ] + }, + "min": { + "mean": 0.16033506020903587, + "seed_sd": 0.004772566869652116, + "per_seed": [ + 0.15373621322214603, + 0.1648622889071703, + 0.16240667849779128 + ] + }, + "ci_vs_logit_persist": { + "point": 0.021344024688005447, + "ci_lo": -0.018825750797986984, + "ci_hi": 0.08027508109807968, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "is_shuffled": false, + "capability_top10_rate": 0.02314296379245987, + "capability_mean_rank": 4447.954460619634, + "arms": { + "logit": { + "persist": { + "mean": 0.03133960859073947, + "seed_sd": 0.0012472971224492292, + "per_seed": [ + 0.031733551621437074, + 0.03263167429249734, + 0.029653599858283995 + ] + }, + "min": { + "mean": 0.11887628249824048, + "seed_sd": 0.002486558583239482, + "per_seed": [ + 0.1222337793558836, + 0.11810300163924695, + 0.11629206649959087 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.0287941564184924, + "seed_sd": 0.0017705102577080987, + "per_seed": [ + 0.03068833714351058, + 0.02926520546898246, + 0.02642892664298415 + ] + }, + "min": { + "mean": 0.1507422794898351, + "seed_sd": 0.0010832533849985677, + "per_seed": [ + 0.1521053597331047, + 0.14945520982146263, + 0.15066626891493798 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.02549306258248786, + "seed_sd": 0.0004883206176151186, + "per_seed": [ + 0.025341645814478397, + 0.026152286399155854, + 0.024985255533829333 + ] + }, + "min": { + "mean": 0.11619785837829114, + "seed_sd": 0.0036822469253014645, + "per_seed": [ + 0.12139833569526673, + 0.11383147947490216, + 0.11336375996470452 + ] + }, + "ci_vs_logit_persist": { + "point": -0.006240491289645433, + "ci_lo": -0.036989498883485794, + "ci_hi": 0.022498106583952904, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.036288712111612165, + "seed_sd": 0.0004109425886821703, + "per_seed": [ + 0.03661795160733163, + 0.03653883533552289, + 0.03570934939198196 + ] + }, + "min": { + "mean": 0.1264076568186283, + "seed_sd": 0.0031150765432869467, + "per_seed": [ + 0.13078852146863937, + 0.12461918406188488, + 0.12381526492536069 + ] + }, + "ci_vs_logit_persist": { + "point": 0.004555158317089081, + "ci_lo": -0.032625067979097366, + "ci_hi": 0.04584131017327309, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04547049587902924, + "seed_sd": 0.001014317309138442, + "per_seed": [ + 0.04687285046093166, + 0.0445079555734992, + 0.045030681602656844 + ] + }, + "min": { + "mean": 0.14767617098987101, + "seed_sd": 0.001519346917268398, + "per_seed": [ + 0.14695448726415633, + 0.14978972561657428, + 0.14628430008888244 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013736945576965809, + "ci_lo": -0.03402138873934746, + "ci_hi": 0.07279022783041, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04594817009444038, + "seed_sd": 0.001959298398339249, + "per_seed": [ + 0.048588567320257424, + 0.04535564049147069, + 0.043900302471593024 + ] + }, + "min": { + "mean": 0.1427968057493369, + "seed_sd": 0.0007363139990125478, + "per_seed": [ + 0.14319085218012334, + 0.1434345178306103, + 0.14176504723727704 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014214618131518364, + "ci_lo": -0.031123919412493706, + "ci_hi": 0.07101155817508698, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04761502892555048, + "seed_sd": 0.0014062784579078286, + "per_seed": [ + 0.04959265971556306, + 0.04644411192275584, + 0.04680831513833254 + ] + }, + "min": { + "mean": 0.14846517232557138, + "seed_sd": 0.0016306080186266531, + "per_seed": [ + 0.15068337209522725, + 0.14790198095142842, + 0.14681016393005847 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01588147133588791, + "ci_lo": -0.03258395195007324, + "ci_hi": 0.07852833718061447, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + }, + "SHUFFLED_Pile-CC": { + "containment_k32": 0.9276060773863086, + "is_shuffled": true, + "capability_top10_rate": 0.006718924972004479, + "capability_mean_rank": 4539.5617767823815, + "arms": { + "logit": { + "persist": { + "mean": 0.020351312107717, + "seed_sd": 0.000482612292566415, + "per_seed": [ + 0.020097647863440215, + 0.021026881667785345, + 0.01992940679192543 + ] + }, + "min": { + "mean": 0.09142902692159016, + "seed_sd": 0.003582667498722964, + "per_seed": [ + 0.08644843250513076, + 0.09311401732265949, + 0.09472463093698025 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.02114899874043961, + "seed_sd": 0.002019063471942637, + "per_seed": [ + 0.022680402873083948, + 0.0224704063963145, + 0.01829618695192039 + ] + }, + "min": { + "mean": 0.11579039630790551, + "seed_sd": 0.005090350330952079, + "per_seed": [ + 0.10880363546311855, + 0.12078590169548989, + 0.1177816517651081 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.020016702295591434, + "seed_sd": 0.0009777985327617273, + "per_seed": [ + 0.020015397132374345, + 0.021214908082038166, + 0.018819801672361792 + ] + }, + "min": { + "mean": 0.08711997345089913, + "seed_sd": 0.004120376933501696, + "per_seed": [ + 0.08290965184569359, + 0.09271385222673416, + 0.08573641628026962 + ] + }, + "ci_vs_logit_persist": { + "point": -8.094674558378756e-05, + "ci_lo": -0.01984638348221779, + "ci_hi": 0.02272208034992218, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.028413841927734513, + "seed_sd": 0.0021243530746435806, + "per_seed": [ + 0.028896221611648797, + 0.030740685644559562, + 0.025604618526995183 + ] + }, + "min": { + "mean": 0.09340037666261196, + "seed_sd": 0.0035719766328911524, + "per_seed": [ + 0.09174824655055999, + 0.09836061336100102, + 0.09009227007627488 + ] + }, + "ci_vs_logit_persist": { + "point": 0.008316193707287312, + "ci_lo": -0.017632903531193733, + "ci_hi": 0.041986677795648575, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.03509347875757764, + "seed_sd": 0.0021110244575648406, + "per_seed": [ + 0.03446822033729404, + 0.03793423480819911, + 0.03287798112723976 + ] + }, + "min": { + "mean": 0.10783494816472133, + "seed_sd": 0.005252777684393637, + "per_seed": [ + 0.10120301451534033, + 0.11404929421842099, + 0.10825253576040268 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014995826408267021, + "ci_lo": -0.018552424386143684, + "ci_hi": 0.0634133443236351, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.033768538812485836, + "seed_sd": 0.0022614134406113017, + "per_seed": [ + 0.034108439553529026, + 0.03635255589615553, + 0.030844620987772943 + ] + }, + "min": { + "mean": 0.10412973879526058, + "seed_sd": 0.0045382061828327346, + "per_seed": [ + 0.09992853049188852, + 0.11043217107653618, + 0.10202851481735706 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013670889660716057, + "ci_lo": -0.017414817586541176, + "ci_hi": 0.056603047996759415, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.03425933218871554, + "seed_sd": 0.002765070688948554, + "per_seed": [ + 0.03651014384813607, + 0.03590318781789392, + 0.030364664900116623 + ] + }, + "min": { + "mean": 0.11891029054919879, + "seed_sd": 0.00466408272762991, + "per_seed": [ + 0.1129979819059372, + 0.12439902387559414, + 0.11933386586606502 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014161686412990093, + "ci_lo": -0.01854432187974453, + "ci_hi": 0.05955292657017708, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + } + } + } + }, + "crossings": { + "Github": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [ + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "USPTO_Backgrounds", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "crosses": false + }, + "Wikipedia_en": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "StackExchange": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "Pile-CC": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "USPTO_Backgrounds": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + } + }, + "controls": { + "C1_Q0_reproduces_e33": { + "logit_e33": 0.02843950316309929, + "logit_here": 0.028439503419212996, + "abs_diff": 2.561137066314778e-10, + "tolerance": 1e-06, + "fires": true, + "note": "Q0 is the no-prefix rung and MUST equal E33 exactly. A mismatch means the prefixing path altered the base measurement and the ladder is void." + }, + "C2_shuf_below_everything": { + "per_rung": { + "Q0": false, + "Pile-CC": false, + "StackExchange": false, + "Wikipedia_en": false, + "USPTO_Backgrounds": false, + "Github": false, + "CONTROL_PubMed_2023": false, + "OOD_Wikipedia_2023": false, + "TRAP_FineWeb": false, + "OOD_News_2024": false, + "OOD_arXiv_2023": false, + "OOD_CommonPile": false, + "SHUFFLED_Pile-CC": false + }, + "fires": false + }, + "C3_capability": { + "REGISTERED_FORM_IS_DEGENERATE": "the pre-registered rule is top-k accuracy < 50% of Q0, but Q0's top-10 rate is 0.020156774916013438. The eval targets are latent concepts, not emitted tokens, so this quantity is floored at Q0 and the rule cannot fire. NOT reinterpreted silently \u2014 adjudication below uses mean rank, which is not floored, and the operator is asked to confirm.", + "registered_top10_rate": { + "q0": 0.020156774916013438, + "per_rung": { + "Q0": 0.020156774916013438, + "Pile-CC": 0.022396416573348264, + "StackExchange": 0.01866368047779022, + "Wikipedia_en": 0.026502426278462114, + "USPTO_Backgrounds": 0.024262784621127288, + "Github": 0.020530048525569243, + "CONTROL_PubMed_2023": 0.019036954087346025, + "OOD_Wikipedia_2023": 0.03471444568868981, + "TRAP_FineWeb": 0.021649869354236656, + "OOD_News_2024": 0.017543859649122806, + "OOD_arXiv_2023": 0.02314296379245987, + "OOD_CommonPile": 0.02314296379245987, + "SHUFFLED_Pile-CC": 0.006718924972004479 + }, + "usable": false + }, + "adjudicated_mean_rank": { + "q0": 4224.63045912654, + "per_rung": { + "Q0": 4224.63045912654, + "Pile-CC": 4309.144083613289, + "StackExchange": 4674.715192235909, + "Wikipedia_en": 4231.8185890257555, + "USPTO_Backgrounds": 4211.015304217992, + "Github": 4797.154535274356, + "CONTROL_PubMed_2023": 4409.058230683091, + "OOD_Wikipedia_2023": 4096.241508025383, + "TRAP_FineWeb": 4213.683837252706, + "OOD_News_2024": 4370.228070175438, + "OOD_arXiv_2023": 4401.499066815976, + "OOD_CommonPile": 4447.954460619634, + "SHUFFLED_Pile-CC": 4539.5617767823815 + }, + "rule": "a rung collapses if its mean target rank exceeds 2x Q0's" + }, + "collapsed_rungs": [], + "note": "a rung whose capability collapsed cannot distinguish lens failure from model failure" + }, + "C5_self_distance": { + "value": 0.0, + "fires": true, + "note": "D_act(J,J)=0 by construction; measured 0.0 in t23" + } + }, + "VERDICT": "REJECT S3 (the publishable null) \u2014 no fitting corpus produces a J-lens curve that goes strictly below the logit lens at any measured rung. Shift in the READ distribution degrades both lenses together; the averaged derivative is no more shift-fragile than not fitting at all.", + "provenance": { + "schema_version": 1, + "experiment": "E36", + "utc": "2026-08-15T03:55:17+00:00", + "script": { + "path": "experiments/t36_qladder.py", + "sha256": "8c2e219a8d73a34c7370fc7bc3548afcb60c731409604224dc8985a7481832e7", + "n_lines": 452 + }, + "git": { + "commit": "1a6f5df9d531bd57fe46f1d2bb026a8a811a6705", + "commit_short": "1a6f5df9d531", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "pythia/results/e49_derangement_stability.json", + "ythia/results/e48b_exposure_growth.json" + ], + "dirty_diff_sha256": "404196310590940350cb26e00a5497d449f2583514e39405aeff8e9af7df35aa", + "describe": "1a6f5df-dirty" + }, + "argv": [ + "pythia/t36_qladder.py", + "--device", + "cpu", + "--seeds", + "3", + "--n-boot", + "4000" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48b_exposure_growth.json", + "sha256": "30ba5ecd395f1cdff653201c634a135a6d0d41e4b12c359a60c307cd85ad7d95", + "bytes": 32715, + "exists": true + }, + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n400_s0.pt", + "sha256": "4c87ed9a5f778ef0280a17ff0df5e623e8732fb7b706936fcc7caf624979511a", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n400_s0.pt", + "sha256": "5e2af80098d34c57e6cdbfcf680a9c9303e18daae91b1577d3e9222e7eb4952b", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n400_s0.pt", + "sha256": "3c3bbbca13ef5891fb19ce2c552334ebf12263c905f25545bc36a831b4bd2a00", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n400_s0.pt", + "sha256": "4754d23b9e6eb16ac8159661cc393328374d8cc8d286bbb9c971ee95166fca62", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0.pt", + "sha256": "4bc59371a760c85c5570f68e00a6e41a65a1ff7168079fc9ab5a695e2d70987b", + "bytes": 46145173, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "1811c47b45a5b200ccd14b4795f47534786f6c1ec9748bacfeca374d1ea3b714", + "path_migration": { + "date": "2026-08-15", + "reason": "the programme directory `pythia/` was dissolved; its contents moved to the repo root", + "rule": "re-point provenance.script.path and provenance.inputs[].path at where each file lives now, resolving by unique basename; refuse if the basename is ambiguous", + "not_touched": [ + "provenance.argv", + "provenance.git.dirty_files", + "every field outside the provenance block (payload_sha256 must not move)" + ], + "script_path_before": "t36_qladder.py", + "script_path_after": "experiments/t36_qladder.py", + "script_sha256_before": "bc7b644ab5714963a7e9f413fbe9b5176b535872661e3809cb00d54f46c97f89", + "script_content_changed": true, + "script_sha256_on_disk_now": "8c2e219a8d73a34c7370fc7bc3548afcb60c731409604224dc8985a7481832e7", + "restamp_required": "content differs; run --restamp-scripts , which proves the diff is path-only before recording it", + "script_restamps": [ + { + "sha256_before": "bc7b644ab5714963a7e9f413fbe9b5176b535872661e3809cb00d54f46c97f89", + "sha256_after": "8c2e219a8d73a34c7370fc7bc3548afcb60c731409604224dc8985a7481832e7", + "verified": "every differing line vs the pre-move blob is a path line (contains a separator, a path-construction call, or a moved directory name)", + "baseline_commit": "88d3697" + } + ], + "inputs_refreshed": [ + { + "path": "results/e48b_exposure_growth.json", + "sha256_before": "6c155a10efa6afa52b8c2bd79aaf613b971972bf5698fb5124d9477f609f9dda", + "sha256_after": "30ba5ecd395f1cdff653201c634a135a6d0d41e4b12c359a60c307cd85ad7d95", + "payload_sha256_of_input": "a3904599356aee3c0265c5c1b7b54240e9814b735dd84c0812fd7ad8b6777eef", + "why": "file bytes moved (provenance re-pointed); the input's own payload hash still matches its content, so the science it carries is unchanged", + "pre_existing": false + } + ] + } + } +} \ No newline at end of file diff --git a/results/e36_qladder_410m_c1arm.json b/results/e36_qladder_410m_c1arm.json new file mode 100644 index 0000000000000000000000000000000000000000..b0166e4d997c02af221b36c5793d2a8fe70f577a --- /dev/null +++ b/results/e36_qladder_410m_c1arm.json @@ -0,0 +1,4356 @@ +{ + "experiment": "E36 \u2014 the Q-ladder: vary the READ distribution, hold J^P fixed", + "prereg": "specs/archive/prereg/PREREG_E36_QLADDER.md (2026-08-13, before any E36 number existed)", + "prereg_r4b": "specs/experiments/R4b_e36_rstrip.md", + "status": "PRE-REGISTERED", + "rstrip": false, + "readout_convention": "UNSTRIPPED (LEGACY) \u2014 readout at the trailing-space token, which does not occur in the tokenisation of prompt+target", + "decision_rule_verbatim_R4b": "REJECT STANDS: the fitted operator is flatter on >= 4 of 5 rungs. Section 3.1 is unchanged. | REJECT OVERTURNED: flatter on <= 1 of 5. STOP and alert. | UNCLEAR: 2 or 3 of 5. Report and stop. Do not re-cut.", + "PREREG_C4_WAS_NEVER_CODED": "PREREG_E36_QLADDER.md:133 registers C4, the prefix-length control -- a prefix of the same 128 tokens drawn from the FITTING corpus P -- and :136 calls it load-bearing: 'Without C4 the entire ladder could be a prefix-length artifact.' No C4 key exists in any E36 results file. Surfaced by R4e; recorded here so the omission travels with the result rather than being rediscovered.", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "prefix_tokens": 128, + "n_items": 541, + "n_pairs": 893, + "rungs": [ + "Q0", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "USPTO_Backgrounds", + "Github", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "smoke": false, + "x_axis": { + "source": "e48b_exposure_growth.json", + "k": 32, + "coverage_shards": 20, + "note": "measured containment, not a rung number \u2014 this is what E35/M1 was for" + }, + "unavailable_rung": { + "Q5_non_pile_script": "no non-Latin-script corpus is on disk; the ladder does not extend to script-level shift. Recorded, not silently omitted." + }, + "declared_bias": "we shift the READ distribution, not the FIT distribution. EQ2's bias term is ||J^P - J^Q|| where J^Q is the operator that WOULD have been fitted on Q; we never fit J^Q here. E48 supplies that half.", + "ladder": { + "Q0": { + "containment_k32": null, + "is_shuffled": false, + "capability_top10_rate": 0.020156774916013438, + "capability_mean_rank": 4224.63045912654, + "arms": { + "logit": { + "persist": { + "mean": 0.028439503419212996, + "seed_sd": 0.0, + "per_seed": [ + 0.028439503419212996, + 0.028439503419212996, + 0.028439503419212996 + ] + }, + "min": { + "mean": 0.1247396256774664, + "seed_sd": 0.0, + "per_seed": [ + 0.1247396256774664, + 0.1247396256774664, + 0.1247396256774664 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.02855067129712552, + "seed_sd": 0.0, + "per_seed": [ + 0.02855067129712552, + 0.02855067129712552, + 0.02855067129712552 + ] + }, + "min": { + "mean": 0.15541725233197212, + "seed_sd": 0.0, + "per_seed": [ + 0.15541725233197212, + 0.15541725233197212, + 0.15541725233197212 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.027925857156515122, + "seed_sd": 0.0, + "per_seed": [ + 0.027925857156515122, + 0.027925857156515122, + 0.027925857156515122 + ] + }, + "min": { + "mean": 0.11324172168970108, + "seed_sd": 0.0, + "per_seed": [ + 0.11324172168970108, + 0.11324172168970108, + 0.11324172168970108 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0005136465188115835, + "ci_lo": -0.021382784470915794, + "ci_hi": 0.022756576538085938, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.039340307447128, + "seed_sd": 0.0, + "per_seed": [ + 0.039340307447128, + 0.039340307447128, + 0.039340307447128 + ] + }, + "min": { + "mean": 0.13213929273188113, + "seed_sd": 0.0, + "per_seed": [ + 0.13213929273188113, + 0.13213929273188113, + 0.13213929273188113 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01090080477297306, + "ci_lo": -0.014096681028604507, + "ci_hi": 0.047603681683540344, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04615347860381007, + "seed_sd": 0.0, + "per_seed": [ + 0.04615347860381007, + 0.04615347860381007, + 0.04615347860381007 + ] + }, + "min": { + "mean": 0.14736362285912036, + "seed_sd": 0.0, + "per_seed": [ + 0.14736362285912036, + 0.14736362285912036, + 0.14736362285912036 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01771397516131401, + "ci_lo": -0.021887753158807755, + "ci_hi": 0.074996717274189, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.045389215904287995, + "seed_sd": 0.0, + "per_seed": [ + 0.045389215904287995, + 0.045389215904287995, + 0.045389215904287995 + ] + }, + "min": { + "mean": 0.1445665042847395, + "seed_sd": 0.0, + "per_seed": [ + 0.1445665042847395, + 0.1445665042847395, + 0.1445665042847395 + ] + }, + "ci_vs_logit_persist": { + "point": 0.016949711367487907, + "ci_lo": -0.020029015839099884, + "ci_hi": 0.07102534174919128, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.050897271279245614, + "seed_sd": 0.0, + "per_seed": [ + 0.050897271279245614, + 0.050897271279245614, + 0.050897271279245614 + ] + }, + "min": { + "mean": 0.15696468092501165, + "seed_sd": 0.0, + "per_seed": [ + 0.15696468092501165, + 0.15696468092501165, + 0.15696468092501165 + ] + }, + "ci_vs_logit_persist": { + "point": 0.02245776355266571, + "ci_lo": -0.015598583035171032, + "ci_hi": 0.0812041237950325, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.01587643986567855, + "seed_sd": 0.0, + "per_seed": [ + 0.01587643986567855, + 0.01587643986567855, + 0.01587643986567855 + ] + }, + "min": { + "mean": 0.12092158123850823, + "seed_sd": 0.0, + "per_seed": [ + 0.12092158123850823, + 0.12092158123850823, + 0.12092158123850823 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.021565641462802886, + "seed_sd": 0.0, + "per_seed": [ + 0.021565641462802886, + 0.021565641462802886, + 0.021565641462802886 + ] + }, + "min": { + "mean": 0.14252219274640082, + "seed_sd": 0.0, + "per_seed": [ + 0.14252219274640082, + 0.14252219274640082, + 0.14252219274640082 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.02808863080572337, + "seed_sd": 0.0, + "per_seed": [ + 0.02808863080572337, + 0.02808863080572337, + 0.02808863080572337 + ] + }, + "min": { + "mean": 0.1592821255326271, + "seed_sd": 0.0, + "per_seed": [ + 0.1592821255326271, + 0.1592821255326271, + 0.1592821255326271 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.02855067129712552, + "seed_sd": 0.0, + "per_seed": [ + 0.02855067129712552, + 0.02855067129712552, + 0.02855067129712552 + ] + }, + "min": { + "mean": 0.15541725233197212, + "seed_sd": 0.0, + "per_seed": [ + 0.15541725233197212, + 0.15541725233197212, + 0.15541725233197212 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.027981736697256565, + "seed_sd": 0.0, + "per_seed": [ + 0.027981736697256565, + 0.027981736697256565, + 0.027981736697256565 + ] + }, + "min": { + "mean": 0.13978539556264877, + "seed_sd": 0.0, + "per_seed": [ + 0.13978539556264877, + 0.13978539556264877, + 0.13978539556264877 + ] + } + } + } + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "is_shuffled": false, + "capability_top10_rate": 0.022396416573348264, + "capability_mean_rank": 4309.144083613289, + "arms": { + "logit": { + "persist": { + "mean": 0.03629972389123092, + "seed_sd": 0.0022810950733453132, + "per_seed": [ + 0.03462574060540646, + 0.03952490042429417, + 0.03474853064399212 + ] + }, + "min": { + "mean": 0.1330052327364683, + "seed_sd": 0.0014164883328735025, + "per_seed": [ + 0.1340819675475359, + 0.13392978720366955, + 0.1310039434581995 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.03130553908025225, + "seed_sd": 0.0008823959613629814, + "per_seed": [ + 0.030057793483138084, + 0.031912614777684215, + 0.03194620897993446 + ] + }, + "min": { + "mean": 0.1601704572637876, + "seed_sd": 0.0061335209465566, + "per_seed": [ + 0.16515550389885902, + 0.16382546201348305, + 0.1515304058790207 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.026641206163913012, + "seed_sd": 0.0005490161444991766, + "per_seed": [ + 0.02730230875313282, + 0.02666326742619276, + 0.025958042312413453 + ] + }, + "min": { + "mean": 0.11949207074940205, + "seed_sd": 0.002793334988492231, + "per_seed": [ + 0.12006914354860783, + 0.12258795760571957, + 0.11581911109387874 + ] + }, + "ci_vs_logit_persist": { + "point": -0.00798453576862812, + "ci_lo": -0.042686257511377335, + "ci_hi": 0.022087737917900085, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.042335875960998234, + "seed_sd": 0.001032902757615945, + "per_seed": [ + 0.041822236659936604, + 0.04377695198636502, + 0.04140843923669309 + ] + }, + "min": { + "mean": 0.14075611618657907, + "seed_sd": 0.003580610594648241, + "per_seed": [ + 0.13986529037356377, + 0.14551847018301486, + 0.13688458800315856 + ] + }, + "ci_vs_logit_persist": { + "point": 0.007710135076195002, + "ci_lo": -0.03473234921693802, + "ci_hi": 0.057025715708732605, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04749905125548442, + "seed_sd": 0.0024920584976778066, + "per_seed": [ + 0.047684010956436394, + 0.050454501155763866, + 0.044358641654253 + ] + }, + "min": { + "mean": 0.1542269659539064, + "seed_sd": 0.0020765537534632853, + "per_seed": [ + 0.15365934744477272, + 0.15700606480240822, + 0.1520154856145382 + ] + }, + "ci_vs_logit_persist": { + "point": 0.012873309664428234, + "ci_lo": -0.03922334313392639, + "ci_hi": 0.07404104620218277, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.050609867088496685, + "seed_sd": 0.002171233544999387, + "per_seed": [ + 0.05041661513969302, + 0.05336042842827737, + 0.04805255769751966 + ] + }, + "min": { + "mean": 0.1531935981164376, + "seed_sd": 0.004097844888945463, + "per_seed": [ + 0.15286606140434741, + 0.15836815871298313, + 0.14834657423198222 + ] + }, + "ci_vs_logit_persist": { + "point": 0.015984121710062027, + "ci_lo": -0.036915309727191925, + "ci_hi": 0.07962671667337418, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.050712701166048646, + "seed_sd": 0.0015313698759926331, + "per_seed": [ + 0.050940170791000126, + 0.05246412954293191, + 0.0487338031642139 + ] + }, + "min": { + "mean": 0.17039897752304872, + "seed_sd": 0.005706180472277549, + "per_seed": [ + 0.17443899177014827, + 0.17442871890962125, + 0.16232922188937665 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01608695462346077, + "ci_lo": -0.03724583610892296, + "ci_hi": 0.08263824135065079, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.017510719985390704, + "seed_sd": 0.0005399085152294292, + "per_seed": [ + 0.01677953377366066, + 0.017685848264954986, + 0.018066777917556464 + ] + }, + "min": { + "mean": 0.12547955388824145, + "seed_sd": 0.004844007144690792, + "per_seed": [ + 0.1293361470103264, + 0.12845448330044745, + 0.1186480313539505 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.0240809575188905, + "seed_sd": 0.0012375640350872552, + "per_seed": [ + 0.02310459497384727, + 0.02582706734538078, + 0.023311210237443446 + ] + }, + "min": { + "mean": 0.14174381991227467, + "seed_sd": 0.004999068249353136, + "per_seed": [ + 0.14452686309814453, + 0.14598053321242332, + 0.13472406342625617 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.02935713321591417, + "seed_sd": 0.0005810289098783817, + "per_seed": [ + 0.029764199838973583, + 0.029771753982640804, + 0.028535445826128124 + ] + }, + "min": { + "mean": 0.15776805753509204, + "seed_sd": 0.005010915659240234, + "per_seed": [ + 0.15973466262221336, + 0.1626807928085327, + 0.15088871717453003 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.03130553908025225, + "seed_sd": 0.0008823959613629814, + "per_seed": [ + 0.030057793483138084, + 0.031912614777684215, + 0.03194620897993446 + ] + }, + "min": { + "mean": 0.1601704572637876, + "seed_sd": 0.0061335209465566, + "per_seed": [ + 0.16515550389885902, + 0.16382546201348305, + 0.1515304058790207 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.029187689512036742, + "seed_sd": 0.000663583937974395, + "per_seed": [ + 0.02867567278444767, + 0.028762599779292942, + 0.03012479597236961 + ] + }, + "min": { + "mean": 0.15325938239693643, + "seed_sd": 0.00444267989202959, + "per_seed": [ + 0.15648338198661804, + 0.15631755143404008, + 0.14697721377015113 + ] + } + } + } + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "is_shuffled": false, + "capability_top10_rate": 0.01866368047779022, + "capability_mean_rank": 4674.715192235909, + "arms": { + "logit": { + "persist": { + "mean": 0.028078026658234497, + "seed_sd": 0.0016248270852042151, + "per_seed": [ + 0.02840878739953041, + 0.025943371653556823, + 0.029881920921616255 + ] + }, + "min": { + "mean": 0.12258193418383599, + "seed_sd": 0.004003446205338022, + "per_seed": [ + 0.12605063132941724, + 0.11697234660387039, + 0.12472282461822033 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.02416364283611377, + "seed_sd": 0.001003349309462458, + "per_seed": [ + 0.024971047695726157, + 0.022749426774680614, + 0.02477045403793454 + ] + }, + "min": { + "mean": 0.14806132937471073, + "seed_sd": 0.00276723905758561, + "per_seed": [ + 0.15189499780535698, + 0.14546357095241547, + 0.1468254193663597 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.022569557776053746, + "seed_sd": 0.0012074208994716435, + "per_seed": [ + 0.02368360187392682, + 0.020891831861808895, + 0.023133239592425525 + ] + }, + "min": { + "mean": 0.10596823145945868, + "seed_sd": 0.005845325709225712, + "per_seed": [ + 0.10839929915964604, + 0.09791024327278137, + 0.1115951519459486 + ] + }, + "ci_vs_logit_persist": { + "point": -0.005839229095727205, + "ci_lo": -0.03438413888216019, + "ci_hi": 0.024198899045586586, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.03214704647349814, + "seed_sd": 0.0024661763503202403, + "per_seed": [ + 0.03469139921944588, + 0.028809019061736763, + 0.03294072113931179 + ] + }, + "min": { + "mean": 0.11322029903531075, + "seed_sd": 0.005204709409931878, + "per_seed": [ + 0.1151746816933155, + 0.1060974758118391, + 0.11838873960077763 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0037382624577730894, + "ci_lo": -0.03161531314253807, + "ci_hi": 0.04482903704047203, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04227183082451423, + "seed_sd": 0.0014374277309662056, + "per_seed": [ + 0.044263992365449666, + 0.04092536880634725, + 0.04162613130174577 + ] + }, + "min": { + "mean": 0.13252984347442787, + "seed_sd": 0.004803867667689962, + "per_seed": [ + 0.13294915668666363, + 0.12644789181649685, + 0.1381924819201231 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01386304385960102, + "ci_lo": -0.0319400429725647, + "ci_hi": 0.07346367090940475, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.041621484948943056, + "seed_sd": 0.0025450721298467087, + "per_seed": [ + 0.044255858799442646, + 0.03818035263102502, + 0.04242824341636151 + ] + }, + "min": { + "mean": 0.12788050323724748, + "seed_sd": 0.004018488968736439, + "per_seed": [ + 0.12957906983792783, + 0.1223345685750246, + 0.13172787129879 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013212698511779308, + "ci_lo": -0.030285555869340897, + "ci_hi": 0.06918405741453171, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04389908245454232, + "seed_sd": 0.0018030786304362044, + "per_seed": [ + 0.04644289943389594, + 0.04278009617701173, + 0.04247425175271928 + ] + }, + "min": { + "mean": 0.15009770318865776, + "seed_sd": 0.007101317957691461, + "per_seed": [ + 0.15323154553771018, + 0.14026777110993863, + 0.15679379291832446 + ] + }, + "ci_vs_logit_persist": { + "point": 0.015490291640162468, + "ci_lo": -0.03153032809495926, + "ci_hi": 0.07946377247571945, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.013367696618661285, + "seed_sd": 0.0007077392068739216, + "per_seed": [ + 0.014238849538378418, + 0.012505316198803485, + 0.013358924118801952 + ] + }, + "min": { + "mean": 0.1157922625541687, + "seed_sd": 0.003459292460774911, + "per_seed": [ + 0.12021465674042701, + 0.11176946982741356, + 0.11539266109466553 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.017424028366804123, + "seed_sd": 0.001383474096797694, + "per_seed": [ + 0.0188205074518919, + 0.015539034642279148, + 0.017912543006241323 + ] + }, + "min": { + "mean": 0.13535516386230786, + "seed_sd": 0.0028179940015110533, + "per_seed": [ + 0.13902485370635986, + 0.1321743369102478, + 0.13486630097031593 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.024409827768492202, + "seed_sd": 0.001020187774467825, + "per_seed": [ + 0.024791516363620758, + 0.02301403118763119, + 0.025423935754224657 + ] + }, + "min": { + "mean": 0.14287620633840561, + "seed_sd": 0.0038404266117902553, + "per_seed": [ + 0.1441191703081131, + 0.13767601326107978, + 0.14683343544602395 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.02416364283611377, + "seed_sd": 0.001003349309462458, + "per_seed": [ + 0.024971047695726157, + 0.022749426774680614, + 0.02477045403793454 + ] + }, + "min": { + "mean": 0.14806132937471073, + "seed_sd": 0.00276723905758561, + "per_seed": [ + 0.15189499780535698, + 0.14546357095241547, + 0.1468254193663597 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.025389814594139654, + "seed_sd": 0.0007424380292847766, + "per_seed": [ + 0.02643975168466568, + 0.024858106579631566, + 0.02487158551812172 + ] + }, + "min": { + "mean": 0.14123825455705324, + "seed_sd": 0.00500903185177366, + "per_seed": [ + 0.14498427286744117, + 0.1341584123671055, + 0.14457207843661307 + ] + } + } + } + }, + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "is_shuffled": false, + "capability_top10_rate": 0.026502426278462114, + "capability_mean_rank": 4231.8185890257555, + "arms": { + "logit": { + "persist": { + "mean": 0.03157440964908649, + "seed_sd": 0.002796284928918886, + "per_seed": [ + 0.034713071095757185, + 0.027921712095849215, + 0.03208844575565308 + ] + }, + "min": { + "mean": 0.1278687487045924, + "seed_sd": 0.006448430626530321, + "per_seed": [ + 0.1357261061668396, + 0.11993134990334511, + 0.12794879004359244 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.0318799717972676, + "seed_sd": 0.0008508815518589679, + "per_seed": [ + 0.03191518913954496, + 0.032904029544442895, + 0.030820696707814933 + ] + }, + "min": { + "mean": 0.15736184120178223, + "seed_sd": 0.004065740714376282, + "per_seed": [ + 0.1624331034719944, + 0.15247948989272117, + 0.15717293024063111 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.028430391071985164, + "seed_sd": 0.00040201361994103454, + "per_seed": [ + 0.028182579111307858, + 0.028111166879534723, + 0.028997427225112914 + ] + }, + "min": { + "mean": 0.11221431779364745, + "seed_sd": 0.005428481955499573, + "per_seed": [ + 0.11970514766871929, + 0.10701349042356015, + 0.10992431528866291 + ] + }, + "ci_vs_logit_persist": { + "point": -0.006282678805291653, + "ci_lo": -0.03783251345157623, + "ci_hi": 0.022103214636445045, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04342051934606085, + "seed_sd": 0.002331506175030251, + "per_seed": [ + 0.046479623904451726, + 0.04082546669524163, + 0.0429564674384892 + ] + }, + "min": { + "mean": 0.13656094384690126, + "seed_sd": 0.003168943329982689, + "per_seed": [ + 0.13985242135822773, + 0.13228120915591718, + 0.13754920102655888 + ] + }, + "ci_vs_logit_persist": { + "point": 0.008707449771463871, + "ci_lo": -0.03267471864819527, + "ci_hi": 0.05781780183315277, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04891945105046034, + "seed_sd": 0.002041292329157774, + "per_seed": [ + 0.0517880929633975, + 0.04720498416572809, + 0.04776527602225542 + ] + }, + "min": { + "mean": 0.14881497162083784, + "seed_sd": 0.0009700772687800834, + "per_seed": [ + 0.1501594215631485, + 0.147906281799078, + 0.14837921150028704 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014206379652023315, + "ci_lo": -0.03510032221674919, + "ci_hi": 0.07447437942028046, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.05241671356682976, + "seed_sd": 0.002162919922204846, + "per_seed": [ + 0.05540219666436315, + 0.050347342249006034, + 0.051500601787120105 + ] + }, + "min": { + "mean": 0.14973365664482116, + "seed_sd": 0.004798387607280762, + "per_seed": [ + 0.15460128299891948, + 0.1432051584124565, + 0.1513945285230875 + ] + }, + "ci_vs_logit_persist": { + "point": 0.017703641206026077, + "ci_lo": -0.033347200602293015, + "ci_hi": 0.08149312436580658, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.05023885692159335, + "seed_sd": 0.002192391318925561, + "per_seed": [ + 0.05247340011410415, + 0.04726013909094036, + 0.050983031559735534 + ] + }, + "min": { + "mean": 0.16274620306988558, + "seed_sd": 0.001623798532968072, + "per_seed": [ + 0.1649680022150278, + 0.16213808469474317, + 0.16113252229988576 + ] + }, + "ci_vs_logit_persist": { + "point": 0.015525788068771362, + "ci_lo": -0.03374544531106949, + "ci_hi": 0.07918965816497803, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.016466317985517284, + "seed_sd": 0.0005837444743219453, + "per_seed": [ + 0.01671387501992285, + 0.015660503506660463, + 0.017024575429968537 + ] + }, + "min": { + "mean": 0.12197338367501895, + "seed_sd": 0.003764059589058818, + "per_seed": [ + 0.126987062394619, + 0.12101557180285454, + 0.11791751682758331 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.02480899039655924, + "seed_sd": 0.001482323894372705, + "per_seed": [ + 0.026464467123150825, + 0.0250950176268816, + 0.022867486439645292 + ] + }, + "min": { + "mean": 0.14390204747517904, + "seed_sd": 0.0035830819765241113, + "per_seed": [ + 0.14896516650915145, + 0.1415475018322468, + 0.14119347408413888 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.029116893908940254, + "seed_sd": 0.0008819007347518028, + "per_seed": [ + 0.027869851095601916, + 0.0297573423711583, + 0.02972348826006055 + ] + }, + "min": { + "mean": 0.15497935265302656, + "seed_sd": 0.003666313017360248, + "per_seed": [ + 0.15923285260796546, + 0.150284805893898, + 0.15542039945721625 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.0318799717972676, + "seed_sd": 0.0008508815518589679, + "per_seed": [ + 0.03191518913954496, + 0.032904029544442895, + 0.030820696707814933 + ] + }, + "min": { + "mean": 0.15736184120178223, + "seed_sd": 0.004065740714376282, + "per_seed": [ + 0.1624331034719944, + 0.15247948989272117, + 0.15717293024063111 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.029010930075310172, + "seed_sd": 0.001116090357883598, + "per_seed": [ + 0.029257540684193374, + 0.030237763258628546, + 0.027537486283108592 + ] + }, + "min": { + "mean": 0.15219910989205043, + "seed_sd": 0.0030193587223089494, + "per_seed": [ + 0.15514956414699554, + 0.15339706540107728, + 0.14805070012807847 + ] + } + } + } + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "is_shuffled": false, + "capability_top10_rate": 0.024262784621127288, + "capability_mean_rank": 4211.015304217992, + "arms": { + "logit": { + "persist": { + "mean": 0.028793133182140686, + "seed_sd": 0.0010285076269501952, + "per_seed": [ + 0.030163079011254014, + 0.027684865775518118, + 0.02853145475964993 + ] + }, + "min": { + "mean": 0.12128662702937922, + "seed_sd": 0.0021928223516823725, + "per_seed": [ + 0.1199255108833313, + 0.11955404877662659, + 0.12438032142817974 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.03226851496535043, + "seed_sd": 0.000694903377751064, + "per_seed": [ + 0.032909794221632184, + 0.03130296701565385, + 0.03259278365876526 + ] + }, + "min": { + "mean": 0.15766812091072402, + "seed_sd": 0.0030098658474654143, + "per_seed": [ + 0.16132253482937814, + 0.1539507068693638, + 0.1577311210334301 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.027415493588584167, + "seed_sd": 0.0005045354860205193, + "per_seed": [ + 0.026976512838155033, + 0.028122124611400068, + 0.027147843316197396 + ] + }, + "min": { + "mean": 0.11763123249014219, + "seed_sd": 0.003588175000510387, + "per_seed": [ + 0.11652602553367615, + 0.11389473490417004, + 0.12247293703258037 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0027475852984935045, + "ci_lo": -0.025122208520770073, + "ci_hi": 0.022981896996498108, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04146475471400966, + "seed_sd": 0.0006228269274547368, + "per_seed": [ + 0.04157824940048158, + 0.042164452536962924, + 0.04065156220458448 + ] + }, + "min": { + "mean": 0.13566014481087527, + "seed_sd": 0.004365688431446649, + "per_seed": [ + 0.13475896269083024, + 0.13082114644348622, + 0.14140032529830932 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01130167581140995, + "ci_lo": -0.019091835245490074, + "ci_hi": 0.05440139397978783, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04922135318629444, + "seed_sd": 0.0012973659708061676, + "per_seed": [ + 0.047551121283322575, + 0.04939883570186794, + 0.0507141025736928 + ] + }, + "min": { + "mean": 0.15273317725708088, + "seed_sd": 0.004483918300540694, + "per_seed": [ + 0.15177303440868856, + 0.14778490830212831, + 0.15864158906042575 + ] + }, + "ci_vs_logit_persist": { + "point": 0.019058268517255783, + "ci_lo": -0.022475916892290115, + "ci_hi": 0.07973159849643707, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.049197832588106394, + "seed_sd": 0.0013624558082363772, + "per_seed": [ + 0.04733125129714608, + 0.0497171975672245, + 0.0505450488999486 + ] + }, + "min": { + "mean": 0.14771586476514736, + "seed_sd": 0.0037996570167676273, + "per_seed": [ + 0.14641902185976505, + 0.1438482327386737, + 0.15288033969700338 + ] + }, + "ci_vs_logit_persist": { + "point": 0.019034752622246742, + "ci_lo": -0.02064024843275547, + "ci_hi": 0.07573997229337692, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.05094143764581531, + "seed_sd": 0.002022776255880077, + "per_seed": [ + 0.04855143851600587, + 0.05077505416702479, + 0.053497820254415275 + ] + }, + "min": { + "mean": 0.16123308278620244, + "seed_sd": 0.0013256078180656381, + "per_seed": [ + 0.15953294970095158, + 0.16139903329312802, + 0.1627672653645277 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020778357982635498, + "ci_lo": -0.02221827767789364, + "ci_hi": 0.08543560653924942, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.0179284755140543, + "seed_sd": 0.0004370027466213808, + "per_seed": [ + 0.017374484450556338, + 0.01796824219636619, + 0.018442699895240368 + ] + }, + "min": { + "mean": 0.12493583324054877, + "seed_sd": 0.0021331171463262904, + "per_seed": [ + 0.12671928405761718, + 0.12193703390657902, + 0.1261511817574501 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.024518597087201974, + "seed_sd": 0.000257761890199154, + "per_seed": [ + 0.024882586672902107, + 0.02431941293179989, + 0.024353791656903923 + ] + }, + "min": { + "mean": 0.14265149037043254, + "seed_sd": 0.0028476549832648123, + "per_seed": [ + 0.14327458441257476, + 0.13889429047703744, + 0.1457855962216854 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.030165471827300887, + "seed_sd": 0.00021552193001693533, + "per_seed": [ + 0.030232056020759047, + 0.02987459583673626, + 0.030389763624407352 + ] + }, + "min": { + "mean": 0.15930220832427344, + "seed_sd": 0.0003690633205064254, + "per_seed": [ + 0.15981256365776061, + 0.15914171263575555, + 0.15895234867930413 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.03226851496535043, + "seed_sd": 0.000694903377751064, + "per_seed": [ + 0.032909794221632184, + 0.03130296701565385, + 0.03259278365876526 + ] + }, + "min": { + "mean": 0.15766812091072402, + "seed_sd": 0.0030098658474654143, + "per_seed": [ + 0.16132253482937814, + 0.1539507068693638, + 0.1577311210334301 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.03100951196004947, + "seed_sd": 0.00023489592748023703, + "per_seed": [ + 0.031021289853379132, + 0.03071611630730331, + 0.031291129719465974 + ] + }, + "min": { + "mean": 0.15175020781656107, + "seed_sd": 0.003501655314930296, + "per_seed": [ + 0.15670035928487777, + 0.149155193567276, + 0.1493950705975294 + ] + } + } + } + }, + "Github": { + "containment_k32": 0.9340004374453194, + "is_shuffled": false, + "capability_top10_rate": 0.020530048525569243, + "capability_mean_rank": 4797.154535274356, + "arms": { + "logit": { + "persist": { + "mean": 0.023829053342342376, + "seed_sd": 0.0019133100599684207, + "per_seed": [ + 0.02512250132858753, + 0.025240575522184373, + 0.021124083176255228 + ] + }, + "min": { + "mean": 0.11852239407598972, + "seed_sd": 0.0028139239754922797, + "per_seed": [ + 0.12220506258308887, + 0.11798709072172642, + 0.11537502892315388 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.025754503525483113, + "seed_sd": 0.0006269310714382005, + "per_seed": [ + 0.024879138544201852, + 0.026314112078398465, + 0.02607025995384902 + ] + }, + "min": { + "mean": 0.153716907898585, + "seed_sd": 0.003182965315677801, + "per_seed": [ + 0.15006403177976607, + 0.15326534807682038, + 0.15782134383916854 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.023965128883719444, + "seed_sd": 0.0012537102566416024, + "per_seed": [ + 0.02224739659577608, + 0.02520443024113774, + 0.02444355981424451 + ] + }, + "min": { + "mean": 0.10658646300435067, + "seed_sd": 0.0008441858676514157, + "per_seed": [ + 0.10766854919493199, + 0.10648221708834171, + 0.10560862272977829 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0011573706287890673, + "ci_lo": -0.023954618722200394, + "ci_hi": 0.026057662442326546, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.03312810758749644, + "seed_sd": 0.00043170887726855836, + "per_seed": [ + 0.03328255987726152, + 0.033562415605410935, + 0.03253934727981687 + ] + }, + "min": { + "mean": 0.11906188415984313, + "seed_sd": 0.002329785482299106, + "per_seed": [ + 0.11745987385511399, + 0.11736949346959591, + 0.12235628515481949 + ] + }, + "ci_vs_logit_persist": { + "point": 0.008005605079233646, + "ci_lo": -0.02340543456375599, + "ci_hi": 0.047453757375478745, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.045316207610691585, + "seed_sd": 0.0007238705837581695, + "per_seed": [ + 0.04473803648725152, + 0.04487367062829435, + 0.04633691571652889 + ] + }, + "min": { + "mean": 0.14035971301297348, + "seed_sd": 0.0019086486372587475, + "per_seed": [ + 0.14288209639489652, + 0.13826629780232907, + 0.13993074484169482 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020193712785840034, + "ci_lo": -0.021595049649477005, + "ci_hi": 0.08058124035596848, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04261825208862623, + "seed_sd": 0.0007243694688608191, + "per_seed": [ + 0.043500224407762286, + 0.042628553323447706, + 0.04172597853466868 + ] + }, + "min": { + "mean": 0.13725972324609756, + "seed_sd": 0.001667425277131875, + "per_seed": [ + 0.13809503726661204, + 0.13493231534957886, + 0.13875181712210177 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01749575138092041, + "ci_lo": -0.021068150177598, + "ci_hi": 0.07054220139980316, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.044759471978371344, + "seed_sd": 0.0001851358354110141, + "per_seed": [ + 0.04489713786169887, + 0.04449776862747967, + 0.044883509445935485 + ] + }, + "min": { + "mean": 0.1492717611292998, + "seed_sd": 0.0034139530565961015, + "per_seed": [ + 0.14786469452083112, + 0.1459755774587393, + 0.153975011408329 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01963697373867035, + "ci_lo": -0.02300436422228813, + "ci_hi": 0.08291710913181305, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.015060939228472611, + "seed_sd": 0.0013084710562746454, + "per_seed": [ + 0.013212827476672828, + 0.015904329298064113, + 0.01606566091068089 + ] + }, + "min": { + "mean": 0.11447235867381096, + "seed_sd": 0.004438263516314416, + "per_seed": [ + 0.10873512774705887, + 0.11513634026050568, + 0.11954560801386833 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.01978080750753482, + "seed_sd": 0.0009572910558582751, + "per_seed": [ + 0.019833887182176112, + 0.0209258034825325, + 0.01858273185789585 + ] + }, + "min": { + "mean": 0.13359517653783162, + "seed_sd": 0.0028601067538628483, + "per_seed": [ + 0.13050860315561294, + 0.13287459164857865, + 0.13740233480930328 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.026343628438189624, + "seed_sd": 0.0013518333534012857, + "per_seed": [ + 0.024583000573329628, + 0.026578711974434555, + 0.027869172766804694 + ] + }, + "min": { + "mean": 0.15187092858056228, + "seed_sd": 0.002140176447950562, + "per_seed": [ + 0.15278118960559367, + 0.14891597777605056, + 0.15391561836004258 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.025754503525483113, + "seed_sd": 0.0006269310714382005, + "per_seed": [ + 0.024879138544201852, + 0.026314112078398465, + 0.02607025995384902 + ] + }, + "min": { + "mean": 0.153716907898585, + "seed_sd": 0.003182965315677801, + "per_seed": [ + 0.15006403177976607, + 0.15326534807682038, + 0.15782134383916854 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.02653873189507673, + "seed_sd": 0.0006116615916095771, + "per_seed": [ + 0.026223682868294417, + 0.025998579594306646, + 0.02739393322262913 + ] + }, + "min": { + "mean": 0.14194749568899473, + "seed_sd": 0.00490245238826048, + "per_seed": [ + 0.13896340653300285, + 0.13801990300416947, + 0.14885917752981187 + ] + } + } + } + }, + "CONTROL_PubMed_2023": { + "containment_k32": 0.7927273508312213, + "is_shuffled": false, + "capability_top10_rate": 0.019036954087346025, + "capability_mean_rank": 4409.058230683091, + "arms": { + "logit": { + "persist": { + "mean": 0.030234563273067276, + "seed_sd": 0.0015639236905520023, + "per_seed": [ + 0.028575583430938422, + 0.02979731510858983, + 0.032330791279673574 + ] + }, + "min": { + "mean": 0.12175768092274666, + "seed_sd": 0.0024778191012028417, + "per_seed": [ + 0.12339883185923099, + 0.11825580932199956, + 0.12361840158700943 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.03168034661406031, + "seed_sd": 0.000345372082924927, + "per_seed": [ + 0.03132576560601592, + 0.03214854439720512, + 0.031566729838959875 + ] + }, + "min": { + "mean": 0.15922437235713005, + "seed_sd": 0.0028119111024592805, + "per_seed": [ + 0.1608014225959778, + 0.15527436807751654, + 0.1615973263978958 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.02758992228967448, + "seed_sd": 0.00031880767791986667, + "per_seed": [ + 0.02714625564403832, + 0.02774227587506175, + 0.027881235349923372 + ] + }, + "min": { + "mean": 0.11301399742563566, + "seed_sd": 0.0028545590193116235, + "per_seed": [ + 0.11472423039376736, + 0.10899200737476349, + 0.11532575450837612 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0009856612887233496, + "ci_lo": -0.02284880541265011, + "ci_hi": 0.023544693365693092, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.0420196181240802, + "seed_sd": 0.0005438403250338024, + "per_seed": [ + 0.042205870035104454, + 0.04128025218378752, + 0.04257273215334863 + ] + }, + "min": { + "mean": 0.13920330504576364, + "seed_sd": 0.0030836327908162217, + "per_seed": [ + 0.14034029543399812, + 0.13498876728117465, + 0.1422808524221182 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01344403438270092, + "ci_lo": -0.01404797937721014, + "ci_hi": 0.05441785231232643, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04744758574912945, + "seed_sd": 0.001083414083904069, + "per_seed": [ + 0.04734843308106065, + 0.04882128657773137, + 0.046173037588596345 + ] + }, + "min": { + "mean": 0.15173541915913424, + "seed_sd": 0.0034089159953771065, + "per_seed": [ + 0.1532831747084856, + 0.14700750522315503, + 0.15491557754576207 + ] + }, + "ci_vs_logit_persist": { + "point": 0.018872002139687538, + "ci_lo": -0.01906522363424301, + "ci_hi": 0.07450412958860397, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04918796766238908, + "seed_sd": 0.0003472843140221594, + "per_seed": [ + 0.04967892593704164, + 0.048931105574592945, + 0.04895387147553265 + ] + }, + "min": { + "mean": 0.15158311314880848, + "seed_sd": 0.002551027148333135, + "per_seed": [ + 0.15480516105890274, + 0.14856659471988679, + 0.15137758366763593 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020612385123968124, + "ci_lo": -0.01704348437488079, + "ci_hi": 0.07595079392194748, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04972566197781513, + "seed_sd": 0.00010373794261556514, + "per_seed": [ + 0.04978115267585963, + 0.04981553009711206, + 0.0495803031604737 + ] + }, + "min": { + "mean": 0.16385552746554216, + "seed_sd": 0.0024719234343604985, + "per_seed": [ + 0.16457470841705799, + 0.160533219948411, + 0.1664586540311575 + ] + }, + "ci_vs_logit_persist": { + "point": 0.021150080487132072, + "ci_lo": -0.018501339480280876, + "ci_hi": 0.08187014609575272, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.0170674793732663, + "seed_sd": 0.0005536813962763951, + "per_seed": [ + 0.016287137940526008, + 0.017513731005601586, + 0.017401569173671305 + ] + }, + "min": { + "mean": 0.12491961494088173, + "seed_sd": 0.001551269863606981, + "per_seed": [ + 0.1261227920651436, + 0.12272933796048165, + 0.12590671479701995 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.02441145104045669, + "seed_sd": 0.0005081229242212095, + "per_seed": [ + 0.02389377439394593, + 0.025101904384791852, + 0.024238674342632292 + ] + }, + "min": { + "mean": 0.1392166250695785, + "seed_sd": 0.0012347270874702948, + "per_seed": [ + 0.13947861790657043, + 0.13759052120149134, + 0.14058073610067368 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.02870412642757098, + "seed_sd": 0.00042622104464347464, + "per_seed": [ + 0.029238933208398522, + 0.028195926756598055, + 0.02867751931771636 + ] + }, + "min": { + "mean": 0.15977732290824254, + "seed_sd": 0.0006131332497078674, + "per_seed": [ + 0.16045141369104385, + 0.1589679431170225, + 0.15991261191666126 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.03168034661406031, + "seed_sd": 0.000345372082924927, + "per_seed": [ + 0.03132576560601592, + 0.03214854439720512, + 0.031566729838959875 + ] + }, + "min": { + "mean": 0.15922437235713005, + "seed_sd": 0.0028119111024592805, + "per_seed": [ + 0.1608014225959778, + 0.15527436807751654, + 0.1615973263978958 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.029943198803812267, + "seed_sd": 0.00035028242943592746, + "per_seed": [ + 0.03042350709438324, + 0.029598046420142056, + 0.029808042896911503 + ] + }, + "min": { + "mean": 0.15041070543229582, + "seed_sd": 0.0023855038943509646, + "per_seed": [ + 0.15229294523596765, + 0.14704495705664158, + 0.1518942140042782 + ] + } + } + } + }, + "OOD_Wikipedia_2023": { + "containment_k32": 0.26957534299550123, + "is_shuffled": false, + "capability_top10_rate": 0.03471444568868981, + "capability_mean_rank": 4096.241508025383, + "arms": { + "logit": { + "persist": { + "mean": 0.03236095819156617, + "seed_sd": 0.0006463349208366876, + "per_seed": [ + 0.03165938209276646, + 0.03220433469396085, + 0.0332191577879712 + ] + }, + "min": { + "mean": 0.13364263189335665, + "seed_sd": 0.0014895803356980816, + "per_seed": [ + 0.13501964025199414, + 0.1343347653746605, + 0.1315734900534153 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.031634360264676316, + "seed_sd": 0.0003194579934647776, + "per_seed": [ + 0.03163924559485167, + 0.03124068593606353, + 0.03202314926311374 + ] + }, + "min": { + "mean": 0.16317480032642684, + "seed_sd": 0.005029954083515421, + "per_seed": [ + 0.1668417438864708, + 0.166620135307312, + 0.15606252178549768 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.030502137060587606, + "seed_sd": 0.0005318527627935567, + "per_seed": [ + 0.03125416897237301, + 0.0301378277130425, + 0.03011441449634731 + ] + }, + "min": { + "mean": 0.11896193102002144, + "seed_sd": 0.0035537538119840856, + "per_seed": [ + 0.1194708239287138, + 0.1230375561863184, + 0.11437741294503212 + ] + }, + "ci_vs_logit_persist": { + "point": -0.0011572458315640688, + "ci_lo": -0.025759171694517136, + "ci_hi": 0.02485557645559311, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04532940947295477, + "seed_sd": 0.0008049523985396637, + "per_seed": [ + 0.04500848348252475, + 0.04643574636429548, + 0.04454399857204407 + ] + }, + "min": { + "mean": 0.14300246437390646, + "seed_sd": 0.0018359509600638763, + "per_seed": [ + 0.1453766044229269, + 0.14090508222579956, + 0.1427257064729929 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013670025393366814, + "ci_lo": -0.020116280764341354, + "ci_hi": 0.05685994774103165, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.051152476606269676, + "seed_sd": 0.0005932088673280165, + "per_seed": [ + 0.05078884782269597, + 0.05198902431875467, + 0.05067955767735839 + ] + }, + "min": { + "mean": 0.15769512820988893, + "seed_sd": 0.004761395154268062, + "per_seed": [ + 0.15863485671579838, + 0.16299969106912612, + 0.1514508368447423 + ] + }, + "ci_vs_logit_persist": { + "point": 0.019493091851472855, + "ci_lo": -0.02235272154211998, + "ci_hi": 0.07470886409282684, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.05297815694163243, + "seed_sd": 0.0008157224864377802, + "per_seed": [ + 0.052032429119572045, + 0.05402312567457557, + 0.05287891603074968 + ] + }, + "min": { + "mean": 0.15353299925724664, + "seed_sd": 0.001669077283205774, + "per_seed": [ + 0.15557737164199353, + 0.15353264212608336, + 0.15148898400366306 + ] + }, + "ci_vs_logit_persist": { + "point": 0.021318774670362473, + "ci_lo": -0.0209893099963665, + "ci_hi": 0.07794060558080673, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.05419266736134887, + "seed_sd": 0.001034816868778697, + "per_seed": [ + 0.05373580693267286, + 0.05562514401972294, + 0.0532170511316508 + ] + }, + "min": { + "mean": 0.169710739950339, + "seed_sd": 0.001374380223636731, + "per_seed": [ + 0.16799281686544418, + 0.17135706208646298, + 0.16978234089910985 + ] + }, + "ci_vs_logit_persist": { + "point": 0.022533288225531578, + "ci_lo": -0.021409669890999794, + "ci_hi": 0.085838183760643, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.017151434541059038, + "seed_sd": 0.0005099245402746264, + "per_seed": [ + 0.017588791088201106, + 0.016436194092966615, + 0.01742931844200939 + ] + }, + "min": { + "mean": 0.13191584646701812, + "seed_sd": 0.005987460429807287, + "per_seed": [ + 0.1324428305029869, + 0.1389712505042553, + 0.12433345839381219 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.024703472697486482, + "seed_sd": 0.0009234072468616345, + "per_seed": [ + 0.024226003140211106, + 0.02388957338407636, + 0.02599484156817198 + ] + }, + "min": { + "mean": 0.14536115974187852, + "seed_sd": 0.0049015792137517495, + "per_seed": [ + 0.15042544305324554, + 0.14692816361784936, + 0.13872987255454064 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.03023966608258585, + "seed_sd": 0.0006272482495585672, + "per_seed": [ + 0.02945051728747785, + 0.030985087575390934, + 0.03028339338488877 + ] + }, + "min": { + "mean": 0.16369788149992626, + "seed_sd": 0.003030060256484177, + "per_seed": [ + 0.16464764326810838, + 0.16684175208210944, + 0.15960424914956092 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.031634360264676316, + "seed_sd": 0.0003194579934647776, + "per_seed": [ + 0.03163924559485167, + 0.03124068593606353, + 0.03202314926311374 + ] + }, + "min": { + "mean": 0.16317480032642684, + "seed_sd": 0.005029954083515421, + "per_seed": [ + 0.1668417438864708, + 0.166620135307312, + 0.15606252178549768 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.0300429904833436, + "seed_sd": 0.0004771235916662172, + "per_seed": [ + 0.029431943059898914, + 0.030596377397887408, + 0.03010065099224448 + ] + }, + "min": { + "mean": 0.15795367546379566, + "seed_sd": 0.0030669476318221, + "per_seed": [ + 0.16022774055600167, + 0.16001519300043582, + 0.15361809283494948 + ] + } + } + } + }, + "TRAP_FineWeb": { + "containment_k32": 0.06309446548070848, + "is_shuffled": false, + "capability_top10_rate": 0.021649869354236656, + "capability_mean_rank": 4213.683837252706, + "arms": { + "logit": { + "persist": { + "mean": 0.035998970402094224, + "seed_sd": 0.0023369168548651445, + "per_seed": [ + 0.03898334268014878, + 0.03327718588989228, + 0.03573638263624161 + ] + }, + "min": { + "mean": 0.13256995032231014, + "seed_sd": 0.00239487878653843, + "per_seed": [ + 0.13594319708645344, + 0.13114614114165307, + 0.13062051273882389 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.030792023916728795, + "seed_sd": 0.0013693613215323738, + "per_seed": [ + 0.03268995825201273, + 0.030176387797109783, + 0.02950972570106387 + ] + }, + "min": { + "mean": 0.15908302664756777, + "seed_sd": 0.0017480497707151356, + "per_seed": [ + 0.1604225791990757, + 0.1602126158773899, + 0.15661388486623765 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.028455864824354648, + "seed_sd": 0.0008500278521134297, + "per_seed": [ + 0.027818697644397615, + 0.02789164916612208, + 0.02965724766254425 + ] + }, + "min": { + "mean": 0.11889895846446355, + "seed_sd": 0.0026057068605158753, + "per_seed": [ + 0.12236944548785686, + 0.11609070412814618, + 0.11823672577738761 + ] + }, + "ci_vs_logit_persist": { + "point": -0.010527478531002998, + "ci_lo": -0.046978045254945755, + "ci_hi": 0.023138413205742836, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04054118523684641, + "seed_sd": 0.001242298611190451, + "per_seed": [ + 0.04229722085874528, + 0.0397102092159912, + 0.039616125635802746 + ] + }, + "min": { + "mean": 0.13712344678739707, + "seed_sd": 0.0020742287795043157, + "per_seed": [ + 0.14001811109483242, + 0.1352646216750145, + 0.13608760759234428 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0015578410821035504, + "ci_lo": -0.04127275198698044, + "ci_hi": 0.048159196972846985, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04694484616629779, + "seed_sd": 0.0010435078307278341, + "per_seed": [ + 0.048248788248747586, + 0.04689134331420064, + 0.045694406935945155 + ] + }, + "min": { + "mean": 0.15072562620043756, + "seed_sd": 0.0028031406941793026, + "per_seed": [ + 0.1528418455272913, + 0.14676448218524457, + 0.15257055088877677 + ] + }, + "ci_vs_logit_persist": { + "point": 0.007961506024003029, + "ci_lo": -0.04653144255280495, + "ci_hi": 0.07018264383077621, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04928968380360554, + "seed_sd": 0.0006379626847678697, + "per_seed": [ + 0.05010079345665872, + 0.04922628453932702, + 0.048541973414830866 + ] + }, + "min": { + "mean": 0.15071169324219227, + "seed_sd": 0.0024284392989413757, + "per_seed": [ + 0.15404304228723048, + 0.1483231347054243, + 0.14976890273392202 + ] + }, + "ci_vs_logit_persist": { + "point": 0.010306340642273426, + "ci_lo": -0.042944543063640594, + "ci_hi": 0.07253025472164154, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.049697506008669734, + "seed_sd": 0.0017622222859018243, + "per_seed": [ + 0.052007290977053346, + 0.04773217467591166, + 0.049353052373044194 + ] + }, + "min": { + "mean": 0.16596383278568586, + "seed_sd": 0.003502461972567475, + "per_seed": [ + 0.16932464428246022, + 0.16113230511546134, + 0.167434548959136 + ] + }, + "ci_vs_logit_persist": { + "point": 0.010714161209762096, + "ci_lo": -0.04303861781954765, + "ci_hi": 0.07641012221574783, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.01743227130888651, + "seed_sd": 0.000667657228511234, + "per_seed": [ + 0.017036032862961292, + 0.016888167127035558, + 0.018372613936662674 + ] + }, + "min": { + "mean": 0.12653878753383954, + "seed_sd": 0.0007940085906954136, + "per_seed": [ + 0.1272184781730175, + 0.1269730158150196, + 0.1254248686134815 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.023290704687436423, + "seed_sd": 0.0006943921310440697, + "per_seed": [ + 0.024123162403702735, + 0.022423331439495087, + 0.023325620219111443 + ] + }, + "min": { + "mean": 0.1451292984187603, + "seed_sd": 0.00307241162799276, + "per_seed": [ + 0.1472642995417118, + 0.14733912870287896, + 0.14078446701169015 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.02890734188258648, + "seed_sd": 0.00021537221130894048, + "per_seed": [ + 0.02906961878761649, + 0.028602982824668288, + 0.02904942403547466 + ] + }, + "min": { + "mean": 0.15716540664434434, + "seed_sd": 0.0025672619651035647, + "per_seed": [ + 0.1603202149271965, + 0.15714416429400443, + 0.15403184071183204 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.030792023916728795, + "seed_sd": 0.0013693613215323738, + "per_seed": [ + 0.03268995825201273, + 0.030176387797109783, + 0.02950972570106387 + ] + }, + "min": { + "mean": 0.15908302664756777, + "seed_sd": 0.0017480497707151356, + "per_seed": [ + 0.1604225791990757, + 0.1602126158773899, + 0.15661388486623765 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.029335595954520007, + "seed_sd": 0.001349171388058859, + "per_seed": [ + 0.029707744251936674, + 0.027528866264037787, + 0.03077017734758556 + ] + }, + "min": { + "mean": 0.15327000158528487, + "seed_sd": 0.002481950728064471, + "per_seed": [ + 0.1567534752190113, + 0.1519013300538063, + 0.151155199483037 + ] + } + } + } + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "is_shuffled": false, + "capability_top10_rate": 0.017543859649122806, + "capability_mean_rank": 4370.228070175438, + "arms": { + "logit": { + "persist": { + "mean": 0.036840112763457, + "seed_sd": 0.0006125081892508191, + "per_seed": [ + 0.037061250093393026, + 0.03600423510652036, + 0.03745485309045762 + ] + }, + "min": { + "mean": 0.12556015662848952, + "seed_sd": 0.0012670002291384453, + "per_seed": [ + 0.12654305137693883, + 0.12636615782976152, + 0.12377126067876816 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.030533886918177208, + "seed_sd": 0.0007017004080437123, + "per_seed": [ + 0.030557604413479565, + 0.03138118665665388, + 0.029662869684398175 + ] + }, + "min": { + "mean": 0.15122126365701358, + "seed_sd": 0.004345917130561741, + "per_seed": [ + 0.15460145026445388, + 0.1539765328168869, + 0.14508580788969994 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.027587826767315467, + "seed_sd": 0.0015278288845761718, + "per_seed": [ + 0.028825605846941472, + 0.028502664342522622, + 0.025435210112482308 + ] + }, + "min": { + "mean": 0.11221578580637773, + "seed_sd": 0.002388753706827494, + "per_seed": [ + 0.11437908075749874, + 0.1133812103420496, + 0.10888706631958485 + ] + }, + "ci_vs_logit_persist": { + "point": -0.009473425336182117, + "ci_lo": -0.05272747203707695, + "ci_hi": 0.029177239164710045, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.04160388757785161, + "seed_sd": 0.001092341834007453, + "per_seed": [ + 0.04066372397355735, + 0.04101242083124816, + 0.043135517928749326 + ] + }, + "min": { + "mean": 0.13462657121320565, + "seed_sd": 0.002486481636740192, + "per_seed": [ + 0.13735937923192978, + 0.13517659790813924, + 0.13134373649954795 + ] + }, + "ci_vs_logit_persist": { + "point": 0.004542634356766939, + "ci_lo": -0.044434886425733566, + "ci_hi": 0.05856844782829285, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04644438338776429, + "seed_sd": 0.0009598411331915246, + "per_seed": [ + 0.045579977240413426, + 0.045970195811241864, + 0.04778297711163759 + ] + }, + "min": { + "mean": 0.14397299848496914, + "seed_sd": 0.0030620801082744294, + "per_seed": [ + 0.14578439369797708, + 0.14647371210157872, + 0.13966088965535164 + ] + }, + "ci_vs_logit_persist": { + "point": 0.009383130818605423, + "ci_lo": -0.04959649592638016, + "ci_hi": 0.0770169347524643, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.049888986706112824, + "seed_sd": 0.0003970836175988191, + "per_seed": [ + 0.04968298217281699, + 0.0495395680423826, + 0.05044440990313888 + ] + }, + "min": { + "mean": 0.14638157797356446, + "seed_sd": 0.0015567421245738348, + "per_seed": [ + 0.14661192260682582, + 0.14816255308687687, + 0.1443702582269907 + ] + }, + "ci_vs_logit_persist": { + "point": 0.012827733531594276, + "ci_lo": -0.04676033928990364, + "ci_hi": 0.08359326422214508, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.048988748838504154, + "seed_sd": 0.0009010528224351549, + "per_seed": [ + 0.0498417885042727, + 0.04938203799538314, + 0.04774242001585662 + ] + }, + "min": { + "mean": 0.15854955079654853, + "seed_sd": 0.0032592837034044037, + "per_seed": [ + 0.1596286602318287, + 0.16189085133373737, + 0.15412914082407952 + ] + }, + "ci_vs_logit_persist": { + "point": 0.011927495710551739, + "ci_lo": -0.04547123983502388, + "ci_hi": 0.08188861608505249, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.017159060520740847, + "seed_sd": 0.000655968746642691, + "per_seed": [ + 0.017377324285916984, + 0.01626908730249852, + 0.017830769973807038 + ] + }, + "min": { + "mean": 0.1184150109688441, + "seed_sd": 0.005270963206546685, + "per_seed": [ + 0.12299019321799279, + 0.12122402265667916, + 0.11103081703186035 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.02402511574327946, + "seed_sd": 0.001106068947480953, + "per_seed": [ + 0.023224143125116826, + 0.02326202355325222, + 0.025589180551469325 + ] + }, + "min": { + "mean": 0.13570216620961825, + "seed_sd": 0.0014437580866237143, + "per_seed": [ + 0.13714331462979318, + 0.13623417913913727, + 0.1337290048599243 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.027585911316176256, + "seed_sd": 0.0004894312223930928, + "per_seed": [ + 0.02814105988945812, + 0.027666348102502524, + 0.02695032595656812 + ] + }, + "min": { + "mean": 0.14927442769209545, + "seed_sd": 0.0048364365160703335, + "per_seed": [ + 0.15266186445951463, + 0.1527266427874565, + 0.14243477582931519 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.030533886918177208, + "seed_sd": 0.0007017004080437123, + "per_seed": [ + 0.030557604413479565, + 0.03138118665665388, + 0.029662869684398175 + ] + }, + "min": { + "mean": 0.15122126365701358, + "seed_sd": 0.004345917130561741, + "per_seed": [ + 0.15460145026445388, + 0.1539765328168869, + 0.14508580788969994 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.02963750227354467, + "seed_sd": 0.00039773486330009136, + "per_seed": [ + 0.029135168995708227, + 0.0301078389165923, + 0.02966949890833348 + ] + }, + "min": { + "mean": 0.14556186000506083, + "seed_sd": 0.004154347252949874, + "per_seed": [ + 0.14889819622039796, + 0.1480817161500454, + 0.13970566764473916 + ] + } + } + } + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "is_shuffled": false, + "capability_top10_rate": 0.02314296379245987, + "capability_mean_rank": 4401.499066815976, + "arms": { + "logit": { + "persist": { + "mean": 0.028438278171233834, + "seed_sd": 0.001665133926724567, + "per_seed": [ + 0.026223077508620917, + 0.028854017960838974, + 0.030237739044241606 + ] + }, + "min": { + "mean": 0.12067306488752365, + "seed_sd": 0.001669345484239769, + "per_seed": [ + 0.12042249478399754, + 0.12283132374286651, + 0.1187653761357069 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.029294955373431245, + "seed_sd": 0.0006186343222731774, + "per_seed": [ + 0.030160497408360243, + 0.02897259439341724, + 0.028751774318516256 + ] + }, + "min": { + "mean": 0.1527389742434025, + "seed_sd": 0.002334612894349757, + "per_seed": [ + 0.15006249621510506, + 0.15575142204761505, + 0.15240300446748734 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.026148018380627035, + "seed_sd": 0.0008004233053879947, + "per_seed": [ + 0.02531201713718474, + 0.025905078276991844, + 0.027226959727704524 + ] + }, + "min": { + "mean": 0.11367069917420546, + "seed_sd": 0.0015096672898677378, + "per_seed": [ + 0.11164908520877362, + 0.11527602337300777, + 0.114086988940835 + ] + }, + "ci_vs_logit_persist": { + "point": -7.505864050472155e-05, + "ci_lo": -0.0221811905503273, + "ci_hi": 0.02505931816995144, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.03783603221333275, + "seed_sd": 0.0008523357632408745, + "per_seed": [ + 0.037666665157303214, + 0.03688717789482325, + 0.03895425358787179 + ] + }, + "min": { + "mean": 0.13496138068536917, + "seed_sd": 0.0009107089046214201, + "per_seed": [ + 0.13396995030343534, + 0.13474513702094554, + 0.13616905473172664 + ] + }, + "ci_vs_logit_persist": { + "point": 0.011612953618168831, + "ci_lo": -0.01801110990345478, + "ci_hi": 0.05204121768474579, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04517050000528495, + "seed_sd": 0.0012563456946498783, + "per_seed": [ + 0.044943356141448024, + 0.043757994938641784, + 0.04681014893576503 + ] + }, + "min": { + "mean": 0.15203597210347652, + "seed_sd": 0.00233295535119527, + "per_seed": [ + 0.15067538246512413, + 0.1553192686289549, + 0.15011326521635054 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01894741877913475, + "ci_lo": -0.01935446634888649, + "ci_hi": 0.07368820160627365, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04685056325979531, + "seed_sd": 0.0013995171028735106, + "per_seed": [ + 0.047398817632347344, + 0.04492945852689445, + 0.04822341362014413 + ] + }, + "min": { + "mean": 0.14826099450389543, + "seed_sd": 0.0019149920702472346, + "per_seed": [ + 0.14597206562757492, + 0.15065901204943657, + 0.14815190583467483 + ] + }, + "ci_vs_logit_persist": { + "point": 0.02062748745083809, + "ci_lo": -0.018101608380675316, + "ci_hi": 0.07591471076011658, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04756710189394653, + "seed_sd": 0.0008653622927034748, + "per_seed": [ + 0.048763201525434854, + 0.047193299932405355, + 0.04674480422399938 + ] + }, + "min": { + "mean": 0.16033506020903587, + "seed_sd": 0.004772566869652116, + "per_seed": [ + 0.15373621322214603, + 0.1648622889071703, + 0.16240667849779128 + ] + }, + "ci_vs_logit_persist": { + "point": 0.021344024688005447, + "ci_lo": -0.018825750797986984, + "ci_hi": 0.08027508109807968, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.01620841259912898, + "seed_sd": 0.0006576441938368478, + "per_seed": [ + 0.016172033385373652, + 0.017031432152725755, + 0.015421772259287536 + ] + }, + "min": { + "mean": 0.1189204715192318, + "seed_sd": 0.0009472038984793707, + "per_seed": [ + 0.1177787035703659, + 0.11888466998934746, + 0.12009804099798202 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.02177908221880595, + "seed_sd": 0.0010360535920715473, + "per_seed": [ + 0.022809381876140834, + 0.0203617281280458, + 0.022166136652231216 + ] + }, + "min": { + "mean": 0.13699827666083972, + "seed_sd": 0.001159462364650511, + "per_seed": [ + 0.13589717522263528, + 0.13649658635258674, + 0.13860106840729713 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.02838003942742944, + "seed_sd": 0.0004874301272370249, + "per_seed": [ + 0.028032427188009024, + 0.029069361300207673, + 0.028038329794071615 + ] + }, + "min": { + "mean": 0.1553345274180174, + "seed_sd": 0.00290994833265384, + "per_seed": [ + 0.15202872268855572, + 0.15911003202199936, + 0.1548648275434971 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.029294955373431245, + "seed_sd": 0.0006186343222731774, + "per_seed": [ + 0.030160497408360243, + 0.02897259439341724, + 0.028751774318516256 + ] + }, + "min": { + "mean": 0.1527389742434025, + "seed_sd": 0.002334612894349757, + "per_seed": [ + 0.15006249621510506, + 0.15575142204761505, + 0.15240300446748734 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.02827020320110023, + "seed_sd": 0.0010578264472489956, + "per_seed": [ + 0.02976585174910724, + 0.027550159394741057, + 0.02749459845945239 + ] + }, + "min": { + "mean": 0.14998891750971477, + "seed_sd": 0.0019444484587589404, + "per_seed": [ + 0.14945998899638652, + 0.15259036608040333, + 0.14791639745235444 + ] + } + } + } + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "is_shuffled": false, + "capability_top10_rate": 0.02314296379245987, + "capability_mean_rank": 4447.954460619634, + "arms": { + "logit": { + "persist": { + "mean": 0.03133960859073947, + "seed_sd": 0.0012472971224492292, + "per_seed": [ + 0.031733551621437074, + 0.03263167429249734, + 0.029653599858283995 + ] + }, + "min": { + "mean": 0.11887628249824048, + "seed_sd": 0.002486558583239482, + "per_seed": [ + 0.1222337793558836, + 0.11810300163924695, + 0.11629206649959087 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.0287941564184924, + "seed_sd": 0.0017705102577080987, + "per_seed": [ + 0.03068833714351058, + 0.02926520546898246, + 0.02642892664298415 + ] + }, + "min": { + "mean": 0.1507422794898351, + "seed_sd": 0.0010832533849985677, + "per_seed": [ + 0.1521053597331047, + 0.14945520982146263, + 0.15066626891493798 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.02549306258248786, + "seed_sd": 0.0004883206176151186, + "per_seed": [ + 0.025341645814478397, + 0.026152286399155854, + 0.024985255533829333 + ] + }, + "min": { + "mean": 0.11619785837829114, + "seed_sd": 0.0036822469253014645, + "per_seed": [ + 0.12139833569526673, + 0.11383147947490216, + 0.11336375996470452 + ] + }, + "ci_vs_logit_persist": { + "point": -0.006240491289645433, + "ci_lo": -0.036989498883485794, + "ci_hi": 0.022498106583952904, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.036288712111612165, + "seed_sd": 0.0004109425886821703, + "per_seed": [ + 0.03661795160733163, + 0.03653883533552289, + 0.03570934939198196 + ] + }, + "min": { + "mean": 0.1264076568186283, + "seed_sd": 0.0031150765432869467, + "per_seed": [ + 0.13078852146863937, + 0.12461918406188488, + 0.12381526492536069 + ] + }, + "ci_vs_logit_persist": { + "point": 0.004555158317089081, + "ci_lo": -0.032625067979097366, + "ci_hi": 0.04584131017327309, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.04547049587902924, + "seed_sd": 0.001014317309138442, + "per_seed": [ + 0.04687285046093166, + 0.0445079555734992, + 0.045030681602656844 + ] + }, + "min": { + "mean": 0.14767617098987101, + "seed_sd": 0.001519346917268398, + "per_seed": [ + 0.14695448726415633, + 0.14978972561657428, + 0.14628430008888244 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013736945576965809, + "ci_lo": -0.03402138873934746, + "ci_hi": 0.07279022783041, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.04594817009444038, + "seed_sd": 0.001959298398339249, + "per_seed": [ + 0.048588567320257424, + 0.04535564049147069, + 0.043900302471593024 + ] + }, + "min": { + "mean": 0.1427968057493369, + "seed_sd": 0.0007363139990125478, + "per_seed": [ + 0.14319085218012334, + 0.1434345178306103, + 0.14176504723727704 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014214618131518364, + "ci_lo": -0.031123919412493706, + "ci_hi": 0.07101155817508698, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.04761502892555048, + "seed_sd": 0.0014062784579078286, + "per_seed": [ + 0.04959265971556306, + 0.04644411192275584, + 0.04680831513833254 + ] + }, + "min": { + "mean": 0.14846517232557138, + "seed_sd": 0.0016306080186266531, + "per_seed": [ + 0.15068337209522725, + 0.14790198095142842, + 0.14681016393005847 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01588147133588791, + "ci_lo": -0.03258395195007324, + "ci_hi": 0.07852833718061447, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.01598271708159397, + "seed_sd": 0.000777202775754075, + "per_seed": [ + 0.015771681279875337, + 0.017022399930283427, + 0.015154070034623147 + ] + }, + "min": { + "mean": 0.11489870188136896, + "seed_sd": 0.0034120073281580614, + "per_seed": [ + 0.11885747164487839, + 0.11530862599611283, + 0.11053000800311566 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.021565832135577997, + "seed_sd": 0.0015760037229555312, + "per_seed": [ + 0.023704786039888858, + 0.01995382523164153, + 0.0210388851352036 + ] + }, + "min": { + "mean": 0.13434698258837063, + "seed_sd": 0.001185314725469481, + "per_seed": [ + 0.13602032884955406, + 0.1335962787270546, + 0.13342434018850327 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.029046525332766276, + "seed_sd": 0.0005461273471141931, + "per_seed": [ + 0.029802078963257372, + 0.028530052257701756, + 0.028807444777339698 + ] + }, + "min": { + "mean": 0.15180259943008423, + "seed_sd": 0.002243813011760901, + "per_seed": [ + 0.15450985133647918, + 0.15188254788517952, + 0.14901539906859398 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.0287941564184924, + "seed_sd": 0.0017705102577080987, + "per_seed": [ + 0.03068833714351058, + 0.02926520546898246, + 0.02642892664298415 + ] + }, + "min": { + "mean": 0.1507422794898351, + "seed_sd": 0.0010832533849985677, + "per_seed": [ + 0.1521053597331047, + 0.14945520982146263, + 0.15066626891493798 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.0288393650824825, + "seed_sd": 0.002163347844412154, + "per_seed": [ + 0.031801176350563766, + 0.028022437123581767, + 0.02669448177330196 + ] + }, + "min": { + "mean": 0.13879335001111032, + "seed_sd": 0.0008497220659118813, + "per_seed": [ + 0.1381180226802826, + 0.13827020525932313, + 0.1399918220937252 + ] + } + } + } + }, + "SHUFFLED_Pile-CC": { + "containment_k32": 0.9276060773863086, + "is_shuffled": true, + "capability_top10_rate": 0.006718924972004479, + "capability_mean_rank": 4539.5617767823815, + "arms": { + "logit": { + "persist": { + "mean": 0.020351312107717, + "seed_sd": 0.000482612292566415, + "per_seed": [ + 0.020097647863440215, + 0.021026881667785345, + 0.01992940679192543 + ] + }, + "min": { + "mean": 0.09142902692159016, + "seed_sd": 0.003582667498722964, + "per_seed": [ + 0.08644843250513076, + 0.09311401732265949, + 0.09472463093698025 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.02114899874043961, + "seed_sd": 0.002019063471942637, + "per_seed": [ + 0.022680402873083948, + 0.0224704063963145, + 0.01829618695192039 + ] + }, + "min": { + "mean": 0.11579039630790551, + "seed_sd": 0.005090350330952079, + "per_seed": [ + 0.10880363546311855, + 0.12078590169548989, + 0.1177816517651081 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.020016702295591434, + "seed_sd": 0.0009777985327617273, + "per_seed": [ + 0.020015397132374345, + 0.021214908082038166, + 0.018819801672361792 + ] + }, + "min": { + "mean": 0.08711997345089913, + "seed_sd": 0.004120376933501696, + "per_seed": [ + 0.08290965184569359, + 0.09271385222673416, + 0.08573641628026962 + ] + }, + "ci_vs_logit_persist": { + "point": -8.094674558378756e-05, + "ci_lo": -0.01984638348221779, + "ci_hi": 0.02272208034992218, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.028413841927734513, + "seed_sd": 0.0021243530746435806, + "per_seed": [ + 0.028896221611648797, + 0.030740685644559562, + 0.025604618526995183 + ] + }, + "min": { + "mean": 0.09340037666261196, + "seed_sd": 0.0035719766328911524, + "per_seed": [ + 0.09174824655055999, + 0.09836061336100102, + 0.09009227007627488 + ] + }, + "ci_vs_logit_persist": { + "point": 0.008316193707287312, + "ci_lo": -0.017632903531193733, + "ci_hi": 0.041986677795648575, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.03509347875757764, + "seed_sd": 0.0021110244575648406, + "per_seed": [ + 0.03446822033729404, + 0.03793423480819911, + 0.03287798112723976 + ] + }, + "min": { + "mean": 0.10783494816472133, + "seed_sd": 0.005252777684393637, + "per_seed": [ + 0.10120301451534033, + 0.11404929421842099, + 0.10825253576040268 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014995826408267021, + "ci_lo": -0.018552424386143684, + "ci_hi": 0.0634133443236351, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.033768538812485836, + "seed_sd": 0.0022614134406113017, + "per_seed": [ + 0.034108439553529026, + 0.03635255589615553, + 0.030844620987772943 + ] + }, + "min": { + "mean": 0.10412973879526058, + "seed_sd": 0.0045382061828327346, + "per_seed": [ + 0.09992853049188852, + 0.11043217107653618, + 0.10202851481735706 + ] + }, + "ci_vs_logit_persist": { + "point": 0.013670889660716057, + "ci_lo": -0.017414817586541176, + "ci_hi": 0.056603047996759415, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.03425933218871554, + "seed_sd": 0.002765070688948554, + "per_seed": [ + 0.03651014384813607, + 0.03590318781789392, + 0.030364664900116623 + ] + }, + "min": { + "mean": 0.11891029054919879, + "seed_sd": 0.00466408272762991, + "per_seed": [ + 0.1129979819059372, + 0.12439902387559414, + 0.11933386586606502 + ] + }, + "ci_vs_logit_persist": { + "point": 0.014161686412990093, + "ci_lo": -0.01854432187974453, + "ci_hi": 0.05955292657017708, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.012213658029213547, + "seed_sd": 0.0010317602906604268, + "per_seed": [ + 0.01204364150762558, + 0.013553702039644122, + 0.011043630540370941 + ] + }, + "min": { + "mean": 0.08803458561499913, + "seed_sd": 0.0026288171828149384, + "per_seed": [ + 0.0849973052740097, + 0.09140989370644093, + 0.08769655786454678 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.016165042187397678, + "seed_sd": 0.0017890802475037835, + "per_seed": [ + 0.01606782111339271, + 0.01840320136398077, + 0.014024104084819555 + ] + }, + "min": { + "mean": 0.1049199666827917, + "seed_sd": 0.003330751967607705, + "per_seed": [ + 0.10152823738753795, + 0.10944655425846576, + 0.1037851084023714 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.022130672191269695, + "seed_sd": 0.0009317529902869157, + "per_seed": [ + 0.022605451196432112, + 0.022957793902605772, + 0.0208287714747712 + ] + }, + "min": { + "mean": 0.11448084997634093, + "seed_sd": 0.004934851027272841, + "per_seed": [ + 0.10770402140915394, + 0.11931324414908887, + 0.11642528437078 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.02114899874043961, + "seed_sd": 0.002019063471942637, + "per_seed": [ + 0.022680402873083948, + 0.0224704063963145, + 0.01829618695192039 + ] + }, + "min": { + "mean": 0.11579039630790551, + "seed_sd": 0.005090350330952079, + "per_seed": [ + 0.10880363546311855, + 0.12078590169548989, + 0.1177816517651081 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.020185331348329782, + "seed_sd": 0.0013919509527218776, + "per_seed": [ + 0.021655184775590898, + 0.020584397297352553, + 0.0183164119720459 + ] + }, + "min": { + "mean": 0.10957249738276005, + "seed_sd": 0.00424440750920036, + "per_seed": [ + 0.1044944766908884, + 0.11488328911364079, + 0.10933972634375096 + ] + } + } + } + } + }, + "crossings": { + "Github": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [ + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "USPTO_Backgrounds", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "crosses": false + }, + "Wikipedia_en": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "StackExchange": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "Pile-CC": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "USPTO_Backgrounds": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + } + }, + "controls": { + "C1_Q0_reproduces_e33": { + "logit_e33": 0.02843950316309929, + "logit_here": 0.028439503419212996, + "abs_diff": 2.561137066314778e-10, + "tolerance": 1e-06, + "fires": true, + "note": "Q0 is the no-prefix rung and MUST equal E33 exactly. A mismatch means the prefixing path altered the base measurement and the ladder is void." + }, + "C2_shuf_below_everything": { + "per_rung": { + "Q0": false, + "Pile-CC": false, + "StackExchange": false, + "Wikipedia_en": false, + "USPTO_Backgrounds": false, + "Github": false, + "CONTROL_PubMed_2023": false, + "OOD_Wikipedia_2023": false, + "TRAP_FineWeb": false, + "OOD_News_2024": false, + "OOD_arXiv_2023": false, + "OOD_CommonPile": false, + "SHUFFLED_Pile-CC": false + }, + "fires": false, + "WHY_IT_DOES_NOT_FIRE_AND_WHY_THAT_IS_NOT_LIKE_FOR_LIKE": "this control builds its derangement from PILE-CC's operator and then compares it to EVERY corpus's operator (R4d / F-7). 'A derangement beats Github's operator' is therefore a comparison between two different corpora's operators, not between an operator and its own derangement. C2b below is the like-for-like form." + }, + "C2b_own_derangement_below_its_own_operator": { + "per_rung_per_corpus": { + "Q0": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Pile-CC": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "StackExchange": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Wikipedia_en": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "USPTO_Backgrounds": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Github": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "CONTROL_PubMed_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_Wikipedia_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "TRAP_FineWeb": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_News_2024": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_arXiv_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_CommonPile": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "SHUFFLED_Pile-CC": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + } + }, + "n_cells": 65, + "n_cells_where_derangement_wins": 0, + "required": "0 cells where the derangement is at or above its own operator", + "fires": true, + "like_for_like_evidence_already_on_disk": "e33_logit_baseline_410m_v2: Github 0.027926 vs its own jshuf 0.018142; e54 C2: 0 of 15 and 0 of 120" + }, + "C3_capability": { + "REGISTERED_FORM_IS_DEGENERATE": "the pre-registered rule is top-k accuracy < 50% of Q0, but Q0's top-10 rate is 0.020156774916013438. The eval targets are latent concepts, not emitted tokens, so this quantity is floored at Q0 and the rule cannot fire. NOT reinterpreted silently \u2014 adjudication below uses mean rank, which is not floored, and the operator is asked to confirm.", + "registered_top10_rate": { + "q0": 0.020156774916013438, + "per_rung": { + "Q0": 0.020156774916013438, + "Pile-CC": 0.022396416573348264, + "StackExchange": 0.01866368047779022, + "Wikipedia_en": 0.026502426278462114, + "USPTO_Backgrounds": 0.024262784621127288, + "Github": 0.020530048525569243, + "CONTROL_PubMed_2023": 0.019036954087346025, + "OOD_Wikipedia_2023": 0.03471444568868981, + "TRAP_FineWeb": 0.021649869354236656, + "OOD_News_2024": 0.017543859649122806, + "OOD_arXiv_2023": 0.02314296379245987, + "OOD_CommonPile": 0.02314296379245987, + "SHUFFLED_Pile-CC": 0.006718924972004479 + }, + "usable": false + }, + "adjudicated_mean_rank": { + "q0": 4224.63045912654, + "per_rung": { + "Q0": 4224.63045912654, + "Pile-CC": 4309.144083613289, + "StackExchange": 4674.715192235909, + "Wikipedia_en": 4231.8185890257555, + "USPTO_Backgrounds": 4211.015304217992, + "Github": 4797.154535274356, + "CONTROL_PubMed_2023": 4409.058230683091, + "OOD_Wikipedia_2023": 4096.241508025383, + "TRAP_FineWeb": 4213.683837252706, + "OOD_News_2024": 4370.228070175438, + "OOD_arXiv_2023": 4401.499066815976, + "OOD_CommonPile": 4447.954460619634, + "SHUFFLED_Pile-CC": 4539.5617767823815 + }, + "rule": "a rung collapses if its mean target rank exceeds 2x Q0's" + }, + "collapsed_rungs": [], + "note": "a rung whose capability collapsed cannot distinguish lens failure from model failure" + }, + "C5_self_distance": { + "value": 0.0, + "fires": true, + "note": "D_act(J,J)=0 by construction; measured 0.0 in t23" + } + }, + "VERDICT": "REJECT S3 (the publishable null) \u2014 no fitting corpus produces a J-lens curve that goes strictly below the logit lens at any measured rung. Shift in the READ distribution degrades both lenses together; the averaged derivative is no more shift-fragile than not fitting at all.", + "provenance": { + "schema_version": 1, + "experiment": "E36", + "utc": "2026-08-20T09:45:45+00:00", + "script": { + "path": "experiments/t36_qladder.py", + "sha256": "ef457ce679e6c6b6883c6c90e1e7f8ead76c94708e623904c6f9b9d208b3ef45", + "n_lines": 594 + }, + "git": { + "commit": "0085188b142f807a95aed2394063dae7009cf1b7", + "commit_short": "0085188b142f", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/e36_qladder_410m_rstrip.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "aee92baa8be53162e66b7c00eec04f9286b40c2afaf4dbb2dbe8c1782597cc28", + "describe": "0085188-dirty" + }, + "argv": [ + "experiments/t36_qladder.py", + "--device", + "cpu", + "--workers", + "3", + "--out", + "results/e36_qladder_410m_c1arm.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48b_exposure_growth.json", + "sha256": "30ba5ecd395f1cdff653201c634a135a6d0d41e4b12c359a60c307cd85ad7d95", + "bytes": 32715, + "exists": true + }, + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n400_s0.pt", + "sha256": "4c87ed9a5f778ef0280a17ff0df5e623e8732fb7b706936fcc7caf624979511a", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n400_s0.pt", + "sha256": "5e2af80098d34c57e6cdbfcf680a9c9303e18daae91b1577d3e9222e7eb4952b", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n400_s0.pt", + "sha256": "3c3bbbca13ef5891fb19ce2c552334ebf12263c905f25545bc36a831b4bd2a00", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n400_s0.pt", + "sha256": "4754d23b9e6eb16ac8159661cc393328374d8cc8d286bbb9c971ee95166fca62", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0.pt", + "sha256": "4bc59371a760c85c5570f68e00a6e41a65a1ff7168079fc9ab5a695e2d70987b", + "bytes": 46145173, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "de912bf57dcf05f686d1596f921eba020dae82bd81b9e1517380708f3339280c" + } +} \ No newline at end of file diff --git a/results/e36_qladder_410m_rstrip.json b/results/e36_qladder_410m_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..dd169f911c0e19d2ec5648c90676a35babbf013c --- /dev/null +++ b/results/e36_qladder_410m_rstrip.json @@ -0,0 +1,4316 @@ +{ + "experiment": "E36 \u2014 the Q-ladder: vary the READ distribution, hold J^P fixed", + "prereg": "docs/experiments/preregs/superseded/PREREG_E36_QLADDER.md (2026-08-13, before any E36 number existed)", + "prereg_r4b": "docs/experiments/preregs/R4b_e36_rstrip.md", + "status": "PRE-REGISTERED", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`. jacobian-lens/data/evaluations/README.md; src/anchor_evals.py:228", + "decision_rule_verbatim_R4b": "REJECT STANDS: the fitted operator is flatter on >= 4 of 5 rungs. Section 3.1 is unchanged. | REJECT OVERTURNED: flatter on <= 1 of 5. STOP and alert. | UNCLEAR: 2 or 3 of 5. Report and stop. Do not re-cut.", + "PREREG_C4_WAS_NEVER_CODED": "PREREG_E36_QLADDER.md:133 registers C4, the prefix-length control -- a prefix of the same 128 tokens drawn from the FITTING corpus P -- and :136 calls it load-bearing: 'Without C4 the entire ladder could be a prefix-length artifact.' No C4 key exists in any E36 results file. Surfaced by R4e; recorded here so the omission travels with the result rather than being rediscovered.", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "prefix_tokens": 128, + "n_items": 541, + "n_pairs": 893, + "rungs": [ + "Q0", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "USPTO_Backgrounds", + "Github", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "smoke": false, + "x_axis": { + "source": "e48b_exposure_growth.json", + "k": 32, + "coverage_shards": 20, + "note": "measured containment, not a rung number \u2014 this is what E35/M1 was for" + }, + "unavailable_rung": { + "Q5_non_pile_script": "no non-Latin-script corpus is on disk; the ladder does not extend to script-level shift. Recorded, not silently omitted." + }, + "declared_bias": "we shift the READ distribution, not the FIT distribution. EQ2's bias term is ||J^P - J^Q|| where J^Q is the operator that WOULD have been fitted on Q; we never fit J^Q here. E48 supplies that half.", + "ladder": { + "Q0": { + "containment_k32": null, + "is_shuffled": false, + "capability_top10_rate": 0.06942889137737962, + "capability_mean_rank": 3124.912653975364, + "arms": { + "logit": { + "persist": { + "mean": 0.08317913140635938, + "seed_sd": 0.0, + "per_seed": [ + 0.08317913140635938, + 0.08317913140635938, + 0.08317913140635938 + ] + }, + "min": { + "mean": 0.19810852520167826, + "seed_sd": 0.0, + "per_seed": [ + 0.19810852520167826, + 0.19810852520167826, + 0.19810852520167826 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08682075466495007, + "seed_sd": 0.0, + "per_seed": [ + 0.08682075466495007, + 0.08682075466495007, + 0.08682075466495007 + ] + }, + "min": { + "mean": 0.27424257025122645, + "seed_sd": 0.0, + "per_seed": [ + 0.27424257025122645, + 0.27424257025122645, + 0.27424257025122645 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.10069348337128758, + "seed_sd": 0.0, + "per_seed": [ + 0.10069348337128758, + 0.10069348337128758, + 0.10069348337128758 + ] + }, + "min": { + "mean": 0.24063372761011123, + "seed_sd": 0.0, + "per_seed": [ + 0.24063372761011123, + 0.24063372761011123, + 0.24063372761011123 + ] + }, + "ci_vs_logit_persist": { + "point": 0.017514346167445183, + "ci_lo": -0.010132446885108948, + "ci_hi": 0.04753753915429115, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10518511298578233, + "seed_sd": 0.0, + "per_seed": [ + 0.10518511298578233, + 0.10518511298578233, + 0.10518511298578233 + ] + }, + "min": { + "mean": 0.24231685139238834, + "seed_sd": 0.0, + "per_seed": [ + 0.24231685139238834, + 0.24231685139238834, + 0.24231685139238834 + ] + }, + "ci_vs_logit_persist": { + "point": 0.022005977109074593, + "ci_lo": -0.0019287060713395476, + "ci_hi": 0.05520264431834221, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.1172195934690535, + "seed_sd": 0.0, + "per_seed": [ + 0.1172195934690535, + 0.1172195934690535, + 0.1172195934690535 + ] + }, + "min": { + "mean": 0.26687923111021516, + "seed_sd": 0.0, + "per_seed": [ + 0.26687923111021516, + 0.26687923111021516, + 0.26687923111021516 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03404046222567558, + "ci_lo": 0.002237852895632386, + "ci_hi": 0.08252142369747162, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.11135949518065899, + "seed_sd": 0.0, + "per_seed": [ + 0.11135949518065899, + 0.11135949518065899, + 0.11135949518065899 + ] + }, + "min": { + "mean": 0.2618883144110441, + "seed_sd": 0.0, + "per_seed": [ + 0.2618883144110441, + 0.2618883144110441, + 0.2618883144110441 + ] + }, + "ci_vs_logit_persist": { + "point": 0.028180357068777084, + "ci_lo": -0.003070030827075243, + "ci_hi": 0.07631673663854599, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.12077938998118043, + "seed_sd": 0.0, + "per_seed": [ + 0.12077938998118043, + 0.12077938998118043, + 0.12077938998118043 + ] + }, + "min": { + "mean": 0.2621975053101778, + "seed_sd": 0.0, + "per_seed": [ + 0.2621975053101778, + 0.2621975053101778, + 0.2621975053101778 + ] + }, + "ci_vs_logit_persist": { + "point": 0.037600256502628326, + "ci_lo": 0.0052227480337023735, + "ci_hi": 0.08806127309799194, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.08321978356689215, + "seed_sd": 0.0, + "per_seed": [ + 0.08321978356689215, + 0.08321978356689215, + 0.08321978356689215 + ] + }, + "min": { + "mean": 0.23048616349697112, + "seed_sd": 0.0, + "per_seed": [ + 0.23048616349697112, + 0.23048616349697112, + 0.23048616349697112 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.08163625672459603, + "seed_sd": 0.0, + "per_seed": [ + 0.08163625672459603, + 0.08163625672459603, + 0.08163625672459603 + ] + }, + "min": { + "mean": 0.2478497453033924, + "seed_sd": 0.0, + "per_seed": [ + 0.2478497453033924, + 0.2478497453033924, + 0.2478497453033924 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.0877299316925928, + "seed_sd": 0.0, + "per_seed": [ + 0.0877299316925928, + 0.0877299316925928, + 0.0877299316925928 + ] + }, + "min": { + "mean": 0.267060199379921, + "seed_sd": 0.0, + "per_seed": [ + 0.267060199379921, + 0.267060199379921, + 0.267060199379921 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08682075466495007, + "seed_sd": 0.0, + "per_seed": [ + 0.08682075466495007, + 0.08682075466495007, + 0.08682075466495007 + ] + }, + "min": { + "mean": 0.27424257025122645, + "seed_sd": 0.0, + "per_seed": [ + 0.27424257025122645, + 0.27424257025122645, + 0.27424257025122645 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.09018322564661503, + "seed_sd": 0.0, + "per_seed": [ + 0.09018322564661503, + 0.09018322564661503, + 0.09018322564661503 + ] + }, + "min": { + "mean": 0.25691011399030683, + "seed_sd": 0.0, + "per_seed": [ + 0.25691011399030683, + 0.25691011399030683, + 0.25691011399030683 + ] + } + } + } + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "is_shuffled": false, + "capability_top10_rate": 0.07801418439716312, + "capability_mean_rank": 3019.8409854423294, + "arms": { + "logit": { + "persist": { + "mean": 0.08086661498527974, + "seed_sd": 0.0021998873920887458, + "per_seed": [ + 0.07940697956364602, + 0.08397579293232411, + 0.07921707245986909 + ] + }, + "min": { + "mean": 0.20627504698932173, + "seed_sd": 0.0032399405941399215, + "per_seed": [ + 0.21083327420055867, + 0.20439935140311719, + 0.2035925153642893 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08836938155194124, + "seed_sd": 0.002330483112489207, + "per_seed": [ + 0.08519333917647601, + 0.09071982875466347, + 0.08919497672468424 + ] + }, + "min": { + "mean": 0.2775926634669304, + "seed_sd": 0.005697794262257156, + "per_seed": [ + 0.28321693018078803, + 0.27977783009409907, + 0.2697832301259041 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.09989952286705375, + "seed_sd": 0.0012831309902949053, + "per_seed": [ + 0.0987907842732966, + 0.10169794270768762, + 0.09920984162017703 + ] + }, + "min": { + "mean": 0.23854513478775818, + "seed_sd": 0.0015901467127034945, + "per_seed": [ + 0.23812507279217243, + 0.24066841192543506, + 0.2368419196456671 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020492542535066605, + "ci_lo": -0.0005576495314016938, + "ci_hi": 0.047156982123851776, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.1067790173071747, + "seed_sd": 0.0017067091704993612, + "per_seed": [ + 0.10666529561858624, + 0.10892384003382176, + 0.1047479162691161 + ] + }, + "min": { + "mean": 0.2444565334667762, + "seed_sd": 0.004115384685076093, + "per_seed": [ + 0.24709751680493355, + 0.24762753285467626, + 0.23864455074071883 + ] + }, + "ci_vs_logit_persist": { + "point": 0.02737203799188137, + "ci_lo": 0.0017164096934720874, + "ci_hi": 0.06457997858524323, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.1151930189691484, + "seed_sd": 0.003219008576951259, + "per_seed": [ + 0.1144216456450522, + 0.11946416078135372, + 0.11169325048103929 + ] + }, + "min": { + "mean": 0.27579693173368774, + "seed_sd": 0.0019146981436855507, + "per_seed": [ + 0.27582685425877573, + 0.27812684401869775, + 0.2734370969235897 + ] + }, + "ci_vs_logit_persist": { + "point": 0.035786036401987076, + "ci_lo": 0.004759902600198984, + "ci_hi": 0.08430489152669907, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.11517221195002397, + "seed_sd": 0.00437218681522132, + "per_seed": [ + 0.11302485624328255, + 0.1212674108799547, + 0.11122436872683465 + ] + }, + "min": { + "mean": 0.27111224047839644, + "seed_sd": 0.0047988576337443575, + "per_seed": [ + 0.27377016432583334, + 0.27519115544855594, + 0.2643754016608 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03576522693037987, + "ci_lo": 0.002816121093928814, + "ci_hi": 0.0879293903708458, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.11731845908798277, + "seed_sd": 0.0032655763087503964, + "per_seed": [ + 0.11835227617993951, + 0.12069954997859896, + 0.11290355110540987 + ] + }, + "min": { + "mean": 0.2714436214417219, + "seed_sd": 0.004536518578682427, + "per_seed": [ + 0.27428063862025737, + 0.2750084351748228, + 0.2650417905300856 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03791147097945213, + "ci_lo": 0.004563671536743641, + "ci_hi": 0.09098520874977112, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.08007450182922185, + "seed_sd": 0.002089841937325202, + "per_seed": [ + 0.07735197609290481, + 0.08243178136181087, + 0.08043974803294987 + ] + }, + "min": { + "mean": 0.24019063835342724, + "seed_sd": 0.003019628280511557, + "per_seed": [ + 0.24416115805506705, + 0.2395667977631092, + 0.2368439592421055 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.083067851079007, + "seed_sd": 0.0030261024096004635, + "per_seed": [ + 0.08033571224659682, + 0.08728655204176902, + 0.08158128894865513 + ] + }, + "min": { + "mean": 0.2502322519818942, + "seed_sd": 0.005808153049338236, + "per_seed": [ + 0.25344939306378367, + 0.25516886785626414, + 0.24207849502563478 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.08648697358245651, + "seed_sd": 0.003926687503954713, + "per_seed": [ + 0.08448808554094285, + 0.0919732480077073, + 0.08299958719871939 + ] + }, + "min": { + "mean": 0.2731296184162299, + "seed_sd": 0.0036764650873873715, + "per_seed": [ + 0.2730236954987049, + 0.2776843771338463, + 0.2686807826161385 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08836938155194124, + "seed_sd": 0.002330483112489207, + "per_seed": [ + 0.08519333917647601, + 0.09071982875466347, + 0.08919497672468424 + ] + }, + "min": { + "mean": 0.2775926634669304, + "seed_sd": 0.005697794262257156, + "per_seed": [ + 0.28321693018078803, + 0.27977783009409907, + 0.2697832301259041 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.08542103620711713, + "seed_sd": 0.0030585139176426126, + "per_seed": [ + 0.08202834129333496, + 0.08944095354527234, + 0.08479381378274412 + ] + }, + "min": { + "mean": 0.27069795404871305, + "seed_sd": 0.005367308829141551, + "per_seed": [ + 0.2742910757660866, + 0.2746918261051178, + 0.26311096027493475 + ] + } + } + } + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "is_shuffled": false, + "capability_top10_rate": 0.06270996640537514, + "capability_mean_rank": 3130.946248600224, + "arms": { + "logit": { + "persist": { + "mean": 0.06526795584553231, + "seed_sd": 0.0014831304461918116, + "per_seed": [ + 0.06695626005530357, + 0.06334596537053586, + 0.06550164211075753 + ] + }, + "min": { + "mean": 0.1939402662217617, + "seed_sd": 0.0033251814287801945, + "per_seed": [ + 0.1969407718628645, + 0.18930425643920898, + 0.19557577036321164 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.07021416208396355, + "seed_sd": 0.0007374888669363916, + "per_seed": [ + 0.07105098403990269, + 0.07033485360443592, + 0.06925664860755205 + ] + }, + "min": { + "mean": 0.2535843513906002, + "seed_sd": 0.004680705809440288, + "per_seed": [ + 0.2600083999335766, + 0.248989500105381, + 0.251755154132843 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.08316785292699932, + "seed_sd": 0.001180273311162483, + "per_seed": [ + 0.08194298360031098, + 0.0827982799615711, + 0.08476229521911591 + ] + }, + "min": { + "mean": 0.22392050127188365, + "seed_sd": 0.004742006916078918, + "per_seed": [ + 0.2271083001047373, + 0.21721696406602858, + 0.22743623964488507 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01621158979833126, + "ci_lo": -0.007110326085239649, + "ci_hi": 0.04314352199435234, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.08932039183564484, + "seed_sd": 0.002287620114599491, + "per_seed": [ + 0.08985154654365032, + 0.08629108245950193, + 0.09181854650378227 + ] + }, + "min": { + "mean": 0.22070264940460524, + "seed_sd": 0.004530491575118153, + "per_seed": [ + 0.22081239745020867, + 0.2150998931378126, + 0.2261956576257944 + ] + }, + "ci_vs_logit_persist": { + "point": 0.02236413210630417, + "ci_lo": -0.0009431808139197528, + "ci_hi": 0.053652770817279816, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.10094482178489367, + "seed_sd": 0.000572587038223978, + "per_seed": [ + 0.1017530114389956, + 0.10058438456617295, + 0.10049706934951246 + ] + }, + "min": { + "mean": 0.2503357617805402, + "seed_sd": 0.0044537234357113365, + "per_seed": [ + 0.2515347760170698, + 0.24438132680952548, + 0.25509118251502516 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03398855775594711, + "ci_lo": 0.0028232831973582506, + "ci_hi": 0.08294752240180969, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.09803160264467199, + "seed_sd": 0.00214398718865389, + "per_seed": [ + 0.09976591444574297, + 0.0950105796335265, + 0.09931831385474651 + ] + }, + "min": { + "mean": 0.23863771334290504, + "seed_sd": 0.003240750980277118, + "per_seed": [ + 0.23799834363162517, + 0.2350271176546812, + 0.24288767874240874 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03107534348964691, + "ci_lo": 0.001411188393831253, + "ci_hi": 0.0766524076461792, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.10184808447957039, + "seed_sd": 0.0009031758749590254, + "per_seed": [ + 0.10228330618701875, + 0.10267043830826879, + 0.10059050894342363 + ] + }, + "min": { + "mean": 0.252543918043375, + "seed_sd": 0.005146247118674712, + "per_seed": [ + 0.2534764088690281, + 0.2458267819136381, + 0.25832856334745885 + ] + }, + "ci_vs_logit_persist": { + "point": 0.034891821444034576, + "ci_lo": 0.0009394892840646207, + "ci_hi": 0.08712325245141983, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.06264353417791427, + "seed_sd": 0.0011789678546419234, + "per_seed": [ + 0.06100974848959595, + 0.06317231932189316, + 0.06374853472225368 + ] + }, + "min": { + "mean": 0.22403105174501736, + "seed_sd": 0.004610479163800373, + "per_seed": [ + 0.23009200021624565, + 0.21891888231039047, + 0.223082272708416 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.06371138778825601, + "seed_sd": 0.0009896726255240332, + "per_seed": [ + 0.06458775065839291, + 0.06421828251332044, + 0.06232813019305468 + ] + }, + "min": { + "mean": 0.23286404212315878, + "seed_sd": 0.003585309162408412, + "per_seed": [ + 0.23698765337467192, + 0.22824713662266732, + 0.23335733637213707 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.0706400263045604, + "seed_sd": 0.0005751722148003918, + "per_seed": [ + 0.06985205551609397, + 0.0708591942442581, + 0.07120882915332913 + ] + }, + "min": { + "mean": 0.2519735765953859, + "seed_sd": 0.005629116896427742, + "per_seed": [ + 0.2564854800701141, + 0.2440376177430153, + 0.25539763197302817 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.07021416208396355, + "seed_sd": 0.0007374888669363916, + "per_seed": [ + 0.07105098403990269, + 0.07033485360443592, + 0.06925664860755205 + ] + }, + "min": { + "mean": 0.2535843513906002, + "seed_sd": 0.004680705809440288, + "per_seed": [ + 0.2600083999335766, + 0.248989500105381, + 0.251755154132843 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.0678905524003009, + "seed_sd": 0.0015857183969934874, + "per_seed": [ + 0.06782671781256795, + 0.06986378319561481, + 0.06598115619271994 + ] + }, + "min": { + "mean": 0.25049916009108225, + "seed_sd": 0.00588003365117762, + "per_seed": [ + 0.2562478296458721, + 0.24242128878831865, + 0.252828361839056 + ] + } + } + } + }, + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "is_shuffled": false, + "capability_top10_rate": 0.07204180664427025, + "capability_mean_rank": 2939.3516237402014, + "arms": { + "logit": { + "persist": { + "mean": 0.07577144653381158, + "seed_sd": 0.0014940148885986928, + "per_seed": [ + 0.0776408253936097, + 0.07398400406818836, + 0.07568951013963669 + ] + }, + "min": { + "mean": 0.20590047190586727, + "seed_sd": 0.0012763207976937917, + "per_seed": [ + 0.20761289298534394, + 0.20553843006491662, + 0.20455009266734123 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08514144808674852, + "seed_sd": 0.00042352989452602613, + "per_seed": [ + 0.08476794548332692, + 0.08492268957197666, + 0.08573370920494199 + ] + }, + "min": { + "mean": 0.27109660059213636, + "seed_sd": 0.0010203547725377603, + "per_seed": [ + 0.2709772400557995, + 0.2699108891189098, + 0.27240167260169984 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.09716132419804732, + "seed_sd": 0.002206427060381279, + "per_seed": [ + 0.09516720799729228, + 0.09607989583164453, + 0.10023686876520514 + ] + }, + "min": { + "mean": 0.2400886919349432, + "seed_sd": 0.0012747718919220747, + "per_seed": [ + 0.24159727208316326, + 0.24018921442329882, + 0.2384795892983675 + ] + }, + "ci_vs_logit_persist": { + "point": 0.019520502537488937, + "ci_lo": -0.0027302494272589684, + "ci_hi": 0.046593379229307175, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10764469669666142, + "seed_sd": 0.0018259067604536993, + "per_seed": [ + 0.10917605101130903, + 0.10507843110244722, + 0.108679607976228 + ] + }, + "min": { + "mean": 0.24269976727664472, + "seed_sd": 0.003356894983346067, + "per_seed": [ + 0.24235893674194814, + 0.23876945190131665, + 0.24697091318666936 + ] + }, + "ci_vs_logit_persist": { + "point": 0.030003875494003296, + "ci_lo": 0.0017477483488619328, + "ci_hi": 0.06822838634252548, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.115503211474667, + "seed_sd": 0.0012213082603927874, + "per_seed": [ + 0.11701137777417898, + 0.11402011075988412, + 0.11547814588993788 + ] + }, + "min": { + "mean": 0.2724491686870654, + "seed_sd": 0.004171927942248548, + "per_seed": [ + 0.2671830505132675, + 0.27277821525931356, + 0.27738624028861525 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0378623902797699, + "ci_lo": 0.0070224544033408165, + "ci_hi": 0.08547209948301315, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.1151967967239519, + "seed_sd": 0.0011067058334437445, + "per_seed": [ + 0.1166127041913569, + 0.1139112607575953, + 0.11506642522290349 + ] + }, + "min": { + "mean": 0.26569442798693976, + "seed_sd": 0.004937507672255364, + "per_seed": [ + 0.2648704808205366, + 0.2601014614105225, + 0.27211134172976015 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03755597025156021, + "ci_lo": 0.004251394886523485, + "ci_hi": 0.08973889797925949, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.11267888868848483, + "seed_sd": 0.002688113728197998, + "per_seed": [ + 0.11262538670562208, + 0.10941371223889292, + 0.11599756712093949 + ] + }, + "min": { + "mean": 0.26523289096852143, + "seed_sd": 0.0018516073633958305, + "per_seed": [ + 0.26632466800510884, + 0.26262576691806316, + 0.26674823798239233 + ] + }, + "ci_vs_logit_persist": { + "point": 0.035038061439991, + "ci_lo": 0.001977217849344015, + "ci_hi": 0.08776047825813293, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.07674371267203242, + "seed_sd": 0.002953691334227744, + "per_seed": [ + 0.07468890678137541, + 0.07462154924869538, + 0.08092068198602646 + ] + }, + "min": { + "mean": 0.23852406839529672, + "seed_sd": 0.0020096980239819114, + "per_seed": [ + 0.241331647336483, + 0.23750297278165816, + 0.23673758506774903 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.07977201230823994, + "seed_sd": 0.0022602368337765123, + "per_seed": [ + 0.08269383795559407, + 0.07943368889391422, + 0.07718851007521152 + ] + }, + "min": { + "mean": 0.24874673212567966, + "seed_sd": 0.0017739330332369927, + "per_seed": [ + 0.24962188303470612, + 0.24627302289009095, + 0.2503452904522419 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.08285898054794719, + "seed_sd": 0.002007207765018327, + "per_seed": [ + 0.08264860697090626, + 0.08051261033397168, + 0.08541572433896363 + ] + }, + "min": { + "mean": 0.2718578740954399, + "seed_sd": 0.0019349662056385025, + "per_seed": [ + 0.2709145940840244, + 0.27010492235422134, + 0.27455410584807394 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08514144808674852, + "seed_sd": 0.00042352989452602613, + "per_seed": [ + 0.08476794548332692, + 0.08492268957197666, + 0.08573370920494199 + ] + }, + "min": { + "mean": 0.27109660059213636, + "seed_sd": 0.0010203547725377603, + "per_seed": [ + 0.2709772400557995, + 0.2699108891189098, + 0.27240167260169984 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.07904477208697547, + "seed_sd": 0.0012038918571838626, + "per_seed": [ + 0.080739073920995, + 0.07834267073776573, + 0.0780525716021657 + ] + }, + "min": { + "mean": 0.2673769851525625, + "seed_sd": 0.000532086409492499, + "per_seed": [ + 0.267519137263298, + 0.26794584542512895, + 0.2666659727692604 + ] + } + } + } + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "is_shuffled": false, + "capability_top10_rate": 0.07502799552071668, + "capability_mean_rank": 2924.0399402762223, + "arms": { + "logit": { + "persist": { + "mean": 0.07759068199278166, + "seed_sd": 0.0011666562984703965, + "per_seed": [ + 0.0764145004330203, + 0.07717673580627889, + 0.0791808097390458 + ] + }, + "min": { + "mean": 0.2046105229606231, + "seed_sd": 0.0029174879033121185, + "per_seed": [ + 0.20232056677341462, + 0.20278318226337433, + 0.20872781984508038 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08899192212459942, + "seed_sd": 0.0023441637311779045, + "per_seed": [ + 0.09067804652731866, + 0.08567693885415792, + 0.09062078099232167 + ] + }, + "min": { + "mean": 0.2771342965463797, + "seed_sd": 0.002634108153879408, + "per_seed": [ + 0.28021539077162744, + 0.2737804837524891, + 0.27740701511502264 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.09564396337761233, + "seed_sd": 0.0010399742956467269, + "per_seed": [ + 0.09440810857340694, + 0.09557137142401188, + 0.09695241013541818 + ] + }, + "min": { + "mean": 0.23496665929754576, + "seed_sd": 0.003485415856822407, + "per_seed": [ + 0.23197077214717865, + 0.23307481221854687, + 0.23985439352691174 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0192294679582119, + "ci_lo": -0.004338638391345739, + "ci_hi": 0.04692116752266884, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.1069752536015585, + "seed_sd": 0.0007360129532765529, + "per_seed": [ + 0.10777486762963236, + 0.10715253900270909, + 0.10599835417233408 + ] + }, + "min": { + "mean": 0.24053661984701952, + "seed_sd": 0.0037254963523599516, + "per_seed": [ + 0.2397593751549721, + 0.23641238249838353, + 0.24543810188770293 + ] + }, + "ci_vs_logit_persist": { + "point": 0.030560756102204323, + "ci_lo": 0.0023352575954049826, + "ci_hi": 0.06618619710206985, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.11768047941538196, + "seed_sd": 0.0012497164648554815, + "per_seed": [ + 0.11605207053944469, + 0.11789978868328035, + 0.11908957902342081 + ] + }, + "min": { + "mean": 0.26992320250719787, + "seed_sd": 0.003544502467952927, + "per_seed": [ + 0.2676765266805887, + 0.26716587264090774, + 0.2749272082000971 + ] + }, + "ci_vs_logit_persist": { + "point": 0.041265975683927536, + "ci_lo": 0.007857851684093475, + "ci_hi": 0.09114450961351395, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.11413911652440827, + "seed_sd": 0.0009688123448619102, + "per_seed": [ + 0.1132082880474627, + 0.11373386159539223, + 0.11547519993036985 + ] + }, + "min": { + "mean": 0.2648810582235456, + "seed_sd": 0.0029500720964950426, + "per_seed": [ + 0.26342818327248096, + 0.26222057174891233, + 0.26899441964924337 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03772461414337158, + "ci_lo": 0.005051273852586746, + "ci_hi": 0.08566340059041977, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.11762098555918783, + "seed_sd": 0.0016068394776396606, + "per_seed": [ + 0.11615267894230782, + 0.11685315470676869, + 0.11985712302848697 + ] + }, + "min": { + "mean": 0.26637390640874703, + "seed_sd": 0.0028916387530319287, + "per_seed": [ + 0.26364439763128755, + 0.26510148458182814, + 0.2703758370131254 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0412064865231514, + "ci_lo": 0.005755121819674969, + "ci_hi": 0.09506615996360779, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.07828068247375389, + "seed_sd": 0.002112953736200954, + "per_seed": [ + 0.07789396999869495, + 0.07590797161683441, + 0.08104010580573232 + ] + }, + "min": { + "mean": 0.23646583072841168, + "seed_sd": 0.0014998509547624879, + "per_seed": [ + 0.23660235553979875, + 0.23456444256007672, + 0.23823069408535957 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.08320455975675335, + "seed_sd": 0.0015414949743434445, + "per_seed": [ + 0.08389658052474261, + 0.0810682587325573, + 0.08464884001296014 + ] + }, + "min": { + "mean": 0.24728001082936923, + "seed_sd": 0.0026231422456474062, + "per_seed": [ + 0.24617122560739518, + 0.24476858228445053, + 0.250900224596262 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.08710450015496463, + "seed_sd": 0.0012686014536365164, + "per_seed": [ + 0.08694369222503155, + 0.08563744488637895, + 0.08873236335348338 + ] + }, + "min": { + "mean": 0.27541386634111403, + "seed_sd": 0.002181433332958399, + "per_seed": [ + 0.27821816951036454, + 0.2751252181828022, + 0.2728982113301754 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08899192212459942, + "seed_sd": 0.0023441637311779045, + "per_seed": [ + 0.09067804652731866, + 0.08567693885415792, + 0.09062078099232167 + ] + }, + "min": { + "mean": 0.2771342965463797, + "seed_sd": 0.002634108153879408, + "per_seed": [ + 0.28021539077162744, + 0.2737804837524891, + 0.27740701511502264 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.08503780318424106, + "seed_sd": 0.00188935459304277, + "per_seed": [ + 0.08572751712054014, + 0.08245738986879587, + 0.08692850256338716 + ] + }, + "min": { + "mean": 0.26753689236938955, + "seed_sd": 0.004635220650472959, + "per_seed": [ + 0.27263310849666594, + 0.26141816228628156, + 0.2685594063252211 + ] + } + } + } + }, + "Github": { + "containment_k32": 0.9340004374453194, + "is_shuffled": false, + "capability_top10_rate": 0.06532288167226577, + "capability_mean_rank": 3151.3322135125045, + "arms": { + "logit": { + "persist": { + "mean": 0.05914952543874582, + "seed_sd": 0.0009684725906379451, + "per_seed": [ + 0.05975621789693832, + 0.05990959443151951, + 0.057782763987779616 + ] + }, + "min": { + "mean": 0.1873030010610819, + "seed_sd": 0.0019305041766259563, + "per_seed": [ + 0.18631163276731968, + 0.18559569604694842, + 0.19000167436897755 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.06625801414096108, + "seed_sd": 0.0018347112421706203, + "per_seed": [ + 0.06572897005826235, + 0.06872238442301751, + 0.06432268794160337 + ] + }, + "min": { + "mean": 0.25727134744326274, + "seed_sd": 0.006193945233178092, + "per_seed": [ + 0.2528333656489849, + 0.25295002683997153, + 0.26603064984083175 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.07580642585332195, + "seed_sd": 0.004797449142676534, + "per_seed": [ + 0.07094239043071866, + 0.08233467927202583, + 0.07414220785722136 + ] + }, + "min": { + "mean": 0.21830543279647827, + "seed_sd": 0.003875721589255199, + "per_seed": [ + 0.21551329381763934, + 0.215616799518466, + 0.22378620505332947 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01605021208524704, + "ci_lo": -0.004181141033768654, + "ci_hi": 0.04061052203178406, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.08365243996183078, + "seed_sd": 0.0035953916642723236, + "per_seed": [ + 0.080784767633304, + 0.08872256590984762, + 0.08144998634234071 + ] + }, + "min": { + "mean": 0.22284991753598055, + "seed_sd": 0.00253269326534501, + "per_seed": [ + 0.22005704939365386, + 0.22230426035821438, + 0.22618844285607337 + ] + }, + "ci_vs_logit_persist": { + "point": 0.023896221071481705, + "ci_lo": 0.0007319613359868526, + "ci_hi": 0.05715494975447655, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.09797171006600063, + "seed_sd": 0.00307450567240169, + "per_seed": [ + 0.0949385860003531, + 0.1021862336434424, + 0.09679031055420637 + ] + }, + "min": { + "mean": 0.2571307464192311, + "seed_sd": 0.004889540507049013, + "per_seed": [ + 0.25683421827852726, + 0.251296079531312, + 0.26326194144785403 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03821549564599991, + "ci_lo": 0.006458164658397436, + "ci_hi": 0.08842137455940247, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.09229047736153007, + "seed_sd": 0.0032600480607009696, + "per_seed": [ + 0.0917640001513064, + 0.09652032488957048, + 0.08858710704371334 + ] + }, + "min": { + "mean": 0.24890574192007384, + "seed_sd": 0.0038373560064633733, + "per_seed": [ + 0.248624474927783, + 0.24435290992259978, + 0.2537398409098387 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03253426402807236, + "ci_lo": 0.004598523490130901, + "ci_hi": 0.07671785354614258, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.09317711807476978, + "seed_sd": 0.0025733475647324493, + "per_seed": [ + 0.0920425501652062, + 0.09673902108334005, + 0.09074978297576308 + ] + }, + "min": { + "mean": 0.2490005947649479, + "seed_sd": 0.004415958001782838, + "per_seed": [ + 0.24406534545123576, + 0.2481540810316801, + 0.2547823578119278 + ] + }, + "ci_vs_logit_persist": { + "point": 0.033420901745557785, + "ci_lo": 0.00030709756538271904, + "ci_hi": 0.08880078047513962, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.05663115277420729, + "seed_sd": 0.0016901108860377075, + "per_seed": [ + 0.05425179440062493, + 0.058017561491578815, + 0.057624102430418135 + ] + }, + "min": { + "mean": 0.21537433589498203, + "seed_sd": 0.005362937947767536, + "per_seed": [ + 0.20795998126268386, + 0.21769870296120644, + 0.22046432346105577 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.05983376583705346, + "seed_sd": 0.004150241683603618, + "per_seed": [ + 0.05626813508570194, + 0.06565408874303102, + 0.05757907368242741 + ] + }, + "min": { + "mean": 0.23277453730503717, + "seed_sd": 0.0053405437053454345, + "per_seed": [ + 0.2280175007879734, + 0.23007269501686095, + 0.24023341611027718 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.06531875274764995, + "seed_sd": 0.0016432780249716627, + "per_seed": [ + 0.06344507371541112, + 0.06744619680102915, + 0.06506498772650957 + ] + }, + "min": { + "mean": 0.2547997443626324, + "seed_sd": 0.007253955771600285, + "per_seed": [ + 0.247873018309474, + 0.25170985162258147, + 0.26481636315584184 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.06625801414096108, + "seed_sd": 0.0018347112421706203, + "per_seed": [ + 0.06572897005826235, + 0.06872238442301751, + 0.06432268794160337 + ] + }, + "min": { + "mean": 0.25727134744326274, + "seed_sd": 0.006193945233178092, + "per_seed": [ + 0.2528333656489849, + 0.25295002683997153, + 0.26603064984083175 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.06380579161923379, + "seed_sd": 0.0008681817884099598, + "per_seed": [ + 0.06374985061120242, + 0.06489595908205956, + 0.06277156516443938 + ] + }, + "min": { + "mean": 0.24446982716520627, + "seed_sd": 0.008714870491163615, + "per_seed": [ + 0.23416783437132835, + 0.2437620557844639, + 0.25547959133982656 + ] + } + } + } + }, + "CONTROL_PubMed_2023": { + "containment_k32": 0.7927273508312213, + "is_shuffled": false, + "capability_top10_rate": 0.0683090705487122, + "capability_mean_rank": 3107.5569242254574, + "arms": { + "logit": { + "persist": { + "mean": 0.07458710451610387, + "seed_sd": 0.0024374158337087015, + "per_seed": [ + 0.07801449547987431, + 0.0731915871379897, + 0.07255523093044758 + ] + }, + "min": { + "mean": 0.1959550288816293, + "seed_sd": 0.0021303530609239323, + "per_seed": [ + 0.1956109929829836, + 0.19353497512638568, + 0.19871911853551866 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08562730956667414, + "seed_sd": 0.0013783270340776044, + "per_seed": [ + 0.08647896572947503, + 0.08368303179740906, + 0.08671993117313832 + ] + }, + "min": { + "mean": 0.26551368658741314, + "seed_sd": 0.0030281507860926184, + "per_seed": [ + 0.26865466982126235, + 0.2614222399890423, + 0.2664641499519348 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.0947382472300281, + "seed_sd": 0.0015259079760868294, + "per_seed": [ + 0.09665577574633062, + 0.09463674956932663, + 0.09292221637442707 + ] + }, + "min": { + "mean": 0.22320082038640976, + "seed_sd": 0.004550319001056915, + "per_seed": [ + 0.2248064536601305, + 0.21700128614902497, + 0.2277947213500738 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0167237501591444, + "ci_lo": -0.0027792768087238073, + "ci_hi": 0.0395326130092144, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10302141247472416, + "seed_sd": 0.0007937480013514457, + "per_seed": [ + 0.1038065719185397, + 0.10193406853359192, + 0.10332359697204083 + ] + }, + "min": { + "mean": 0.23391008526086807, + "seed_sd": 0.0029378694775510743, + "per_seed": [ + 0.23514403104782106, + 0.22985732294619082, + 0.23672890178859235 + ] + }, + "ci_vs_logit_persist": { + "point": 0.025006914511322975, + "ci_lo": 0.0016595759661868215, + "ci_hi": 0.060689158737659454, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.11017052801325918, + "seed_sd": 0.000789697155569425, + "per_seed": [ + 0.11105547593906522, + 0.11031803945079446, + 0.10913806864991785 + ] + }, + "min": { + "mean": 0.2638506319373846, + "seed_sd": 0.0028974691048503118, + "per_seed": [ + 0.2646347675472498, + 0.25997548513114455, + 0.2669416431337595 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03215603157877922, + "ci_lo": 0.002649535657837987, + "ci_hi": 0.0816304162144661, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.11080300064447025, + "seed_sd": 0.0010650398118962356, + "per_seed": [ + 0.11164718992076814, + 0.10930064073763787, + 0.11146117127500474 + ] + }, + "min": { + "mean": 0.2571348571528991, + "seed_sd": 0.0036640602559295916, + "per_seed": [ + 0.26131184995174406, + 0.2523906961083412, + 0.25770202539861203 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03278850391507149, + "ci_lo": 0.0037948330864310265, + "ci_hi": 0.0811409130692482, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.11306450484165301, + "seed_sd": 0.00112601969941998, + "per_seed": [ + 0.1140851716278121, + 0.11361273652873934, + 0.11149560636840761 + ] + }, + "min": { + "mean": 0.2519092973321676, + "seed_sd": 0.004161898913161638, + "per_seed": [ + 0.254751031473279, + 0.24602462984621526, + 0.2549522306770086 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03505001217126846, + "ci_lo": 0.0027071484364569187, + "ci_hi": 0.08861207962036133, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.07499544559977948, + "seed_sd": 0.0009119453705809948, + "per_seed": [ + 0.0737073868047446, + 0.07569555479567498, + 0.07558339519891888 + ] + }, + "min": { + "mean": 0.23083171943823497, + "seed_sd": 0.0046525819973772895, + "per_seed": [ + 0.2370555154979229, + 0.2258710242807865, + 0.22956861853599547 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.07877365835011005, + "seed_sd": 0.0005259562100541973, + "per_seed": [ + 0.07930671032518148, + 0.07895639464259148, + 0.0780578700825572 + ] + }, + "min": { + "mean": 0.2419613285611073, + "seed_sd": 0.002230785990100578, + "per_seed": [ + 0.24390030577778815, + 0.23883664049208164, + 0.24314703941345214 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.08128409089210133, + "seed_sd": 0.0014862706134781429, + "per_seed": [ + 0.08309342449065298, + 0.07945301465224475, + 0.08130583353340626 + ] + }, + "min": { + "mean": 0.2684160863359769, + "seed_sd": 0.005027433672137837, + "per_seed": [ + 0.2755245015025139, + 0.2647375430911779, + 0.2649862144142389 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08562730956667414, + "seed_sd": 0.0013783270340776044, + "per_seed": [ + 0.08647896572947503, + 0.08368303179740906, + 0.08671993117313832 + ] + }, + "min": { + "mean": 0.26551368658741314, + "seed_sd": 0.0030281507860926184, + "per_seed": [ + 0.26865466982126235, + 0.2614222399890423, + 0.2664641499519348 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.08213502019643784, + "seed_sd": 0.0019072516080071004, + "per_seed": [ + 0.08401826033368706, + 0.07952113579958678, + 0.08286566445603967 + ] + }, + "min": { + "mean": 0.25323704319695634, + "seed_sd": 0.003497714117333183, + "per_seed": [ + 0.258134875446558, + 0.250188597664237, + 0.25138765648007394 + ] + } + } + } + }, + "OOD_Wikipedia_2023": { + "containment_k32": 0.26957534299550123, + "is_shuffled": false, + "capability_top10_rate": 0.08212019410227697, + "capability_mean_rank": 2780.15976110489, + "arms": { + "logit": { + "persist": { + "mean": 0.08000354556522021, + "seed_sd": 0.0007621020286966921, + "per_seed": [ + 0.08030142064671963, + 0.07895758401136846, + 0.08075163203757256 + ] + }, + "min": { + "mean": 0.2132010769098997, + "seed_sd": 0.003446230257170176, + "per_seed": [ + 0.21460544876754284, + 0.2165406182408333, + 0.208457163721323 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08869295563393582, + "seed_sd": 0.0005432286657790841, + "per_seed": [ + 0.08944471448194236, + 0.0884541530162096, + 0.08817999940365553 + ] + }, + "min": { + "mean": 0.2791113016506036, + "seed_sd": 0.005601565237012388, + "per_seed": [ + 0.2854490876197815, + 0.28005831092596056, + 0.2718265064060688 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.1035349782090634, + "seed_sd": 0.0016371246202352419, + "per_seed": [ + 0.10376586942002178, + 0.10541459713131189, + 0.10142446807585656 + ] + }, + "min": { + "mean": 0.24271432707707088, + "seed_sd": 0.003838422798403444, + "per_seed": [ + 0.2465036991983652, + 0.2441857557743788, + 0.23745352625846863 + ] + }, + "ci_vs_logit_persist": { + "point": 0.023233551532030106, + "ci_lo": -0.001245902618393302, + "ci_hi": 0.053249794989824295, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10915367560616383, + "seed_sd": 0.0013954982839404224, + "per_seed": [ + 0.10926335281692445, + 0.11080532502382993, + 0.10739234897773713 + ] + }, + "min": { + "mean": 0.24594851831595102, + "seed_sd": 0.0043414311147938495, + "per_seed": [ + 0.250863828882575, + 0.24030466973781586, + 0.2466770563274622 + ] + }, + "ci_vs_logit_persist": { + "point": 0.028852254152297974, + "ci_lo": 0.0024707582779228687, + "ci_hi": 0.0649031549692154, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.11865560815980036, + "seed_sd": 0.001527463823848413, + "per_seed": [ + 0.11877911174669861, + 0.12046154979616404, + 0.11672616293653845 + ] + }, + "min": { + "mean": 0.27674824812759957, + "seed_sd": 0.004130809542222062, + "per_seed": [ + 0.2766894418746233, + 0.28183658272027967, + 0.2717187197878957 + ] + }, + "ci_vs_logit_persist": { + "point": 0.038354188203811646, + "ci_lo": 0.006439499091356993, + "ci_hi": 0.08481526374816895, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.11782668561985096, + "seed_sd": 0.0006357685789779205, + "per_seed": [ + 0.11734391278587282, + 0.11872495887801052, + 0.11741118519566954 + ] + }, + "min": { + "mean": 0.26906866431236265, + "seed_sd": 0.002411552330188776, + "per_seed": [ + 0.2713257264345884, + 0.2657259449362755, + 0.2701543215662241 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03752527013421059, + "ci_lo": 0.0036111739464104176, + "ci_hi": 0.0862661674618721, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.11937384071449439, + "seed_sd": 0.001911012951821589, + "per_seed": [ + 0.1187131294514984, + 0.12197367893531919, + 0.11743471375666559 + ] + }, + "min": { + "mean": 0.27033022890488306, + "seed_sd": 0.001562678360281275, + "per_seed": [ + 0.2717500597238541, + 0.27108694426715374, + 0.2681536827236414 + ] + }, + "ci_vs_logit_persist": { + "point": 0.039072420448064804, + "ci_lo": 0.002445467747747898, + "ci_hi": 0.09244605153799057, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.0825429373808826, + "seed_sd": 0.0019991380769401636, + "per_seed": [ + 0.0843949981732294, + 0.08346683473791927, + 0.07976697923149914 + ] + }, + "min": { + "mean": 0.24414827401439348, + "seed_sd": 0.00360893013122499, + "per_seed": [ + 0.2462806709110737, + 0.24709782302379607, + 0.2390663281083107 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.08297482517858346, + "seed_sd": 0.0003796109702724024, + "per_seed": [ + 0.08333016335964202, + 0.08314566407352686, + 0.0824486481025815 + ] + }, + "min": { + "mean": 0.2530659633378188, + "seed_sd": 0.006477758223495518, + "per_seed": [ + 0.2611837416887283, + 0.2526837959885597, + 0.2453303523361683 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.08794025954169532, + "seed_sd": 0.0021926427420291086, + "per_seed": [ + 0.08814302491955459, + 0.09051855742000044, + 0.08515919628553092 + ] + }, + "min": { + "mean": 0.2787179653843244, + "seed_sd": 0.004144981754223442, + "per_seed": [ + 0.2811732605099678, + 0.2821000851690769, + 0.27288055047392845 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08869295563393582, + "seed_sd": 0.0005432286657790841, + "per_seed": [ + 0.08944471448194236, + 0.0884541530162096, + 0.08817999940365553 + ] + }, + "min": { + "mean": 0.2791113016506036, + "seed_sd": 0.005601565237012388, + "per_seed": [ + 0.2854490876197815, + 0.28005831092596056, + 0.2718265064060688 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.08619984535810848, + "seed_sd": 0.0006074213219840745, + "per_seed": [ + 0.08584853939246387, + 0.08569661758374422, + 0.08705437909811735 + ] + }, + "min": { + "mean": 0.274014351144433, + "seed_sd": 0.004110849087958322, + "per_seed": [ + 0.2794879786670208, + 0.27297405488789084, + 0.26958101987838745 + ] + } + } + } + }, + "TRAP_FineWeb": { + "containment_k32": 0.06309446548070848, + "is_shuffled": false, + "capability_top10_rate": 0.07092198581560284, + "capability_mean_rank": 2948.4147069802166, + "arms": { + "logit": { + "persist": { + "mean": 0.0795899466999496, + "seed_sd": 0.0025989293483814962, + "per_seed": [ + 0.08182283889036626, + 0.07594520177226513, + 0.08100179943721741 + ] + }, + "min": { + "mean": 0.20299336810906726, + "seed_sd": 0.002802612086887318, + "per_seed": [ + 0.20158933587372302, + 0.20048548206686972, + 0.20690528638660907 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08802314458880574, + "seed_sd": 0.0015567850162907348, + "per_seed": [ + 0.08990342915058136, + 0.08609115208964795, + 0.0880748525261879 + ] + }, + "min": { + "mean": 0.2750706220666568, + "seed_sd": 0.001378590845172854, + "per_seed": [ + 0.2765483982861042, + 0.27323041558265687, + 0.2754330523312092 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.0983587772740672, + "seed_sd": 0.002435330951356177, + "per_seed": [ + 0.09786534733138978, + 0.09565360294654965, + 0.10155738154426217 + ] + }, + "min": { + "mean": 0.23675129810969034, + "seed_sd": 0.0019420311406396255, + "per_seed": [ + 0.2388120550662279, + 0.23414860032498835, + 0.23729323893785476 + ] + }, + "ci_vs_logit_persist": { + "point": 0.01653594709932804, + "ci_lo": -0.002749744337052107, + "ci_hi": 0.03909606114029884, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10457686848628024, + "seed_sd": 0.0013028571268157657, + "per_seed": [ + 0.10620027284603566, + 0.10451986740808934, + 0.10301046520471573 + ] + }, + "min": { + "mean": 0.24428771920502185, + "seed_sd": 0.0017504090506176664, + "per_seed": [ + 0.24672008417546748, + 0.24267329275608063, + 0.24346978068351746 + ] + }, + "ci_vs_logit_persist": { + "point": 0.022754034027457237, + "ci_lo": 0.0002955104282591492, + "ci_hi": 0.055787838995456696, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.1169947596732527, + "seed_sd": 0.0014920460995238812, + "per_seed": [ + 0.11885219449177384, + 0.11693307990208268, + 0.11519900462590157 + ] + }, + "min": { + "mean": 0.2765649271508058, + "seed_sd": 0.0035912338140722164, + "per_seed": [ + 0.273892093077302, + 0.2741613697260618, + 0.28164131864905356 + ] + }, + "ci_vs_logit_persist": { + "point": 0.035171933472156525, + "ci_lo": 0.005130950827151537, + "ci_hi": 0.08154710382223129, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.11200698917576422, + "seed_sd": 0.0011150225171679868, + "per_seed": [ + 0.11297756037674844, + 0.11259799650870264, + 0.11044541064184159 + ] + }, + "min": { + "mean": 0.27004300641516843, + "seed_sd": 0.0016986497278932678, + "per_seed": [ + 0.2691035162657499, + 0.2685980360955, + 0.2724274668842554 + ] + }, + "ci_vs_logit_persist": { + "point": 0.030184155330061913, + "ci_lo": -0.0009237008052878082, + "ci_hi": 0.07910424470901489, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.11576574464949468, + "seed_sd": 0.0024589463195423874, + "per_seed": [ + 0.11839113484602422, + 0.11247817734256386, + 0.11642792175989598 + ] + }, + "min": { + "mean": 0.2681752033531666, + "seed_sd": 0.0024270113462753363, + "per_seed": [ + 0.2668032143265009, + 0.2661365307867527, + 0.27158586494624615 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03394290804862976, + "ci_lo": 0.002497356850653887, + "ci_hi": 0.08511165529489517, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.07800471356604248, + "seed_sd": 0.0016051413227644337, + "per_seed": [ + 0.07708899369463325, + 0.07666373753454536, + 0.08026140946894884 + ] + }, + "min": { + "mean": 0.2381704437235991, + "seed_sd": 0.0005329244656381395, + "per_seed": [ + 0.23825520575046538, + 0.23747950717806815, + 0.2387766182422638 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.0821227254346013, + "seed_sd": 0.0015299371011673782, + "per_seed": [ + 0.08428394310176372, + 0.08095315061509609, + 0.0811310825869441 + ] + }, + "min": { + "mean": 0.24952786366144816, + "seed_sd": 0.0016516548338833018, + "per_seed": [ + 0.2510794185101986, + 0.2502641886472702, + 0.24723998382687568 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.08601364324179789, + "seed_sd": 0.001040938469684036, + "per_seed": [ + 0.08550386130809784, + 0.08507253206335008, + 0.08746453635394573 + ] + }, + "min": { + "mean": 0.271577147146066, + "seed_sd": 0.0009151307913204917, + "per_seed": [ + 0.2728273034095764, + 0.27124194577336314, + 0.27066219225525856 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08802314458880574, + "seed_sd": 0.0015567850162907348, + "per_seed": [ + 0.08990342915058136, + 0.08609115208964795, + 0.0880748525261879 + ] + }, + "min": { + "mean": 0.2750706220666568, + "seed_sd": 0.001378590845172854, + "per_seed": [ + 0.2765483982861042, + 0.27323041558265687, + 0.2754330523312092 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.08444404101464897, + "seed_sd": 0.0026204805348973167, + "per_seed": [ + 0.08460120521485806, + 0.08115892622154205, + 0.0875719916075468 + ] + }, + "min": { + "mean": 0.26908543296158316, + "seed_sd": 0.0009861434943212119, + "per_seed": [ + 0.2703584484755993, + 0.26795567870140075, + 0.26894217170774937 + ] + } + } + } + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "is_shuffled": false, + "capability_top10_rate": 0.07204180664427025, + "capability_mean_rank": 3115.691675998507, + "arms": { + "logit": { + "persist": { + "mean": 0.07684011658808837, + "seed_sd": 0.001751493594113689, + "per_seed": [ + 0.07681916456203908, + 0.07470553654711694, + 0.07899564865510911 + ] + }, + "min": { + "mean": 0.19724800152083238, + "seed_sd": 0.0013320360378951766, + "per_seed": [ + 0.19853582866489888, + 0.19541345536708832, + 0.19779472053050995 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08884706540654103, + "seed_sd": 0.0017995901297006647, + "per_seed": [ + 0.0899215169250965, + 0.08631185255944729, + 0.09030782673507928 + ] + }, + "min": { + "mean": 0.26579767515261965, + "seed_sd": 0.00434593525083939, + "per_seed": [ + 0.2701214350759983, + 0.2674185924232006, + 0.2598529979586601 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.09928013977284234, + "seed_sd": 0.0005801813237393571, + "per_seed": [ + 0.09991122027859092, + 0.0994187125004828, + 0.09851048653945327 + ] + }, + "min": { + "mean": 0.22897644005715848, + "seed_sd": 0.003917473677941729, + "per_seed": [ + 0.23329056464135647, + 0.22982955165207386, + 0.2238092038780451 + ] + }, + "ci_vs_logit_persist": { + "point": 0.022460972890257835, + "ci_lo": -0.0025930493138730526, + "ci_hi": 0.05142374709248543, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10512595524390539, + "seed_sd": 6.642083575565186e-05, + "per_seed": [ + 0.10505095324479044, + 0.10521243191324174, + 0.10511448057368397 + ] + }, + "min": { + "mean": 0.2396896065523227, + "seed_sd": 0.0017980858823376258, + "per_seed": [ + 0.24095516204833983, + 0.2409669194370508, + 0.23714673817157744 + ] + }, + "ci_vs_logit_persist": { + "point": 0.028306787833571434, + "ci_lo": 0.0008101174025796354, + "ci_hi": 0.06765178591012955, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.11473581977188588, + "seed_sd": 0.0018726509028712721, + "per_seed": [ + 0.11299091381952167, + 0.11388295842334628, + 0.11733358707278967 + ] + }, + "min": { + "mean": 0.26859935087462267, + "seed_sd": 0.003571633643836793, + "per_seed": [ + 0.2713373266160488, + 0.2709062937647104, + 0.26355443224310876 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03791665658354759, + "ci_lo": 0.004731585271656513, + "ci_hi": 0.08828982710838318, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.11509880359905461, + "seed_sd": 0.0010130128767998309, + "per_seed": [ + 0.116063751000911, + 0.11369929662905634, + 0.1155333631671965 + ] + }, + "min": { + "mean": 0.2656327939281861, + "seed_sd": 0.0028700825884618543, + "per_seed": [ + 0.2657771985977888, + 0.2690734852105379, + 0.26204769797623156 + ] + }, + "ci_vs_logit_persist": { + "point": 0.038279641419649124, + "ci_lo": 0.0030695470049977303, + "ci_hi": 0.09241834282875061, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.11405457643171152, + "seed_sd": 0.000759181768186063, + "per_seed": [ + 0.11427508825436235, + 0.11485430202446878, + 0.11303433901630341 + ] + }, + "min": { + "mean": 0.2623630706220865, + "seed_sd": 0.0021456008316961973, + "per_seed": [ + 0.2635163523256779, + 0.2642170388251543, + 0.2593558207154274 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03723541647195816, + "ci_lo": 0.003358369693160057, + "ci_hi": 0.09113697707653046, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.07864857548071692, + "seed_sd": 0.0021079200877713524, + "per_seed": [ + 0.07869367881212383, + 0.07604465500917286, + 0.08120739262085408 + ] + }, + "min": { + "mean": 0.22813203086455663, + "seed_sd": 0.0049004540924145355, + "per_seed": [ + 0.23468756154179574, + 0.22680117040872574, + 0.22290736064314842 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.08349030707031488, + "seed_sd": 0.0013303519375145083, + "per_seed": [ + 0.08351919502019882, + 0.08184671346098185, + 0.08510501272976398 + ] + }, + "min": { + "mean": 0.24087788636485735, + "seed_sd": 0.002137802916050193, + "per_seed": [ + 0.24227171316742896, + 0.2425043836236, + 0.2378575623035431 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.08492612466216087, + "seed_sd": 0.001969426422651843, + "per_seed": [ + 0.0852398195071146, + 0.08237258007284254, + 0.08716597440652549 + ] + }, + "min": { + "mean": 0.2645803511142731, + "seed_sd": 0.0055111752777544, + "per_seed": [ + 0.26968929916620255, + 0.2671232849359512, + 0.2569284692406654 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08884706540654103, + "seed_sd": 0.0017995901297006647, + "per_seed": [ + 0.0899215169250965, + 0.08631185255944729, + 0.09030782673507928 + ] + }, + "min": { + "mean": 0.26579767515261965, + "seed_sd": 0.00434593525083939, + "per_seed": [ + 0.2701214350759983, + 0.2674185924232006, + 0.2598529979586601 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.08544803943174581, + "seed_sd": 0.0014964563066751746, + "per_seed": [ + 0.08371593076735735, + 0.0852610401576385, + 0.08736714737024158 + ] + }, + "min": { + "mean": 0.2600932650268078, + "seed_sd": 0.002688260276088376, + "per_seed": [ + 0.26356557905673983, + 0.2596977762877941, + 0.25701643973588945 + ] + } + } + } + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "is_shuffled": false, + "capability_top10_rate": 0.06942889137737962, + "capability_mean_rank": 3023.849570735349, + "arms": { + "logit": { + "persist": { + "mean": 0.07045147494257739, + "seed_sd": 0.0009721312166622767, + "per_seed": [ + 0.0693399629322812, + 0.07030654239933938, + 0.07170791949611158 + ] + }, + "min": { + "mean": 0.19340261866648992, + "seed_sd": 0.0007952649827854935, + "per_seed": [ + 0.1927247617393732, + 0.19296433925628662, + 0.19451875500380994 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08128419645751517, + "seed_sd": 0.00014557509600992107, + "per_seed": [ + 0.0811225475743413, + 0.08147543100640178, + 0.0812546107918024 + ] + }, + "min": { + "mean": 0.2632633405427138, + "seed_sd": 0.004202327416581266, + "per_seed": [ + 0.257327476888895, + 0.2664832159876823, + 0.26597932875156405 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.09161601231123011, + "seed_sd": 0.0004546603425951726, + "per_seed": [ + 0.09114987244829535, + 0.09146553399041295, + 0.09223263049498201 + ] + }, + "min": { + "mean": 0.21727704343696436, + "seed_sd": 0.0026661614016379944, + "per_seed": [ + 0.21546604819595813, + 0.22104660235345364, + 0.21531847976148127 + ] + }, + "ci_vs_logit_persist": { + "point": 0.022276053205132484, + "ci_lo": -0.0010243094293400645, + "ci_hi": 0.05137227848172188, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10010122263338417, + "seed_sd": 0.0004575233097804, + "per_seed": [ + 0.09971623620949685, + 0.09984334881883114, + 0.1007440828718245 + ] + }, + "min": { + "mean": 0.23036332887907823, + "seed_sd": 0.0010680231146243287, + "per_seed": [ + 0.22934415452182294, + 0.2299075286835432, + 0.23183830343186856 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03076126053929329, + "ci_lo": 0.003821216756477952, + "ci_hi": 0.0650964304804802, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.10856484451020758, + "seed_sd": 0.0008859629622237418, + "per_seed": [ + 0.10760067356750369, + 0.10835398240014911, + 0.10973987756296992 + ] + }, + "min": { + "mean": 0.2563386093825102, + "seed_sd": 0.004435562786996305, + "per_seed": [ + 0.2558373026549816, + 0.2620043199509382, + 0.25117420554161074 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03922487795352936, + "ci_lo": 0.007593361660838127, + "ci_hi": 0.084592804312706, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.10916116856969893, + "seed_sd": 0.0014456596647823535, + "per_seed": [ + 0.10844473736360669, + 0.10786108910106122, + 0.11117767924442887 + ] + }, + "min": { + "mean": 0.25261220609148344, + "seed_sd": 0.004507725293860214, + "per_seed": [ + 0.24720141738653184, + 0.25823683068156245, + 0.252398370206356 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03982120379805565, + "ci_lo": 0.006961393170058727, + "ci_hi": 0.08678563684225082, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.10990942271115879, + "seed_sd": 0.0009380811879615334, + "per_seed": [ + 0.11116077690385281, + 0.10966529375873506, + 0.1089021974708885 + ] + }, + "min": { + "mean": 0.24985794002811113, + "seed_sd": 0.004917077133433733, + "per_seed": [ + 0.24311369843780994, + 0.2517627928406, + 0.25469732880592344 + ] + }, + "ci_vs_logit_persist": { + "point": 0.04056946188211441, + "ci_lo": 0.005845213308930397, + "ci_hi": 0.09058152884244919, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.07240477312201013, + "seed_sd": 0.0012279405765813437, + "per_seed": [ + 0.07149670703802258, + 0.07157688608858734, + 0.07414072623942047 + ] + }, + "min": { + "mean": 0.22585741604367893, + "seed_sd": 0.002324492840003141, + "per_seed": [ + 0.2237498328089714, + 0.22909603491425515, + 0.2247263804078102 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.07519606448089083, + "seed_sd": 0.0014668032477569498, + "per_seed": [ + 0.07427326180040836, + 0.07404855499044061, + 0.07726637665182352 + ] + }, + "min": { + "mean": 0.23793021912376086, + "seed_sd": 0.001500430888085689, + "per_seed": [ + 0.23592469468712807, + 0.2383326955139637, + 0.2395332671701908 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.0798563212621957, + "seed_sd": 0.0009895765606382128, + "per_seed": [ + 0.07845734562724829, + 0.0805881051113829, + 0.08052351304795594 + ] + }, + "min": { + "mean": 0.26348761431872847, + "seed_sd": 0.004438617617590149, + "per_seed": [ + 0.25791943185031413, + 0.2687813863158226, + 0.2637620247900486 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08128419645751517, + "seed_sd": 0.00014557509600992107, + "per_seed": [ + 0.0811225475743413, + 0.08147543100640178, + 0.0812546107918024 + ] + }, + "min": { + "mean": 0.2632633405427138, + "seed_sd": 0.004202327416581266, + "per_seed": [ + 0.257327476888895, + 0.2664832159876823, + 0.26597932875156405 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.07719090627506375, + "seed_sd": 0.0005012601681693165, + "per_seed": [ + 0.07762867035344242, + 0.07648915303871036, + 0.07745489543303848 + ] + }, + "min": { + "mean": 0.25332979957262675, + "seed_sd": 0.0025788858944989705, + "per_seed": [ + 0.24996582381427288, + 0.2562319222837687, + 0.2537916526198387 + ] + } + } + } + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "is_shuffled": false, + "capability_top10_rate": 0.07054871220604703, + "capability_mean_rank": 2915.5222097797687, + "arms": { + "logit": { + "persist": { + "mean": 0.0724533821378524, + "seed_sd": 0.0004272272880772997, + "per_seed": [ + 0.07197563499212264, + 0.0723719387082383, + 0.07301257271319628 + ] + }, + "min": { + "mean": 0.18854948890705903, + "seed_sd": 0.0019688213841248963, + "per_seed": [ + 0.18821407072246074, + 0.19111094139516355, + 0.18632345460355282 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.08192070548733076, + "seed_sd": 0.0010870772158478037, + "per_seed": [ + 0.0834509139880538, + 0.08128386568278075, + 0.08102733679115773 + ] + }, + "min": { + "mean": 0.25526002670327824, + "seed_sd": 0.001641092485155529, + "per_seed": [ + 0.25724452510476115, + 0.25530993938446045, + 0.2532256156206131 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.09262961894273758, + "seed_sd": 0.00131484935296154, + "per_seed": [ + 0.09355246005579829, + 0.0935662386007607, + 0.09077015817165375 + ] + }, + "min": { + "mean": 0.21830062282582124, + "seed_sd": 0.006609570539269488, + "per_seed": [ + 0.2252133905887604, + 0.22029307968914508, + 0.20939539819955827 + ] + }, + "ci_vs_logit_persist": { + "point": 0.020653974264860153, + "ci_lo": -0.0004522601666394621, + "ci_hi": 0.04654961824417114, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.09759563927849134, + "seed_sd": 0.0003791414525796786, + "per_seed": [ + 0.09813039056025445, + 0.09736222187057138, + 0.09729430540464819 + ] + }, + "min": { + "mean": 0.22473898629347483, + "seed_sd": 0.002442633044545982, + "per_seed": [ + 0.22808485105633736, + 0.22381008081138135, + 0.22232202701270581 + ] + }, + "ci_vs_logit_persist": { + "point": 0.0256199948489666, + "ci_lo": 0.0034273744095116854, + "ci_hi": 0.055345676839351654, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.10815058833298584, + "seed_sd": 0.001722116264849758, + "per_seed": [ + 0.11056414446793497, + 0.10722592147067189, + 0.10666169906035065 + ] + }, + "min": { + "mean": 0.25362065248191357, + "seed_sd": 0.0026651021626132266, + "per_seed": [ + 0.2567349553108215, + 0.25390197671949866, + 0.2502250254154205 + ] + }, + "ci_vs_logit_persist": { + "point": 0.036174945533275604, + "ci_lo": 0.006669916212558746, + "ci_hi": 0.08263116329908371, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.10728055983781815, + "seed_sd": 0.0016035623969192783, + "per_seed": [ + 0.10946682216599583, + 0.10670926324091852, + 0.10566559410654008 + ] + }, + "min": { + "mean": 0.24602157374223074, + "seed_sd": 0.004115758270685337, + "per_seed": [ + 0.2492201928049326, + 0.24863365665078163, + 0.24021087177097797 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03530491888523102, + "ci_lo": 0.006728854496032, + "ci_hi": 0.07981326431035995, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.10964172089006752, + "seed_sd": 0.0013336800364678301, + "per_seed": [ + 0.11149031566455961, + 0.10904157883487642, + 0.10839326817076653 + ] + }, + "min": { + "mean": 0.24412576643129189, + "seed_sd": 0.004645882028209517, + "per_seed": [ + 0.24665332175791263, + 0.24811412431299687, + 0.23760985322296618 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03766607493162155, + "ci_lo": 0.005471596959978342, + "ci_hi": 0.08726012706756592, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.07468990227983643, + "seed_sd": 0.001615180338600573, + "per_seed": [ + 0.07241271005477756, + 0.07598344343714417, + 0.07567355334758759 + ] + }, + "min": { + "mean": 0.22321442291140556, + "seed_sd": 0.0036510136949046185, + "per_seed": [ + 0.22781812250614167, + 0.22293724939227105, + 0.21888789683580398 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.07591690821573138, + "seed_sd": 0.0013684341236919232, + "per_seed": [ + 0.07655753195285797, + 0.07401510374620557, + 0.07717808894813061 + ] + }, + "min": { + "mean": 0.23378101587295533, + "seed_sd": 0.002647276679867049, + "per_seed": [ + 0.2374240890145302, + 0.23270655646920205, + 0.23121240213513375 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.083286849836198, + "seed_sd": 0.0015144344342971053, + "per_seed": [ + 0.0854217985412106, + 0.08236689837649465, + 0.08207185259088874 + ] + }, + "min": { + "mean": 0.2565431396166484, + "seed_sd": 0.0019839152128879275, + "per_seed": [ + 0.25844736546278, + 0.25737549364566803, + 0.253806559741497 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.08192070548733076, + "seed_sd": 0.0010870772158478037, + "per_seed": [ + 0.0834509139880538, + 0.08128386568278075, + 0.08102733679115773 + ] + }, + "min": { + "mean": 0.25526002670327824, + "seed_sd": 0.001641092485155529, + "per_seed": [ + 0.25724452510476115, + 0.25530993938446045, + 0.2532256156206131 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.08087106676151355, + "seed_sd": 0.002292791523161221, + "per_seed": [ + 0.08409913796931505, + 0.07952161887660622, + 0.07899244343861937 + ] + }, + "min": { + "mean": 0.24747320339083673, + "seed_sd": 0.00023825866174508858, + "per_seed": [ + 0.24758554473519326, + 0.24714192301034926, + 0.24769214242696763 + ] + } + } + } + }, + "SHUFFLED_Pile-CC": { + "containment_k32": 0.9276060773863086, + "is_shuffled": true, + "capability_top10_rate": 0.05300485255692422, + "capability_mean_rank": 2744.460619634192, + "arms": { + "logit": { + "persist": { + "mean": 0.06958344173617662, + "seed_sd": 0.0013783167964091432, + "per_seed": [ + 0.06908768590074033, + 0.06819874334614724, + 0.07146389596164227 + ] + }, + "min": { + "mean": 0.1842905692756176, + "seed_sd": 0.003037108651238929, + "per_seed": [ + 0.18027709499001504, + 0.1849725190550089, + 0.18762209378182887 + ] + } + }, + "shuf": { + "persist": { + "mean": 0.07425369514773289, + "seed_sd": 0.0017306714846263506, + "per_seed": [ + 0.0760330656543374, + 0.07481941916048526, + 0.07190860062837601 + ] + }, + "min": { + "mean": 0.23691394018630185, + "seed_sd": 0.0018219645711057297, + "per_seed": [ + 0.23433729372918605, + 0.238200806081295, + 0.23820372074842452 + ] + } + }, + "J|Github": { + "persist": { + "mean": 0.10129859369869033, + "seed_sd": 0.00042802746880296117, + "per_seed": [ + 0.10164948988240212, + 0.10155030502937734, + 0.10069598618429154 + ] + }, + "min": { + "mean": 0.2190749297539393, + "seed_sd": 0.0026292580634847133, + "per_seed": [ + 0.21544356346130372, + 0.22158298641443253, + 0.22019823938608168 + ] + }, + "ci_vs_logit_persist": { + "point": 0.032210905104875565, + "ci_lo": 0.0006047620554454625, + "ci_hi": 0.07619215548038483, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Wikipedia_en": { + "persist": { + "mean": 0.10069140825265398, + "seed_sd": 0.0007500908225475252, + "per_seed": [ + 0.10149236768484116, + 0.10089325930457563, + 0.09968859776854515 + ] + }, + "min": { + "mean": 0.21330228534837564, + "seed_sd": 0.0023014785148488892, + "per_seed": [ + 0.2130067877471447, + 0.21625711731612682, + 0.2106429509818554 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03160371631383896, + "ci_lo": 0.0030301266815513372, + "ci_hi": 0.06437204033136368, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|StackExchange": { + "persist": { + "mean": 0.10878449132821212, + "seed_sd": 0.0018444529002009163, + "per_seed": [ + 0.11091255496721715, + 0.10902679737191648, + 0.10641412164550275 + ] + }, + "min": { + "mean": 0.24478809777647256, + "seed_sd": 0.004726871176992899, + "per_seed": [ + 0.23811438586562872, + 0.24845833592116834, + 0.24779157154262066 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03969680145382881, + "ci_lo": 0.00666072266176343, + "ci_hi": 0.07961485534906387, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|Pile-CC": { + "persist": { + "mean": 0.10181608270698538, + "seed_sd": 0.0011156658639072425, + "per_seed": [ + 0.10336326584219932, + 0.1013103226898238, + 0.10077465958893299 + ] + }, + "min": { + "mean": 0.2334883584951361, + "seed_sd": 0.004236587217393799, + "per_seed": [ + 0.2305895695462823, + 0.23947876319289207, + 0.23039674274623395 + ] + }, + "ci_vs_logit_persist": { + "point": 0.032728392630815506, + "ci_lo": 0.003999152220785618, + "ci_hi": 0.06937381625175476, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "J|USPTO_Backgrounds": { + "persist": { + "mean": 0.10729005644097925, + "seed_sd": 0.0027072896219653133, + "per_seed": [ + 0.11111863902769983, + 0.10540009986143559, + 0.10535143043380231 + ] + }, + "min": { + "mean": 0.24142746279637017, + "seed_sd": 0.0038873473828989684, + "per_seed": [ + 0.23677336797118187, + 0.24628862850368022, + 0.24122039191424846 + ] + }, + "ci_vs_logit_persist": { + "point": 0.03820237144827843, + "ci_lo": 0.007150859106332064, + "ci_hi": 0.07562210410833359, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + } + }, + "shufown|Github": { + "persist": { + "mean": 0.08087335017820199, + "seed_sd": 0.0008193993253789582, + "per_seed": [ + 0.07994176764041186, + 0.08193599851801991, + 0.08074228437617421 + ] + }, + "min": { + "mean": 0.21811652841667334, + "seed_sd": 0.001646291165895501, + "per_seed": [ + 0.21791328079998493, + 0.2202267412096262, + 0.2162095632404089 + ] + } + }, + "shufown|Wikipedia_en": { + "persist": { + "mean": 0.07421657151232162, + "seed_sd": 0.0012470330546857254, + "per_seed": [ + 0.07435555178672075, + 0.07566962875425816, + 0.07262453399598598 + ] + }, + "min": { + "mean": 0.2183329896380504, + "seed_sd": 0.0004640303326965738, + "per_seed": [ + 0.21822186894714832, + 0.21782843805849553, + 0.21894866190850734 + ] + } + }, + "shufown|StackExchange": { + "persist": { + "mean": 0.07820263085110735, + "seed_sd": 0.0010242982691094577, + "per_seed": [ + 0.07914561536163092, + 0.07868343349546195, + 0.07677884369622916 + ] + }, + "min": { + "mean": 0.24362073975304763, + "seed_sd": 0.004564433521311145, + "per_seed": [ + 0.23785513453185558, + 0.24901738949120045, + 0.24398969523608685 + ] + } + }, + "shufown|Pile-CC": { + "persist": { + "mean": 0.07425369514773289, + "seed_sd": 0.0017306714846263506, + "per_seed": [ + 0.0760330656543374, + 0.07481941916048526, + 0.07190860062837601 + ] + }, + "min": { + "mean": 0.23691394018630185, + "seed_sd": 0.0018219645711057297, + "per_seed": [ + 0.23433729372918605, + 0.238200806081295, + 0.23820372074842452 + ] + } + }, + "shufown|USPTO_Backgrounds": { + "persist": { + "mean": 0.07257553214828173, + "seed_sd": 0.0007337989985251888, + "per_seed": [ + 0.07348471619188786, + 0.07168766222894192, + 0.07255421802401543 + ] + }, + "min": { + "mean": 0.2372163210064173, + "seed_sd": 0.002492529273571302, + "per_seed": [ + 0.23829566352069378, + 0.23958273269236088, + 0.23377056680619718 + ] + } + } + } + } + }, + "crossings": { + "Github": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "Wikipedia_en": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "StackExchange": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "Pile-CC": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + }, + "USPTO_Backgrounds": { + "rungs_ci_strictly_below_logit": [], + "rungs_point_below_logit": [], + "crosses": false + } + }, + "controls": { + "C1_Q0_reproduces_e33": { + "logit_e33": 0.02843950316309929, + "logit_here": 0.08317913140635938, + "abs_diff": 0.05473962824326009, + "tolerance": 1e-06, + "fires": false, + "note": "Q0 is the no-prefix rung and MUST equal E33 exactly. A mismatch means the prefixing path altered the base measurement and the ladder is void." + }, + "C2_shuf_below_everything": { + "per_rung": { + "Q0": false, + "Pile-CC": false, + "StackExchange": false, + "Wikipedia_en": false, + "USPTO_Backgrounds": false, + "Github": false, + "CONTROL_PubMed_2023": false, + "OOD_Wikipedia_2023": false, + "TRAP_FineWeb": false, + "OOD_News_2024": false, + "OOD_arXiv_2023": false, + "OOD_CommonPile": false, + "SHUFFLED_Pile-CC": false + }, + "fires": false, + "WHY_IT_DOES_NOT_FIRE_AND_WHY_THAT_IS_NOT_LIKE_FOR_LIKE": "this control builds its derangement from PILE-CC's operator and then compares it to EVERY corpus's operator (R4d / F-7). 'A derangement beats Github's operator' is therefore a comparison between two different corpora's operators, not between an operator and its own derangement. C2b below is the like-for-like form." + }, + "C2b_own_derangement_below_its_own_operator": { + "per_rung_per_corpus": { + "Q0": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Pile-CC": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "StackExchange": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Wikipedia_en": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "USPTO_Backgrounds": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Github": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "CONTROL_PubMed_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_Wikipedia_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "TRAP_FineWeb": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_News_2024": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_arXiv_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_CommonPile": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "SHUFFLED_Pile-CC": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + } + }, + "n_cells": 65, + "n_cells_where_derangement_wins": 0, + "required": "0 cells where the derangement is at or above its own operator", + "fires": true, + "like_for_like_evidence_already_on_disk": "e33_logit_baseline_410m_v2: Github 0.027926 vs its own jshuf 0.018142; e54 C2: 0 of 15 and 0 of 120" + }, + "C3_capability": { + "REGISTERED_FORM_IS_DEGENERATE": "the pre-registered rule is top-k accuracy < 50% of Q0, but Q0's top-10 rate is 0.06942889137737962. The eval targets are latent concepts, not emitted tokens, so this quantity is floored at Q0 and the rule cannot fire. NOT reinterpreted silently \u2014 adjudication below uses mean rank, which is not floored, and the operator is asked to confirm.", + "registered_top10_rate": { + "q0": 0.06942889137737962, + "per_rung": { + "Q0": 0.06942889137737962, + "Pile-CC": 0.07801418439716312, + "StackExchange": 0.06270996640537514, + "Wikipedia_en": 0.07204180664427025, + "USPTO_Backgrounds": 0.07502799552071668, + "Github": 0.06532288167226577, + "CONTROL_PubMed_2023": 0.0683090705487122, + "OOD_Wikipedia_2023": 0.08212019410227697, + "TRAP_FineWeb": 0.07092198581560284, + "OOD_News_2024": 0.07204180664427025, + "OOD_arXiv_2023": 0.06942889137737962, + "OOD_CommonPile": 0.07054871220604703, + "SHUFFLED_Pile-CC": 0.05300485255692422 + }, + "usable": true + }, + "adjudicated_mean_rank": { + "q0": 3124.912653975364, + "per_rung": { + "Q0": 3124.912653975364, + "Pile-CC": 3019.8409854423294, + "StackExchange": 3130.946248600224, + "Wikipedia_en": 2939.3516237402014, + "USPTO_Backgrounds": 2924.0399402762223, + "Github": 3151.3322135125045, + "CONTROL_PubMed_2023": 3107.5569242254574, + "OOD_Wikipedia_2023": 2780.15976110489, + "TRAP_FineWeb": 2948.4147069802166, + "OOD_News_2024": 3115.691675998507, + "OOD_arXiv_2023": 3023.849570735349, + "OOD_CommonPile": 2915.5222097797687, + "SHUFFLED_Pile-CC": 2744.460619634192 + }, + "rule": "a rung collapses if its mean target rank exceeds 2x Q0's" + }, + "collapsed_rungs": [], + "note": "a rung whose capability collapsed cannot distinguish lens failure from model failure" + }, + "C5_self_distance": { + "value": 0.0, + "fires": true, + "note": "D_act(J,J)=0 by construction; measured 0.0 in t23" + } + }, + "VERDICT": "REJECT S3 (the publishable null) \u2014 no fitting corpus produces a J-lens curve that goes strictly below the logit lens at any measured rung. Shift in the READ distribution degrades both lenses together; the averaged derivative is no more shift-fragile than not fitting at all.", + "provenance": { + "schema_version": 1, + "experiment": "E36", + "utc": "2026-08-30T07:53:00+00:00", + "script": { + "path": "experiments/t36_qladder.py", + "sha256": "82d9806c59be9d5bf4058b6edc3910e81b8122718f2e7cea03d7e91916b2b9f6", + "n_lines": 594 + }, + "git": { + "commit": "614809b36083400b4752d74e5be0e2670f9c6b26", + "commit_short": "614809b36083", + "git_repository_present": true, + "branch": "pythia", + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": "614809b" + }, + "argv": [ + "experiments/t36_qladder.py", + "--device", + "cpu", + "--workers", + "4", + "--rstrip" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48b_exposure_growth.json", + "sha256": "768fe0cf44eb5309ba6688b4ef329b92ac8740cfaa3d8bb6671fcab4d440945d", + "bytes": 34923, + "exists": true + }, + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n400_s0.pt", + "sha256": "4c87ed9a5f778ef0280a17ff0df5e623e8732fb7b706936fcc7caf624979511a", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n400_s0.pt", + "sha256": "5e2af80098d34c57e6cdbfcf680a9c9303e18daae91b1577d3e9222e7eb4952b", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n400_s0.pt", + "sha256": "3c3bbbca13ef5891fb19ce2c552334ebf12263c905f25545bc36a831b4bd2a00", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n400_s0.pt", + "sha256": "4754d23b9e6eb16ac8159661cc393328374d8cc8d286bbb9c971ee95166fca62", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0.pt", + "sha256": "4bc59371a760c85c5570f68e00a6e41a65a1ff7168079fc9ab5a695e2d70987b", + "bytes": 46145173, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "1318d73053a6a271698afa57e9711b4c730c3782ca1d2b39f52d439cf4e42d8d" + } +} \ No newline at end of file diff --git a/results/e37_rank_ablation_160m_wikitext.json b/results/e37_rank_ablation_160m_wikitext.json new file mode 100644 index 0000000000000000000000000000000000000000..a52097e92cff5d8a2cc5875e41b6ba931b8cbae1 --- /dev/null +++ b/results/e37_rank_ablation_160m_wikitext.json @@ -0,0 +1,381 @@ +{ + "model_short": "160m", + "experiment": "E37 \u2014 unembedding rank ablation: is rank the CAUSE of the read collapse?", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-160m-deduped", + "band": [ + 4, + 5, + 6, + 7, + 8, + 9 + ], + "lens": "lens_160m_n200_db128_pen.pt", + "ranks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + "full" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "d_model": 768, + "top1_share": 0.9151984453201294, + "by_rank": { + "top_1": { + "rank": 1, + "mode": "top", + "logit": { + "min": 0.014868400432169436, + "persist": 0.014868400432169436 + }, + "jlens": { + "min": 0.014868400432169436, + "persist": 0.014868400432169436 + }, + "gap": { + "min": 0.0, + "persist": 0.0 + } + }, + "top_2": { + "rank": 2, + "mode": "top", + "logit": { + "min": 0.03579655124672822, + "persist": 0.031626181756811485 + }, + "jlens": { + "min": 0.04725164803011077, + "persist": 0.01966991291514465 + }, + "gap": { + "min": 0.011455096783382551, + "persist": -0.011956268841666836 + } + }, + "top_4": { + "rank": 4, + "mode": "top", + "logit": { + "min": 0.04190716543900115, + "persist": 0.022724459400134428 + }, + "jlens": { + "min": 0.005705815420619079, + "persist": 0.0005952381129775729 + }, + "gap": { + "min": -0.036201350018382074, + "persist": -0.022129221287156855 + } + }, + "top_8": { + "rank": 8, + "mode": "top", + "logit": { + "min": 0.049728919858379024, + "persist": 0.023037526942789554 + }, + "jlens": { + "min": 0.05780268966087273, + "persist": 0.0047467970954520365 + }, + "gap": { + "min": 0.008073769802493705, + "persist": -0.018290729847337518 + } + }, + "top_16": { + "rank": 16, + "mode": "top", + "logit": { + "min": 0.07468391986829895, + "persist": 0.03497526030987501 + }, + "jlens": { + "min": 0.04451656969530242, + "persist": 0.009464704963777746 + }, + "gap": { + "min": -0.03016735017299653, + "persist": -0.025510555346097263 + } + }, + "top_32": { + "rank": 32, + "mode": "top", + "logit": { + "min": 0.05848017566438232, + "persist": 0.01698361550058637 + }, + "jlens": { + "min": 0.03804796425891774, + "persist": 0.01076414060911962 + }, + "gap": { + "min": -0.020432211405464584, + "persist": -0.006219474891466752 + } + }, + "top_64": { + "rank": 64, + "mode": "top", + "logit": { + "min": 0.0885392565280199, + "persist": 0.018568860819297178 + }, + "jlens": { + "min": 0.04277827351221017, + "persist": 0.020944511917020594 + }, + "gap": { + "min": -0.045760983015809734, + "persist": 0.0023756510977234156 + } + }, + "top_128": { + "rank": 128, + "mode": "top", + "logit": { + "min": 0.09870986158826521, + "persist": 0.027331618618752274 + }, + "jlens": { + "min": 0.05818900330258267, + "persist": 0.030414551523114952 + }, + "gap": { + "min": -0.04052085828568254, + "persist": 0.0030829329043626785 + } + }, + "top_256": { + "rank": 256, + "mode": "top", + "logit": { + "min": 0.11457518472203186, + "persist": 0.035682742137994085 + }, + "jlens": { + "min": 0.07199768349528313, + "persist": 0.03808527147131307 + }, + "gap": { + "min": -0.042577501226748735, + "persist": 0.002402529333318984 + } + }, + "top_512": { + "rank": 512, + "mode": "top", + "logit": { + "min": 0.09978947394660541, + "persist": 0.024061174690723417 + }, + "jlens": { + "min": 0.06875899561813899, + "persist": 0.03310696495962995 + }, + "gap": { + "min": -0.03103047832846642, + "persist": 0.00904579026890653 + } + }, + "top_full": { + "rank": null, + "mode": "top", + "logit": { + "min": 0.09125682400273424, + "persist": 0.023089128892336573 + }, + "jlens": { + "min": 0.05549318290182523, + "persist": 0.02568470990019185 + }, + "gap": { + "min": -0.035763641100909016, + "persist": 0.0025955810078552785 + } + }, + "random_1": { + "rank": 1, + "mode": "random", + "logit": { + "min": 0.0007402707423482623, + "persist": 0.0007402707423482623 + }, + "jlens": { + "min": 0.0007402707423482623, + "persist": 0.0007402707423482623 + }, + "gap": { + "min": 0.0, + "persist": 0.0 + } + }, + "random_2": { + "rank": 2, + "mode": "random", + "logit": { + "min": 0.0, + "persist": 0.0 + }, + "jlens": { + "min": 0.0006541267569575991, + "persist": 0.0 + }, + "gap": { + "min": 0.0006541267569575991, + "persist": 0.0 + } + }, + "random_4": { + "rank": 4, + "mode": "random", + "logit": { + "min": 0.0011904762259551459, + "persist": 0.0 + }, + "jlens": { + "min": 0.0007402707423482623, + "persist": 0.0 + }, + "gap": { + "min": -0.00045020548360688356, + "persist": 0.0 + } + }, + "random_8": { + "rank": 8, + "mode": "random", + "logit": { + "min": 0.004856457933783531, + "persist": 0.0019789677911571093 + }, + "jlens": { + "min": 0.0024759141728281976, + "persist": 0.0013112133634941919 + }, + "gap": { + "min": -0.0023805437609553336, + "persist": -0.0006677544276629175 + } + }, + "random_16": { + "rank": 16, + "mode": "random", + "logit": { + "min": 0.0060660534937466894, + "persist": 0.00027739251298563824 + }, + "jlens": { + "min": 0.004250158023621355, + "persist": 0.0004224251423563276 + }, + "gap": { + "min": -0.0018158954701253342, + "persist": 0.00014503262937068937 + } + }, + "random_32": { + "rank": 32, + "mode": "random", + "logit": { + "min": 0.006582951492496899, + "persist": 0.0008928571428571428 + }, + "jlens": { + "min": 0.006571544600384576, + "persist": 0.00029761905648878647 + }, + "gap": { + "min": -1.1406892112323513e-05, + "persist": -0.0005952380863683564 + } + }, + "random_64": { + "rank": 64, + "mode": "random", + "logit": { + "min": 0.010517988487013748, + "persist": 0.0015302422323397228 + }, + "jlens": { + "min": 0.005148411888097014, + "persist": 0.0008867832965084485 + }, + "gap": { + "min": -0.005369576598916734, + "persist": -0.0006434589358312743 + } + }, + "random_128": { + "rank": 128, + "mode": "random", + "logit": { + "min": 0.015817268645124775, + "persist": 0.003411288426390716 + }, + "jlens": { + "min": 0.019308368781847614, + "persist": 0.007346778442817075 + }, + "gap": { + "min": 0.0034911001367228384, + "persist": 0.003935490016426358 + } + }, + "random_256": { + "rank": 256, + "mode": "random", + "logit": { + "min": 0.032052502860980374, + "persist": 0.0073210031326328 + }, + "jlens": { + "min": 0.020247709431818552, + "persist": 0.005178781093231269 + }, + "gap": { + "min": -0.011804793429161822, + "persist": -0.0021422220394015307 + } + }, + "random_512": { + "rank": 512, + "mode": "random", + "logit": { + "min": 0.05450193698384932, + "persist": 0.014540606843573707 + }, + "jlens": { + "min": 0.038169501588812896, + "persist": 0.01737505021904196 + }, + "gap": { + "min": -0.016332435395036426, + "persist": 0.0028344433754682544 + } + } + }, + "gap_ratio_lowrank_over_full_min": -0.3203000709872182, + "gap_ratio_lowrank_over_full_persist": 0.0, + "VERDICT": { + "min": "ACCEPT \u2014 rank is the causal bottleneck", + "persist": "ACCEPT \u2014 rank is the causal bottleneck" + } +} \ No newline at end of file diff --git a/results/e37_rank_ablation_410m.json b/results/e37_rank_ablation_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..5b28052d774ae09c03d0680711e5612938d5afd0 --- /dev/null +++ b/results/e37_rank_ablation_410m.json @@ -0,0 +1,387 @@ +{ + "experiment": "E37 \u2014 unembedding rank ablation: is rank the CAUSE of the read collapse?", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "lens": "e28_USPTO_Backgrounds_410m_n400_s0.pt", + "ranks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + "full" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "d_model": 1024, + "top1_share": 0.09707066416740417, + "by_rank": { + "top_1": { + "rank": 1, + "mode": "top", + "logit": { + "min": 0.044816138675170286, + "persist": 0.044816138675170286 + }, + "jlens": { + "min": 0.044816138675170286, + "persist": 0.004036917617278439 + }, + "gap": { + "min": 0.0, + "persist": -0.04077922105789185 + } + }, + "top_2": { + "rank": 2, + "mode": "top", + "logit": { + "min": 0.050930884666740894, + "persist": 0.04585509968123266 + }, + "jlens": { + "min": 0.05782601737550327, + "persist": 0.03618984557688236 + }, + "gap": { + "min": 0.0068951327087623734, + "persist": -0.009665254104350297 + } + }, + "top_4": { + "rank": 4, + "mode": "top", + "logit": { + "min": 0.05982658934912511, + "persist": 0.04682154400008065 + }, + "jlens": { + "min": 0.06537730254765067, + "persist": 0.02605997523558991 + }, + "gap": { + "min": 0.005550713198525563, + "persist": -0.02076156876449074 + } + }, + "top_8": { + "rank": 8, + "mode": "top", + "logit": { + "min": 0.05813455539090293, + "persist": 0.04444857755942004 + }, + "jlens": { + "min": 0.06818360627761909, + "persist": 0.02389509632651295 + }, + "gap": { + "min": 0.010049050886716156, + "persist": -0.02055348123290709 + } + }, + "top_16": { + "rank": 16, + "mode": "top", + "logit": { + "min": 0.07149410971573421, + "persist": 0.043704661673733165 + }, + "jlens": { + "min": 0.07448017775480235, + "persist": 0.016689935592668396 + }, + "gap": { + "min": 0.002986068039068146, + "persist": -0.02701472608106477 + } + }, + "top_32": { + "rank": 32, + "mode": "top", + "logit": { + "min": 0.0787029292434454, + "persist": 0.048510608156876904 + }, + "jlens": { + "min": 0.10756424742617778, + "persist": 0.011408340771283422 + }, + "gap": { + "min": 0.02886131818273238, + "persist": -0.03710226738559348 + } + }, + "top_64": { + "rank": 64, + "mode": "top", + "logit": { + "min": 0.07608962117561273, + "persist": 0.04795391535652535 + }, + "jlens": { + "min": 0.12727906169103725, + "persist": 0.019707938895693847 + }, + "gap": { + "min": 0.051189440515424525, + "persist": -0.028245976460831504 + } + }, + "top_128": { + "rank": 128, + "mode": "top", + "logit": { + "min": 0.09110439246786492, + "persist": 0.04560314690960305 + }, + "jlens": { + "min": 0.14797603221876282, + "persist": 0.042308352062744754 + }, + "gap": { + "min": 0.0568716397508979, + "persist": -0.0032947948468582955 + } + }, + "top_256": { + "rank": 256, + "mode": "top", + "logit": { + "min": 0.11014232670090028, + "persist": 0.050225482375494075 + }, + "jlens": { + "min": 0.16522453241050244, + "persist": 0.060520668780165064 + }, + "gap": { + "min": 0.055082205709602164, + "persist": 0.010295186404670989 + } + }, + "top_512": { + "rank": 512, + "mode": "top", + "logit": { + "min": 0.12660836593380997, + "persist": 0.033668833864586696 + }, + "jlens": { + "min": 0.16090655622205563, + "persist": 0.054852031197931084 + }, + "gap": { + "min": 0.034298190288245656, + "persist": 0.021183197333344388 + } + }, + "top_full": { + "rank": null, + "mode": "top", + "logit": { + "min": 0.12473962224487747, + "persist": 0.02843950316309929 + }, + "jlens": { + "min": 0.15696467062724487, + "persist": 0.05089726820588112 + }, + "gap": { + "min": 0.032225048382367394, + "persist": 0.02245776504278183 + } + }, + "random_1": { + "rank": 1, + "mode": "random", + "logit": { + "min": 0.005752837844192981, + "persist": 0.0025033237146479745 + }, + "jlens": { + "min": 0.005897870420345238, + "persist": 0.0016180203163198062 + }, + "gap": { + "min": 0.00014503257615225678, + "persist": -0.0008853033983281682 + } + }, + "random_2": { + "rank": 2, + "mode": "random", + "logit": { + "min": 0.006921798443155629, + "persist": 0.00137565088059221 + }, + "jlens": { + "min": 0.007326654344797135, + "persist": 0.0005689376965165138 + }, + "gap": { + "min": 0.00040485590164150535, + "persist": -0.0008067131840756962 + } + }, + "random_4": { + "rank": 4, + "mode": "random", + "logit": { + "min": 0.016177748356546673, + "persist": 0.007272727255310331 + }, + "jlens": { + "min": 0.02014524628009115, + "persist": 0.0008006393111177853 + }, + "gap": { + "min": 0.003967497923544477, + "persist": -0.006472087944192545 + } + }, + "random_8": { + "rank": 8, + "mode": "random", + "logit": { + "min": 0.014812702499330045, + "persist": 0.0005076142027974129 + }, + "jlens": { + "min": 0.0049971484978284155, + "persist": 0.0001450326293706894 + }, + "gap": { + "min": -0.00981555400150163, + "persist": -0.0003625815734267235 + } + }, + "random_16": { + "rank": 16, + "mode": "random", + "logit": { + "min": 0.018663861096969672, + "persist": 0.00326175508754594 + }, + "jlens": { + "min": 0.02432235883814948, + "persist": 0.0034531539572136742 + }, + "gap": { + "min": 0.005658497741179807, + "persist": 0.00019139886966773427 + } + }, + "random_32": { + "rank": 32, + "mode": "random", + "logit": { + "min": 0.01320039802896125, + "persist": 7.25163146853447e-05 + }, + "jlens": { + "min": 0.02424657020185675, + "persist": 0.00029761905648878647 + }, + "gap": { + "min": 0.011046172172895498, + "persist": 0.00022510274180344178 + } + }, + "random_64": { + "rank": 64, + "mode": "random", + "logit": { + "min": 0.024362531144704137, + "persist": 0.0006602006299155099 + }, + "jlens": { + "min": 0.01635445955076388, + "persist": 0.001528237307710307 + }, + "gap": { + "min": -0.008008071593940257, + "persist": 0.0008680366777947972 + } + }, + "random_128": { + "rank": 128, + "mode": "random", + "logit": { + "min": 0.02679472959467343, + "persist": 0.0034124573160495076 + }, + "jlens": { + "min": 0.023935706434505325, + "persist": 0.001069953026516097 + }, + "gap": { + "min": -0.002859023160168103, + "persist": -0.0023425042895334107 + } + }, + "random_256": { + "rank": 256, + "mode": "random", + "logit": { + "min": 0.08813109185014452, + "persist": 0.006402370546545301 + }, + "jlens": { + "min": 0.05672641670597451, + "persist": 0.004546203996453967 + }, + "gap": { + "min": -0.03140467514417001, + "persist": -0.0018561665500913347 + } + }, + "random_512": { + "rank": 512, + "mode": "random", + "logit": { + "min": 0.05877692249736616, + "persist": 0.004606830887496471 + }, + "jlens": { + "min": 0.07533051847879375, + "persist": 0.01568285296005862 + }, + "gap": { + "min": 0.016553595981427596, + "persist": 0.011076022072562149 + } + } + }, + "gap_ratio_lowrank_over_full_min": 0.21396811036396113, + "gap_ratio_lowrank_over_full_persist": -0.4303747094128948, + "VERDICT": { + "min": "ACCEPT \u2014 rank is the causal bottleneck", + "persist": "ACCEPT \u2014 rank is the causal bottleneck" + } +} \ No newline at end of file diff --git a/results/e37_rank_ablation_410m_wikitext.json b/results/e37_rank_ablation_410m_wikitext.json new file mode 100644 index 0000000000000000000000000000000000000000..21ced0bfbdf9f48b12071f6a90954b3766b2a55f --- /dev/null +++ b/results/e37_rank_ablation_410m_wikitext.json @@ -0,0 +1,388 @@ +{ + "model_short": "410m", + "experiment": "E37 \u2014 unembedding rank ablation: is rank the CAUSE of the read collapse?", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "lens": "lens_410m_n200_db128_pen.pt", + "ranks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + "full" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "d_model": 1024, + "top1_share": 0.09707066416740417, + "by_rank": { + "top_1": { + "rank": 1, + "mode": "top", + "logit": { + "min": 0.044816138675170286, + "persist": 0.044816138675170286 + }, + "jlens": { + "min": 0.044816138675170286, + "persist": 0.026114839501678942 + }, + "gap": { + "min": 0.0, + "persist": -0.018701299173491344 + } + }, + "top_2": { + "rank": 2, + "mode": "top", + "logit": { + "min": 0.050930884666740894, + "persist": 0.04585509968123266 + }, + "jlens": { + "min": 0.05736571321530002, + "persist": 0.043497878100190845 + }, + "gap": { + "min": 0.006434828548559125, + "persist": -0.0023572215810418115 + } + }, + "top_4": { + "rank": 4, + "mode": "top", + "logit": { + "min": 0.059899105690419674, + "persist": 0.04682154400008065 + }, + "jlens": { + "min": 0.0582200244867376, + "persist": 0.028344681592924255 + }, + "gap": { + "min": -0.001679081203682077, + "persist": -0.018476862407156396 + } + }, + "top_8": { + "rank": 8, + "mode": "top", + "logit": { + "min": 0.05898345961634602, + "persist": 0.04444857755942004 + }, + "jlens": { + "min": 0.06165012275534017, + "persist": 0.024208775455398217 + }, + "gap": { + "min": 0.002666663138994149, + "persist": -0.020239802104021824 + } + }, + "top_16": { + "rank": 16, + "mode": "top", + "logit": { + "min": 0.07178565497909273, + "persist": 0.043704661673733165 + }, + "jlens": { + "min": 0.07327612319162913, + "persist": 0.023347367638988155 + }, + "gap": { + "min": 0.0014904682125364, + "persist": -0.02035729403474501 + } + }, + "top_32": { + "rank": 32, + "mode": "top", + "logit": { + "min": 0.08202619185405119, + "persist": 0.04928982904446976 + }, + "jlens": { + "min": 0.11245432245944227, + "persist": 0.02001671857599701 + }, + "gap": { + "min": 0.030428130605391082, + "persist": -0.02927311046847275 + } + }, + "top_64": { + "rank": 64, + "mode": "top", + "logit": { + "min": 0.08112067114561797, + "persist": 0.04745208740766559 + }, + "jlens": { + "min": 0.131429607208286, + "persist": 0.027107547942016808 + }, + "gap": { + "min": 0.050308936062668036, + "persist": -0.020344539465648785 + } + }, + "top_128": { + "rank": 128, + "mode": "top", + "logit": { + "min": 0.08865202215633222, + "persist": 0.04291814916900226 + }, + "jlens": { + "min": 0.1409369697528226, + "persist": 0.04970484301447869 + }, + "gap": { + "min": 0.05228494759649038, + "persist": 0.006786693845476428 + } + }, + "top_256": { + "rank": 256, + "mode": "top", + "logit": { + "min": 0.10863874865961927, + "persist": 0.04080786981752941 + }, + "jlens": { + "min": 0.161116580106318, + "persist": 0.057609349861741065 + }, + "gap": { + "min": 0.05247783144669872, + "persist": 0.016801480044211656 + } + }, + "top_512": { + "rank": 512, + "mode": "top", + "logit": { + "min": 0.1200619030211653, + "persist": 0.023672968095966748 + }, + "jlens": { + "min": 0.15150082273674864, + "persist": 0.048822183295020034 + }, + "gap": { + "min": 0.03143891971558334, + "persist": 0.025149215199053287 + } + }, + "top_full": { + "rank": null, + "mode": "top", + "logit": { + "min": 0.12056936134717294, + "persist": 0.019692221363740308 + }, + "jlens": { + "min": 0.12939210507486548, + "persist": 0.03477372769266367 + }, + "gap": { + "min": 0.008822743727692539, + "persist": 0.015081506328923359 + } + }, + "random_1": { + "rank": 1, + "mode": "random", + "logit": { + "min": 0.005897870420345238, + "persist": 0.0022057046847684045 + }, + "jlens": { + "min": 0.0036695044487714767, + "persist": 0.0010378897988370485 + }, + "gap": { + "min": -0.0022283659715737615, + "persist": -0.001167814885931356 + } + }, + "random_2": { + "rank": 2, + "mode": "random", + "logit": { + "min": 0.006921798443155629, + "persist": 0.00137565088059221 + }, + "jlens": { + "min": 0.006253762649638312, + "persist": 0.00027739251298563824 + }, + "gap": { + "min": -0.0006680357935173169, + "persist": -0.0010982583676065719 + } + }, + "random_4": { + "rank": 4, + "mode": "random", + "logit": { + "min": 0.015616076519446714, + "persist": 0.006493506527372769 + }, + "jlens": { + "min": 0.01900563112327031, + "persist": 0.00043509788811206817 + }, + "gap": { + "min": 0.0033895546038235948, + "persist": -0.006058408639260701 + } + }, + "random_8": { + "rank": 8, + "mode": "random", + "logit": { + "min": 0.015267739045832838, + "persist": 0.00043509788811206817 + }, + "jlens": { + "min": 0.003875163942575455, + "persist": 0.0001450326293706894 + }, + "gap": { + "min": -0.011392575103257384, + "persist": -0.00029006525874137874 + } + }, + "random_16": { + "rank": 16, + "mode": "random", + "logit": { + "min": 0.019043974152633123, + "persist": 0.003781235590577126 + }, + "jlens": { + "min": 0.024863950881574833, + "persist": 0.008451828573431287 + }, + "gap": { + "min": 0.00581997672894171, + "persist": 0.00467059298285416 + } + }, + "random_32": { + "rank": 32, + "mode": "random", + "logit": { + "min": 0.011417954681175096, + "persist": 7.25163146853447e-05 + }, + "jlens": { + "min": 0.0146968703184809, + "persist": 0.0011702496824519974 + }, + "gap": { + "min": 0.003278915637305804, + "persist": 0.0010977333677666527 + } + }, + "random_64": { + "rank": 64, + "mode": "random", + "logit": { + "min": 0.023268866858312063, + "persist": 0.0008448502847126552 + }, + "jlens": { + "min": 0.011754340039832252, + "persist": 0.000436577812901565 + }, + "gap": { + "min": -0.01151452681847981, + "persist": -0.0004082724718110902 + } + }, + "random_128": { + "rank": 128, + "mode": "random", + "logit": { + "min": 0.02538522420717137, + "persist": 0.0028576722900782313 + }, + "jlens": { + "min": 0.030879194768411774, + "persist": 0.0032090610159294947 + }, + "gap": { + "min": 0.005493970561240402, + "persist": 0.00035138872585126343 + } + }, + "random_256": { + "rank": 256, + "mode": "random", + "logit": { + "min": 0.08098250542368207, + "persist": 0.005146931883479868 + }, + "jlens": { + "min": 0.03562966247222253, + "persist": 0.004893309649612223 + }, + "gap": { + "min": -0.04535284295145954, + "persist": -0.0002536222338676451 + } + }, + "random_512": { + "rank": 512, + "mode": "random", + "logit": { + "min": 0.055071425038789, + "persist": 0.0031664841675332614 + }, + "jlens": { + "min": 0.06163427127259118, + "persist": 0.012995834701827594 + }, + "gap": { + "min": 0.006562846233802183, + "persist": 0.009829350534294332 + } + } + }, + "gap_ratio_lowrank_over_full_min": 0.7293455128206542, + "gap_ratio_lowrank_over_full_persist": -0.1562988158895723, + "VERDICT": { + "min": "UNCLEAR \u2014 report and stop", + "persist": "ACCEPT \u2014 rank is the causal bottleneck" + } +} \ No newline at end of file diff --git a/results/e37_rank_ablation_70m_wikitext.json b/results/e37_rank_ablation_70m_wikitext.json new file mode 100644 index 0000000000000000000000000000000000000000..8ea55a785a33f8171d5abe3f3785b3f3b449cc5e --- /dev/null +++ b/results/e37_rank_ablation_70m_wikitext.json @@ -0,0 +1,344 @@ +{ + "model_short": "70m", + "experiment": "E37 \u2014 unembedding rank ablation: is rank the CAUSE of the read collapse?", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-70m-deduped", + "band": [ + 2, + 3 + ], + "lens": "lens_70m_n200_db128_pen.pt", + "ranks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + "full" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "d_model": 512, + "top1_share": 0.954581618309021, + "by_rank": { + "top_1": { + "rank": 1, + "mode": "top", + "logit": { + "min": 0.01026481922183718, + "persist": 0.01026481922183718 + }, + "jlens": { + "min": 0.01026481922183718, + "persist": 0.01026481922183718 + }, + "gap": { + "min": 0.0, + "persist": 0.0 + } + }, + "top_2": { + "rank": 2, + "mode": "top", + "logit": { + "min": 0.012157118134200572, + "persist": 0.012157118134200572 + }, + "jlens": { + "min": 0.02922552430203983, + "persist": 0.02922552430203983 + }, + "gap": { + "min": 0.017068406167839258, + "persist": 0.017068406167839258 + } + }, + "top_4": { + "rank": 4, + "mode": "top", + "logit": { + "min": 0.012269806329693113, + "persist": 0.012269806329693113 + }, + "jlens": { + "min": 0.06286027505993842, + "persist": 0.06286027505993842 + }, + "gap": { + "min": 0.05059046873024531, + "persist": 0.05059046873024531 + } + }, + "top_8": { + "rank": 8, + "mode": "top", + "logit": { + "min": 0.03633122377629791, + "persist": 0.03633122377629791 + }, + "jlens": { + "min": 0.05184787628906114, + "persist": 0.05184787628906114 + }, + "gap": { + "min": 0.015516652512763227, + "persist": 0.015516652512763227 + } + }, + "top_16": { + "rank": 16, + "mode": "top", + "logit": { + "min": 0.046939611621201034, + "persist": 0.046939611621201034 + }, + "jlens": { + "min": 0.0252133257953184, + "persist": 0.0252133257953184 + }, + "gap": { + "min": -0.021726285825882635, + "persist": -0.021726285825882635 + } + }, + "top_32": { + "rank": 32, + "mode": "top", + "logit": { + "min": 0.03625549442533936, + "persist": 0.03625549442533936 + }, + "jlens": { + "min": 0.03753852123128516, + "persist": 0.03753852123128516 + }, + "gap": { + "min": 0.0012830268059458028, + "persist": 0.0012830268059458028 + } + }, + "top_64": { + "rank": 64, + "mode": "top", + "logit": { + "min": 0.03183021721030985, + "persist": 0.03183021721030985 + }, + "jlens": { + "min": 0.0398478374417339, + "persist": 0.0398478374417339 + }, + "gap": { + "min": 0.008017620231424057, + "persist": 0.008017620231424057 + } + }, + "top_128": { + "rank": 128, + "mode": "top", + "logit": { + "min": 0.03403322414628097, + "persist": 0.03403322414628097 + }, + "jlens": { + "min": 0.03716093538595097, + "persist": 0.03716093538595097 + }, + "gap": { + "min": 0.0031277112396700044, + "persist": 0.0031277112396700044 + } + }, + "top_256": { + "rank": 256, + "mode": "top", + "logit": { + "min": 0.021691799270255226, + "persist": 0.021691799270255226 + }, + "jlens": { + "min": 0.04466124065220356, + "persist": 0.04466124065220356 + }, + "gap": { + "min": 0.022969441381948334, + "persist": 0.022969441381948334 + } + }, + "top_full": { + "rank": null, + "mode": "top", + "logit": { + "min": 0.036518192291259764, + "persist": 0.036518192291259764 + }, + "jlens": { + "min": 0.0420132593650903, + "persist": 0.0420132593650903 + }, + "gap": { + "min": 0.005495067073830538, + "persist": 0.005495067073830538 + } + }, + "random_1": { + "rank": 1, + "mode": "random", + "logit": { + "min": 0.0002915451835308756, + "persist": 0.0002915451835308756 + }, + "jlens": { + "min": 0.0002915451835308756, + "persist": 0.0002915451835308756 + }, + "gap": { + "min": 0.0, + "persist": 0.0 + } + }, + "random_2": { + "rank": 2, + "mode": "random", + "logit": { + "min": 0.0014502164774707386, + "persist": 0.0014502164774707386 + }, + "jlens": { + "min": 0.0011904762259551459, + "persist": 0.0011904762259551459 + }, + "gap": { + "min": -0.0002597402515155927, + "persist": -0.0002597402515155927 + } + }, + "random_4": { + "rank": 4, + "mode": "random", + "logit": { + "min": 0.0, + "persist": 0.0 + }, + "jlens": { + "min": 7.25163146853447e-05, + "persist": 7.25163146853447e-05 + }, + "gap": { + "min": 7.25163146853447e-05, + "persist": 7.25163146853447e-05 + } + }, + "random_8": { + "rank": 8, + "mode": "random", + "logit": { + "min": 0.0012569186676825795, + "persist": 0.0012569186676825795 + }, + "jlens": { + "min": 0.003553207058991705, + "persist": 0.003553207058991705 + }, + "gap": { + "min": 0.0022962883913091254, + "persist": 0.0022962883913091254 + } + }, + "random_16": { + "rank": 16, + "mode": "random", + "logit": { + "min": 0.0029525576425450187, + "persist": 0.0029525576425450187 + }, + "jlens": { + "min": 0.0010389610060623712, + "persist": 0.0010389610060623712 + }, + "gap": { + "min": -0.0019135966364826474, + "persist": -0.0019135966364826474 + } + }, + "random_32": { + "rank": 32, + "mode": "random", + "logit": { + "min": 0.00249722669167178, + "persist": 0.00249722669167178 + }, + "jlens": { + "min": 0.0005830903670617512, + "persist": 0.0005830903670617512 + }, + "gap": { + "min": -0.0019141363246100288, + "persist": -0.0019141363246100288 + } + }, + "random_64": { + "rank": 64, + "mode": "random", + "logit": { + "min": 0.0033228284546307158, + "persist": 0.0033228284546307158 + }, + "jlens": { + "min": 0.004734734259545803, + "persist": 0.004734734259545803 + }, + "gap": { + "min": 0.0014119058049150875, + "persist": 0.0014119058049150875 + } + }, + "random_128": { + "rank": 128, + "mode": "random", + "logit": { + "min": 0.005005642399191856, + "persist": 0.005005642399191856 + }, + "jlens": { + "min": 0.005152427990521703, + "persist": 0.005152427990521703 + }, + "gap": { + "min": 0.00014678559132984704, + "persist": 0.00014678559132984704 + } + }, + "random_256": { + "rank": 256, + "mode": "random", + "logit": { + "min": 0.02029156277754477, + "persist": 0.02029156277754477 + }, + "jlens": { + "min": 0.032011948046939714, + "persist": 0.032011948046939714 + }, + "gap": { + "min": 0.011720385269394944, + "persist": 0.011720385269394944 + } + } + }, + "gap_ratio_lowrank_over_full_min": 3.1061324527092804, + "gap_ratio_lowrank_over_full_persist": 3.1061324527092804, + "VERDICT": { + "min": "REJECT \u2014 rank is not the mechanism; retract S1's mechanistic claim", + "persist": "REJECT \u2014 rank is not the mechanism; retract S1's mechanistic claim" + } +} \ No newline at end of file diff --git a/results/e38_jgeometry.json b/results/e38_jgeometry.json new file mode 100644 index 0000000000000000000000000000000000000000..6672ab9815a8ba7410f791d6ccea0e0ba3cb49d6 --- /dev/null +++ b/results/e38_jgeometry.json @@ -0,0 +1,631 @@ +{ + "experiment": "E38-E42 \u2014 J geometry: is the SLM read failure a property of J, not W_U?", + "prereg": "docs/experiments/preregs/superseded/PREREG_E38_JGEOMETRY.md", + "boundary": "E38/E41 touch no eval data and run at all 6 scales; E39/E40 stop at 1b because 1.4b/2.8b are the last clean cells of PREREG_PYTHIA_T7_v2 \u00a72", + "E38": { + "70m": { + "rho_I_per_layer": { + "2": 0.8775066137313843, + "3": 0.681338369846344 + }, + "rho_I_band_median": 0.7794224917888641, + "d_model": 512, + "band": [ + 2, + 3 + ] + }, + "160m": { + "rho_I_per_layer": { + "4": 0.910070538520813, + "5": 0.8664129376411438, + "6": 0.8136516809463501, + "7": 0.6939327120780945, + "8": 0.5913311839103699, + "9": 0.44346362352371216 + }, + "rho_I_band_median": 0.7537921965122223, + "d_model": 768, + "band": [ + 4, + 5, + 6, + 7, + 8, + 9 + ] + }, + "410m": { + "rho_I_per_layer": { + "8": 0.9257386922836304, + "9": 0.9082852005958557, + "10": 0.8883463144302368, + "11": 0.8613531589508057, + "12": 0.8195936679840088, + "13": 0.7797642350196838, + "14": 0.7474753856658936, + "15": 0.719423770904541, + "16": 0.6370239853858948, + "17": 0.558649480342865, + "18": 0.5136339664459229, + "19": 0.44926247000694275, + "20": 0.3968655467033386 + }, + "rho_I_band_median": 0.7474753856658936, + "d_model": 1024, + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + "1b": { + "rho_I_per_layer": { + "5": 0.9036093950271606, + "6": 0.8773133158683777, + "7": 0.8445247411727905, + "8": 0.8094492554664612, + "9": 0.7089056968688965, + "10": 0.6619659662246704, + "11": 0.535653293132782, + "12": 0.4540800154209137, + "13": 0.33653751015663147 + }, + "rho_I_band_median": 0.7089056968688965, + "d_model": 2048, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ] + }, + "1.4b": { + "rho_I_per_layer": { + "8": 0.8804674744606018, + "9": 0.8604248762130737, + "10": 0.8244580030441284, + "11": 0.7763474583625793, + "12": 0.7209271788597107, + "13": 0.664571225643158, + "14": 0.5920743942260742, + "15": 0.5389373898506165, + "16": 0.47405102849006653, + "17": 0.42703258991241455, + "18": 0.37328365445137024, + "19": 0.3073745667934418, + "20": 0.28337958455085754 + }, + "rho_I_band_median": 0.5920743942260742, + "d_model": 2048, + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + "2.8b": { + "rho_I_per_layer": { + "11": 0.7774393558502197, + "12": 0.7495322823524475, + "13": 0.703403115272522, + "14": 0.6430928111076355, + "15": 0.6066815257072449, + "16": 0.5602889657020569, + "17": 0.5100991129875183, + "18": 0.47178763151168823, + "19": 0.4493074119091034, + "20": 0.4155581295490265, + "21": 0.3797268271446228, + "22": 0.3474956154823303, + "23": 0.3178030550479889, + "24": 0.29778072237968445, + "25": 0.2759111523628235, + "26": 0.25979921221733093, + "27": 0.22977542877197266 + }, + "rho_I_band_median": 0.4493074119091034, + "d_model": 2560, + "band": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27 + ] + }, + "by_corpus_410m": { + "Github": { + "rho_I_band_median": 0.7184840440750122 + }, + "Wikipedia_en": { + "rho_I_band_median": 0.7176766991615295 + }, + "StackExchange": { + "rho_I_band_median": 0.7231940627098083 + }, + "Pile-CC": { + "rho_I_band_median": 0.6994985938072205 + }, + "USPTO_Backgrounds": { + "rho_I_band_median": 0.7485703825950623 + } + } + }, + "E39": { + "70m": { + "n_concept_tokens": 1310, + "mean_cos_I": 0.9585257768630981, + "eff_rank_I": 1, + "per_layer": { + "2": { + "mean_cos_J": 0.9692184329032898, + "eff_rank_J": 1, + "spread_gain": -0.01069265604019165 + }, + "3": { + "mean_cos_J": 0.9622145891189575, + "eff_rank_J": 1, + "spread_gain": -0.003688812255859375 + } + }, + "spread_gain_band_median": -0.007190734148025513 + }, + "160m": { + "n_concept_tokens": 1310, + "mean_cos_I": 0.921846866607666, + "eff_rank_I": 1, + "per_layer": { + "4": { + "mean_cos_J": 0.9358569979667664, + "eff_rank_J": 1, + "spread_gain": -0.014010131359100342 + }, + "5": { + "mean_cos_J": 0.9418216943740845, + "eff_rank_J": 1, + "spread_gain": -0.019974827766418457 + }, + "6": { + "mean_cos_J": 0.9321896433830261, + "eff_rank_J": 1, + "spread_gain": -0.010342776775360107 + }, + "7": { + "mean_cos_J": 0.9413935542106628, + "eff_rank_J": 1, + "spread_gain": -0.019546687602996826 + }, + "8": { + "mean_cos_J": 0.9385942220687866, + "eff_rank_J": 1, + "spread_gain": -0.016747355461120605 + }, + "9": { + "mean_cos_J": 0.938626766204834, + "eff_rank_J": 1, + "spread_gain": -0.01677989959716797 + } + }, + "spread_gain_band_median": -0.016763627529144287 + }, + "410m": { + "n_concept_tokens": 1310, + "mean_cos_I": 0.030666086822748184, + "eff_rank_I": 371, + "per_layer": { + "8": { + "mean_cos_J": 0.10044567286968231, + "eff_rank_J": 168, + "spread_gain": -0.06977958604693413 + }, + "9": { + "mean_cos_J": 0.09548469632863998, + "eff_rank_J": 182, + "spread_gain": -0.0648186095058918 + }, + "10": { + "mean_cos_J": 0.09692367911338806, + "eff_rank_J": 191, + "spread_gain": -0.06625759229063988 + }, + "11": { + "mean_cos_J": 0.09258390963077545, + "eff_rank_J": 204, + "spread_gain": -0.06191782280802727 + }, + "12": { + "mean_cos_J": 0.0784233883023262, + "eff_rank_J": 227, + "spread_gain": -0.04775730147957802 + }, + "13": { + "mean_cos_J": 0.07444868236780167, + "eff_rank_J": 244, + "spread_gain": -0.04378259554505348 + }, + "14": { + "mean_cos_J": 0.07194945216178894, + "eff_rank_J": 252, + "spread_gain": -0.041283365339040756 + }, + "15": { + "mean_cos_J": 0.07019051164388657, + "eff_rank_J": 261, + "spread_gain": -0.03952442482113838 + }, + "16": { + "mean_cos_J": 0.0643744096159935, + "eff_rank_J": 288, + "spread_gain": -0.033708322793245316 + }, + "17": { + "mean_cos_J": 0.06355012208223343, + "eff_rank_J": 308, + "spread_gain": -0.032884035259485245 + }, + "18": { + "mean_cos_J": 0.06801048666238785, + "eff_rank_J": 318, + "spread_gain": -0.037344399839639664 + }, + "19": { + "mean_cos_J": 0.06699850410223007, + "eff_rank_J": 333, + "spread_gain": -0.03633241727948189 + }, + "20": { + "mean_cos_J": 0.05749906599521637, + "eff_rank_J": 342, + "spread_gain": -0.026832979172468185 + } + }, + "spread_gain_band_median": -0.041283365339040756 + }, + "1b": { + "n_concept_tokens": 1310, + "mean_cos_I": 0.019916435703635216, + "eff_rank_I": 508, + "per_layer": { + "5": { + "mean_cos_J": 0.02734173648059368, + "eff_rank_J": 307, + "spread_gain": -0.007425300776958466 + }, + "6": { + "mean_cos_J": 0.02941753715276718, + "eff_rank_J": 322, + "spread_gain": -0.009501101449131966 + }, + "7": { + "mean_cos_J": 0.026158984750509262, + "eff_rank_J": 335, + "spread_gain": -0.006242549046874046 + }, + "8": { + "mean_cos_J": 0.02683248184621334, + "eff_rank_J": 340, + "spread_gain": -0.006916046142578125 + }, + "9": { + "mean_cos_J": 0.019896788522601128, + "eff_rank_J": 390, + "spread_gain": 1.9647181034088135e-05 + }, + "10": { + "mean_cos_J": 0.01869310438632965, + "eff_rank_J": 389, + "spread_gain": 0.0012233313173055649 + }, + "11": { + "mean_cos_J": 0.01785549893975258, + "eff_rank_J": 434, + "spread_gain": 0.002060936763882637 + }, + "12": { + "mean_cos_J": 0.01755559630692005, + "eff_rank_J": 442, + "spread_gain": 0.002360839396715164 + }, + "13": { + "mean_cos_J": 0.01641128957271576, + "eff_rank_J": 469, + "spread_gain": 0.0035051461309194565 + } + }, + "spread_gain_band_median": 1.9647181034088135e-05 + } + }, + "E40": { + "70m": { + "act_cos_per_layer": { + "2": 0.5457611083984375, + "3": 0.6813090443611145 + }, + "act_cos_band_median": 0.613535076379776, + "n_activations": 541 + }, + "160m": { + "act_cos_per_layer": { + "4": 0.41877439618110657, + "5": 0.4936310350894928, + "6": 0.5747991800308228, + "7": 0.6295773386955261, + "8": 0.7001177668571472, + "9": 0.7638267278671265 + }, + "act_cos_band_median": 0.6021882593631744, + "n_activations": 541 + }, + "410m": { + "act_cos_per_layer": { + "8": 0.34123513102531433, + "9": 0.36684760451316833, + "10": 0.3739292621612549, + "11": 0.40823984146118164, + "12": 0.47148656845092773, + "13": 0.4754537045955658, + "14": 0.4960428774356842, + "15": 0.4374483823776245, + "16": 0.5739069581031799, + "17": 0.6946588158607483, + "18": 0.7569411993026733, + "19": 0.8296056389808655, + "20": 0.8607757687568665 + }, + "act_cos_band_median": 0.4754537045955658, + "n_activations": 541 + }, + "1b": { + "act_cos_per_layer": { + "5": 0.3916463553905487, + "6": 0.4406467080116272, + "7": 0.5441496968269348, + "8": 0.60288405418396, + "9": 0.7141939401626587, + "10": 0.7486943602561951, + "11": 0.8453687429428101, + "12": 0.9058312773704529, + "13": 0.9443002343177795 + }, + "act_cos_band_median": 0.7141939401626587, + "n_activations": 541 + } + }, + "E41": { + "70m": { + "layer_cos_median": 0.7500197291374207, + "band": [ + 2, + 3 + ] + }, + "160m": { + "layer_cos_median": 0.7828831672668457, + "band": [ + 4, + 5, + 6, + 7, + 8, + 9 + ] + }, + "410m": { + "layer_cos_median": 0.7752204239368439, + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + "1b": { + "layer_cos_median": 0.7716220319271088, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ] + }, + "1.4b": { + "layer_cos_median": 0.8131929934024811, + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + "2.8b": { + "layer_cos_median": 0.9046958684921265, + "band": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27 + ] + }, + "by_corpus_410m": { + "Github": { + "layer_cos_median": 0.7846129536628723 + }, + "Wikipedia_en": { + "layer_cos_median": 0.7826382219791412 + }, + "StackExchange": { + "layer_cos_median": 0.7724171876907349 + }, + "Pile-CC": { + "layer_cos_median": 0.7917884886264801 + }, + "USPTO_Backgrounds": { + "layer_cos_median": 0.7758865356445312 + } + } + }, + "E42": { + "gap_by_scale": { + "70m": 0.005495067073830538, + "160m": -0.035763641100909016, + "410m": 0.008822743727692539 + }, + "gap_by_corpus_410m": { + "Github": -0.0003554493695700246, + "Wikipedia_en": 0.010165466311741804, + "StackExchange": 0.01723609993766461, + "Pile-CC": 0.017276245121050762, + "USPTO_Backgrounds": 0.021295761683630565 + }, + "scale": { + "rho_I": { + "r": 0.2655500096599166, + "n": 3, + "models": [ + "70m", + "160m", + "410m" + ] + }, + "layer_cos": { + "r": -0.6297857110956958, + "n": 3, + "models": [ + "70m", + "160m", + "410m" + ] + }, + "spread_gain": { + "r": -0.309772888112111, + "n": 3, + "models": [ + "70m", + "160m", + "410m" + ] + }, + "act_cos": { + "r": -0.49389183554479377, + "n": 3, + "models": [ + "70m", + "160m", + "410m" + ] + } + }, + "corpus": { + "rho_I": { + "r": 0.3136416931265416, + "loo": { + "Github": 0.49120022117418083, + "Wikipedia_en": 0.29803495604418107, + "StackExchange": 0.31097763582816784, + "Pile-CC": 0.7293881076009121, + "USPTO_Backgrounds": -0.33741342018888004 + }, + "worst_loo": [ + "Wikipedia_en", + 0.29803495604418107 + ], + "survives": false + }, + "layer_cos": { + "r": -0.33094121553738315, + "loo": { + "Github": -0.2772349313528112, + "Wikipedia_en": -0.32152957507270685, + "StackExchange": -0.21095005219178334, + "Pile-CC": -0.8584707904033608, + "USPTO_Backgrounds": -0.14422605915130887 + }, + "worst_loo": [ + "USPTO_Backgrounds", + -0.14422605915130887 + ], + "survives": false + } + } + }, + "controls": { + "E38_C1_random_matrix_rho": 0.9999992847442627, + "E38_C2_scaled_identity_rho": 1.2887501554814662e-07, + "E38_C1_fires": true, + "E38_C2_fires": true + } +} \ No newline at end of file diff --git a/results/e44_dread_410m.json b/results/e44_dread_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..655989002834b19b5c7f2e5b50895d5b88feca2f --- /dev/null +++ b/results/e44_dread_410m.json @@ -0,0 +1,127 @@ +{ + "experiment": "E44 / M5 \u2014 D_read, the read-relevant operator distance (predictor #18)", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_concept_rows": 1310, + "control_C1_dread_self": 0.0, + "control_C1_fires": true, + "by_corpus": { + "Github": { + "dread_vs_I": 0.816644081886027, + "dread_vs_Jeval": 0.20005170618083593, + "dact_vs_I": 0.8717041649984807, + "dread_random_rows_vs_I": 0.6370016649598689 + }, + "Wikipedia_en": { + "dread_vs_I": 0.9786600243224018, + "dread_vs_Jeval": 0.15553372884856118, + "dact_vs_I": 0.9431615762992673, + "dread_random_rows_vs_I": 0.6517485960063247 + }, + "StackExchange": { + "dread_vs_I": 0.9400196500214478, + "dread_vs_Jeval": 0.1491226058565036, + "dact_vs_I": 0.9193266342520802, + "dread_random_rows_vs_I": 0.6255374004610355 + }, + "Pile-CC": { + "dread_vs_I": 0.8858230747622643, + "dread_vs_Jeval": 0.12221865743670757, + "dact_vs_I": 0.9419957989334248, + "dread_random_rows_vs_I": 0.6243191844934851 + }, + "USPTO_Backgrounds": { + "dread_vs_I": 1.0667946988783044, + "dread_vs_Jeval": 0.18076446952476025, + "dact_vs_I": 0.9305944554069845, + "dread_random_rows_vs_I": 0.8735875853894413 + } + }, + "gap_vs_logit": { + "Github": -0.0003554493695700246, + "Wikipedia_en": 0.010165466311741804, + "StackExchange": 0.01723609993766461, + "Pile-CC": 0.017276245121050762, + "USPTO_Backgrounds": 0.021295761683630565 + }, + "correlations": { + "dread_vs_I": { + "r": 0.7316797681402211, + "loo": { + "Github": 0.3042927219600453, + "Wikipedia_en": 0.8183541360269996, + "StackExchange": 0.755814514866576, + "Pile-CC": 0.8897003220270652, + "USPTO_Backgrounds": 0.5922453949509309 + }, + "worst_loo": [ + "Github", + 0.3042927219600453 + ], + "survives": false + }, + "dread_vs_Jeval": { + "r": -0.5462143495794671, + "loo": { + "Github": 0.2704874836823881, + "Wikipedia_en": -0.5825645400241775, + "StackExchange": -0.5164248903295823, + "Pile-CC": -0.5302063467491597, + "USPTO_Backgrounds": -0.9363562835051303 + }, + "worst_loo": [ + "Github", + 0.2704874836823881 + ], + "survives": false + }, + "dact_vs_I": { + "r": 0.7549528612491698, + "loo": { + "Github": -0.5089565539965326, + "Wikipedia_en": 0.9356715359298814, + "StackExchange": 0.7953168732744653, + "Pile-CC": 0.7323623152168787, + "USPTO_Backgrounds": 0.7948011329819488 + }, + "worst_loo": [ + "Github", + -0.5089565539965326 + ], + "survives": false + }, + "dread_random_rows_vs_I": { + "r": 0.48807592860992544, + "loo": { + "Github": 0.6121122466182216, + "Wikipedia_en": 0.47202068506420203, + "StackExchange": 0.6173101317043055, + "Pile-CC": 0.6218017959592831, + "USPTO_Backgrounds": -0.51079391739967 + }, + "worst_loo": [ + "Wikipedia_en", + 0.47202068506420203 + ], + "survives": false + } + }, + "control_C2_r_dread_vs_dact": 0.6321166618830985, + "control_C2_distinct": true, + "VERDICT": "NULL \u2014 D_read does not predict the corpus floor. Predictor #18 fails." +} \ No newline at end of file diff --git a/results/e45_disagreement_geometry.json b/results/e45_disagreement_geometry.json new file mode 100644 index 0000000000000000000000000000000000000000..29f6c059e882b29c0723b331286a92f82a21926c --- /dev/null +++ b/results/e45_disagreement_geometry.json @@ -0,0 +1,141 @@ +{ + "experiment": "E45 \u2014 between-corpus operator disagreement, resolved per readout direction", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_concept_tokens_per_set": { + "multihop": 199, + "multilingual": 369, + "order-ops": 129, + "poetry": 346, + "typo": 371 + }, + "control_C1_zero_delta": 0.0, + "control_C1_fires": true, + "note": "Delta_P = J^P - mean_corpora(J); s_c is the per-concept term of D_read, which t39 summed away", + "pre_norm": { + "s_by_cell": { + "Github|multihop": 0.06453294306993484, + "Github|multilingual": 0.05565283074975014, + "Github|order-ops": 0.06409358233213425, + "Github|poetry": 0.060621537268161774, + "Github|typo": 0.05715227872133255, + "Wikipedia_en|multihop": 0.02521834336221218, + "Wikipedia_en|multilingual": 0.024450715631246567, + "Wikipedia_en|order-ops": 0.02195018157362938, + "Wikipedia_en|poetry": 0.024187589064240456, + "Wikipedia_en|typo": 0.021993717178702354, + "StackExchange|multihop": 0.02671734429895878, + "StackExchange|multilingual": 0.029058223590254784, + "StackExchange|order-ops": 0.032539475709199905, + "StackExchange|poetry": 0.030990630388259888, + "StackExchange|typo": 0.030744435265660286, + "Pile-CC|multihop": 0.014427314512431622, + "Pile-CC|multilingual": 0.016316192224621773, + "Pile-CC|order-ops": 0.012237297371029854, + "Pile-CC|poetry": 0.01584223099052906, + "Pile-CC|typo": 0.013465631753206253, + "USPTO_Backgrounds|multihop": 0.030059009790420532, + "USPTO_Backgrounds|multilingual": 0.03093351423740387, + "USPTO_Backgrounds|order-ops": 0.02836107648909092, + "USPTO_Backgrounds|poetry": 0.027069009840488434, + "USPTO_Backgrounds|typo": 0.025927294045686722 + }, + "r": 0.6169298718606291, + "n_cells": 25, + "loo_by_set": { + "multihop": 0.53696139646654, + "multilingual": 0.6102176309761381, + "order-ops": 0.6438304506744612, + "poetry": 0.7443380738386328, + "typo": 0.5400063308388493 + }, + "loo_by_corpus": { + "Github": 0.2024534246986471, + "Wikipedia_en": 0.7175949997961392, + "StackExchange": 0.6568928234545184, + "Pile-CC": 0.5710293024383641, + "USPTO_Backgrounds": 0.6397680950558284 + }, + "worst_loo_set": [ + "multihop", + 0.53696139646654 + ], + "worst_loo_corpus": [ + "Github", + 0.2024534246986471 + ], + "survives": false + }, + "post_norm": { + "s_by_cell": { + "Github|multihop": 1.0400294065475464, + "Github|multilingual": 1.0481507778167725, + "Github|order-ops": 1.0506365299224854, + "Github|poetry": 1.1820619106292725, + "Github|typo": 1.1752766370773315, + "Wikipedia_en|multihop": 0.7885950207710266, + "Wikipedia_en|multilingual": 0.6339923739433289, + "Wikipedia_en|order-ops": 0.8369501829147339, + "Wikipedia_en|poetry": 0.6120684742927551, + "Wikipedia_en|typo": 0.6616728901863098, + "StackExchange|multihop": 0.9026137590408325, + "StackExchange|multilingual": 0.7817927002906799, + "StackExchange|order-ops": 0.8961479663848877, + "StackExchange|poetry": 0.7012752890586853, + "StackExchange|typo": 0.7113168835639954, + "Pile-CC|multihop": 0.4335557222366333, + "Pile-CC|multilingual": 0.4763605296611786, + "Pile-CC|order-ops": 0.3831879198551178, + "Pile-CC|poetry": 0.5129296779632568, + "Pile-CC|typo": 0.45020848512649536, + "USPTO_Backgrounds|multihop": 0.8309922218322754, + "USPTO_Backgrounds|multilingual": 0.9021438360214233, + "USPTO_Backgrounds|order-ops": 0.776004433631897, + "USPTO_Backgrounds|poetry": 0.8589208722114563, + "USPTO_Backgrounds|typo": 0.85048508644104 + }, + "r": 0.49614222895095655, + "n_cells": 25, + "loo_by_set": { + "multihop": 0.49062894769317705, + "multilingual": 0.46145585077221873, + "order-ops": 0.5004320756100146, + "poetry": 0.6396445587743138, + "typo": 0.38987876875470867 + }, + "loo_by_corpus": { + "Github": 0.1524035039262419, + "Wikipedia_en": 0.5857116880646671, + "StackExchange": 0.5761449664159827, + "Pile-CC": 0.3944336398355985, + "USPTO_Backgrounds": 0.5574966724513039 + }, + "worst_loo_set": [ + "typo", + 0.38987876875470867 + ], + "worst_loo_corpus": [ + "Github", + 0.1524035039262419 + ], + "survives": false + }, + "control_C2_random_delta_r": 0.030950976174021295, + "control_C2_fires": true, + "VERDICT": "NULL \u2014 disagreement geometry does not explain which concepts move" +} \ No newline at end of file diff --git a/results/e46_disagreement_ablation.json b/results/e46_disagreement_ablation.json new file mode 100644 index 0000000000000000000000000000000000000000..b75d304439c54db1869a4f76818381b541eff9b3 --- /dev/null +++ b/results/e46_disagreement_ablation.json @@ -0,0 +1,624 @@ +{ + "experiment": "E46 \u2014 ablate the between-corpus disagreement subspace and test the SELECTIVE prediction", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "baseline_read": { + "Github": { + "multihop": 0.004160887694784573, + "multilingual": 0.028281363126422678, + "order-ops": 0.019480519528899874, + "poetry": 0.004373177752963134, + "typo": 0.08333333488553762 + }, + "Wikipedia_en": { + "multihop": 0.020804438207830702, + "multilingual": 0.028281362461192266, + "order-ops": 0.018181818138275827, + "poetry": 0.001457725917654378, + "typo": 0.12797618830310448 + }, + "StackExchange": { + "multihop": 0.013869625649281911, + "multilingual": 0.03879622795752117, + "order-ops": 0.0025974025151559283, + "poetry": 0.004373177752963134, + "typo": 0.17113095468708447 + }, + "Pile-CC": { + "multihop": 0.016643550779138292, + "multilingual": 0.036983320356479714, + "order-ops": 0.005194805030311857, + "poetry": 0.001457725917654378, + "typo": 0.16666666977107525 + }, + "USPTO_Backgrounds": { + "multihop": 0.019417475908994675, + "multilingual": 0.04024655504950455, + "order-ops": 0.010389610060623713, + "poetry": 0.004373177752963134, + "typo": 0.18005952225731953 + } + }, + "flip_grid_z": { + "Github|multihop": -1.8579416470957204, + "Github|multilingual": -1.2390257234477016, + "Github|order-ops": 1.3010019509565711, + "Github|poetry": 0.9323741379864844, + "Github|typo": -1.7157719637135673, + "Wikipedia_en|multihop": 0.953223990524682, + "Wikipedia_en|multilingual": -1.1588343573383737, + "Wikipedia_en|order-ops": 1.0245430215214906, + "Wikipedia_en|poetry": -1.4448049644938015, + "Wikipedia_en|typo": -0.5484653056060967, + "StackExchange|multihop": -0.1491938795838039, + "StackExchange|multilingual": 0.9401232954204871, + "StackExchange|order-ops": -1.239660959137691, + "StackExchange|poetry": 0.8873802117565733, + "StackExchange|typo": 0.6710718552524666, + "Pile-CC|multihop": 0.351181362730617, + "Pile-CC|multilingual": 0.4325677762551715, + "Pile-CC|order-ops": -0.7992369362284336, + "Pile-CC|poetry": -0.9573707636697681, + "Pile-CC|typo": 0.6626294990112414, + "USPTO_Backgrounds|multihop": 0.7027301734242256, + "USPTO_Backgrounds|multilingual": 1.0251690091104182, + "USPTO_Backgrounds|order-ops": -0.28664707711193643, + "USPTO_Backgrounds|poetry": 0.5824213784205114, + "USPTO_Backgrounds|typo": 0.9305359150559548 + }, + "arms": { + "Github->USPTO_Backgrounds|k=4|top": { + "pair": [ + "Github", + "USPTO_Backgrounds" + ], + "k": 4, + "arm": "top", + "read": { + "multihop": 0.002773925129856382, + "multilingual": 0.018491660510855063, + "order-ops": 0.0, + "poetry": 0.001457725917654378, + "typo": 0.06250000172959906 + }, + "movement": { + "multihop": -0.0909090909090909, + "multilingual": -0.8181818293012457, + "order-ops": 2.1428570592281804, + "poetry": NaN, + "typo": -0.21538462046335455 + }, + "flip": { + "multihop": 2.560671820519946, + "multilingual": 2.2641947325581198, + "order-ops": 1.5876490280685076, + "poetry": 0.34995275956597294, + "typo": 2.6463078787695222 + } + }, + "Github->USPTO_Backgrounds|k=4|random": { + "pair": [ + "Github", + "USPTO_Backgrounds" + ], + "k": 4, + "arm": "random", + "read": { + "multihop": 0.005547850259712764, + "multilingual": 0.029006526273276125, + "order-ops": 0.022077922310147966, + "poetry": 0.004373177752963134, + "typo": 0.08779762006763901 + }, + "movement": { + "multihop": 0.09090909090909093, + "multilingual": 0.06060606060606062, + "order-ops": -0.28571429407718196, + "poetry": NaN, + "typo": 0.0461538421331777 + }, + "flip": { + "multihop": 2.560671820519946, + "multilingual": 2.2641947325581198, + "order-ops": 1.5876490280685076, + "poetry": 0.34995275956597294, + "typo": 2.6463078787695222 + } + }, + "Github->USPTO_Backgrounds|k=4|bottom": { + "pair": [ + "Github", + "USPTO_Backgrounds" + ], + "k": 4, + "arm": "bottom", + "read": { + "multihop": 0.004160887694784573, + "multilingual": 0.027918781686042036, + "order-ops": 0.022077922310147966, + "poetry": 0.004373177752963134, + "typo": 0.08630952358778034 + }, + "movement": { + "multihop": -0.0, + "multilingual": -0.030303019183603064, + "order-ops": -0.28571429407718196, + "poetry": NaN, + "typo": 0.030769213416872065 + }, + "flip": { + "multihop": 2.560671820519946, + "multilingual": 2.2641947325581198, + "order-ops": 1.5876490280685076, + "poetry": 0.34995275956597294, + "typo": 2.6463078787695222 + } + }, + "Github->USPTO_Backgrounds|k=32|top": { + "pair": [ + "Github", + "USPTO_Backgrounds" + ], + "k": 32, + "arm": "top", + "read": { + "multihop": 0.001386962564928191, + "multilingual": 0.01667875237762928, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 0.08779762046677726 + }, + "movement": { + "multihop": -0.1818181818181818, + "multilingual": -0.969697003055252, + "order-ops": 2.1428570592281804, + "poetry": NaN, + "typo": 0.046153846259653276 + }, + "flip": { + "multihop": 2.560671820519946, + "multilingual": 2.2641947325581198, + "order-ops": 1.5876490280685076, + "poetry": 0.34995275956597294, + "typo": 2.6463078787695222 + } + }, + "Github->USPTO_Backgrounds|k=32|random": { + "pair": [ + "Github", + "USPTO_Backgrounds" + ], + "k": 32, + "arm": "random", + "read": { + "multihop": 0.001386962564928191, + "multilingual": 0.021392313231314932, + "order-ops": 0.005194805030311857, + "poetry": 0.004373177752963134, + "typo": 0.07738095322357756 + }, + "movement": { + "multihop": -0.1818181818181818, + "multilingual": -0.5757575757575759, + "order-ops": 1.57142852961409, + "poetry": NaN, + "typo": -0.06153847084948333 + }, + "flip": { + "multihop": 2.560671820519946, + "multilingual": 2.2641947325581198, + "order-ops": 1.5876490280685076, + "poetry": 0.34995275956597294, + "typo": 2.6463078787695222 + } + }, + "Github->USPTO_Backgrounds|k=32|bottom": { + "pair": [ + "Github", + "USPTO_Backgrounds" + ], + "k": 32, + "arm": "bottom", + "read": { + "multihop": 0.008321775123476982, + "multilingual": 0.02755619997956923, + "order-ops": 0.024675324692257812, + "poetry": 0.004373177752963134, + "typo": 0.08184523680912596 + }, + "movement": { + "multihop": 0.272727255286141, + "multilingual": -0.06060606060606062, + "order-ops": -0.5714285442491586, + "poetry": NaN, + "typo": -0.01538464522220781 + }, + "flip": { + "multihop": 2.560671820519946, + "multilingual": 2.2641947325581198, + "order-ops": 1.5876490280685076, + "poetry": 0.34995275956597294, + "typo": 2.6463078787695222 + } + }, + "Github->StackExchange|k=4|top": { + "pair": [ + "Github", + "StackExchange" + ], + "k": 4, + "arm": "top", + "read": { + "multihop": 0.001386962564928191, + "multilingual": 0.02393038397920983, + "order-ops": 0.0, + "poetry": 0.001457725917654378, + "typo": 0.07738095162702459 + }, + "movement": { + "multihop": -0.2857142857142857, + "multilingual": -0.4137931601692591, + "order-ops": 1.153846147784291, + "poetry": NaN, + "typo": -0.06779663585376793 + }, + "flip": { + "multihop": 1.7087477675119165, + "multilingual": 2.1791490188681886, + "order-ops": 2.5406629100942624, + "poetry": 0.04499392622991105, + "typo": 2.386843818966034 + } + }, + "Github->StackExchange|k=4|random": { + "pair": [ + "Github", + "StackExchange" + ], + "k": 4, + "arm": "random", + "read": { + "multihop": 0.002773925129856382, + "multilingual": 0.026468454993196895, + "order-ops": 0.020779220653431758, + "poetry": 0.004373177752963134, + "typo": 0.08184523680912596 + }, + "movement": { + "multihop": -0.14285714285714285, + "multilingual": -0.17241383149919084, + "order-ops": -0.07692306601172388, + "poetry": NaN, + "typo": -0.016949184724771367 + }, + "flip": { + "multihop": 1.7087477675119165, + "multilingual": 2.1791490188681886, + "order-ops": 2.5406629100942624, + "poetry": 0.04499392622991105, + "typo": 2.386843818966034 + } + }, + "Github->StackExchange|k=4|bottom": { + "pair": [ + "Github", + "StackExchange" + ], + "k": 4, + "arm": "bottom", + "read": { + "multihop": 0.005547850259712764, + "multilingual": 0.028281363126422678, + "order-ops": 0.01688311727983611, + "poetry": 0.004373177752963134, + "typo": 0.0833333335550768 + }, + "movement": { + "multihop": 0.14285714285714288, + "multilingual": -0.0, + "order-ops": 0.15384613202344755, + "poetry": NaN, + "typo": -1.5153723163015596e-08 + }, + "flip": { + "multihop": 1.7087477675119165, + "multilingual": 2.1791490188681886, + "order-ops": 2.5406629100942624, + "poetry": 0.04499392622991105, + "typo": 2.386843818966034 + } + }, + "Github->StackExchange|k=32|top": { + "pair": [ + "Github", + "StackExchange" + ], + "k": 32, + "arm": "top", + "read": { + "multihop": 0.001386962564928191, + "multilingual": 0.01522842635001455, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 0.07738095348966974 + }, + "movement": { + "multihop": -0.2857142857142857, + "multilingual": -1.241379417242065, + "order-ops": 1.153846147784291, + "poetry": NaN, + "typo": -0.06779661463855556 + }, + "flip": { + "multihop": 1.7087477675119165, + "multilingual": 2.1791490188681886, + "order-ops": 2.5406629100942624, + "poetry": 0.04499392622991105, + "typo": 2.386843818966034 + } + }, + "Github->StackExchange|k=32|random": { + "pair": [ + "Github", + "StackExchange" + ], + "k": 32, + "arm": "random", + "read": { + "multihop": 0.001386962564928191, + "multilingual": 0.02538071040596281, + "order-ops": 0.007792207811559949, + "poetry": 0.004373177752963134, + "typo": 0.08630952212427344 + }, + "movement": { + "multihop": -0.2857142857142857, + "multilingual": -0.2758621025617917, + "order-ops": 0.6923076886705745, + "poetry": NaN, + "typo": 0.03389826791959773 + }, + "flip": { + "multihop": 1.7087477675119165, + "multilingual": 2.1791490188681886, + "order-ops": 2.5406629100942624, + "poetry": 0.04499392622991105, + "typo": 2.386843818966034 + } + }, + "Github->StackExchange|k=32|bottom": { + "pair": [ + "Github", + "StackExchange" + ], + "k": 32, + "arm": "bottom", + "read": { + "multihop": 0.002773925129856382, + "multilingual": 0.027556200378707478, + "order-ops": 0.018181817872183665, + "poetry": 0.004373177752963134, + "typo": 0.08035714299018894 + }, + "movement": { + "multihop": -0.14285714285714285, + "multilingual": -0.0689654845177351, + "order-ops": 0.07692309753341055, + "poetry": NaN, + "typo": -0.033898320957628646 + }, + "flip": { + "multihop": 1.7087477675119165, + "multilingual": 2.1791490188681886, + "order-ops": 2.5406629100942624, + "poetry": 0.04499392622991105, + "typo": 2.386843818966034 + } + }, + "Wikipedia_en->USPTO_Backgrounds|k=4|top": { + "pair": [ + "Wikipedia_en", + "USPTO_Backgrounds" + ], + "k": 4, + "arm": "top", + "read": { + "multihop": 0.024965325902615274, + "multilingual": 0.027193618805280755, + "order-ops": 0.011688311185155596, + "poetry": 0.0, + "typo": 0.10267857382340091 + }, + "movement": { + "multihop": -3.0000005755574546, + "multilingual": -0.09090899689939215, + "order-ops": 0.833333361790411, + "poetry": -0.49999999999999994, + "typo": -0.48571419222014406 + }, + "flip": { + "multihop": 0.2504938171004564, + "multilingual": 2.184003366448792, + "order-ops": 1.311190098633427, + "poetry": 2.0272263429143127, + "typo": 1.4790012206620515 + } + }, + "Wikipedia_en->USPTO_Backgrounds|k=4|random": { + "pair": [ + "Wikipedia_en", + "USPTO_Backgrounds" + ], + "k": 4, + "arm": "random", + "read": { + "multihop": 0.01941747564290251, + "multilingual": 0.03045685256698302, + "order-ops": 0.0194805200610842, + "poetry": 0.001457725917654378, + "typo": 0.12351190578192472 + }, + "movement": { + "multihop": 1.0000001918524866, + "multilingual": 0.18181822730674582, + "order-ops": -0.16666674065506842, + "poetry": -0.0, + "typo": -0.08571422338485807 + }, + "flip": { + "multihop": 0.2504938171004564, + "multilingual": 2.184003366448792, + "order-ops": 1.311190098633427, + "poetry": 2.0272263429143127, + "typo": 1.4790012206620515 + } + }, + "Wikipedia_en->USPTO_Backgrounds|k=4|bottom": { + "pair": [ + "Wikipedia_en", + "USPTO_Backgrounds" + ], + "k": 4, + "arm": "bottom", + "read": { + "multihop": 0.020804438207830702, + "multilingual": 0.028643944433757236, + "order-ops": 0.019480519528899874, + "poetry": 0.0, + "typo": 0.13095237806971585 + }, + "movement": { + "multihop": 0.0, + "multilingual": 0.030303061976548862, + "order-ops": -0.1666666723580822, + "poetry": -0.49999999999999994, + "typo": 0.057142842837742645 + }, + "flip": { + "multihop": 0.2504938171004564, + "multilingual": 2.184003366448792, + "order-ops": 1.311190098633427, + "poetry": 2.0272263429143127, + "typo": 1.4790012206620515 + } + }, + "Wikipedia_en->USPTO_Backgrounds|k=32|top": { + "pair": [ + "Wikipedia_en", + "USPTO_Backgrounds" + ], + "k": 32, + "arm": "top", + "read": { + "multihop": 0.008321775123476982, + "multilingual": 0.02102973112570388, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 0.07142857196075576 + }, + "movement": { + "multihop": 9.000001726672366, + "multilingual": -0.6060605612459468, + "order-ops": 2.333333242270685, + "poetry": -0.49999999999999994, + "typo": -1.0857142208303732 + }, + "flip": { + "multihop": 0.2504938171004564, + "multilingual": 2.184003366448792, + "order-ops": 1.311190098633427, + "poetry": 2.0272263429143127, + "typo": 1.4790012206620515 + } + }, + "Wikipedia_en->USPTO_Backgrounds|k=32|random": { + "pair": [ + "Wikipedia_en", + "USPTO_Backgrounds" + ], + "k": 32, + "arm": "random", + "read": { + "multihop": 0.0152565882142101, + "multilingual": 0.026831036965761865, + "order-ops": 0.02337662343467985, + "poetry": 0.0, + "typo": 0.11458333408726114 + }, + "movement": { + "multihop": 4.000000575557456, + "multilingual": -0.12121204775651438, + "order-ops": -0.6666666552838358, + "poetry": -0.49999999999999994, + "typo": -0.25714279787881106 + }, + "flip": { + "multihop": 0.2504938171004564, + "multilingual": 2.184003366448792, + "order-ops": 1.311190098633427, + "poetry": 2.0272263429143127, + "typo": 1.4790012206620515 + } + }, + "Wikipedia_en->USPTO_Backgrounds|k=32|bottom": { + "pair": [ + "Wikipedia_en", + "USPTO_Backgrounds" + ], + "k": 32, + "arm": "bottom", + "read": { + "multihop": 0.020804437941738536, + "multilingual": 0.029006526140230044, + "order-ops": 0.010389610060623713, + "poetry": 0.001457725917654378, + "typo": 0.12351190471755606 + }, + "movement": { + "multihop": 1.9185248646345173e-07, + "multilingual": 0.060606101714244465, + "order-ops": 1.0, + "poetry": -0.0, + "typo": -0.08571424382073616 + }, + "flip": { + "multihop": 0.2504938171004564, + "multilingual": 2.184003366448792, + "order-ops": 1.311190098633427, + "poetry": 2.0272263429143127, + "typo": 1.4790012206620515 + } + } + }, + "correlations": { + "top": { + "r": -0.3754312320927198, + "n_cells": 26 + }, + "random": { + "r": -0.5423222614582651, + "n_cells": 26 + }, + "bottom": { + "r": 0.005944158774703814, + "n_cells": 26 + } + }, + "control_C1_random_inert": false, + "VERDICT": "NULL \u2014 ablating the disagreement subspace does not selectively move the flipping sets" +} \ No newline at end of file diff --git a/results/e47_ablation_pairs.json b/results/e47_ablation_pairs.json new file mode 100644 index 0000000000000000000000000000000000000000..ba5e208555081322b904e84e2dcd3191a576bc5a --- /dev/null +++ b/results/e47_ablation_pairs.json @@ -0,0 +1,418 @@ +{ + "experiment": "E47 \u2014 is the disagreement knockout a law about the operator or a fact about Github?", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "rank": 4, + "baseline_read": { + "Github": { + "multihop": 0.004160887694784573, + "multilingual": 0.028281363126422678, + "order-ops": 0.019480519528899874, + "poetry": 0.004373177752963134, + "typo": 0.08333333488553762 + }, + "Wikipedia_en": { + "multihop": 0.020804438207830702, + "multilingual": 0.028281362461192266, + "order-ops": 0.018181818138275827, + "poetry": 0.001457725917654378, + "typo": 0.12797618830310448 + }, + "StackExchange": { + "multihop": 0.013869625649281911, + "multilingual": 0.03879622795752117, + "order-ops": 0.0025974025151559283, + "poetry": 0.004373177752963134, + "typo": 0.17113095468708447 + }, + "Pile-CC": { + "multihop": 0.016643550779138292, + "multilingual": 0.036983320356479714, + "order-ops": 0.005194805030311857, + "poetry": 0.001457725917654378, + "typo": 0.16666666977107525 + }, + "USPTO_Backgrounds": { + "multihop": 0.019417475908994675, + "multilingual": 0.04024655504950455, + "order-ops": 0.010389610060623713, + "poetry": 0.004373177752963134, + "typo": 0.18005952225731953 + } + }, + "pairs": { + "Github->USPTO_Backgrounds": { + "delta_rel_fro": 0.7316238295128771, + "disagree": { + "multihop": -0.015256588214210103, + "multilingual": -0.011965191923081871, + "order-ops": 0.00909090946827616, + "poetry": 0.0, + "typo": -0.0967261873717819 + }, + "arms": { + "top": { + "read": { + "multihop": 0.002773925129856382, + "multilingual": 0.018491660510855063, + "order-ops": 0.0, + "poetry": 0.001457725917654378, + "typo": 0.06250000172959906 + }, + "move": { + "multihop": 0.0013869625649281909, + "multilingual": 0.009789702615567615, + "order-ops": 0.019480519528899874, + "poetry": 0.0029154518353087565, + "typo": 0.020833333155938563 + }, + "selectivity_r": 0.6156383234735877, + "source": "reused" + }, + "random": { + "read": { + "multihop": 0.005547850259712764, + "multilingual": 0.029006526273276125, + "order-ops": 0.022077922310147966, + "poetry": 0.004373177752963134, + "typo": 0.08779762006763901 + }, + "move": { + "multihop": -0.0013869625649281913, + "multilingual": -0.000725163146853447, + "order-ops": -0.0025974027812480927, + "poetry": 0.0, + "typo": -0.004464285182101388 + }, + "selectivity_r": 0.8683138881706304, + "source": "reused" + }, + "bottom": { + "read": { + "multihop": 0.004160887694784573, + "multilingual": 0.027918781686042036, + "order-ops": 0.022077922310147966, + "poetry": 0.004373177752963134, + "typo": 0.08630952358778034 + }, + "move": { + "multihop": 0.0, + "multilingual": 0.0003625814403806424, + "order-ops": -0.0025974027812480927, + "poetry": 0.0, + "typo": -0.002976188702242713 + }, + "selectivity_r": 0.6761858448923304, + "source": "reused" + } + } + }, + "Github->StackExchange": { + "delta_rel_fro": 0.5310294397534902, + "disagree": { + "multihop": -0.009708737954497337, + "multilingual": -0.010514864831098491, + "order-ops": 0.016883117013743947, + "poetry": 0.0, + "typo": -0.08779761980154685 + }, + "arms": { + "top": { + "read": { + "multihop": 0.001386962564928191, + "multilingual": 0.02393038397920983, + "order-ops": 0.0, + "poetry": 0.001457725917654378, + "typo": 0.07738095162702459 + }, + "move": { + "multihop": 0.0027739251298563817, + "multilingual": 0.004350979147212847, + "order-ops": 0.019480519528899874, + "poetry": 0.0029154518353087565, + "typo": 0.005952383258513036 + }, + "selectivity_r": 0.037886105164915744, + "source": "reused" + }, + "random": { + "read": { + "multihop": 0.002773925129856382, + "multilingual": 0.026468454993196895, + "order-ops": 0.020779220653431758, + "poetry": 0.004373177752963134, + "typo": 0.08184523680912596 + }, + "move": { + "multihop": 0.0013869625649281909, + "multilingual": 0.001812908133225783, + "order-ops": -0.0012987011245318841, + "poetry": 0.0, + "typo": 0.001488098076411662 + }, + "selectivity_r": 0.35789987398865764, + "source": "reused" + }, + "bottom": { + "read": { + "multihop": 0.005547850259712764, + "multilingual": 0.028281363126422678, + "order-ops": 0.01688311727983611, + "poetry": 0.004373177752963134, + "typo": 0.0833333335550768 + }, + "move": { + "multihop": -0.0013869625649281913, + "multilingual": 0.0, + "order-ops": 0.002597402249063765, + "poetry": 0.0, + "typo": 1.3304608248443373e-09 + }, + "selectivity_r": -0.2527131013218424, + "source": "reused" + } + } + }, + "Wikipedia_en->USPTO_Backgrounds": { + "delta_rel_fro": 0.5396574029140836, + "disagree": { + "multihop": 0.0013869622988360274, + "multilingual": -0.011965192588312284, + "order-ops": 0.007792208077652114, + "poetry": -0.0029154518353087565, + "typo": -0.05208333395421505 + }, + "arms": { + "top": { + "read": { + "multihop": 0.024965325902615274, + "multilingual": 0.027193618805280755, + "order-ops": 0.011688311185155596, + "poetry": 0.0, + "typo": 0.10267857382340091 + }, + "move": { + "multihop": -0.004160887694784572, + "multilingual": 0.0010877436559115113, + "order-ops": 0.006493506953120232, + "poetry": 0.001457725917654378, + "typo": 0.025297614479703565 + }, + "selectivity_r": 0.9498469042702207, + "source": "reused" + }, + "random": { + "read": { + "multihop": 0.01941747564290251, + "multilingual": 0.03045685256698302, + "order-ops": 0.0194805200610842, + "poetry": 0.001457725917654378, + "typo": 0.12351190578192472 + }, + "move": { + "multihop": 0.001386962564928193, + "multilingual": -0.0021754901057907533, + "order-ops": -0.0012987019228083742, + "poetry": 0.0, + "typo": 0.004464282521179752 + }, + "selectivity_r": 0.9271887199762486, + "source": "reused" + }, + "bottom": { + "read": { + "multihop": 0.020804438207830702, + "multilingual": 0.028643944433757236, + "order-ops": 0.019480519528899874, + "poetry": 0.0, + "typo": 0.13095237806971585 + }, + "move": { + "multihop": 0.0, + "multilingual": -0.00036258197256497024, + "order-ops": -0.0012987013906240463, + "poetry": 0.001457725917654378, + "typo": -0.0029761897666113757 + }, + "selectivity_r": 0.830333233055002, + "source": "reused" + } + } + }, + "Wikipedia_en->StackExchange": { + "delta_rel_fro": 0.5406615660357557, + "disagree": { + "multihop": 0.006934812558548791, + "multilingual": -0.010514865496328903, + "order-ops": 0.015584415623119898, + "poetry": -0.0029154518353087565, + "typo": -0.04315476638398 + }, + "arms": { + "top": { + "read": { + "multihop": 0.024965325902615274, + "multilingual": 0.030819433741271496, + "order-ops": 0.005194805030311857, + "poetry": 0.0, + "typo": 0.11309523587780339 + }, + "move": { + "multihop": -0.004160887694784572, + "multilingual": -0.00253807128007923, + "order-ops": 0.01298701310796397, + "poetry": 0.001457725917654378, + "typo": 0.014880952425301089 + }, + "selectivity_r": 0.8368777257857798, + "source": "scored" + }, + "random": { + "read": { + "multihop": 0.01941747564290251, + "multilingual": 0.027556199713477066, + "order-ops": 0.018181818138275827, + "poetry": 0.001457725917654378, + "typo": 0.1264880955485361 + }, + "move": { + "multihop": 0.001386962564928193, + "multilingual": 0.0007251627477152002, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 0.0014880927545683764 + }, + "selectivity_r": 0.5328194939641212, + "source": "scored" + }, + "bottom": { + "read": { + "multihop": 0.019417475908994675, + "multilingual": 0.029006526406322206, + "order-ops": 0.02337662343467985, + "poetry": 0.0, + "typo": 0.13392857209380185 + }, + "move": { + "multihop": 0.0013869622988360274, + "multilingual": -0.0007251639451299405, + "order-ops": -0.005194805296404023, + "poetry": 0.001457725917654378, + "typo": -0.005952383790697374 + }, + "selectivity_r": 0.8136550604202794, + "source": "scored" + } + } + }, + "Pile-CC->StackExchange": { + "delta_rel_fro": 0.4222514252541443, + "disagree": { + "multihop": 0.002773925129856381, + "multilingual": -0.0018129076010414552, + "order-ops": 0.0025974025151559283, + "poetry": -0.0029154518353087565, + "typo": -0.004464284916009226 + }, + "arms": { + "top": { + "read": { + "multihop": 0.01803051307797432, + "multilingual": 0.03517041302153042, + "order-ops": 0.0038961039057799746, + "poetry": 0.0, + "typo": 0.16369047548089707 + }, + "move": { + "multihop": -0.0013869622988360274, + "multilingual": 0.001812907334949293, + "order-ops": 0.001298701124531882, + "poetry": 0.001457725917654378, + "typo": 0.0029761942901781746 + }, + "selectivity_r": 0.7558589460641069, + "source": "scored" + }, + "random": { + "read": { + "multihop": 0.0152565882142101, + "multilingual": 0.03517041288848434, + "order-ops": 0.005194805030311857, + "poetry": 0.001457725917654378, + "typo": 0.16220238405678952 + }, + "move": { + "multihop": 0.0013869625649281913, + "multilingual": 0.0018129074679953741, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 0.004464285714285726 + }, + "selectivity_r": 0.6701019020353106, + "source": "scored" + }, + "bottom": { + "read": { + "multihop": 0.016643550779138292, + "multilingual": 0.03625815720962627, + "order-ops": 0.007792207811559949, + "poetry": 0.001457725917654378, + "typo": 0.16666666764233792 + }, + "move": { + "multihop": 0.0, + "multilingual": 0.000725163146853447, + "order-ops": -0.0025974027812480927, + "poetry": 0.0, + "typo": 2.1287373253020547e-09 + }, + "selectivity_r": -0.37171084011405064, + "source": "scored" + } + } + } + }, + "summary": { + "selectivity_top": { + "Github->USPTO_Backgrounds": 0.6156383234735877, + "Github->StackExchange": 0.037886105164915744, + "Wikipedia_en->USPTO_Backgrounds": 0.9498469042702207, + "Wikipedia_en->StackExchange": 0.8368777257857798, + "Pile-CC->StackExchange": 0.7558589460641069 + }, + "selectivity_random": { + "Github->USPTO_Backgrounds": 0.8683138881706304, + "Github->StackExchange": 0.35789987398865764, + "Wikipedia_en->USPTO_Backgrounds": 0.9271887199762486, + "Wikipedia_en->StackExchange": 0.5328194939641212, + "Pile-CC->StackExchange": 0.6701019020353106 + }, + "selectivity_bottom": { + "Github->USPTO_Backgrounds": 0.6761858448923304, + "Github->StackExchange": -0.2527131013218424, + "Wikipedia_en->USPTO_Backgrounds": 0.830333233055002, + "Wikipedia_en->StackExchange": 0.8136550604202794, + "Pile-CC->StackExchange": -0.37171084011405064 + }, + "n_pairs_top_ge_0.5": 4, + "n_pairs_random_lt_0.3": 0, + "n_nonGithub_pairs_top_ge_0.5": "3/3", + "r_magnitude_vs_delta": 0.6255854541806675 + }, + "VERDICT": "REJECT \u2014 top-k behaves like random-k; any rank-k ablation does this" +} \ No newline at end of file diff --git a/results/e48/lens_INSTREAM_Github_410m_n200_s0.pt b/results/e48/lens_INSTREAM_Github_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2373b11421b0e941f1cb35e06e5f73ea5826319 --- /dev/null +++ b/results/e48/lens_INSTREAM_Github_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb +size 27268330 diff --git a/results/e48/lens_INSTREAM_Github_410m_n200_s1.pt b/results/e48/lens_INSTREAM_Github_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fb45f23db5cf53d0f3ad67cd25d1ec3bb7fbcfa --- /dev/null +++ b/results/e48/lens_INSTREAM_Github_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b +size 27268330 diff --git a/results/e48/lens_INSTREAM_Github_410m_n200_s2.pt b/results/e48/lens_INSTREAM_Github_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..a213b89634f998c68b695bcc69e70e3155e832ea --- /dev/null +++ b/results/e48/lens_INSTREAM_Github_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847 +size 27268330 diff --git a/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt b/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..90f7cd861fe90355cd87653e17dc16eeb4719e7c --- /dev/null +++ b/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20 +size 27268349 diff --git a/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt b/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a2d704480794f3396bf3d01cd05414ef93c700d --- /dev/null +++ b/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2 +size 27268349 diff --git a/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt b/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..80e57919b44fd24aa954f1deb806613eb888f5c4 --- /dev/null +++ b/results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028 +size 27268349 diff --git a/results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt b/results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc39bb89d051b4250e2ffbe9a5535e541fa7e38c --- /dev/null +++ b/results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79 +size 27268463 diff --git a/results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt b/results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfb58f347ba22de07d1a56b113fc34c29e11589a --- /dev/null +++ b/results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3 +size 27268463 diff --git a/results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt b/results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..b03bed896474f11620d5499733d51142058e150b --- /dev/null +++ b/results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0 +size 27268463 diff --git a/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt b/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..7769004eb26f4accf205459d2aca1915ea04b49d --- /dev/null +++ b/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942 +size 27268539 diff --git a/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt b/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..0746d200d72d82812409af9828b711893cd825b6 --- /dev/null +++ b/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f +size 27268539 diff --git a/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt b/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd2504c4333a0cd434062c7409deae88839c3dc8 --- /dev/null +++ b/results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1 +size 27268539 diff --git a/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt b/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5507a3744a92445cf370dad07f8e078bb7edc0ef --- /dev/null +++ b/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a +size 27268444 diff --git a/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt b/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d34b4aa84a65357293c48d6061908ca2897f8f5 --- /dev/null +++ b/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc +size 27268444 diff --git a/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt b/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5003e0a30b9de70c0e518eb8b228af5cfc2d9a80 --- /dev/null +++ b/results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5 +size 27268444 diff --git a/results/e48/lens_OOD_CommonPile_410m_n200_s0.pt b/results/e48/lens_OOD_CommonPile_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5c27ed436510190959db1b6bd79ec9f0fec980a --- /dev/null +++ b/results/e48/lens_OOD_CommonPile_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c +size 27268311 diff --git a/results/e48/lens_OOD_CommonPile_410m_n200_s1.pt b/results/e48/lens_OOD_CommonPile_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..f137496d4c700e42fd1c0f79fb8cb372df1f7961 --- /dev/null +++ b/results/e48/lens_OOD_CommonPile_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8 +size 27268311 diff --git a/results/e48/lens_OOD_CommonPile_410m_n200_s2.pt b/results/e48/lens_OOD_CommonPile_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..b44bf3162d8c00f763346847e6b49a4ef88e1e97 --- /dev/null +++ b/results/e48/lens_OOD_CommonPile_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153 +size 27268311 diff --git a/results/e48/lens_OOD_News_2024_410m_n200_s0.pt b/results/e48/lens_OOD_News_2024_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5353b4eb79109d326d5d99ffbb93131c56726ed --- /dev/null +++ b/results/e48/lens_OOD_News_2024_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189 +size 27268292 diff --git a/results/e48/lens_OOD_News_2024_410m_n200_s1.pt b/results/e48/lens_OOD_News_2024_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..85fecc03dfb105eb6ad73bde889d394163641d0f --- /dev/null +++ b/results/e48/lens_OOD_News_2024_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd +size 27268292 diff --git a/results/e48/lens_OOD_News_2024_410m_n200_s2.pt b/results/e48/lens_OOD_News_2024_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..b911519ac1483f0c8038c1748b01959562a99a4a --- /dev/null +++ b/results/e48/lens_OOD_News_2024_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed +size 27268292 diff --git a/results/e48/lens_OOD_News_2024_dedup_410m_n200_s0.pt b/results/e48/lens_OOD_News_2024_dedup_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4949ab89a85cc2bc65092bbd92c1f8974cb23138 --- /dev/null +++ b/results/e48/lens_OOD_News_2024_dedup_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c26bdd4ea65788a2917cbfed900332ea4750eef1c91fa7e2a18c88ef28ee01 +size 27268406 diff --git a/results/e48/lens_OOD_News_2024_dedup_410m_n200_s1.pt b/results/e48/lens_OOD_News_2024_dedup_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..e68503fc8fb257e6b4d14b8d02f8d56756a39a58 --- /dev/null +++ b/results/e48/lens_OOD_News_2024_dedup_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb77a3ac3d40c7ff137e40aa29630ab00e6c488967834e4aaad26ffc76aa8d9e +size 27268406 diff --git a/results/e48/lens_OOD_News_2024_dedup_410m_n200_s2.pt b/results/e48/lens_OOD_News_2024_dedup_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4192b06830d2901c6cf8bfdeac298d899875b69 --- /dev/null +++ b/results/e48/lens_OOD_News_2024_dedup_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0436f3ab4edfe92100cc91c8cbfe700dec104267141738bee857f820d89b1872 +size 27268406 diff --git a/results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt b/results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb927cf0fecb92583539d583b6155bb8c8c81bb6 --- /dev/null +++ b/results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60 +size 27268311 diff --git a/results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt b/results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..62cb17674bb05e9a9e322ffe1a8074de6ecea64e --- /dev/null +++ b/results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818 +size 27268311 diff --git a/results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt b/results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..951dd493962208f66c8f0d0e48ba2b206f6ceaf5 --- /dev/null +++ b/results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c +size 27268311 diff --git a/results/e48/tv_INSTREAM_Github_s0.json b/results/e48/tv_INSTREAM_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..7858de022cfa94e45723aa14fccf869f1fd25270 --- /dev/null +++ b/results/e48/tv_INSTREAM_Github_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Github", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017070308011690422, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11131254503769535, + "best1L": 0.07251631641494376, + "mean": 0.031014669952648028, + "persist": 0.025380710272916725 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023976024206993834, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004933841659554413, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26190476225955145, + "best1L": 0.15922619242753303, + "mean": 0.09111721587500402, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 1.173735567248103, + "dispersion_mean": 0.509869190632461, + "dispersion_slope": -0.10006680126257518, + "stable_rank": 16.407350540161133, + "top8_spectral_share": 0.17215149104595184, + "condition_number": 7906.27734375, + "unembed_alignment": 0.1145876795053482 + }, + "corpus": { + "mean_prompt_tokens": 2232.67, + "type_token_ratio": 0.030734501740069066, + "eval_vocab_overlap": 0.48778625954198473, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 1.173735567248103, + "10": 1.062397060438708, + "11": 0.9485893559091794, + "12": 0.7891666280846766, + "13": 0.684179135252394, + "14": 0.5564734682868888, + "15": 0.46302663912659436, + "16": 0.33620745504258165, + "17": 0.24683216840335787, + "18": 0.17548044427024292, + "19": 0.11337013312373385, + "20": 0.05391227279225558, + "21": 0.02492915024327665 + } + } + }, + "runtime_s": 453.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Github_s1.json b/results/e48/tv_INSTREAM_Github_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..94690b5fbece5723a6303016cc0fb404798fd674 --- /dev/null +++ b/results/e48/tv_INSTREAM_Github_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Github", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06657420311655317, + "mean": 0.018884028906800916, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12291515405688967, + "best1L": 0.08701958001724311, + "mean": 0.03394321372200336, + "persist": 0.0304568527000291 + }, + "order-ops": { + "min": 0.07402597500809602, + "best1L": 0.07142857063029494, + "mean": 0.019980019789987376, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2827380991407803, + "best1L": 0.1666666679084301, + "mean": 0.10222069413534232, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 1.2071625333762765, + "dispersion_mean": 0.5224776953335424, + "dispersion_slope": -0.10364656969581171, + "stable_rank": 15.211066246032715, + "top8_spectral_share": 0.1794212907552719, + "condition_number": 4810.6533203125, + "unembed_alignment": 0.12286306917667389 + }, + "corpus": { + "mean_prompt_tokens": 2467.21, + "type_token_ratio": 0.031033029211133222, + "eval_vocab_overlap": 0.5282442748091603, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 1.2071625333762765, + "10": 1.0958360366714484, + "11": 0.9796544461797566, + "12": 0.8132945337635189, + "13": 0.7054911600374625, + "14": 0.5729771640267951, + "15": 0.47668611847629755, + "16": 0.33544154436494233, + "17": 0.2439156176980619, + "18": 0.1722703461901497, + "19": 0.11151089354171068, + "20": 0.05300469346766302, + "21": 0.02496495154196686 + } + } + }, + "runtime_s": 454.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Github_s2.json b/results/e48/tv_INSTREAM_Github_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..f0e4fb84ba27f82c3b211abf75a6a48465fbf983 --- /dev/null +++ b/results/e48/tv_INSTREAM_Github_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Github", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017283686785958707, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07759245777768749, + "mean": 0.03207452408969402, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.08051948089684759, + "mean": 0.024175824597477913, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004373177886009216, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762259551455, + "best1L": 0.1592261932258095, + "mean": 0.09397893625178508, + "persist": 0.08184523760740246 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 1.259307189452949, + "dispersion_mean": 0.5406536262727959, + "dispersion_slope": -0.10741288961761643, + "stable_rank": 15.241456985473633, + "top8_spectral_share": 0.17799267172813416, + "condition_number": 4878.96728515625, + "unembed_alignment": 0.11666719615459442 + }, + "corpus": { + "mean_prompt_tokens": 2495.045, + "type_token_ratio": 0.02932011246290147, + "eval_vocab_overlap": 0.5290076335877862, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 1.259307189452949, + "10": 1.1382846859451943, + "11": 1.0096814100988638, + "12": 0.835951481190036, + "13": 0.7254275574709099, + "14": 0.5888547970550113, + "15": 0.4918080194736144, + "16": 0.3498732018921718, + "17": 0.2545045472934352, + "18": 0.17941154952228938, + "19": 0.11512079753469731, + "20": 0.05482091088786913, + "21": 0.02545099372930459 + } + } + }, + "runtime_s": 453.2, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Pile-CC_s0.json b/results/e48/tv_INSTREAM_Pile-CC_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..673980ddd06a65f0a8a53b9015787c0655a9d4d8 --- /dev/null +++ b/results/e48/tv_INSTREAM_Pile-CC_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Pile-CC", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17403915843793324, + "best1L": 0.1468455387013299, + "mean": 0.04392815341374704, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014085914241150022, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.008746355905064515, + "mean": 0.004036779648491314, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000319310595, + "best1L": 0.20535714338932717, + "mean": 0.16037087807697908, + "persist": 0.16517857063029492 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7788201376979573, + "dispersion_mean": 0.3280527797025537, + "dispersion_slope": -0.06492695644437091, + "stable_rank": 20.16012954711914, + "top8_spectral_share": 0.13443322479724884, + "condition_number": 11802.7880859375, + "unembed_alignment": 0.1931973695755005 + }, + "corpus": { + "mean_prompt_tokens": 1027.01, + "type_token_ratio": 0.09424445721073797, + "eval_vocab_overlap": 0.749618320610687, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.7788201376979573, + "10": 0.6974078914105855, + "11": 0.6251130369894873, + "12": 0.49081316275749387, + "13": 0.4168993187984826, + "14": 0.34904761899865666, + "15": 0.29155637185227884, + "16": 0.20932457790580888, + "17": 0.15360017543944016, + "18": 0.11165619862299439, + "19": 0.0754686697936176, + "20": 0.0442499558352948, + "21": 0.020729020031100418 + } + } + }, + "runtime_s": 496.3, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Pile-CC_s1.json b/results/e48/tv_INSTREAM_Pile-CC_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..a11e94678322db8a5e133c35b5f916c15b132480 --- /dev/null +++ b/results/e48/tv_INSTREAM_Pile-CC_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Pile-CC", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03126000187226704, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.1754894863281931, + "best1L": 0.14865844590323313, + "mean": 0.04373291613800185, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014185814153669136, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.005830903803663594, + "mean": 0.0033639829738863875, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3482142889073917, + "best1L": 0.2038690485060215, + "mean": 0.16128663186516082, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7798155363686027, + "dispersion_mean": 0.3263895317819609, + "dispersion_slope": -0.06498898154721662, + "stable_rank": 19.8128662109375, + "top8_spectral_share": 0.13467296957969666, + "condition_number": 18120.126953125, + "unembed_alignment": 0.19587922096252441 + }, + "corpus": { + "mean_prompt_tokens": 1244.555, + "type_token_ratio": 0.0835720398053923, + "eval_vocab_overlap": 0.7816793893129771, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.7798155363686027, + "10": 0.6978277142978796, + "11": 0.6216049678860679, + "12": 0.4899189761368765, + "13": 0.41530246161973516, + "14": 0.3453981024953795, + "15": 0.28814559402795814, + "16": 0.20616838919244365, + "17": 0.1512831256370529, + "18": 0.11021044981108413, + "19": 0.07420517769795827, + "20": 0.04307023486101095, + "21": 0.0201131831334417 + } + } + }, + "runtime_s": 478.0, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Pile-CC_s2.json b/results/e48/tv_INSTREAM_Pile-CC_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..dadbca5da5eea602c10af586eb04cbc1218abdcb --- /dev/null +++ b/results/e48/tv_INSTREAM_Pile-CC_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Pile-CC", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.10957003970231328, + "best1L": 0.0915395300835371, + "mean": 0.031580070299761634, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17077592281358583, + "best1L": 0.1432197242975235, + "mean": 0.04270095457988126, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013786213838362269, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20386904424854688, + "mean": 0.15785255948347704, + "persist": 0.16666666551360063 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8069925038170475, + "dispersion_mean": 0.3406058631873397, + "dispersion_slope": -0.06726667927840727, + "stable_rank": 20.590688705444336, + "top8_spectral_share": 0.1330161690711975, + "condition_number": 182962.96875, + "unembed_alignment": 0.19227716326713562 + }, + "corpus": { + "mean_prompt_tokens": 1350.62, + "type_token_ratio": 0.07824184448623595, + "eval_vocab_overlap": 0.7801526717557252, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.8069925038170475, + "10": 0.7236455932067546, + "11": 0.6507782295770467, + "12": 0.5069230366624464, + "13": 0.4325145756560667, + "14": 0.36211958542654904, + "15": 0.3014650083905833, + "16": 0.2186631538109518, + "17": 0.16021510312053397, + "18": 0.1176376790551146, + "19": 0.07944710578317682, + "20": 0.04596457662479728, + "21": 0.021510070304347515 + } + } + }, + "runtime_s": 483.3, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_StackExchange_s0.json b/results/e48/tv_INSTREAM_StackExchange_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..a8b81d059e5e6cdf70d68178d1d08cbaa671afd5 --- /dev/null +++ b/results/e48/tv_INSTREAM_StackExchange_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "StackExchange", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03019310820049473, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17258882841893605, + "best1L": 0.14213197944419725, + "mean": 0.04468120661165033, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010889111219772271, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22619047920618737, + "mean": 0.1605998176549162, + "persist": 0.17261905063475882 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5724424030243304, + "dispersion_mean": 0.24495708683617634, + "dispersion_slope": -0.04791163640146435, + "stable_rank": 20.334392547607422, + "top8_spectral_share": 0.13948950171470642, + "condition_number": 3002.712646484375, + "unembed_alignment": 0.15559512376785278 + }, + "corpus": { + "mean_prompt_tokens": 634.71, + "type_token_ratio": 0.08512549038143404, + "eval_vocab_overlap": 0.483206106870229, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5724424030243304, + "10": 0.5188572305881252, + "11": 0.4623004468568678, + "12": 0.370539460191556, + "13": 0.3138637077863185, + "14": 0.26061888150776946, + "15": 0.2161555568558931, + "16": 0.15658256653833225, + "17": 0.11753507870677082, + "18": 0.08548108141787733, + "19": 0.05773789738737819, + "20": 0.03482525470681331, + "21": 0.017502563302259688 + } + } + }, + "runtime_s": 475.7, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_StackExchange_s1.json b/results/e48/tv_INSTREAM_StackExchange_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..f1bb7b086886477c7b0fb98fa656c1e42b4f54a4 --- /dev/null +++ b/results/e48/tv_INSTREAM_StackExchange_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "StackExchange", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029979728694472994, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17077592760324478, + "best1L": 0.14140681496688298, + "mean": 0.044625424912997654, + "persist": 0.040246556379965374 + }, + "order-ops": { + "min": 0.07012987043708563, + "best1L": 0.05584415593849761, + "mean": 0.010589410609100014, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35863094883305685, + "best1L": 0.22470238325851305, + "mean": 0.1533882782927581, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5878502415941018, + "dispersion_mean": 0.2495487391630521, + "dispersion_slope": -0.04896059740664867, + "stable_rank": 19.765357971191406, + "top8_spectral_share": 0.14270330965518951, + "condition_number": 3743.6552734375, + "unembed_alignment": 0.15273356437683105 + }, + "corpus": { + "mean_prompt_tokens": 687.88, + "type_token_ratio": 0.07611065883584346, + "eval_vocab_overlap": 0.4618320610687023, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5878502415941018, + "10": 0.5297160197594637, + "11": 0.4711118304539421, + "12": 0.375743448506507, + "13": 0.31830964047087573, + "14": 0.26446156908904095, + "15": 0.21929931731213567, + "16": 0.1589821613129372, + "17": 0.11927409876374252, + "18": 0.08755129101135864, + "19": 0.05923407288714819, + "20": 0.03510869345568252, + "21": 0.01749122450274159 + } + } + }, + "runtime_s": 484.3, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_StackExchange_s2.json b/results/e48/tv_INSTREAM_StackExchange_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..02b5b4a709ab404ba69ab1ee873f75a95286f207 --- /dev/null +++ b/results/e48/tv_INSTREAM_StackExchange_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "StackExchange", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952981744494, + "mean": 0.030193108267017772, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17041334403412683, + "best1L": 0.14031906904918806, + "mean": 0.04398393431412322, + "persist": 0.038796229021889825 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15544871666601726, + "persist": 0.16815476066299848 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5744015717620247, + "dispersion_mean": 0.24362816357472725, + "dispersion_slope": -0.047853942193451446, + "stable_rank": 19.812728881835938, + "top8_spectral_share": 0.1424541473388672, + "condition_number": 3020.126220703125, + "unembed_alignment": 0.15600988268852234 + }, + "corpus": { + "mean_prompt_tokens": 677.565, + "type_token_ratio": 0.07989639370392508, + "eval_vocab_overlap": 0.49694656488549616, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5744015717620247, + "10": 0.5179816800191642, + "11": 0.45961539066076823, + "12": 0.3683881378552847, + "13": 0.31076970898714656, + "14": 0.2576772532648013, + "15": 0.21319225389043453, + "16": 0.15447083506766116, + "17": 0.11587805229659949, + "18": 0.08506788646069109, + "19": 0.0576619381987808, + "20": 0.03470386097880974, + "21": 0.017357557029287016 + } + } + }, + "runtime_s": 458.0, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_USPTO_Backgrounds_s0.json b/results/e48/tv_INSTREAM_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..ec2981e452607a223d2c90d5e84723dbca5f7e50 --- /dev/null +++ b/results/e48/tv_INSTREAM_USPTO_Backgrounds_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "USPTO_Backgrounds", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11650485305913857, + "best1L": 0.08183078920202595, + "mean": 0.030086418081607138, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829561965806, + "best1L": 0.1102248023131064, + "mean": 0.03982819164437907, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.12987013161182404, + "best1L": 0.101298702614648, + "mean": 0.0257742257256593, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20089285767504148, + "mean": 0.16380494407245091, + "persist": 0.17857142950275115 + }, + "association": { + "min": 0.020186335779726505, + "best1L": 0.007763975166848728, + "mean": 0.002269469658910696, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5678543452985556, + "dispersion_mean": 0.26716563902248763, + "dispersion_slope": -0.046723130548143435, + "stable_rank": 19.01803207397461, + "top8_spectral_share": 0.14545170962810516, + "condition_number": 1922.5693359375, + "unembed_alignment": 0.15122197568416595 + }, + "corpus": { + "mean_prompt_tokens": 900.0, + "type_token_ratio": 0.0683, + "eval_vocab_overlap": 0.4083969465648855, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5678543452985556, + "10": 0.5206929673031068, + "11": 0.4735036978972943, + "12": 0.3883015748975181, + "13": 0.3481453187941136, + "14": 0.3003240816238169, + "15": 0.26070403628571365, + "16": 0.1908779307432443, + "17": 0.15029800131770402, + "18": 0.11677430548540067, + "19": 0.0819478211862445, + "20": 0.04944810929905842, + "21": 0.024281117160569226 + } + } + }, + "runtime_s": 479.2, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_USPTO_Backgrounds_s1.json b/results/e48/tv_INSTREAM_USPTO_Backgrounds_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..c6d09b083926d723079a97b6393ee302fe57301a --- /dev/null +++ b/results/e48/tv_INSTREAM_USPTO_Backgrounds_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "USPTO_Backgrounds", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11650485412350722, + "best1L": 0.08183079026639462, + "mean": 0.030619865283370018, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14829586765595845, + "best1L": 0.11348803634090084, + "mean": 0.04049757309257984, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.11558441511754479, + "best1L": 0.09220779274723359, + "mean": 0.023676323363490934, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004373177752963134, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190447807312, + "best1L": 0.19940476172736712, + "mean": 0.16071428917348385, + "persist": 0.1726190499695284 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5515950193069236, + "dispersion_mean": 0.25509312099812753, + "dispersion_slope": -0.04551014316656823, + "stable_rank": 19.068532943725586, + "top8_spectral_share": 0.14585871994495392, + "condition_number": 1634.020263671875, + "unembed_alignment": 0.15165270864963531 + }, + "corpus": { + "mean_prompt_tokens": 856.155, + "type_token_ratio": 0.07197294882351911, + "eval_vocab_overlap": 0.40458015267175573, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5515950193069236, + "10": 0.5048057926852118, + "11": 0.4585864981839043, + "12": 0.3711285495225797, + "13": 0.3307662512777294, + "14": 0.2843771834359889, + "15": 0.24593570927496888, + "16": 0.17808548692734655, + "17": 0.13958051972654767, + "18": 0.10821624987660729, + "19": 0.07551021517576124, + "20": 0.045280371958854024, + "21": 0.02234272562323477 + } + } + }, + "runtime_s": 479.7, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_USPTO_Backgrounds_s2.json b/results/e48/tv_INSTREAM_USPTO_Backgrounds_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..2eac2e869b871c5e8150c95a2672da020407112b --- /dev/null +++ b/results/e48/tv_INSTREAM_USPTO_Backgrounds_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "USPTO_Backgrounds", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.12205270544758864, + "best1L": 0.08183079212903976, + "mean": 0.030513175097959384, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829540678433, + "best1L": 0.11203771057937827, + "mean": 0.040134992849613936, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.12597402770604407, + "best1L": 0.09870130089776856, + "mean": 0.023476523580029607, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142857142855, + "best1L": 0.20982143176453455, + "mean": 0.16597985476255417, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.00776397529989481, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5319501001345138, + "dispersion_mean": 0.24916745921921962, + "dispersion_slope": -0.04385243178511833, + "stable_rank": 18.735549926757812, + "top8_spectral_share": 0.14682801067829132, + "condition_number": 2053.4150390625, + "unembed_alignment": 0.14986512064933777 + }, + "corpus": { + "mean_prompt_tokens": 724.32, + "type_token_ratio": 0.07828031809145129, + "eval_vocab_overlap": 0.4015267175572519, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5319501001345138, + "10": 0.48882941106779304, + "11": 0.44472351869907295, + "12": 0.3621343341394191, + "13": 0.32373662515922824, + "14": 0.27895288413295616, + "15": 0.2413678891104479, + "16": 0.17625660224064016, + "17": 0.13896314796764478, + "18": 0.1078579999033729, + "19": 0.0760147693385376, + "20": 0.04572582005709582, + "21": 0.022663867899132738 + } + } + }, + "runtime_s": 471.5, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Wikipedia_en_s0.json b/results/e48/tv_INSTREAM_Wikipedia_en_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..43bbc5cfc7b48ec26984abeabc5c04bf77f696be --- /dev/null +++ b/results/e48/tv_INSTREAM_Wikipedia_en_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Wikipedia_en", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028272697569004128, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.1305293690945421, + "best1L": 0.10224800556898117, + "mean": 0.03402688626998237, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.01888111897278577, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.008746355505926269, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381236212595, + "best1L": 0.17708333422030723, + "mean": 0.12774725338178022, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7796180735841666, + "dispersion_mean": 0.35583822927247116, + "dispersion_slope": -0.06519457906568132, + "stable_rank": 21.22879981994629, + "top8_spectral_share": 0.13851161301136017, + "condition_number": 6802.8115234375, + "unembed_alignment": 0.16818183660507202 + }, + "corpus": { + "mean_prompt_tokens": 1188.27, + "type_token_ratio": 0.08580120679643516, + "eval_vocab_overlap": 0.7618320610687023, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.7796180735841666, + "10": 0.7102423350503225, + "11": 0.64534225867373, + "12": 0.5322098616625642, + "13": 0.464237880030488, + "14": 0.39581156741869594, + "15": 0.3367397084749186, + "16": 0.25510420554259383, + "17": 0.18934799950675807, + "18": 0.14133822456939038, + "19": 0.09648430841691759, + "20": 0.05448278363653508, + "21": 0.024937773975044864 + } + } + }, + "runtime_s": 460.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Wikipedia_en_s1.json b/results/e48/tv_INSTREAM_Wikipedia_en_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..dbd73e378d82ee493f129f6d4ba0cb8dd42a3e29 --- /dev/null +++ b/results/e48/tv_INSTREAM_Wikipedia_en_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Wikipedia_en", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.02848607660936458, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13741841805832727, + "best1L": 0.10986221954226494, + "mean": 0.035254086168216806, + "persist": 0.028643944300711155 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.020279720591913377, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.032069970055350235, + "best1L": 0.008746355505926269, + "mean": 0.005045974494091102, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18154761993459292, + "mean": 0.12820513200547015, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0011944576565708434, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.768932716745316, + "dispersion_mean": 0.34876658858251314, + "dispersion_slope": -0.0644543284930622, + "stable_rank": 21.30463981628418, + "top8_spectral_share": 0.13821963965892792, + "condition_number": 14298.96484375, + "unembed_alignment": 0.17085270583629608 + }, + "corpus": { + "mean_prompt_tokens": 1070.71, + "type_token_ratio": 0.09227054944849679, + "eval_vocab_overlap": 0.7770992366412214, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.768932716745316, + "10": 0.6995601493464163, + "11": 0.6352141977728959, + "12": 0.5253840314579814, + "13": 0.45504575690665916, + "14": 0.38664654459709047, + "15": 0.3282095780140393, + "16": 0.2482729691736257, + "17": 0.18297772040687957, + "18": 0.13538047788835073, + "19": 0.09171348357585637, + "20": 0.052535726738626554, + "21": 0.024092298948933367 + } + } + }, + "runtime_s": 490.2, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_INSTREAM_Wikipedia_en_s2.json b/results/e48/tv_INSTREAM_Wikipedia_en_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..29ca8dab84fb247d9d853ed8853fbfb40a1d6188 --- /dev/null +++ b/results/e48/tv_INSTREAM_Wikipedia_en_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Wikipedia_en", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.028166008181869984, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13379260205796786, + "best1L": 0.10659898338573319, + "mean": 0.034250014395053895, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06753246858716011, + "mean": 0.01918081915937364, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.309523805975914, + "best1L": 0.17410714019622123, + "mean": 0.12637362416301454, + "persist": 0.12202380664114441 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.7648865616785044, + "dispersion_mean": 0.3470521074122824, + "dispersion_slope": -0.06411255565719147, + "stable_rank": 21.25417709350586, + "top8_spectral_share": 0.13874824345111847, + "condition_number": 7472.94140625, + "unembed_alignment": 0.16912022233009338 + }, + "corpus": { + "mean_prompt_tokens": 1159.06, + "type_token_ratio": 0.08913688678756923, + "eval_vocab_overlap": 0.7717557251908397, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.7648865616785044, + "10": 0.6955590785546937, + "11": 0.6310631095019359, + "12": 0.5242350942718641, + "13": 0.45364602057087217, + "14": 0.3845936818270433, + "15": 0.32623239715936747, + "16": 0.2464204633131857, + "17": 0.1818547098587644, + "18": 0.13521407019242546, + "19": 0.09198311704745955, + "20": 0.05216603907138771, + "21": 0.023823053312167244 + } + } + }, + "runtime_s": 467.7, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_CommonPile_s0.json b/results/e48/tv_OOD_CommonPile_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..75f37f1174a727565900810ee07e8e918e7ebb23 --- /dev/null +++ b/results/e48/tv_OOD_CommonPile_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_CommonPile", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.1165048535913229, + "best1L": 0.08460471672671181, + "mean": 0.032753653990636976, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.14176939906818525, + "best1L": 0.1105873818908419, + "mean": 0.03522619478670614, + "persist": 0.031544597420309274 + }, + "order-ops": { + "min": 0.09350649187607425, + "best1L": 0.0688311682481851, + "mean": 0.021278721263765225, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.005830903803663594, + "mean": 0.002579053597790854, + "persist": 0.0 + }, + "typo": { + "min": 0.31994047335215975, + "best1L": 0.16964285554630415, + "mean": 0.13507326133549213, + "persist": 0.132440475347851 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.006211180106869766, + "mean": 0.002030578109302691, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.506547719439495, + "dispersion_mean": 0.23942311251441833, + "dispersion_slope": -0.04179304898490714, + "stable_rank": 20.50550079345703, + "top8_spectral_share": 0.1367933452129364, + "condition_number": 3916.047607421875, + "unembed_alignment": 0.1410866230726242 + }, + "corpus": { + "mean_prompt_tokens": 238.825, + "type_token_ratio": 0.16265047629017063, + "eval_vocab_overlap": 0.3351145038167939, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.506547719439495, + "10": 0.46550411204198716, + "11": 0.42451365599027535, + "12": 0.35121593618071684, + "13": 0.31119813901774557, + "14": 0.2695017929973147, + "15": 0.2332824762644623, + "16": 0.17097674262586013, + "17": 0.13594329362318916, + "18": 0.10392959131586701, + "19": 0.07196487216001288, + "20": 0.04597191351310213, + "21": 0.021950217517410112 + } + } + }, + "runtime_s": 499.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_CommonPile_s1.json b/results/e48/tv_OOD_CommonPile_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..7ca09497d93bd9fed0b349a5a3fb596ad87f125c --- /dev/null +++ b/results/e48/tv_OOD_CommonPile_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_CommonPile", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11650485332523074, + "best1L": 0.08737864212266036, + "mean": 0.03296703293121287, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.14394488824265345, + "best1L": 0.11203770791845662, + "mean": 0.035254086035170724, + "persist": 0.031544597420309274 + }, + "order-ops": { + "min": 0.09610389758433614, + "best1L": 0.07272727308528763, + "mean": 0.022577422371666347, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.013119533524981566, + "best1L": 0.005830903803663594, + "mean": 0.0022426552937499116, + "persist": 0.0 + }, + "typo": { + "min": 0.3258928528853825, + "best1L": 0.16666666524750845, + "mean": 0.13793498517147132, + "persist": 0.139880950695702 + }, + "association": { + "min": 0.013975155273718494, + "best1L": 0.006211180106869766, + "mean": 0.002030578100987311, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.4875907137988232, + "dispersion_mean": 0.23129285753356324, + "dispersion_slope": -0.04015930498878397, + "stable_rank": 20.418800354003906, + "top8_spectral_share": 0.13731586933135986, + "condition_number": 3213.534912109375, + "unembed_alignment": 0.14248991012573242 + }, + "corpus": { + "mean_prompt_tokens": 238.225, + "type_token_ratio": 0.1625354181970826, + "eval_vocab_overlap": 0.33129770992366414, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.4875907137988232, + "10": 0.4484290650051457, + "11": 0.4096472439420713, + "12": 0.338425658571887, + "13": 0.30038274443329716, + "14": 0.26030501268114103, + "15": 0.22545500636270466, + "16": 0.16549033302900945, + "17": 0.13202708902013005, + "18": 0.10121045297514478, + "19": 0.07060803618946244, + "20": 0.045390213425916745, + "21": 0.021845578501588386 + } + } + }, + "runtime_s": 459.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_CommonPile_s2.json b/results/e48/tv_OOD_CommonPile_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..3cb7fec706df45e20b8cb4a64e0df9d22e03da0e --- /dev/null +++ b/results/e48/tv_OOD_CommonPile_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_CommonPile", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.1165048535913229, + "best1L": 0.08876560388931207, + "mean": 0.03296703369622784, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.14358230627008847, + "best1L": 0.11312545463442802, + "mean": 0.0351704128219613, + "persist": 0.03190717886068991 + }, + "order-ops": { + "min": 0.09610389439123017, + "best1L": 0.0753246753343514, + "mean": 0.02237762214748987, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00728862958827189, + "mean": 0.0025790536643138955, + "persist": 0.0 + }, + "typo": { + "min": 0.3110118976661137, + "best1L": 0.16369047335215978, + "mean": 0.12992215928222453, + "persist": 0.13244047428348235 + }, + "association": { + "min": 0.013975155273718494, + "best1L": 0.006211180106869766, + "mean": 0.002030578100987311, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.505922735059862, + "dispersion_mean": 0.24232577263357624, + "dispersion_slope": -0.04158551843207735, + "stable_rank": 20.26382064819336, + "top8_spectral_share": 0.13729415833950043, + "condition_number": 3623.531494140625, + "unembed_alignment": 0.14170265197753906 + }, + "corpus": { + "mean_prompt_tokens": 241.625, + "type_token_ratio": 0.16086911536471807, + "eval_vocab_overlap": 0.33129770992366414, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.505922735059862, + "10": 0.4656282497412279, + "11": 0.4262258586888553, + "12": 0.35353659112683955, + "13": 0.31500694131985935, + "14": 0.2742591800554801, + "15": 0.23834998022033613, + "16": 0.17526538838225436, + "17": 0.14061089788167594, + "18": 0.10817593691587121, + "19": 0.07561092287647944, + "20": 0.04841328294726786, + "21": 0.02322907902048239 + } + } + }, + "runtime_s": 486.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_News_2024_dedup_s0.json b/results/e48/tv_OOD_News_2024_dedup_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..1126fdb7dfb040b30de605e7f12a63e196da79f5 --- /dev/null +++ b/results/e48/tv_OOD_News_2024_dedup_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_News_2024_dedup", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11511789262294769, + "best1L": 0.08599167982382434, + "mean": 0.030833244656345675, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.14684553976569856, + "best1L": 0.11167512647807598, + "mean": 0.035449322180024216, + "persist": 0.025743292777666023 + }, + "order-ops": { + "min": 0.08571428619325161, + "best1L": 0.07662337619279112, + "mean": 0.012987012958287128, + "persist": 0.0 + }, + "poetry": { + "min": 0.030612245867294923, + "best1L": 0.01020408168967281, + "mean": 0.00504597436104502, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31994047867400305, + "best1L": 0.18154762099896157, + "mean": 0.1292353474668094, + "persist": 0.12500000199569122 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.003105590119957924, + "mean": 0.0015527950433482016, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.550341928136247, + "dispersion_mean": 0.2444948973229944, + "dispersion_slope": -0.046049886117039064, + "stable_rank": 18.355375289916992, + "top8_spectral_share": 0.15226894617080688, + "condition_number": 3022.866455078125, + "unembed_alignment": 0.1667533814907074 + }, + "corpus": { + "mean_prompt_tokens": 886.465, + "type_token_ratio": 0.08101842712346229, + "eval_vocab_overlap": 0.6221374045801527, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.550341928136247, + "10": 0.49825287123389367, + "11": 0.44880601741544934, + "12": 0.3694730226682155, + "13": 0.317223691000841, + "14": 0.26842883013750474, + "15": 0.22800757726262702, + "16": 0.1671233800153054, + "17": 0.12396152917959162, + "18": 0.09173292135813063, + "19": 0.06316534800583397, + "20": 0.035649885692464856, + "21": 0.01626666309282207 + } + } + }, + "runtime_s": 451.6, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_News_2024_dedup_s1.json b/results/e48/tv_OOD_News_2024_dedup_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..081d537b1a3c150f01b0ea9035d337fddaae0721 --- /dev/null +++ b/results/e48/tv_OOD_News_2024_dedup_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_News_2024_dedup", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11095700412988663, + "best1L": 0.08599167982382434, + "mean": 0.029873039506907974, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.14612037635275296, + "best1L": 0.10913705533104283, + "mean": 0.035142522903957536, + "persist": 0.02574329264461994 + }, + "order-ops": { + "min": 0.08441558387130499, + "best1L": 0.07792207824864558, + "mean": 0.013186813057732902, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.029154519018317972, + "best1L": 0.010204081822718893, + "mean": 0.005045974494091102, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3214285767504147, + "best1L": 0.18749999840344703, + "mean": 0.13003662840596267, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0017916865846408264, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5687554264087672, + "dispersion_mean": 0.25049299483637033, + "dispersion_slope": -0.04761994330613776, + "stable_rank": 18.394371032714844, + "top8_spectral_share": 0.15196868777275085, + "condition_number": 5385.1982421875, + "unembed_alignment": 0.16688726842403412 + }, + "corpus": { + "mean_prompt_tokens": 826.425, + "type_token_ratio": 0.08548265117826784, + "eval_vocab_overlap": 0.6030534351145038, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5687554264087672, + "10": 0.5146394338358812, + "11": 0.46306623655168316, + "12": 0.37857090399292154, + "13": 0.32379009611394016, + "14": 0.2736918848223462, + "15": 0.23136530711219974, + "16": 0.169270261729744, + "17": 0.12507315467335064, + "18": 0.09250999761450329, + "19": 0.06348110116117872, + "20": 0.03589196055180334, + "21": 0.01630316830449558 + } + } + }, + "runtime_s": 452.0, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_News_2024_dedup_s2.json b/results/e48/tv_OOD_News_2024_dedup_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..6e269d0af5c7b1af2b8f28029a68e000a274554c --- /dev/null +++ b/results/e48/tv_OOD_News_2024_dedup_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_News_2024_dedup", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11234396696090698, + "best1L": 0.08599167902554784, + "mean": 0.03029979732153671, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1522842629679612, + "best1L": 0.11566352312053953, + "mean": 0.035839795268007686, + "persist": 0.02574329264461994 + }, + "order-ops": { + "min": 0.08961038956684726, + "best1L": 0.08311688181545053, + "mean": 0.01348651378482048, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.026239067981285707, + "best1L": 0.010204081822718893, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31547619083098005, + "best1L": 0.18601190352014133, + "mean": 0.12946428145681108, + "persist": 0.1264880926215223 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.003105590119957924, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5530691710109317, + "dispersion_mean": 0.24213086563801725, + "dispersion_slope": -0.04637569121070787, + "stable_rank": 18.451370239257812, + "top8_spectral_share": 0.15225350856781006, + "condition_number": 22639.4921875, + "unembed_alignment": 0.16733910143375397 + }, + "corpus": { + "mean_prompt_tokens": 786.795, + "type_token_ratio": 0.08850462954136719, + "eval_vocab_overlap": 0.6114503816793894, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5530691710109317, + "10": 0.49932435890347515, + "11": 0.4497947315662776, + "12": 0.3669299025279201, + "13": 0.31351877562271097, + "14": 0.2640693220817308, + "15": 0.22258114219993314, + "16": 0.16188305530216718, + "17": 0.11938113257707039, + "18": 0.08799323891394994, + "19": 0.060072140173159885, + "20": 0.033682764970022386, + "21": 0.015401517444874837 + } + } + }, + "runtime_s": 451.7, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_News_2024_s0.json b/results/e48/tv_OOD_News_2024_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..5dd3d4f2841951d5fd3089c88bd3b83b63de3411 --- /dev/null +++ b/results/e48/tv_OOD_News_2024_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_News_2024", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11511789262294769, + "best1L": 0.08599167982382434, + "mean": 0.03093993430957198, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1464829561965806, + "best1L": 0.11167512647807598, + "mean": 0.035421431463743956, + "persist": 0.025743292777666023 + }, + "order-ops": { + "min": 0.08571428619325161, + "best1L": 0.07662337619279112, + "mean": 0.012887112646629768, + "persist": 0.0 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.01020408168967281, + "mean": 0.004933841792600495, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3199404829314777, + "best1L": 0.18154762099896157, + "mean": 0.1291208786091634, + "persist": 0.12500000199569122 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.551074782316003, + "dispersion_mean": 0.24473418561879343, + "dispersion_slope": -0.046120528363436816, + "stable_rank": 18.34921646118164, + "top8_spectral_share": 0.15227602422237396, + "condition_number": 2915.62451171875, + "unembed_alignment": 0.16683748364448547 + }, + "corpus": { + "mean_prompt_tokens": 883.785, + "type_token_ratio": 0.08116227362989867, + "eval_vocab_overlap": 0.6213740458015267, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.551074782316003, + "10": 0.49895033908252784, + "11": 0.4494376945418247, + "12": 0.3699413478179418, + "13": 0.3175360510282699, + "14": 0.2686371028749118, + "15": 0.22811321673199653, + "16": 0.16713609052463432, + "17": 0.1239563509437032, + "18": 0.09171527178930283, + "19": 0.06314248497303229, + "20": 0.035638376477346405, + "21": 0.016265303942820566 + } + } + }, + "runtime_s": 472.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_News_2024_s1.json b/results/e48/tv_OOD_News_2024_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..2ca8c3f8aab5012e53978bdca61f038b4df73da4 --- /dev/null +++ b/results/e48/tv_OOD_News_2024_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_News_2024", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11234396589653832, + "best1L": 0.0873786426548447, + "mean": 0.030513175829712833, + "persist": 0.01664355051304613 + }, + "multilingual": { + "min": 0.14684553976569856, + "best1L": 0.1105873810925654, + "mean": 0.03533775798444237, + "persist": 0.025380710139870644 + }, + "order-ops": { + "min": 0.08831168844231538, + "best1L": 0.08051948063075542, + "mean": 0.013686313875950873, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.010204081822718893, + "mean": 0.004933841593031373, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31845238485506605, + "best1L": 0.18303571481789863, + "mean": 0.12946428544819355, + "persist": 0.12202380983425039 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0017916865181177855, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.5818125810786197, + "dispersion_mean": 0.2557544275320255, + "dispersion_slope": -0.04875426025963626, + "stable_rank": 18.716480255126953, + "top8_spectral_share": 0.15151242911815643, + "condition_number": 7547.69482421875, + "unembed_alignment": 0.16859397292137146 + }, + "corpus": { + "mean_prompt_tokens": 818.42, + "type_token_ratio": 0.0792502627013025, + "eval_vocab_overlap": 0.582442748091603, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5818125810786197, + "10": 0.5269683074709647, + "11": 0.4742956979443894, + "12": 0.3866649058537622, + "13": 0.3301942429077844, + "14": 0.2787155085542182, + "15": 0.2355655354269921, + "16": 0.17178296351674527, + "17": 0.12707808646347557, + "18": 0.09400033307170753, + "19": 0.06455584688853802, + "20": 0.03648165536640072, + "21": 0.01669189337273321 + } + } + }, + "runtime_s": 471.7, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_News_2024_s2.json b/results/e48/tv_OOD_News_2024_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..1908e06c3d729f4ce11d4000224a731b78011e6b --- /dev/null +++ b/results/e48/tv_OOD_News_2024_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_News_2024", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11373092872755867, + "best1L": 0.08460471752498831, + "mean": 0.0309399341765259, + "persist": 0.01664355051304613 + }, + "multilingual": { + "min": 0.1515591016837529, + "best1L": 0.11457577906548977, + "mean": 0.03589557656752212, + "persist": 0.025018129630812576 + }, + "order-ops": { + "min": 0.08961038956684726, + "best1L": 0.08311688181545053, + "mean": 0.013386613739255284, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.010204081822718893, + "mean": 0.004821708958063807, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3184523827263287, + "best1L": 0.18452381076557295, + "mean": 0.1278617217072419, + "persist": 0.12499999880258526 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0017916865181177855, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.5643626988147415, + "dispersion_mean": 0.24787522507695947, + "dispersion_slope": -0.04731820975355668, + "stable_rank": 18.664819717407227, + "top8_spectral_share": 0.15214292705059052, + "condition_number": 19847.302734375, + "unembed_alignment": 0.16673287749290466 + }, + "corpus": { + "mean_prompt_tokens": 806.23, + "type_token_ratio": 0.07863140791089392, + "eval_vocab_overlap": 0.5786259541984733, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.5643626988147415, + "10": 0.5094951246459303, + "11": 0.4590364150868526, + "12": 0.3759866405194264, + "13": 0.3217534257550742, + "14": 0.2711659015594667, + "15": 0.22842896616686936, + "16": 0.1662572200381082, + "17": 0.12276397926371435, + "18": 0.09046739265821396, + "19": 0.06190380708097724, + "20": 0.03482473538458937, + "21": 0.015931619026509296 + } + } + }, + "runtime_s": 477.6, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_arXiv_2023_s0.json b/results/e48/tv_OOD_arXiv_2023_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..f1821844446b86c8b0c95127e2988423d8a11057 --- /dev/null +++ b/results/e48/tv_OOD_arXiv_2023_s0.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_arXiv_2023", + "seed": 0, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11789181535797459, + "best1L": 0.09153952768870763, + "mean": 0.03243358612858823, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.14358230627008847, + "best1L": 0.11022480124873775, + "mean": 0.03472415916621685, + "persist": 0.03226976030107055 + }, + "order-ops": { + "min": 0.09220779075154237, + "best1L": 0.06883116838123117, + "mean": 0.021178821750384356, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.010204081822718893, + "best1L": 0.005830903803663594, + "mean": 0.002018389757722616, + "persist": 0.0 + }, + "typo": { + "min": 0.3437500053218433, + "best1L": 0.17857142803924425, + "mean": 0.14445970579981804, + "persist": 0.1488095269139324 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.45180742254227424, + "dispersion_mean": 0.21006961564829957, + "dispersion_slope": -0.037293115343424034, + "stable_rank": 20.478944778442383, + "top8_spectral_share": 0.13791868090629578, + "condition_number": 2183.10498046875, + "unembed_alignment": 0.14854320883750916 + }, + "corpus": { + "mean_prompt_tokens": 13656.5, + "type_token_ratio": 0.009479368798740527, + "eval_vocab_overlap": 0.7656488549618321, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.45180742254227424, + "10": 0.41341960780049397, + "11": 0.37491187355414735, + "12": 0.3080619874177352, + "13": 0.2727978596921294, + "14": 0.23561235302131206, + "15": 0.20373178914815054, + "16": 0.14759335183060146, + "17": 0.1158883046550611, + "18": 0.08851565885352318, + "19": 0.06102612484038306, + "20": 0.03861634686701347, + "21": 0.01892232320506908 + } + } + }, + "runtime_s": 470.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_arXiv_2023_s1.json b/results/e48/tv_OOD_arXiv_2023_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..ed8e4f61ba1e3ae4ed74e1d90a6b1c24f7474c82 --- /dev/null +++ b/results/e48/tv_OOD_arXiv_2023_s1.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_arXiv_2023", + "seed": 1, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11650485252695424, + "best1L": 0.09153952768870763, + "mean": 0.03211351823327797, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.14430746702211245, + "best1L": 0.11203770898282528, + "mean": 0.03505884962422507, + "persist": 0.03226976030107055 + }, + "order-ops": { + "min": 0.08701298705169133, + "best1L": 0.0649350636771747, + "mean": 0.02047952064978225, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.013119533924119813, + "best1L": 0.005830903803663594, + "mean": 0.0021305225922593047, + "persist": 0.0 + }, + "typo": { + "min": 0.34226190192358835, + "best1L": 0.17559523614389555, + "mean": 0.14377289531486376, + "persist": 0.14732142936970508 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.46491563637845174, + "dispersion_mean": 0.2173062607395302, + "dispersion_slope": -0.03830700452643936, + "stable_rank": 20.590618133544922, + "top8_spectral_share": 0.13677191734313965, + "condition_number": 2422.00830078125, + "unembed_alignment": 0.1480366587638855 + }, + "corpus": { + "mean_prompt_tokens": 12960.48, + "type_token_ratio": 0.00988119267187635, + "eval_vocab_overlap": 0.7717557251908397, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.46491563637845174, + "10": 0.4255462215653238, + "11": 0.3859711840257504, + "12": 0.3188967714862448, + "13": 0.28219366760301323, + "14": 0.2439721837512839, + "15": 0.21122971957002024, + "16": 0.1533247571820779, + "17": 0.1213659470498959, + "18": 0.09265147148591234, + "19": 0.06420564120095554, + "20": 0.04079808862879347, + "21": 0.019910099686169302 + } + } + }, + "runtime_s": 503.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48/tv_OOD_arXiv_2023_s2.json b/results/e48/tv_OOD_arXiv_2023_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..f9ac32d3836734c1bd25e9254a3a4893627389c3 --- /dev/null +++ b/results/e48/tv_OOD_arXiv_2023_s2.json @@ -0,0 +1,102 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "OOD_arXiv_2023", + "seed": 2, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11789181535797459, + "best1L": 0.08876560309103557, + "mean": 0.03222020708822778, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.14176939374634198, + "best1L": 0.10804931234036173, + "mean": 0.034528922755271196, + "persist": 0.032632341741451194 + }, + "order-ops": { + "min": 0.09090909069137913, + "best1L": 0.06883116838123117, + "mean": 0.02107892090654267, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.011661807607327188, + "best1L": 0.005830903803663594, + "mean": 0.002018389757722616, + "persist": 0.0 + }, + "typo": { + "min": 0.33779761620930265, + "best1L": 0.16964285767504148, + "mean": 0.14365842512675694, + "persist": 0.14880952584956372 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.004658385179936886, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.4243287074185154, + "dispersion_mean": 0.19773348903608412, + "dispersion_slope": -0.035039244348540166, + "stable_rank": 20.331480026245117, + "top8_spectral_share": 0.13898056745529175, + "condition_number": 2622.787841796875, + "unembed_alignment": 0.14453446865081787 + }, + "corpus": { + "mean_prompt_tokens": 13437.535, + "type_token_ratio": 0.00941206850810063, + "eval_vocab_overlap": 0.7572519083969466, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "9": 0.4243287074185154, + "10": 0.3883180959719153, + "11": 0.35270748466038426, + "12": 0.2905976505527174, + "13": 0.2570829549288976, + "14": 0.2218186115600737, + "15": 0.1919580649231945, + "16": 0.13877228271858308, + "17": 0.10938228987846971, + "18": 0.08336097901746659, + "19": 0.0576397492267599, + "20": 0.036562051453741654, + "21": 0.01800643515837453 + } + } + }, + "runtime_s": 498.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e48_competence_gate_410m.json b/results/e48_competence_gate_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..da157f4c41e26f822cc4530499934d690baaf6ca --- /dev/null +++ b/results/e48_competence_gate_410m.json @@ -0,0 +1,718 @@ +{ + "experiment": "E48 prerequisite \u2014 model-competence gate on every fitting corpus", + "status": "PRE-REGISTERED (rule in this file's docstring, fixed before the run)", + "model": "EleutherAI/pythia-410m-deduped", + "device": "cpu", + "window": { + "max_seq_len": 128, + "skip_first": 16, + "n_max": 200 + }, + "gate_rule": "PASS iff CE(Q) <= max_instream_CE + (max_instream_CE - min_instream_CE); one in-stream spread above the WORST in-stream corpus", + "uniform_null_ce_nats": 10.825839875788878, + "vocab": 50304, + "why_trainval_ce_is_nan": "trainval.py:89-102 wraps the CE block in `except Exception: pass` and calls `model.forward(...).logits`; jlens' HFLensModel.forward returns the TEXT MODULE output (hidden states), which has no `.logits`. Every fit in the programme therefore recorded model_cross_entropy = NaN.", + "second_bug_avoided_here": "the obvious fix \u2014 unembed the returned last_hidden_state \u2014 is ALSO wrong: gpt_neox's last_hidden_state is already final-layer-normed and jlens' unembed norms again. Measured 6.32 vs the true 3.02 nats at 70M on one English sentence. This script takes logits from the HF head directly. The lens read path is NOT affected: it hooks the transformer blocks, which are pre-norm.", + "cells": { + "Wikipedia_en|s0": { + "ce_nats": 2.4447590004860817, + "ppl": 11.527771076814997, + "bits_per_byte": 0.8125306169496754, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 96366, + "ce_per_doc_sd": 0.5687033895724258, + "corpus": "Wikipedia_en", + "seed": 0 + }, + "Wikipedia_en|s1": { + "ce_nats": 2.4063240059861193, + "ppl": 11.093107904682723, + "bits_per_byte": 0.8066877678685225, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 95538, + "ce_per_doc_sd": 0.5616732367776649, + "corpus": "Wikipedia_en", + "seed": 1 + }, + "Wikipedia_en|s2": { + "ce_nats": 2.402744432741457, + "ppl": 11.053470297610033, + "bits_per_byte": 0.8171369563753036, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 94176, + "ce_per_doc_sd": 0.5389836574241886, + "corpus": "Wikipedia_en", + "seed": 2 + }, + "USPTO_Backgrounds|s0": { + "ce_nats": 2.371612525286975, + "ppl": 10.714656015034388, + "bits_per_byte": 0.6729057633499741, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 112880, + "ce_per_doc_sd": 0.48767188876847756, + "corpus": "USPTO_Backgrounds", + "seed": 0 + }, + "USPTO_Backgrounds|s1": { + "ce_nats": 2.47426736883215, + "ppl": 11.873005392106302, + "bits_per_byte": 0.6907905056374827, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 114717, + "ce_per_doc_sd": 0.4998467460761969, + "corpus": "USPTO_Backgrounds", + "seed": 1 + }, + "USPTO_Backgrounds|s2": { + "ce_nats": 2.393176099244539, + "ppl": 10.948211386748458, + "bits_per_byte": 0.6688327839952949, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 114600, + "ce_per_doc_sd": 0.4974738249596334, + "corpus": "USPTO_Backgrounds", + "seed": 2 + }, + "Pile-CC|s0": { + "ce_nats": 2.905213372857721, + "ppl": 18.26914137526359, + "bits_per_byte": 0.9555896946921513, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 97372, + "ce_per_doc_sd": 0.5550923076506483, + "corpus": "Pile-CC", + "seed": 0 + }, + "Pile-CC|s1": { + "ce_nats": 2.969953924643027, + "ppl": 19.491021519567393, + "bits_per_byte": 0.9614026595136127, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 98940, + "ce_per_doc_sd": 0.6305997119179829, + "corpus": "Pile-CC", + "seed": 1 + }, + "Pile-CC|s2": { + "ce_nats": 2.90814243656021, + "ppl": 18.32273129997899, + "bits_per_byte": 0.9470988703491457, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 98344, + "ce_per_doc_sd": 0.6435494721932618, + "corpus": "Pile-CC", + "seed": 2 + }, + "StackExchange|s0": { + "ce_nats": 2.4950342473897846, + "ppl": 12.122148663430892, + "bits_per_byte": 0.9507725643605275, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 84048, + "ce_per_doc_sd": 0.6105683430280523, + "corpus": "StackExchange", + "seed": 0 + }, + "StackExchange|s1": { + "ce_nats": 2.357038673194679, + "ppl": 10.559634576641699, + "bits_per_byte": 0.9043851077131557, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 83472, + "ce_per_doc_sd": 0.6222820426557922, + "corpus": "StackExchange", + "seed": 1 + }, + "StackExchange|s2": { + "ce_nats": 2.386241854074839, + "ppl": 10.87255641303875, + "bits_per_byte": 0.9066510258144269, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 84295, + "ce_per_doc_sd": 0.5848291041368836, + "corpus": "StackExchange", + "seed": 2 + }, + "Github|s0": { + "ce_nats": 0.9780411115859274, + "ppl": 2.659241978215054, + "bits_per_byte": 0.4187715988375308, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 74801, + "ce_per_doc_sd": 0.8008540416932711, + "corpus": "Github", + "seed": 0 + }, + "Github|s1": { + "ce_nats": 1.1575571220604752, + "ppl": 3.182150168948378, + "bits_per_byte": 0.4977049619256294, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 74490, + "ce_per_doc_sd": 0.9393944167954323, + "corpus": "Github", + "seed": 1 + }, + "Github|s2": { + "ce_nats": 1.0352248014590224, + "ppl": 2.815739146750142, + "bits_per_byte": 0.4409805399523627, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 75187, + "ce_per_doc_sd": 0.8758373273724509, + "corpus": "Github", + "seed": 2 + }, + "OOD_News_2024|s0": { + "ce_nats": 2.812214131398244, + "ppl": 16.64673550435014, + "bits_per_byte": 0.8764485906287259, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 102766, + "ce_per_doc_sd": 0.37859308776404227, + "corpus": "OOD_News_2024", + "seed": 0 + }, + "OOD_News_2024|s1": { + "ce_nats": 2.744364344966304, + "ppl": 15.554723350822988, + "bits_per_byte": 0.8758772565114132, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 100352, + "ce_per_doc_sd": 0.3636236961948326, + "corpus": "OOD_News_2024", + "seed": 1 + }, + "OOD_News_2024|s2": { + "ce_nats": 2.7947089200406463, + "ppl": 16.357866612960237, + "bits_per_byte": 0.8746783728874513, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 102333, + "ce_per_doc_sd": 0.3335318280787767, + "corpus": "OOD_News_2024", + "seed": 2 + }, + "OOD_arXiv_2023|s0": { + "ce_nats": 2.829922255095061, + "ppl": 16.944143452510207, + "bits_per_byte": 0.798543361687886, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 113502, + "ce_per_doc_sd": 0.41267511073466756, + "corpus": "OOD_arXiv_2023", + "seed": 0 + }, + "OOD_arXiv_2023|s1": { + "ce_nats": 2.793526809279983, + "ppl": 16.33854122743907, + "bits_per_byte": 0.7974206906445359, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 112200, + "ce_per_doc_sd": 0.3921334651251922, + "corpus": "OOD_arXiv_2023", + "seed": 1 + }, + "OOD_arXiv_2023|s2": { + "ce_nats": 2.8562353405651746, + "ppl": 17.39591381746838, + "bits_per_byte": 0.8001243727780123, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 114331, + "ce_per_doc_sd": 0.41556986057440554, + "corpus": "OOD_arXiv_2023", + "seed": 2 + }, + "OOD_CommonPile|s0": { + "ce_nats": 2.777735053397514, + "ppl": 16.082553540251343, + "bits_per_byte": 0.8818177168639355, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 100888, + "ce_per_doc_sd": 0.43504115814213323, + "corpus": "OOD_CommonPile", + "seed": 0 + }, + "OOD_CommonPile|s1": { + "ce_nats": 2.8696979014078776, + "ppl": 17.631690886197404, + "bits_per_byte": 0.8837433897065947, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 104001, + "ce_per_doc_sd": 0.40123786984269505, + "corpus": "OOD_CommonPile", + "seed": 1 + }, + "OOD_CommonPile|s2": { + "ce_nats": 2.8099707354296437, + "ppl": 16.609432143778797, + "bits_per_byte": 0.8765768124755634, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 102669, + "ce_per_doc_sd": 0.36806078489157246, + "corpus": "OOD_CommonPile", + "seed": 2 + }, + "OOD_Wikipedia_2023|s0": { + "ce_nats": 2.466197638984199, + "ppl": 11.777578996575858, + "bits_per_byte": 0.776803745994803, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 101682, + "ce_per_doc_sd": 0.4124232733437484, + "corpus": "OOD_Wikipedia_2023", + "seed": 0 + }, + "OOD_Wikipedia_2023|s1": { + "ce_nats": 2.531903154871485, + "ppl": 12.577420151845649, + "bits_per_byte": 0.7974840052228622, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 101684, + "ce_per_doc_sd": 0.41524427273530956, + "corpus": "OOD_Wikipedia_2023", + "seed": 1 + }, + "OOD_Wikipedia_2023|s2": { + "ce_nats": 2.4789074782637863, + "ppl": 11.928225450889085, + "bits_per_byte": 0.7894954115414696, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 100563, + "ce_per_doc_sd": 0.3853947980749275, + "corpus": "OOD_Wikipedia_2023", + "seed": 2 + }, + "TRAP_FineWeb|s0": { + "ce_nats": 3.0642626018352335, + "ppl": 21.418662080376606, + "bits_per_byte": 0.9811028611265628, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 100032, + "ce_per_doc_sd": 0.47309278287933537, + "corpus": "TRAP_FineWeb", + "seed": 0 + }, + "TRAP_FineWeb|s1": { + "ce_nats": 3.184408860765062, + "ppl": 24.153006395210653, + "bits_per_byte": 1.0242295444657659, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 99577, + "ce_per_doc_sd": 0.620379722324631, + "corpus": "TRAP_FineWeb", + "seed": 1 + }, + "TRAP_FineWeb|s2": { + "ce_nats": 3.110387322709367, + "ppl": 22.429730262417632, + "bits_per_byte": 1.0092185729661007, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 98709, + "ce_per_doc_sd": 0.5600991227422809, + "corpus": "TRAP_FineWeb", + "seed": 2 + }, + "CONTROL_PubMed_2023|s0": { + "ce_nats": 2.657386380444776, + "ppl": 14.258972824436512, + "bits_per_byte": 0.8289131839848927, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 102677, + "ce_per_doc_sd": 0.4303503262389787, + "corpus": "CONTROL_PubMed_2023", + "seed": 0 + }, + "CONTROL_PubMed_2023|s1": { + "ce_nats": 2.7434812308646537, + "ppl": 15.540992818990109, + "bits_per_byte": 0.8474163143361065, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 103689, + "ce_per_doc_sd": 0.4818131230345177, + "corpus": "CONTROL_PubMed_2023", + "seed": 1 + }, + "CONTROL_PubMed_2023|s2": { + "ce_nats": 2.662592653841586, + "ppl": 14.333402517864878, + "bits_per_byte": 0.8098871231380694, + "n_docs": 200, + "n_scored_tokens": 22200, + "n_scored_bytes": 105295, + "ce_per_doc_sd": 0.467322426720765, + "corpus": "CONTROL_PubMed_2023", + "seed": 2 + } + }, + "by_corpus": { + "Wikipedia_en": { + "ce_mean": 2.417942479737886, + "ce_seed_sd": 0.0232926528101091, + "ppl_mean": 11.222744519211309, + "bpb_mean": 0.8121184470645005, + "bpb_seed_sd": 0.005236773636815465, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "ce_mean": 2.4130186644545546, + "ce_seed_sd": 0.054127624845418616, + "ppl_mean": 11.16762161708706, + "bpb_mean": 0.6775096843275838, + "bpb_seed_sd": 0.011680430266261903, + "tier": "in-stream" + }, + "Pile-CC": { + "ce_mean": 2.9277699113536526, + "ce_seed_sd": 0.03656177083296005, + "ppl_mean": 18.685912754949932, + "bpb_mean": 0.9546970748516366, + "bpb_seed_sd": 0.007193550844342514, + "tier": "in-stream" + }, + "StackExchange": { + "ce_mean": 2.412771591553101, + "ce_seed_sd": 0.07272251956827648, + "ppl_mean": 11.164862741247552, + "bpb_mean": 0.9206028992960367, + "bpb_seed_sd": 0.026152248732911738, + "tier": "in-stream" + }, + "Github": { + "ce_mean": 1.0569410117018083, + "ce_seed_sd": 0.09170711447132196, + "ppl_mean": 2.8775551049055843, + "bpb_mean": 0.45248570023850765, + "bpb_seed_sd": 0.040704981108051354, + "tier": "in-stream" + }, + "OOD_News_2024": { + "ce_mean": 2.783762465468398, + "ce_seed_sd": 0.03522452314072801, + "ppl_mean": 16.179782442562345, + "bpb_mean": 0.8756680733425302, + "bpb_seed_sd": 0.0009034577530182717, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "ce_mean": 2.826561468313406, + "ce_seed_sd": 0.03148906381317604, + "ppl_mean": 16.88729338315894, + "bpb_mean": 0.798696141703478, + "bpb_seed_sd": 0.0013583006182701558, + "tier": "OOD" + }, + "OOD_CommonPile": { + "ce_mean": 2.8191345634116782, + "ce_seed_sd": 0.046661259745184334, + "ppl_mean": 16.762337652686245, + "bpb_mean": 0.8807126396820312, + "bpb_seed_sd": 0.003708888510801891, + "tier": "OOD" + }, + "OOD_Wikipedia_2023": { + "ce_mean": 2.49233609070649, + "ce_seed_sd": 0.03485038634801425, + "ppl_mean": 12.089485292261289, + "bpb_mean": 0.7879277209197116, + "bpb_seed_sd": 0.01042887917511243, + "tier": "reported-only" + }, + "TRAP_FineWeb": { + "ce_mean": 3.1196862617698873, + "ce_seed_sd": 0.06061050721159486, + "ppl_mean": 22.639275722551698, + "bpb_mean": 1.0048503261861432, + "bpb_seed_sd": 0.02189266746899364, + "tier": "reported-only" + }, + "CONTROL_PubMed_2023": { + "ce_mean": 2.687820088383672, + "ce_seed_sd": 0.04827420022524149, + "ppl_mean": 14.699597146125235, + "bpb_mean": 0.8287388738196896, + "bpb_seed_sd": 0.01876520279719823, + "tier": "reported-only" + } + }, + "instream_ce_range": { + "min": 1.0569410117018083, + "max": 2.9277699113536526, + "argmin": "Github", + "argmax": "Pile-CC" + }, + "gate_threshold_ce": 4.798598811005497, + "gate": { + "OOD_News_2024": { + "ce": 2.783762465468398, + "passes": true, + "margin_nats": 2.014836345537099 + }, + "OOD_arXiv_2023": { + "ce": 2.826561468313406, + "passes": true, + "margin_nats": 1.9720373426920905 + }, + "OOD_CommonPile": { + "ce": 2.8191345634116782, + "passes": true, + "margin_nats": 1.9794642475938184 + } + }, + "gate_threshold_bpb": 1.4569084494647655, + "gate_bpb": { + "OOD_News_2024": { + "bpb": 0.8756680733425302, + "passes": true + }, + "OOD_arXiv_2023": { + "bpb": 0.798696141703478, + "passes": true + }, + "OOD_CommonPile": { + "bpb": 0.8807126396820312, + "passes": true + } + }, + "ce_bpb_agree": true, + "controls": { + "C1_pilecc_lowest_or_second": { + "fires": false, + "pilecc_ce": 2.9277699113536526, + "instream_ces": { + "Wikipedia_en": 2.417942479737886, + "USPTO_Backgrounds": 2.4130186644545546, + "Pile-CC": 2.9277699113536526, + "StackExchange": 2.412771591553101, + "Github": 1.0569410117018083 + } + }, + "C2_far_below_uniform_null": { + "fires": true, + "max_corpus_ce": 3.1196862617698873, + "uniform_null": 10.825839875788878 + }, + "C3_seed_spread_small": { + "fires": true, + "max_seed_sd": 0.09170711447132196, + "between_corpus_spread": 2.0627452500680787, + "ratio": 0.04445876894798098 + } + }, + "VERDICT": "ALL THREE OOD RUNGS PASS \u2014 a negative read on them cannot be attributed to model incompetence; E48 may be scored as registered", + "provenance": { + "schema_version": 1, + "experiment": "E48-gate", + "utc": "2026-08-30T06:47:15+00:00", + "script": { + "path": "experiments/t48_competence_gate.py", + "sha256": "e38436fa6b1fba57151d4531310999847db0e99a5aa7a20d58777b0cf60fb414", + "n_lines": 305 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/t48_competence_gate.py", + "--model", + "410m", + "--device", + "cpu" + ], + "cwd": ".", + "inputs": [ + { + "path": "corpora/Wikipedia_en.jsonl", + "sha256": "b30f4cef11add560f7df8ed07a613e3fcc6a4dda865975bec15f2952253bda77", + "bytes": 10329196, + "exists": true + }, + { + "path": "corpora/USPTO_Backgrounds.jsonl", + "sha256": "b09ac74bdc8b169ee5c9bea6be481262f743947a92107164f30eca2346e3ccb9", + "bytes": 10221054, + "exists": true + }, + { + "path": "corpora/Pile-CC.jsonl", + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "bytes": 13512530, + "exists": true + }, + { + "path": "corpora/StackExchange.jsonl", + "sha256": "bc25e2d5775e9719ebcbdaa3bd115d827f72ebf11f47fedb377f8b0a69a28971", + "bytes": 5980156, + "exists": true + }, + { + "path": "corpora/Github.jsonl", + "sha256": "f2bb459d10cad6c13a8b66c86191bc43df4393ba5b722a5315ea878fad2d1a26", + "bytes": 17486378, + "exists": true + }, + { + "path": "corpora/OOD_News_2024.jsonl", + "sha256": "45aaab463d8987725fe9b7edb5bd316a29c2d547a5a1ffd2c54f41df3be1df80", + "bytes": 9039491, + "exists": true + }, + { + "path": "corpora/OOD_arXiv_2023.jsonl", + "sha256": "565f3541ef189ba624ff56373d7d3df0b0341020578b0c6cffc6391b332ff4af", + "bytes": 115690137, + "exists": true + }, + { + "path": "corpora/OOD_CommonPile.jsonl", + "sha256": "0d7ca5e94b4b1891e1d99a310b263806f370bae861decdfc3cf0f1ebb7a755e4", + "bytes": 2722638, + "exists": true + }, + { + "path": "corpora/OOD_Wikipedia_2023.jsonl", + "sha256": "57fb26cf3997b571543281efe3330efcbab1f37e97f99eb6df4deaa027d98550", + "bytes": 55251339, + "exists": true + }, + { + "path": "corpora/TRAP_FineWeb.jsonl", + "sha256": "6164a3cc1113f6c3b90fd5f67e3eba2e80324a69d9d4c0ae671aefdefc4b9260", + "bytes": 8520540, + "exists": true + }, + { + "path": "corpora/CONTROL_PubMed_2023.jsonl", + "sha256": "cffa12d20484b15c2430fc402a994632a9a27595454d02b457b94ec80c27856a", + "bytes": 2818475, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "682a6cbd921db6dd8f9228d287e422bc0c40faffa0481a952da1be33d90fc12a" + } +} \ No newline at end of file diff --git a/results/e48_crossover_410m.json b/results/e48_crossover_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..03ac515a69fb108cbcb44e830eeadcd074b6cc8d --- /dev/null +++ b/results/e48_crossover_410m.json @@ -0,0 +1,3769 @@ +{ + "experiment": "E48 \u2014 the S3 crossover: does the unfitted logit lens overtake J^P under shift?", + "prereg": "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md + Amendment 1, commits 408fa5c / 5869b63", + "status": "PRE-REGISTERED \u2014 rule fixed before any OOD lens existed", + "rule_verbatim": "CONFIRMED iff all three hold:\n (a) (J^P - logit) CI strictly > 0 for the in-stream English rungs\n (Wikipedia, USPTO, Pile-CC, StackExchange); AND\n (b) (J^P - logit) CI strictly < 0 for >=2 of 3 genuinely-OOD rungs\n (News_2024, arXiv_2023, CommonPile); AND\n (c) those OOD operators read strictly ABOVE the J_shuf floor\n (structured operator, wrong distribution - not a collapsed one).\nNOT REACHED iff (J^P - logit) CI includes or exceeds 0 for all OOD rungs.\n -> positive-framed negative; publishable as registered; NOT to be re-described as a\n weaker crossover, and no hunting for a rung that dips.\nDEGENERACY (excluded either way) iff OOD operators read BELOW J_shuf.\nAMENDMENT 1, clause (d), required for CONFIRMED:\n OOD_News_2024 must be among the rungs going strictly < 0, AND Github must sit AT the\n crossover (CI includes 0) or above - NOT strictly below. If Github goes strictly\n negative -> REGISTER-CONFOUNDED, not CONFIRMED.", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "n_items": 541, + "n_pairs": 893, + "smoke": false, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "excluded_sets": { + "association": "floored \u2014 196/205 ladder cells exactly zero (E28 crit 3)" + }, + "N_per_fit": 200, + "seeds": [ + 0, + 1, + 2 + ], + "n_derangements_per_cell": 5, + "adjudication_aggregation": "persist", + "ci_primary": "three-level: eval set > seed block > item pair (bootstrap.hierarchical_bootstrap_seeded). Chosen and written into this script BEFORE any OOD read was scored, because the pre-registration names both units and the repo's two-level resampler has no seed level.", + "ci_secondary": "two-level on the seed-averaged difference (bootstrap.hierarchical_bootstrap) \u2014 the repo's existing resampler, reported so the seed variance component is visible", + "ci_clause_disagreement": [], + "sensitivity_drop_contested_rung": { + "why": "OOD_CommonPile is 100% arXiv abstracts and arXiv is a Pile component; its OOD status is contested. Declared BEFORE any read was scored.", + "ood_set_used": [ + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "b_ok_needs_both": false, + "b_strictly_negative": [], + "verdict_would_change": false + }, + "contested_rungs": { + "OOD_CommonPile": "100% arXiv abstracts (2400/2400 start with two spaces, 79-col hard wrap, LaTeX escapes, 98.4% scientific vocabulary). arXiv is a Pile component at ~9% weight, so the manifest's 'domains not deliberately represented in the Pile' is false as sampled. k=32 containment 0.0002 IS below the 0.0014 single-line-reflow floor, so absence is not refuted \u2014 but the rung is not register-independent of OOD_arXiv_2023, which is what clause (b)'s '>=2 of 3' was meant to guarantee.", + "OOD_News_2024": "33.4% exact duplicates (2400 -> 1598 distinct), seed-asymmetric: the fitted 200 contain 199/166/161 distinct documents at seeds 0/1/2. This is the clause-(d) load-bearing rung. A deduplicated refit is a separate disclosed robustness arm." + }, + "rstrip_arm": false, + "scoring_path_forks_disclosed": { + "rstrip": "PRIMARY is UNSTRIPPED (matches trainval/t33/E28/E33, and is the only path that can reproduce C0). anchor_evals.py:32-34 calls .rstrip() load-bearing; it changes the readout token on 157/551 items. The stripped path is run separately via --rstrip.", + "passk_pooling": "both this scorer and t22 flat-pool over (item, intermediate) pairs; the anchor README defines a mean over items of the per-item fraction. Inherited unchanged so C0 can fire." + }, + "competence_gate": { + "file": "e48_competence_gate_410m.json", + "verdict": "ALL THREE OOD RUNGS PASS \u2014 a negative read on them cannot be attributed to model incompetence; E48 may be scored as registered", + "failed_rungs": [], + "ce_by_corpus": { + "Wikipedia_en": 2.417942479737886, + "USPTO_Backgrounds": 2.4130186644545546, + "Pile-CC": 2.9277699113536526, + "StackExchange": 2.412771591553101, + "Github": 1.0569410117018083, + "OOD_News_2024": 2.783762465468398, + "OOD_arXiv_2023": 2.826561468313406, + "OOD_CommonPile": 2.8191345634116782, + "OOD_Wikipedia_2023": 2.49233609070649, + "TRAP_FineWeb": 3.1196862617698873, + "CONTROL_PubMed_2023": 2.687820088383672 + } + }, + "controls": { + "C0_reproduces_e33_v2": { + "target": "e33_logit_baseline_410m_v2.json", + "max_abs_diff": 1.9158635844451766e-08, + "per_set": { + "logit_I|min|multihop": { + "this_run": 0.1178918182849884, + "e33_v2": 0.11789181775280408, + "abs_diff": 5.321843243866198e-10 + }, + "logit_I|min|multilingual": { + "this_run": 0.11240029335021973, + "e33_v2": 0.11240029175366674, + "abs_diff": 1.5965529870376471e-09 + }, + "logit_I|min|order-ops": { + "this_run": 0.19740261137485504, + "e33_v2": 0.19740259860243117, + "abs_diff": 1.2772423868545602e-08 + }, + "logit_I|min|poetry": { + "this_run": 0.02040816657245159, + "e33_v2": 0.020408162714115212, + "abs_diff": 3.858336379558569e-09 + }, + "logit_I|min|typo": { + "this_run": 0.1755952388048172, + "e33_v2": 0.17559524040137017, + "abs_diff": 1.5965529731598593e-09 + }, + "logit_I|min|association": { + "this_run": 0.006211180239915848, + "e33_v2": 0.006211180106869766, + "abs_diff": 1.3304608196401668e-10 + }, + "logit_I|persist|multihop": { + "this_run": 0.037447988986968994, + "e33_v2": 0.037447988986968994, + "abs_diff": 0.0 + }, + "logit_I|persist|multilingual": { + "this_run": 0.023205220699310303, + "e33_v2": 0.02320522096540247, + "abs_diff": 2.6609216566275684e-10 + }, + "logit_I|persist|order-ops": { + "this_run": 0.03246753290295601, + "e33_v2": 0.03246753276990993, + "abs_diff": 1.3304608109665494e-10 + }, + "logit_I|persist|poetry": { + "this_run": 0.0014577260008081794, + "e33_v2": 0.001457725917654378, + "abs_diff": 8.315380133593064e-11 + }, + "logit_I|persist|typo": { + "this_run": 0.0476190485060215, + "e33_v2": 0.04761904717556068, + "abs_diff": 1.3304608179054433e-09 + }, + "logit_I|persist|association": { + "this_run": 0.0, + "e33_v2": 0.0, + "abs_diff": 0.0 + }, + "C0_ref_Github_n400_s0|min|multihop": { + "this_run": 0.07628293335437775, + "e33_v2": 0.07628293974058968, + "abs_diff": 6.386211934272801e-09 + }, + "C0_ref_Github_n400_s0|min|multilingual": { + "this_run": 0.11493835598230362, + "e33_v2": 0.11493835970759392, + "abs_diff": 3.725290298461914e-09 + }, + "C0_ref_Github_n400_s0|min|order-ops": { + "this_run": 0.07922078669071198, + "e33_v2": 0.07922077950622354, + "abs_diff": 7.184488434730518e-09 + }, + "C0_ref_Github_n400_s0|min|poetry": { + "this_run": 0.017492711544036865, + "e33_v2": 0.017492711410990784, + "abs_diff": 1.3304608109665494e-10 + }, + "C0_ref_Github_n400_s0|min|typo": { + "this_run": 0.2782738208770752, + "e33_v2": 0.27827380171843935, + "abs_diff": 1.9158635844451766e-08 + }, + "C0_ref_Github_n400_s0|min|association": { + "this_run": 0.004658385645598173, + "e33_v2": 0.004658385046890804, + "abs_diff": 5.987073692717559e-10 + }, + "C0_ref_Github_n400_s0|persist|multihop": { + "this_run": 0.004160888027399778, + "e33_v2": 0.004160887694784573, + "abs_diff": 3.326152053437226e-10 + }, + "C0_ref_Github_n400_s0|persist|multilingual": { + "this_run": 0.02828136458992958, + "e33_v2": 0.028281363126422678, + "abs_diff": 1.4635069024715452e-09 + }, + "C0_ref_Github_n400_s0|persist|order-ops": { + "this_run": 0.019480518996715546, + "e33_v2": 0.019480519528899874, + "abs_diff": 5.321843278560667e-10 + }, + "C0_ref_Github_n400_s0|persist|poetry": { + "this_run": 0.004373178351670504, + "e33_v2": 0.004373177752963134, + "abs_diff": 5.987073692717559e-10 + }, + "C0_ref_Github_n400_s0|persist|typo": { + "this_run": 0.0833333358168602, + "e33_v2": 0.08333333488553762, + "abs_diff": 9.313225746154785e-10 + }, + "C0_ref_Github_n400_s0|persist|association": { + "this_run": 0.0, + "e33_v2": 0.0, + "abs_diff": 0.0 + } + }, + "skipped": false, + "why_skipped": null, + "fp32_tolerance": 1e-06, + "smallest_possible_real_difference": 0.00036258158085569254, + "tolerance_justification": "one pair flipping at one k moves a set mean by >= 1/(394*7) = 3.63e-04; the tolerance is 363x below that", + "fires": true + }, + "C1_identity_equals_logit": { + "max_abs_per_pair_diff": 0.0, + "fires": true + }, + "C2_shuf_floor": { + "Wikipedia_en": { + "jp": 0.038209554925560954, + "shuf_mean_over_draws": 0.01993137374520302, + "ci_vs_shuf": { + "point": 0.018278183415532112, + "ci_lo": 0.005172038450837135, + "ci_hi": 0.03638976812362671, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "USPTO_Backgrounds": { + "jp": 0.04961536889895797, + "shuf_mean_over_draws": 0.027645074779866264, + "ci_vs_shuf": { + "point": 0.021970298141241074, + "ci_lo": 0.006090360693633556, + "ci_hi": 0.04540242254734039, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "Pile-CC": { + "jp": 0.04546590892132372, + "shuf_mean_over_draws": 0.02692041996633634, + "ci_vs_shuf": { + "point": 0.018545489758253098, + "ci_lo": 0.0033147057984024286, + "ci_hi": 0.041924308985471725, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "StackExchange": { + "jp": 0.04572996227070689, + "shuf_mean_over_draws": 0.02639044984825887, + "ci_vs_shuf": { + "point": 0.019339516758918762, + "ci_lo": 0.003664900315925479, + "ci_hi": 0.04347597807645798, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "Github": { + "jp": 0.027896882360801102, + "shuf_mean_over_draws": 0.017129417858086526, + "ci_vs_shuf": { + "point": 0.010767465457320213, + "ci_lo": 0.002376304939389229, + "ci_hi": 0.02325580082833767, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "OOD_News_2024": { + "jp": 0.03376360982656479, + "shuf_mean_over_draws": 0.018717033526627347, + "ci_vs_shuf": { + "point": 0.015046574175357819, + "ci_lo": 0.0019232068443670869, + "ci_hi": 0.03606754541397095, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "OOD_arXiv_2023": { + "jp": 0.04105739537626505, + "shuf_mean_over_draws": 0.024700006493367253, + "ci_vs_shuf": { + "point": 0.016357388347387314, + "ci_lo": 0.0035497164353728294, + "ci_hi": 0.03377021104097366, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "OOD_CommonPile": { + "jp": 0.038239676505327225, + "shuf_mean_over_draws": 0.023558578826487063, + "ci_vs_shuf": { + "point": 0.01468109805136919, + "ci_lo": 0.0032950022723525763, + "ci_hi": 0.029930993914604187, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + } + }, + "C3_norm_matched_random": { + "Wikipedia_en": 0.0, + "USPTO_Backgrounds": 0.0, + "Pile-CC": 0.0, + "StackExchange": 0.0, + "Github": 0.0, + "OOD_News_2024": 0.0, + "OOD_arXiv_2023": 0.0, + "OOD_CommonPile": 0.0 + } + }, + "clauses": { + "a_instream_ci_positive": { + "Wikipedia_en": false, + "USPTO_Backgrounds": false, + "Pile-CC": false, + "StackExchange": false + }, + "a_ok": false, + "b_ood_ci_strictly_negative": [], + "b_ok": false, + "c_above_shuf_floor": { + "OOD_News_2024": true, + "OOD_arXiv_2023": true, + "OOD_CommonPile": true + }, + "c_ok": false, + "d_news_among_negatives": false, + "d_github_strictly_negative": false, + "d_ok": false + }, + "rungs": { + "Wikipedia_en": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.009770049713551998, + "ci_lo": -0.014474375173449516, + "ci_hi": 0.044941700994968414, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.038209554925560954, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.01993137374520302, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.009770051576197147, + "ci_lo": -0.014545489102602005, + "ci_hi": 0.04577590897679329, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.018278183415532112, + "ci_lo": 0.005172038450837135, + "ci_hi": 0.03638976812362671, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.038099304959177974, + "s1": 0.039063173532485965, + "s2": 0.03746618703007698 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.021268022060394288, + "s0|d1": 0.02049652235582471, + "s0|d2": 0.0173626272007823, + "s0|d3": 0.01960282977670431, + "s0|d4": 0.020682007353752852, + "s1|d0": 0.02165838349610567, + "s1|d1": 0.02128908596932888, + "s1|d2": 0.017355072870850564, + "s1|d3": 0.019145069643855095, + "s1|d4": 0.021024363674223424, + "s2|d0": 0.021585867367684843, + "s2|d1": 0.020569039974361658, + "s2|d2": 0.017435143515467644, + "s2|d3": 0.01893507465720177, + "s2|d4": 0.020561485830694438 + }, + "per_set": { + "multihop": { + "jp": 0.02034211903810501, + "logit": 0.037447988986968994, + "shuf": 0.009616274386644363, + "paired_item_ci": { + "point": -0.017105871811509132, + "ci_lo": -0.03883495181798935, + "ci_hi": -0.0004623213899321854, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.02800000086426735, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.02888566628098488, + "logit": 0.023205220699310303, + "shuf": 0.01672709733247757, + "paired_item_ci": { + "point": 0.005680444184690714, + "ci_lo": -0.0014503271086141467, + "ci_hi": 0.013536377809941769, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.1340000033378601, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.017316019162535667, + "logit": 0.03246753290295601, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.015151516534388065, + "ci_lo": -0.029004331678152084, + "ci_hi": -0.0034632037859410048, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.013000000268220901, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0, + "logit": 0.0014577260008081794, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.0014577260008081794, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7540000081062317, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1245039701461792, + "logit": 0.0476190485060215, + "shuf": 0.07331349700689316, + "paired_item_ci": { + "point": 0.0768849179148674, + "ci_lo": 0.0436507947742939, + "ci_hi": 0.11360368877649307, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0010351967066526413, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0010351967066526413, + "ci_lo": 0.0, + "ci_hi": 0.003105590119957924, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7039999961853027, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.00573596078902483, + "ci_lo": -0.0678190141916275, + "ci_hi": 0.07895585894584656, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.1304755862802267, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.15765396803617476, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": 0.005735957529395819, + "ci_lo": -0.06754170358181, + "ci_hi": 0.07908517867326736, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.02717837132513523, + "ci_lo": -0.0590832382440567, + "ci_hi": -0.004270642064511776, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.12926936261355876, + "s1": 0.1321444157510996, + "s2": 0.13001298978924752 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.14269272312521936, + "s0|d1": 0.16346083655953408, + "s0|d2": 0.16865189298987388, + "s0|d3": 0.15862704440951347, + "s0|d4": 0.1680971346795559, + "s1|d0": 0.13943470343947412, + "s1|d1": 0.16312385052442552, + "s1|d2": 0.16388381868600846, + "s1|d3": 0.15627818033099175, + "s1|d4": 0.16377755142748357, + "s2|d0": 0.13662957176566123, + "s2|d1": 0.16075667440891267, + "s2|d2": 0.16358056701719761, + "s2|d3": 0.15247442126274108, + "s2|d4": 0.16334041729569435 + }, + "per_set": { + "multihop": { + "jp": 0.1003236249089241, + "logit": 0.1178918182849884, + "shuf": 0.12436432391405106, + "paired_item_ci": { + "point": -0.01756819523870945, + "ci_lo": -0.03792186826467514, + "ci_hi": 0.0023116024676710367, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.07699999958276749, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.13415518403053284, + "logit": 0.11240029335021973, + "shuf": 0.15242929756641388, + "paired_item_ci": { + "point": 0.02175489440560341, + "ci_lo": 0.006405606400221586, + "ci_hi": 0.03649987652897835, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.004000000189989805, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.07922077924013138, + "logit": 0.19740261137485504, + "shuf": 0.16545455157756805, + "paired_item_ci": { + "point": -0.11818182468414307, + "ci_lo": -0.14373376965522766, + "ci_hi": -0.09437229484319687, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.029154522344470024, + "logit": 0.02040816657245159, + "shuf": 0.03381924331188202, + "paired_item_ci": { + "point": 0.008746356703341007, + "ci_lo": -0.007774539291858673, + "ci_hi": 0.025279410183429718, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.33000001311302185, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3095238208770752, + "logit": 0.1755952388048172, + "shuf": 0.31220242381095886, + "paired_item_ci": { + "point": 0.1339285671710968, + "ci_lo": 0.0887896791100502, + "ci_hi": 0.1855158805847168, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.006211180239915848, + "logit": 0.006211180239915848, + "shuf": 0.017080746591091156, + "paired_item_ci": { + "point": -8.098457321414898e-11, + "ci_lo": -0.009834369644522667, + "ci_hi": 0.009834368713200092, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.9760000109672546, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "USPTO_Backgrounds": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.021175866946578026, + "ci_lo": -0.017530113458633423, + "ci_hi": 0.07830693572759628, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.04961536889895797, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.027645074779866264, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.021175866946578026, + "ci_lo": -0.017563803121447563, + "ci_hi": 0.07935483008623123, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.021970298141241074, + "ci_lo": 0.006090360693633556, + "ci_hi": 0.04540242254734039, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.04964358932338655, + "s1": 0.04933666000142693, + "s2": 0.04986586356535554 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.028034024871885777, + "s0|d1": 0.029616341809742153, + "s0|d2": 0.026408451795578002, + "s0|d3": 0.02468258198350668, + "s0|d4": 0.030097970552742482, + "s1|d0": 0.027221238426864146, + "s1|d1": 0.028286907263100146, + "s1|d2": 0.026038317754864693, + "s1|d3": 0.024035052210092545, + "s1|d4": 0.029575247876346113, + "s2|d0": 0.028279356472194195, + "s2|d1": 0.029642644245177506, + "s2|d2": 0.0264489040710032, + "s2|d3": 0.025748249515891074, + "s2|d4": 0.03056084644049406 + }, + "per_set": { + "multihop": { + "jp": 0.019417475908994675, + "logit": 0.037447988986968994, + "shuf": 0.008229310624301434, + "paired_item_ci": { + "point": -0.01803051307797432, + "ci_lo": -0.034674063324928284, + "ci_hi": -0.005547850392758846, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0020000000949949026, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.039400532841682434, + "logit": 0.023205220699310303, + "shuf": 0.017597293481230736, + "paired_item_ci": { + "point": 0.01619531214237213, + "ci_lo": 0.007372491993010044, + "ci_hi": 0.02622673660516739, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.007792208343744278, + "logit": 0.03246753290295601, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.02467532642185688, + "ci_lo": -0.04112553969025612, + "ci_hi": -0.010822512209415436, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0038872691802680492, + "logit": 0.0014577260008081794, + "shuf": 9.718173532746732e-05, + "paired_item_ci": { + "point": 0.0024295432958751917, + "ci_lo": -0.0029154520016163588, + "ci_hi": 0.009718173183500767, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5590000152587891, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1775793582201004, + "logit": 0.0476190485060215, + "shuf": 0.11230158805847168, + "paired_item_ci": { + "point": 0.1299603134393692, + "ci_lo": 0.0897817388176918, + "ci_hi": 0.1750991940498352, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.030287031084299088, + "ci_lo": -0.037461571395397186, + "ci_hi": 0.11365257948637009, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.1550266459584236, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.15389137640595435, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": 0.03028702735900879, + "ci_lo": -0.03727513551712036, + "ci_hi": 0.11377983540296555, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.0011352775618433952, + "ci_lo": -0.01282324269413948, + "ci_hi": 0.016479473561048508, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.15465880744159222, + "s1": 0.153244324401021, + "s2": 0.1571768194437027 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.14029880315065385, + "s0|d1": 0.1608631432056427, + "s0|d2": 0.16253653764724732, + "s0|d3": 0.1554515317082405, + "s0|d4": 0.1537318855524063, + "s1|d0": 0.13848397210240365, + "s1|d1": 0.15829955972731113, + "s1|d2": 0.16074975728988647, + "s1|d3": 0.15409957990050316, + "s1|d4": 0.14995899870991708, + "s2|d0": 0.1394907593727112, + "s2|d1": 0.15859544463455677, + "s2|d2": 0.1648475132882595, + "s2|d3": 0.15744183510541915, + "s2|d4": 0.15352136306464673 + }, + "per_set": { + "multihop": { + "jp": 0.11835414171218872, + "logit": 0.1178918182849884, + "shuf": 0.1104946881532669, + "paired_item_ci": { + "point": 0.00046231821761466563, + "ci_lo": -0.018042076379060745, + "ci_hi": 0.01849283277988434, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.9679999947547913, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.14708727598190308, + "logit": 0.11240029335021973, + "shuf": 0.14974619448184967, + "paired_item_ci": { + "point": 0.034686971455812454, + "ci_lo": 0.019821124151349068, + "ci_hi": 0.04979453608393669, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.12380951642990112, + "logit": 0.19740261137485504, + "shuf": 0.0981818214058876, + "paired_item_ci": { + "point": -0.07359308004379272, + "ci_lo": -0.10043291002511978, + "ci_hi": -0.0463203527033329, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.02526725083589554, + "logit": 0.02040816657245159, + "shuf": 0.037026237696409225, + "paired_item_ci": { + "point": 0.004859085660427809, + "ci_lo": -0.009730319492518902, + "ci_hi": 0.019934413954615593, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5699999928474426, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3606150448322296, + "logit": 0.1755952388048172, + "shuf": 0.3740079402923584, + "paired_item_ci": { + "point": 0.185019850730896, + "ci_lo": 0.1289682537317276, + "ci_hi": 0.24505209922790527, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.020703934133052826, + "logit": 0.006211180239915848, + "shuf": 0.03322981670498848, + "paired_item_ci": { + "point": 0.014492753893136978, + "ci_lo": 0.0005175983533263206, + "ci_hi": 0.030538301914930344, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.039000000804662704, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": true + } + }, + "Pile-CC": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.017026403918862343, + "ci_lo": -0.02031945064663887, + "ci_hi": 0.07075221091508865, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.04546590892132372, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.02692041996633634, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.017026405781507492, + "ci_lo": -0.02021908387541771, + "ci_hi": 0.07168250530958176, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.018545489758253098, + "ci_lo": 0.0033147057984024286, + "ci_hi": 0.041924308985471725, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.04509159799199551, + "s1": 0.046116814273409544, + "s2": 0.04518931868951768 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.028093868121504783, + "s0|d1": 0.0278983935713768, + "s0|d2": 0.025132926274091007, + "s0|d3": 0.024317564442753792, + "s0|d4": 0.029227183479815722, + "s1|d0": 0.0275650731055066, + "s1|d1": 0.028236466646194457, + "s1|d2": 0.025422991160303353, + "s1|d3": 0.02505783662199974, + "s1|d4": 0.030132714845240115, + "s2|d0": 0.02773788485210389, + "s2|d1": 0.028223793581128122, + "s2|d2": 0.024014964420348405, + "s2|d3": 0.02401994578540325, + "s2|d4": 0.02872468959540129 + }, + "per_set": { + "multihop": { + "jp": 0.01756819151341915, + "logit": 0.037447988986968994, + "shuf": 0.008876560255885124, + "paired_item_ci": { + "point": -0.019879797473549843, + "ci_lo": -0.04392047971487045, + "ci_hi": -0.002311604330316186, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.010999999940395355, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.036379020661115646, + "logit": 0.023205220699310303, + "shuf": 0.02100555971264839, + "paired_item_ci": { + "point": 0.013173798099160194, + "ci_lo": 0.005193980410695076, + "ci_hi": 0.022480057552456856, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0020000000949949026, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.004761904943734407, + "logit": 0.03246753290295601, + "shuf": 0.00025974027812480927, + "paired_item_ci": { + "point": -0.02770562842488289, + "ci_lo": -0.04416666924953461, + "ci_hi": -0.013419914059340954, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0014577260008081794, + "logit": 0.0014577260008081794, + "shuf": 0.00019436347065493464, + "paired_item_ci": { + "point": 0.0, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.004373178351670504, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 1.281999945640564, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1671627014875412, + "logit": 0.0476190485060215, + "shuf": 0.10426587611436844, + "paired_item_ci": { + "point": 0.1195436492562294, + "ci_lo": 0.0798611119389534, + "ci_hi": 0.16520339250564575, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.019781149923801422, + "ci_lo": -0.06539817154407501, + "ci_hi": 0.10520178079605103, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.14452077597379684, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.1636852040886879, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": 0.01978115364909172, + "ci_lo": -0.06420391798019409, + "ci_hi": 0.10555368661880493, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.019164424389600754, + "ci_lo": -0.04153844341635704, + "ci_hi": -0.0007532127201557159, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.1452333066612482, + "s1": 0.1453225638717413, + "s2": 0.14300646334886552 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.1562277466058731, + "s0|d1": 0.17145237997174262, + "s0|d2": 0.16748275458812714, + "s0|d3": 0.15515274554491043, + "s0|d4": 0.16029519215226173, + "s1|d0": 0.15819527208805084, + "s1|d1": 0.17523027956485748, + "s1|d2": 0.17057643383741378, + "s1|d3": 0.15710598155856131, + "s1|d4": 0.1651713326573372, + "s2|d0": 0.1529442362487316, + "s2|d1": 0.1742464482784271, + "s2|d2": 0.1694401055574417, + "s2|d3": 0.15795974880456926, + "s2|d4": 0.16379739716649055 + }, + "per_set": { + "multihop": { + "jp": 0.10957003384828568, + "logit": 0.1178918182849884, + "shuf": 0.12538141012191772, + "paired_item_ci": { + "point": -0.008321776986122131, + "ci_lo": -0.025427648797631264, + "ci_hi": 0.007859453558921814, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.30799999833106995, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.17343486845493317, + "logit": 0.11240029335021973, + "shuf": 0.16792362928390503, + "paired_item_ci": { + "point": 0.061034563928842545, + "ci_lo": 0.04145213961601257, + "ci_hi": 0.08122432231903076, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.07445887476205826, + "logit": 0.19740261137485504, + "shuf": 0.13489177823066711, + "paired_item_ci": { + "point": -0.12294372916221619, + "ci_lo": -0.14979436993598938, + "ci_hi": -0.09740260988473892, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.020894072949886322, + "logit": 0.02040816657245159, + "shuf": 0.03566569834947586, + "paired_item_ci": { + "point": 0.0004859073960687965, + "ci_lo": -0.014577260240912437, + "ci_hi": 0.016034983098506927, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.9869999885559082, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3442460298538208, + "logit": 0.1755952388048172, + "shuf": 0.35456350445747375, + "paired_item_ci": { + "point": 0.1686507910490036, + "ci_lo": 0.12101934105157852, + "ci_hi": 0.2251983880996704, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.009834368713200092, + "logit": 0.006211180239915848, + "shuf": 0.02857142873108387, + "paired_item_ci": { + "point": 0.0036231884732842445, + "ci_lo": -0.00776397529989481, + "ci_hi": 0.01552795059978962, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5950000286102295, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "StackExchange": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.017290454357862473, + "ci_lo": -0.022211669012904167, + "ci_hi": 0.07221946865320206, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.04572996227070689, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.02639044984825887, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.017290456220507622, + "ci_lo": -0.02206658385694027, + "ci_hi": 0.07310322672128677, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.019339516758918762, + "ci_lo": 0.003664900315925479, + "ci_hi": 0.04347597807645798, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.04637858076021075, + "s1": 0.04555068640038371, + "s2": 0.04526062114164233 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.02845876819919795, + "s0|d1": 0.030003434675745667, + "s0|d2": 0.026951538678258658, + "s0|d3": 0.023122804425656795, + "s0|d4": 0.030220342613756657, + "s1|d0": 0.02496741011273116, + "s1|d1": 0.02796662673354149, + "s1|d2": 0.02405541865155101, + "s1|d3": 0.021409607119858266, + "s1|d4": 0.02888483162969351, + "s2|d0": 0.02564123962074518, + "s2|d1": 0.028561867028474807, + "s2|d2": 0.02479568971320987, + "s2|d3": 0.021859811805188655, + "s2|d4": 0.02895734813064337 + }, + "per_set": { + "multihop": { + "jp": 0.013869625516235828, + "logit": 0.037447988986968994, + "shuf": 0.006010171491652727, + "paired_item_ci": { + "point": -0.02357836440205574, + "ci_lo": -0.04715672880411148, + "ci_hi": -0.006934812292456627, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.004000000189989805, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.039158813655376434, + "logit": 0.023205220699310303, + "shuf": 0.020232053473591805, + "paired_item_ci": { + "point": 0.015953589230775833, + "ci_lo": 0.007006889209151268, + "ci_hi": 0.025743290781974792, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0020000000949949026, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.0025974027812480927, + "logit": 0.03246753290295601, + "shuf": 0.00025974027812480927, + "paired_item_ci": { + "point": -0.029870130121707916, + "ci_lo": -0.048051945865154266, + "ci_hi": -0.01428571529686451, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.004373178351670504, + "logit": 0.0014577260008081794, + "shuf": 0.00029154520598240197, + "paired_item_ci": { + "point": 0.0029154520016163588, + "ci_lo": -0.0029154520016163588, + "ci_hi": 0.011661808006465435, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5550000071525574, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1686507910490036, + "logit": 0.0476190485060215, + "shuf": 0.1051587387919426, + "paired_item_ci": { + "point": 0.1210317388176918, + "ci_lo": 0.07935267686843872, + "ci_hi": 0.1671627014875412, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.020821167156100273, + "ci_lo": -0.06813506782054901, + "ci_hi": 0.11441024392843246, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.14556079730391502, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.15189975090324878, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": 0.02082117088139057, + "ci_lo": -0.06827782839536667, + "ci_hi": 0.11422941833734512, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.006338963750749826, + "ci_lo": -0.027583099901676178, + "ci_hi": 0.01089502777904272, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.1469995591789484, + "s1": 0.14416988678276538, + "s2": 0.14551294930279254 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.15713599622249602, + "s0|d1": 0.15554534792900085, + "s0|d2": 0.15769589096307754, + "s0|d3": 0.15036898851394653, + "s0|d4": 0.14294670857489108, + "s1|d0": 0.15702062100172043, + "s1|d1": 0.15585158467292787, + "s1|d2": 0.15864074900746344, + "s1|d3": 0.14701028987765313, + "s1|d4": 0.1404565878212452, + "s2|d0": 0.1567239560186863, + "s2|d1": 0.15567368194460868, + "s2|d2": 0.1558786541223526, + "s2|d3": 0.14785368256270887, + "s2|d4": 0.13969369679689408 + }, + "per_set": { + "multihop": { + "jp": 0.10679612308740616, + "logit": 0.1178918182849884, + "shuf": 0.09616272896528244, + "paired_item_ci": { + "point": -0.011095701716840267, + "ci_lo": -0.029126213863492012, + "ci_hi": 0.005547848530113697, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.16899999976158142, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.17077593505382538, + "logit": 0.11240029335021973, + "shuf": 0.16463620960712433, + "paired_item_ci": { + "point": 0.058375634253025055, + "ci_lo": 0.039400532841682434, + "ci_hi": 0.07893099635839462, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.0701298713684082, + "logit": 0.19740261137485504, + "shuf": 0.11471860855817795, + "paired_item_ci": { + "point": -0.12727272510528564, + "ci_lo": -0.1567099541425705, + "ci_hi": -0.09870130568742752, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.017006803303956985, + "logit": 0.02040816657245159, + "shuf": 0.030806610360741615, + "paired_item_ci": { + "point": -0.003401360707357526, + "ci_lo": -0.017492713406682014, + "ci_hi": 0.011661808006465435, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.625, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3630952537059784, + "logit": 0.1755952388048172, + "shuf": 0.3531745970249176, + "paired_item_ci": { + "point": 0.1875, + "ci_lo": 0.136408731341362, + "ci_hi": 0.2460317462682724, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.011904762126505375, + "logit": 0.006211180239915848, + "shuf": 0.013768116943538189, + "paired_item_ci": { + "point": 0.005693581886589527, + "ci_lo": -0.003623188938945532, + "ci_hi": 0.01552795059978962, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.2770000100135803, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "Github": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": -0.0005426221759989858, + "ci_lo": -0.02176518924534321, + "ci_hi": 0.022351862862706184, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.027896882360801102, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.017129417858086526, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": -0.0005426229326985776, + "ci_lo": -0.021851813420653343, + "ci_hi": 0.022414328530430794, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.010767465457320213, + "ci_lo": 0.002376304939389229, + "ci_hi": 0.02325580082833767, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.02704486474394798, + "s1": 0.02903519310057163, + "s2": 0.027610587328672408 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.015263549471274018, + "s0|d1": 0.019197244732640683, + "s0|d2": 0.019098307471722364, + "s0|d3": 0.015432688873261213, + "s0|d4": 0.015929940342903137, + "s1|d0": 0.01511744516901672, + "s1|d1": 0.019366411282680927, + "s1|d2": 0.019319505034945905, + "s1|d3": 0.015140480641275645, + "s1|d4": 0.01742706182412803, + "s2|d0": 0.015170807018876076, + "s2|d1": 0.019494863017462193, + "s2|d2": 0.018831012677401303, + "s2|d3": 0.015727066714316607, + "s2|d4": 0.0164248816203326 + }, + "per_set": { + "multihop": { + "jp": 0.003698567161336541, + "logit": 0.037447988986968994, + "shuf": 0.0023116045631468296, + "paired_item_ci": { + "point": -0.033749423921108246, + "ci_lo": -0.06426260620355606, + "ci_hi": -0.011095700785517693, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0020000000949949026, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.02803964354097843, + "logit": 0.023205220699310303, + "shuf": 0.01619531214237213, + "paired_item_ci": { + "point": 0.004834420513361692, + "ci_lo": -0.0033871165942400694, + "ci_hi": 0.013536379672586918, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.2630000114440918, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.01904761977493763, + "logit": 0.03246753290295601, + "shuf": 0.015064935199916363, + "paired_item_ci": { + "point": -0.013419914059340954, + "ci_lo": -0.025974025949835777, + "ci_hi": -0.0034632037859410048, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.007000000216066837, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.004373178351670504, + "logit": 0.0014577260008081794, + "shuf": 0.0003887269413098693, + "paired_item_ci": { + "point": 0.0029154520016163588, + "ci_lo": -0.0029154520016163588, + "ci_hi": 0.010204081423580647, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5929999947547913, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.0843254029750824, + "logit": 0.0476190485060215, + "shuf": 0.05168651044368744, + "paired_item_ci": { + "point": 0.0367063544690609, + "ci_lo": 0.0084325410425663, + "ci_hi": 0.06796876341104507, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.006000000052154064, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": -0.012123704887926579, + "ci_lo": -0.07557906210422516, + "ci_hi": 0.05212606117129326, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.11261592544615269, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.11933545954525471, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": -0.012123695574700832, + "ci_lo": -0.07539984583854675, + "ci_hi": 0.05153894051909447, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.006719531957060099, + "ci_lo": -0.03693092614412308, + "ci_hi": 0.018102142959833145, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.10961288511753083, + "s1": 0.1149481400847435, + "s2": 0.11328674778342247 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.11639587208628654, + "s0|d1": 0.11431133672595024, + "s0|d2": 0.13957404792308808, + "s0|d3": 0.1023078877478838, + "s0|d4": 0.11455089561641216, + "s1|d0": 0.12445689216256142, + "s1|d1": 0.11649707406759262, + "s1|d2": 0.13955935537815095, + "s1|d3": 0.1048777874559164, + "s1|d4": 0.11539066918194293, + "s2|d0": 0.1208079494535923, + "s2|d1": 0.11724287122488022, + "s2|d2": 0.14114154726266862, + "s2|d3": 0.10386928357183933, + "s2|d4": 0.11904843114316463 + }, + "per_set": { + "multihop": { + "jp": 0.07720757275819778, + "logit": 0.1178918182849884, + "shuf": 0.07008784264326096, + "paired_item_ci": { + "point": -0.04068423807621002, + "ci_lo": -0.06749884784221649, + "ci_hi": -0.018018944188952446, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.11626783013343811, + "logit": 0.11240029335021973, + "shuf": 0.11131253838539124, + "paired_item_ci": { + "point": 0.0038675342220813036, + "ci_lo": -0.012941143475472927, + "ci_hi": 0.02175791934132576, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.6600000262260437, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.07878787070512772, + "logit": 0.19740261137485504, + "shuf": 0.1400865912437439, + "paired_item_ci": { + "point": -0.11861473321914673, + "ci_lo": -0.14502164721488953, + "ci_hi": -0.094794362783432, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.017006801441311836, + "logit": 0.02040816657245159, + "shuf": 0.028960155323147774, + "paired_item_ci": { + "point": -0.0034013616386801004, + "ci_lo": -0.020408162847161293, + "ci_hi": 0.015549074858427048, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.6449999809265137, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.273809552192688, + "logit": 0.1755952388048172, + "shuf": 0.24623017013072968, + "paired_item_ci": { + "point": 0.0982142761349678, + "ci_lo": 0.0525793619453907, + "ci_hi": 0.14981399476528168, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.005175983998924494, + "logit": 0.006211180239915848, + "shuf": 0.01366459671407938, + "paired_item_ci": { + "point": -0.0010351968230679631, + "ci_lo": -0.010351967997848988, + "ci_hi": 0.00828157365322113, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.8450000286102295, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "OOD_News_2024": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.0053241066634655, + "ci_lo": -0.023551836609840393, + "ci_hi": 0.042993493378162384, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.03376360982656479, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.018717033526627347, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.005324105266481638, + "ci_lo": -0.02342044562101364, + "ci_hi": 0.044015560299158096, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.015046574175357819, + "ci_lo": 0.0019232068443670869, + "ci_hi": 0.03606754541397095, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.034046309324912725, + "s1": 0.033360902103595436, + "s2": 0.03388362142723054 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.019069751794449984, + "s0|d1": 0.020263867918401957, + "s0|d2": 0.017607957217842342, + "s0|d3": 0.01655495883896947, + "s0|d4": 0.01963656535372138, + "s1|d0": 0.01922085862606764, + "s1|d1": 0.021354046650230885, + "s1|d2": 0.0182031930424273, + "s1|d3": 0.016534732282161714, + "s1|d4": 0.01978159798309207, + "s2|d0": 0.01942573320120573, + "s2|d1": 0.020686291344463825, + "s2|d2": 0.016344963852316142, + "s2|d3": 0.01651450674980879, + "s2|d4": 0.01955649657174945 + }, + "per_set": { + "multihop": { + "jp": 0.017105871811509132, + "logit": 0.037447988986968994, + "shuf": 0.007212205324321985, + "paired_item_ci": { + "point": -0.02034211903810501, + "ci_lo": -0.04530744627118111, + "ci_hi": -0.002311605028808117, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.01899999938905239, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.025380710139870644, + "logit": 0.023205220699310303, + "shuf": 0.01653371937572956, + "paired_item_ci": { + "point": 0.00217548874206841, + "ci_lo": -0.005441746674478054, + "ci_hi": 0.010394004173576832, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5789999961853027, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.0008658008882775903, + "logit": 0.03246753290295601, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.03160173073410988, + "ci_lo": -0.05066017806529999, + "ci_hi": -0.016017315909266472, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0014577260008081794, + "logit": 0.0014577260008081794, + "shuf": 9.718173532746732e-05, + "paired_item_ci": { + "point": 0.0, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.004373178351670504, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 1.281999945640564, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1240079402923584, + "logit": 0.0476190485060215, + "shuf": 0.06974206119775772, + "paired_item_ci": { + "point": 0.0763888955116272, + "ci_lo": 0.0401785746216774, + "ci_hi": 0.11806797236204147, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0010351967066526413, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0010351967066526413, + "ci_lo": 0.0, + "ci_hi": 0.003105590119957924, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7039999961853027, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.014667714945971966, + "ci_lo": -0.05951589345932007, + "ci_hi": 0.0878107026219368, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.13940734118223191, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.171519685536623, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": 0.014667719602584839, + "ci_lo": -0.05875428766012192, + "ci_hi": 0.08850378543138504, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.03211233764886856, + "ci_lo": -0.07211270183324814, + "ci_hi": 0.002804108429700136, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.13928202912211418, + "s1": 0.1387300755828619, + "s2": 0.140209923312068 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.1814884640276432, + "s0|d1": 0.18362783789634704, + "s0|d2": 0.17608134001493453, + "s0|d3": 0.16537454649806022, + "s0|d4": 0.17388786785304547, + "s1|d0": 0.16890818774700164, + "s1|d1": 0.17471088767051696, + "s1|d2": 0.17205415070056915, + "s1|d3": 0.1558712527155876, + "s1|d4": 0.1659378670156002, + "s2|d0": 0.17278050258755684, + "s2|d1": 0.17618520706892013, + "s2|d2": 0.17805482000112532, + "s2|d3": 0.15897554568946362, + "s2|d4": 0.16885684728622435 + }, + "per_set": { + "multihop": { + "jp": 0.11373092979192734, + "logit": 0.1178918182849884, + "shuf": 0.17697644233703613, + "paired_item_ci": { + "point": -0.004160889890044928, + "ci_lo": -0.02311604470014572, + "ci_hi": 0.013881194405257702, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.6050000190734863, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.14829587936401367, + "logit": 0.11240029335021973, + "shuf": 0.1547498255968094, + "paired_item_ci": { + "point": 0.03589557111263275, + "ci_lo": 0.01957940310239792, + "ci_hi": 0.05269821733236313, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.08787877857685089, + "logit": 0.19740261137485504, + "shuf": 0.18432900309562683, + "paired_item_ci": { + "point": -0.10952381789684296, + "ci_lo": -0.1355086714029312, + "ci_hi": -0.0852813869714737, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.028182700276374817, + "logit": 0.02040816657245159, + "shuf": 0.03449951484799385, + "paired_item_ci": { + "point": 0.007774537894874811, + "ci_lo": -0.006316813174635172, + "ci_hi": 0.02283770777285099, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.3269999921321869, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3189484179019928, + "logit": 0.1755952388048172, + "shuf": 0.3070436418056488, + "paired_item_ci": { + "point": 0.1433531790971756, + "ci_lo": 0.09621775150299072, + "ci_hi": 0.1969246119260788, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.009834368713200092, + "logit": 0.006211180239915848, + "shuf": 0.022877847775816917, + "paired_item_ci": { + "point": 0.0036231884732842445, + "ci_lo": -0.008281574584543705, + "ci_hi": 0.017080744728446007, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5720000267028809, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "OOD_arXiv_2023": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.012617891654372215, + "ci_lo": -0.020018048584461212, + "ci_hi": 0.05886620655655861, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.04105739537626505, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.024700006493367253, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.012617891654372215, + "ci_lo": -0.019899912178516388, + "ci_hi": 0.05993577092885971, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.016357388347387314, + "ci_lo": 0.0035497164353728294, + "ci_hi": 0.03377021104097366, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.041398055385798216, + "s1": 0.040840694308280946, + "s2": 0.04093343988060951 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.023910248093307017, + "s0|d1": 0.028411714360117912, + "s0|d2": 0.022016710974276064, + "s0|d3": 0.022530001122504473, + "s0|d4": 0.028167221508920193, + "s1|d0": 0.023540113866329194, + "s1|d1": 0.02751885764300823, + "s1|d2": 0.021791608072817326, + "s1|d3": 0.021359750442206858, + "s1|d4": 0.027849376667290926, + "s2|d0": 0.025028206408023834, + "s2|d1": 0.026830878853797913, + "s2|d2": 0.02190631702542305, + "s2|d3": 0.02156462660059333, + "s2|d4": 0.028074479941278697 + }, + "per_set": { + "multihop": { + "jp": 0.018955154344439507, + "logit": 0.037447988986968994, + "shuf": 0.007397133391350508, + "paired_item_ci": { + "point": -0.018492834642529488, + "ci_lo": -0.037447988986968994, + "ci_hi": -0.004149318672716618, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0020000000949949026, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.03239062428474426, + "logit": 0.023205220699310303, + "shuf": 0.015397632494568825, + "paired_item_ci": { + "point": 0.009185399860143661, + "ci_lo": 0.0012055833358317614, + "ci_hi": 0.018370799720287323, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.027000000700354576, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.005627705715596676, + "logit": 0.03246753290295601, + "shuf": 0.000606060610152781, + "paired_item_ci": { + "point": -0.026839828118681908, + "ci_lo": -0.04415584355592728, + "ci_hi": -0.012554111890494823, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0, + "logit": 0.0014577260008081794, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.0014577260008081794, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7540000081062317, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1483134925365448, + "logit": 0.0476190485060215, + "shuf": 0.10009920597076416, + "paired_item_ci": { + "point": 0.1006944477558136, + "ci_lo": 0.0625000074505806, + "ci_hi": 0.1428571492433548, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.01597847044467926, + "ci_lo": -0.06020646169781685, + "ci_hi": 0.09794672578573227, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.1407180955633521, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.14946635514497758, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": 0.01597847230732441, + "ci_lo": -0.06021327152848244, + "ci_hi": 0.09937313199043274, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.008748260326683521, + "ci_lo": -0.02264239639043808, + "ci_hi": 0.004959478974342346, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.14152720663696527, + "s1": 0.14062113035470247, + "s2": 0.14000595565885304 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.13290157429873944, + "s0|d1": 0.1612406000494957, + "s0|d2": 0.15406292527914048, + "s0|d3": 0.1421377941966057, + "s0|d4": 0.1567019246518612, + "s1|d0": 0.13329095840454103, + "s1|d1": 0.16320623829960823, + "s1|d2": 0.15535474717617034, + "s1|d3": 0.14220397919416428, + "s1|d4": 0.15683038160204887, + "s2|d0": 0.1321408160030842, + "s2|d1": 0.16205081194639206, + "s2|d2": 0.1522611804306507, + "s2|d3": 0.14074688032269478, + "s2|d4": 0.15686447992920877 + }, + "per_set": { + "multihop": { + "jp": 0.1178918331861496, + "logit": 0.1178918182849884, + "shuf": 0.12306980788707733, + "paired_item_ci": { + "point": -1.1573717539903328e-09, + "ci_lo": -0.01803051494061947, + "ci_hi": 0.016643550246953964, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.9309999942779541, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.1432197242975235, + "logit": 0.11240029335021973, + "shuf": 0.13705584406852722, + "paired_item_ci": { + "point": 0.03081943280994892, + "ci_lo": 0.015470145270228386, + "ci_hi": 0.047256458550691605, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.09004329144954681, + "logit": 0.19740261137485504, + "shuf": 0.11324675381183624, + "paired_item_ci": { + "point": -0.10735930502414703, + "ci_lo": -0.14027056097984314, + "ci_hi": -0.07575757801532745, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.011661808006465435, + "logit": 0.02040816657245159, + "shuf": 0.03566569834947586, + "paired_item_ci": { + "point": -0.008746356703341007, + "ci_lo": -0.02235179767012596, + "ci_hi": 0.004373177420347929, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.17000000178813934, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3407738208770752, + "logit": 0.1755952388048172, + "shuf": 0.3382936716079712, + "paired_item_ci": { + "point": 0.165178582072258, + "ci_lo": 0.11060267686843872, + "ci_hi": 0.22671134769916534, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.011387163773179054, + "logit": 0.006211180239915848, + "shuf": 0.028778469190001488, + "paired_item_ci": { + "point": 0.005175983998924494, + "ci_lo": -0.0020703936461359262, + "ci_hi": 0.013975156471133232, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.22300000488758087, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "OOD_CommonPile": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.009800171479582787, + "ci_lo": -0.01996418461203575, + "ci_hi": 0.050820767879486084, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.038239676505327225, + "logit_admitted_mean": 0.028439503419212996, + "shuf_admitted_mean": 0.023558578826487063, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.009800171479582787, + "ci_lo": -0.019945479929447174, + "ci_hi": 0.05199887603521347, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.01468109805136919, + "ci_lo": 0.0032950022723525763, + "ci_hi": 0.029930993914604187, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.037979211937636134, + "s1": 0.03894782699644565, + "s2": 0.03779198825359344 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.021971944719552994, + "s0|d1": 0.027253299206495284, + "s0|d2": 0.021275742817670106, + "s0|d3": 0.02162203611806035, + "s0|d4": 0.026433797646313907, + "s1|d0": 0.021899430267512798, + "s1|d1": 0.027723732590675353, + "s1|d2": 0.020811128057539464, + "s1|d3": 0.020954283978790046, + "s1|d4": 0.027029034588485955, + "s2|d0": 0.021674325317144395, + "s2|d1": 0.026810648664832116, + "s2|d2": 0.020181251876056194, + "s2|d3": 0.021304191276431083, + "s2|d4": 0.026433794666081666 + }, + "per_set": { + "multihop": { + "jp": 0.019417475908994675, + "logit": 0.037447988986968994, + "shuf": 0.00813684705644846, + "paired_item_ci": { + "point": -0.01803051307797432, + "ci_lo": -0.03467406705021858, + "ci_hi": -0.005547849927097559, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0010000000474974513, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.031665459275245667, + "logit": 0.023205220699310303, + "shuf": 0.01677544414997101, + "paired_item_ci": { + "point": 0.00846023578196764, + "ci_lo": 0.00012086005881428719, + "ci_hi": 0.017769522964954376, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.04600000008940697, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.005194805562496185, + "logit": 0.03246753290295601, + "shuf": 0.0005194805562496185, + "paired_item_ci": { + "point": -0.027272727340459824, + "ci_lo": -0.04503246769309044, + "ci_hi": -0.012554114684462547, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0, + "logit": 0.0014577260008081794, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.0014577260008081794, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7540000081062317, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1349206417798996, + "logit": 0.0476190485060215, + "shuf": 0.09236112236976624, + "paired_item_ci": { + "point": 0.0873015895485878, + "ci_lo": 0.0525793693959713, + "ci_hi": 0.1264880895614624, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.012836692854762077, + "ci_lo": -0.05727912858128548, + "ci_hi": 0.08582098037004471, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.13757631685584784, + "logit_admitted_mean": 0.1247396256774664, + "shuf_admitted_mean": 0.14552284255623818, + "rand_admitted_mean": 0.027811610139906406, + "hierarchical_ci_vs_logit": { + "point": 0.012836696580052376, + "ci_lo": -0.056966252624988556, + "ci_hi": 0.08607124537229538, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.00794652383774519, + "ci_lo": -0.022086434066295624, + "ci_hi": 0.005812960676848888, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.1372596962377429, + "s1": 0.1391132090240717, + "s2": 0.13635604362934828 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.1396980531513691, + "s0|d1": 0.15505741983652116, + "s0|d2": 0.1550469197332859, + "s0|d3": 0.1395270235836506, + "s0|d4": 0.14095841497182846, + "s1|d0": 0.13861030340194702, + "s1|d1": 0.15715472623705865, + "s1|d2": 0.15290846154093743, + "s1|d3": 0.14020299538969994, + "s1|d4": 0.14162667691707612, + "s2|d0": 0.13895561844110488, + "s2|d1": 0.15443600937724114, + "s2|d2": 0.15316562503576278, + "s2|d3": 0.13669089898467063, + "s2|d4": 0.13880348578095436 + }, + "per_set": { + "multihop": { + "jp": 0.11650485545396805, + "logit": 0.1178918182849884, + "shuf": 0.10725843906402588, + "paired_item_ci": { + "point": -0.0013869631802663207, + "ci_lo": -0.01665510982275009, + "ci_hi": 0.014331946149468422, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.8330000042915344, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.1430988609790802, + "logit": 0.11240029335021973, + "shuf": 0.13766014575958252, + "paired_item_ci": { + "point": 0.03069857321679592, + "ci_lo": 0.014865843579173088, + "ci_hi": 0.0472564660012722, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.095238097012043, + "logit": 0.19740261137485504, + "shuf": 0.1175757646560669, + "paired_item_ci": { + "point": -0.10216449946165085, + "ci_lo": -0.13116882741451263, + "ci_hi": -0.0740259662270546, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.014091352932155132, + "logit": 0.02040816657245159, + "shuf": 0.03615160658955574, + "paired_item_ci": { + "point": -0.0063168141059577465, + "ci_lo": -0.020420312881469727, + "ci_hi": 0.007774537429213524, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.3700000047683716, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3189484179019928, + "logit": 0.1755952388048172, + "shuf": 0.32896825671195984, + "paired_item_ci": { + "point": 0.1433531790971756, + "ci_lo": 0.091269850730896, + "ci_hi": 0.1994047611951828, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.013975156471133232, + "logit": 0.006211180239915848, + "shuf": 0.030745340511202812, + "paired_item_ci": { + "point": 0.00776397529989481, + "ci_lo": -0.0015527952928096056, + "ci_hi": 0.018633542582392693, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.13300000131130219, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + } + }, + "arms_admitted_mean": { + "logit_I": { + "min": 0.1247396256774664, + "persist": 0.028439503419212996 + }, + "C0_ref_Github_n400_s0": { + "min": 0.11324172168970108, + "persist": 0.027925857156515122 + }, + "C1_identity": { + "min": 0.1247396256774664, + "persist": 0.028439503419212996 + }, + "J|Wikipedia_en|s0": { + "min": 0.12926936261355876, + "persist": 0.038099304959177974 + }, + "shuf|Wikipedia_en|s0|d0": { + "min": 0.14269272312521936, + "persist": 0.021268022060394288 + }, + "shuf|Wikipedia_en|s0|d1": { + "min": 0.16346083655953408, + "persist": 0.02049652235582471 + }, + "shuf|Wikipedia_en|s0|d2": { + "min": 0.16865189298987388, + "persist": 0.0173626272007823 + }, + "shuf|Wikipedia_en|s0|d3": { + "min": 0.15862704440951347, + "persist": 0.01960282977670431 + }, + "shuf|Wikipedia_en|s0|d4": { + "min": 0.1680971346795559, + "persist": 0.020682007353752852 + }, + "J|Wikipedia_en|s1": { + "min": 0.1321444157510996, + "persist": 0.039063173532485965 + }, + "shuf|Wikipedia_en|s1|d0": { + "min": 0.13943470343947412, + "persist": 0.02165838349610567 + }, + "shuf|Wikipedia_en|s1|d1": { + "min": 0.16312385052442552, + "persist": 0.02128908596932888 + }, + "shuf|Wikipedia_en|s1|d2": { + "min": 0.16388381868600846, + "persist": 0.017355072870850564 + }, + "shuf|Wikipedia_en|s1|d3": { + "min": 0.15627818033099175, + "persist": 0.019145069643855095 + }, + "shuf|Wikipedia_en|s1|d4": { + "min": 0.16377755142748357, + "persist": 0.021024363674223424 + }, + "J|Wikipedia_en|s2": { + "min": 0.13001298978924752, + "persist": 0.03746618703007698 + }, + "shuf|Wikipedia_en|s2|d0": { + "min": 0.13662957176566123, + "persist": 0.021585867367684843 + }, + "shuf|Wikipedia_en|s2|d1": { + "min": 0.16075667440891267, + "persist": 0.020569039974361658 + }, + "shuf|Wikipedia_en|s2|d2": { + "min": 0.16358056701719761, + "persist": 0.017435143515467644 + }, + "shuf|Wikipedia_en|s2|d3": { + "min": 0.15247442126274108, + "persist": 0.01893507465720177 + }, + "shuf|Wikipedia_en|s2|d4": { + "min": 0.16334041729569435, + "persist": 0.020561485830694438 + }, + "rand|Wikipedia_en": { + "min": 0.027811610139906406, + "persist": 0.0 + }, + "J|USPTO_Backgrounds|s0": { + "min": 0.15465880744159222, + "persist": 0.04964358932338655 + }, + "shuf|USPTO_Backgrounds|s0|d0": { + "min": 0.14029880315065385, + "persist": 0.028034024871885777 + }, + "shuf|USPTO_Backgrounds|s0|d1": { + "min": 0.1608631432056427, + "persist": 0.029616341809742153 + }, + "shuf|USPTO_Backgrounds|s0|d2": { + "min": 0.16253653764724732, + "persist": 0.026408451795578002 + }, + "shuf|USPTO_Backgrounds|s0|d3": { + "min": 0.1554515317082405, + "persist": 0.02468258198350668 + }, + "shuf|USPTO_Backgrounds|s0|d4": { + "min": 0.1537318855524063, + "persist": 0.030097970552742482 + }, + "J|USPTO_Backgrounds|s1": { + "min": 0.153244324401021, + "persist": 0.04933666000142693 + }, + "shuf|USPTO_Backgrounds|s1|d0": { + "min": 0.13848397210240365, + "persist": 0.027221238426864146 + }, + "shuf|USPTO_Backgrounds|s1|d1": { + "min": 0.15829955972731113, + "persist": 0.028286907263100146 + }, + "shuf|USPTO_Backgrounds|s1|d2": { + "min": 0.16074975728988647, + "persist": 0.026038317754864693 + }, + "shuf|USPTO_Backgrounds|s1|d3": { + "min": 0.15409957990050316, + "persist": 0.024035052210092545 + }, + "shuf|USPTO_Backgrounds|s1|d4": { + "min": 0.14995899870991708, + "persist": 0.029575247876346113 + }, + "J|USPTO_Backgrounds|s2": { + "min": 0.1571768194437027, + "persist": 0.04986586356535554 + }, + "shuf|USPTO_Backgrounds|s2|d0": { + "min": 0.1394907593727112, + "persist": 0.028279356472194195 + }, + "shuf|USPTO_Backgrounds|s2|d1": { + "min": 0.15859544463455677, + "persist": 0.029642644245177506 + }, + "shuf|USPTO_Backgrounds|s2|d2": { + "min": 0.1648475132882595, + "persist": 0.0264489040710032 + }, + "shuf|USPTO_Backgrounds|s2|d3": { + "min": 0.15744183510541915, + "persist": 0.025748249515891074 + }, + "shuf|USPTO_Backgrounds|s2|d4": { + "min": 0.15352136306464673, + "persist": 0.03056084644049406 + }, + "rand|USPTO_Backgrounds": { + "min": 0.027811610139906406, + "persist": 0.0 + }, + "J|Pile-CC|s0": { + "min": 0.1452333066612482, + "persist": 0.04509159799199551 + }, + "shuf|Pile-CC|s0|d0": { + "min": 0.1562277466058731, + "persist": 0.028093868121504783 + }, + "shuf|Pile-CC|s0|d1": { + "min": 0.17145237997174262, + "persist": 0.0278983935713768 + }, + "shuf|Pile-CC|s0|d2": { + "min": 0.16748275458812714, + "persist": 0.025132926274091007 + }, + "shuf|Pile-CC|s0|d3": { + "min": 0.15515274554491043, + "persist": 0.024317564442753792 + }, + "shuf|Pile-CC|s0|d4": { + "min": 0.16029519215226173, + "persist": 0.029227183479815722 + }, + "J|Pile-CC|s1": { + "min": 0.1453225638717413, + "persist": 0.046116814273409544 + }, + "shuf|Pile-CC|s1|d0": { + "min": 0.15819527208805084, + "persist": 0.0275650731055066 + }, + "shuf|Pile-CC|s1|d1": { + "min": 0.17523027956485748, + "persist": 0.028236466646194457 + }, + "shuf|Pile-CC|s1|d2": { + "min": 0.17057643383741378, + "persist": 0.025422991160303353 + }, + "shuf|Pile-CC|s1|d3": { + "min": 0.15710598155856131, + "persist": 0.02505783662199974 + }, + "shuf|Pile-CC|s1|d4": { + "min": 0.1651713326573372, + "persist": 0.030132714845240115 + }, + "J|Pile-CC|s2": { + "min": 0.14300646334886552, + "persist": 0.04518931868951768 + }, + "shuf|Pile-CC|s2|d0": { + "min": 0.1529442362487316, + "persist": 0.02773788485210389 + }, + "shuf|Pile-CC|s2|d1": { + "min": 0.1742464482784271, + "persist": 0.028223793581128122 + }, + "shuf|Pile-CC|s2|d2": { + "min": 0.1694401055574417, + "persist": 0.024014964420348405 + }, + "shuf|Pile-CC|s2|d3": { + "min": 0.15795974880456926, + "persist": 0.02401994578540325 + }, + "shuf|Pile-CC|s2|d4": { + "min": 0.16379739716649055, + "persist": 0.02872468959540129 + }, + "rand|Pile-CC": { + "min": 0.027811610139906406, + "persist": 0.0 + }, + "J|StackExchange|s0": { + "min": 0.1469995591789484, + "persist": 0.04637858076021075 + }, + "shuf|StackExchange|s0|d0": { + "min": 0.15713599622249602, + "persist": 0.02845876819919795 + }, + "shuf|StackExchange|s0|d1": { + "min": 0.15554534792900085, + "persist": 0.030003434675745667 + }, + "shuf|StackExchange|s0|d2": { + "min": 0.15769589096307754, + "persist": 0.026951538678258658 + }, + "shuf|StackExchange|s0|d3": { + "min": 0.15036898851394653, + "persist": 0.023122804425656795 + }, + "shuf|StackExchange|s0|d4": { + "min": 0.14294670857489108, + "persist": 0.030220342613756657 + }, + "J|StackExchange|s1": { + "min": 0.14416988678276538, + "persist": 0.04555068640038371 + }, + "shuf|StackExchange|s1|d0": { + "min": 0.15702062100172043, + "persist": 0.02496741011273116 + }, + "shuf|StackExchange|s1|d1": { + "min": 0.15585158467292787, + "persist": 0.02796662673354149 + }, + "shuf|StackExchange|s1|d2": { + "min": 0.15864074900746344, + "persist": 0.02405541865155101 + }, + "shuf|StackExchange|s1|d3": { + "min": 0.14701028987765313, + "persist": 0.021409607119858266 + }, + "shuf|StackExchange|s1|d4": { + "min": 0.1404565878212452, + "persist": 0.02888483162969351 + }, + "J|StackExchange|s2": { + "min": 0.14551294930279254, + "persist": 0.04526062114164233 + }, + "shuf|StackExchange|s2|d0": { + "min": 0.1567239560186863, + "persist": 0.02564123962074518 + }, + "shuf|StackExchange|s2|d1": { + "min": 0.15567368194460868, + "persist": 0.028561867028474807 + }, + "shuf|StackExchange|s2|d2": { + "min": 0.1558786541223526, + "persist": 0.02479568971320987 + }, + "shuf|StackExchange|s2|d3": { + "min": 0.14785368256270887, + "persist": 0.021859811805188655 + }, + "shuf|StackExchange|s2|d4": { + "min": 0.13969369679689408, + "persist": 0.02895734813064337 + }, + "rand|StackExchange": { + "min": 0.027811610139906406, + "persist": 0.0 + }, + "J|Github|s0": { + "min": 0.10961288511753083, + "persist": 0.02704486474394798 + }, + "shuf|Github|s0|d0": { + "min": 0.11639587208628654, + "persist": 0.015263549471274018 + }, + "shuf|Github|s0|d1": { + "min": 0.11431133672595024, + "persist": 0.019197244732640683 + }, + "shuf|Github|s0|d2": { + "min": 0.13957404792308808, + "persist": 0.019098307471722364 + }, + "shuf|Github|s0|d3": { + "min": 0.1023078877478838, + "persist": 0.015432688873261213 + }, + "shuf|Github|s0|d4": { + "min": 0.11455089561641216, + "persist": 0.015929940342903137 + }, + "J|Github|s1": { + "min": 0.1149481400847435, + "persist": 0.02903519310057163 + }, + "shuf|Github|s1|d0": { + "min": 0.12445689216256142, + "persist": 0.01511744516901672 + }, + "shuf|Github|s1|d1": { + "min": 0.11649707406759262, + "persist": 0.019366411282680927 + }, + "shuf|Github|s1|d2": { + "min": 0.13955935537815095, + "persist": 0.019319505034945905 + }, + "shuf|Github|s1|d3": { + "min": 0.1048777874559164, + "persist": 0.015140480641275645 + }, + "shuf|Github|s1|d4": { + "min": 0.11539066918194293, + "persist": 0.01742706182412803 + }, + "J|Github|s2": { + "min": 0.11328674778342247, + "persist": 0.027610587328672408 + }, + "shuf|Github|s2|d0": { + "min": 0.1208079494535923, + "persist": 0.015170807018876076 + }, + "shuf|Github|s2|d1": { + "min": 0.11724287122488022, + "persist": 0.019494863017462193 + }, + "shuf|Github|s2|d2": { + "min": 0.14114154726266862, + "persist": 0.018831012677401303 + }, + "shuf|Github|s2|d3": { + "min": 0.10386928357183933, + "persist": 0.015727066714316607 + }, + "shuf|Github|s2|d4": { + "min": 0.11904843114316463, + "persist": 0.0164248816203326 + }, + "rand|Github": { + "min": 0.027811610139906406, + "persist": 0.0 + }, + "J|OOD_News_2024|s0": { + "min": 0.13928202912211418, + "persist": 0.034046309324912725 + }, + "shuf|OOD_News_2024|s0|d0": { + "min": 0.1814884640276432, + "persist": 0.019069751794449984 + }, + "shuf|OOD_News_2024|s0|d1": { + "min": 0.18362783789634704, + "persist": 0.020263867918401957 + }, + "shuf|OOD_News_2024|s0|d2": { + "min": 0.17608134001493453, + "persist": 0.017607957217842342 + }, + "shuf|OOD_News_2024|s0|d3": { + "min": 0.16537454649806022, + "persist": 0.01655495883896947 + }, + "shuf|OOD_News_2024|s0|d4": { + "min": 0.17388786785304547, + "persist": 0.01963656535372138 + }, + "J|OOD_News_2024|s1": { + "min": 0.1387300755828619, + "persist": 0.033360902103595436 + }, + "shuf|OOD_News_2024|s1|d0": { + "min": 0.16890818774700164, + "persist": 0.01922085862606764 + }, + "shuf|OOD_News_2024|s1|d1": { + "min": 0.17471088767051696, + "persist": 0.021354046650230885 + }, + "shuf|OOD_News_2024|s1|d2": { + "min": 0.17205415070056915, + "persist": 0.0182031930424273 + }, + "shuf|OOD_News_2024|s1|d3": { + "min": 0.1558712527155876, + "persist": 0.016534732282161714 + }, + "shuf|OOD_News_2024|s1|d4": { + "min": 0.1659378670156002, + "persist": 0.01978159798309207 + }, + "J|OOD_News_2024|s2": { + "min": 0.140209923312068, + "persist": 0.03388362142723054 + }, + "shuf|OOD_News_2024|s2|d0": { + "min": 0.17278050258755684, + "persist": 0.01942573320120573 + }, + "shuf|OOD_News_2024|s2|d1": { + "min": 0.17618520706892013, + "persist": 0.020686291344463825 + }, + "shuf|OOD_News_2024|s2|d2": { + "min": 0.17805482000112532, + "persist": 0.016344963852316142 + }, + "shuf|OOD_News_2024|s2|d3": { + "min": 0.15897554568946362, + "persist": 0.01651450674980879 + }, + "shuf|OOD_News_2024|s2|d4": { + "min": 0.16885684728622435, + "persist": 0.01955649657174945 + }, + "rand|OOD_News_2024": { + "min": 0.027811610139906406, + "persist": 0.0 + }, + "J|OOD_arXiv_2023|s0": { + "min": 0.14152720663696527, + "persist": 0.041398055385798216 + }, + "shuf|OOD_arXiv_2023|s0|d0": { + "min": 0.13290157429873944, + "persist": 0.023910248093307017 + }, + "shuf|OOD_arXiv_2023|s0|d1": { + "min": 0.1612406000494957, + "persist": 0.028411714360117912 + }, + "shuf|OOD_arXiv_2023|s0|d2": { + "min": 0.15406292527914048, + "persist": 0.022016710974276064 + }, + "shuf|OOD_arXiv_2023|s0|d3": { + "min": 0.1421377941966057, + "persist": 0.022530001122504473 + }, + "shuf|OOD_arXiv_2023|s0|d4": { + "min": 0.1567019246518612, + "persist": 0.028167221508920193 + }, + "J|OOD_arXiv_2023|s1": { + "min": 0.14062113035470247, + "persist": 0.040840694308280946 + }, + "shuf|OOD_arXiv_2023|s1|d0": { + "min": 0.13329095840454103, + "persist": 0.023540113866329194 + }, + "shuf|OOD_arXiv_2023|s1|d1": { + "min": 0.16320623829960823, + "persist": 0.02751885764300823 + }, + "shuf|OOD_arXiv_2023|s1|d2": { + "min": 0.15535474717617034, + "persist": 0.021791608072817326 + }, + "shuf|OOD_arXiv_2023|s1|d3": { + "min": 0.14220397919416428, + "persist": 0.021359750442206858 + }, + "shuf|OOD_arXiv_2023|s1|d4": { + "min": 0.15683038160204887, + "persist": 0.027849376667290926 + }, + "J|OOD_arXiv_2023|s2": { + "min": 0.14000595565885304, + "persist": 0.04093343988060951 + }, + "shuf|OOD_arXiv_2023|s2|d0": { + "min": 0.1321408160030842, + "persist": 0.025028206408023834 + }, + "shuf|OOD_arXiv_2023|s2|d1": { + "min": 0.16205081194639206, + "persist": 0.026830878853797913 + }, + "shuf|OOD_arXiv_2023|s2|d2": { + "min": 0.1522611804306507, + "persist": 0.02190631702542305 + }, + "shuf|OOD_arXiv_2023|s2|d3": { + "min": 0.14074688032269478, + "persist": 0.02156462660059333 + }, + "shuf|OOD_arXiv_2023|s2|d4": { + "min": 0.15686447992920877, + "persist": 0.028074479941278697 + }, + "rand|OOD_arXiv_2023": { + "min": 0.027811610139906406, + "persist": 0.0 + }, + "J|OOD_CommonPile|s0": { + "min": 0.1372596962377429, + "persist": 0.037979211937636134 + }, + "shuf|OOD_CommonPile|s0|d0": { + "min": 0.1396980531513691, + "persist": 0.021971944719552994 + }, + "shuf|OOD_CommonPile|s0|d1": { + "min": 0.15505741983652116, + "persist": 0.027253299206495284 + }, + "shuf|OOD_CommonPile|s0|d2": { + "min": 0.1550469197332859, + "persist": 0.021275742817670106 + }, + "shuf|OOD_CommonPile|s0|d3": { + "min": 0.1395270235836506, + "persist": 0.02162203611806035 + }, + "shuf|OOD_CommonPile|s0|d4": { + "min": 0.14095841497182846, + "persist": 0.026433797646313907 + }, + "J|OOD_CommonPile|s1": { + "min": 0.1391132090240717, + "persist": 0.03894782699644565 + }, + "shuf|OOD_CommonPile|s1|d0": { + "min": 0.13861030340194702, + "persist": 0.021899430267512798 + }, + "shuf|OOD_CommonPile|s1|d1": { + "min": 0.15715472623705865, + "persist": 0.027723732590675353 + }, + "shuf|OOD_CommonPile|s1|d2": { + "min": 0.15290846154093743, + "persist": 0.020811128057539464 + }, + "shuf|OOD_CommonPile|s1|d3": { + "min": 0.14020299538969994, + "persist": 0.020954283978790046 + }, + "shuf|OOD_CommonPile|s1|d4": { + "min": 0.14162667691707612, + "persist": 0.027029034588485955 + }, + "J|OOD_CommonPile|s2": { + "min": 0.13635604362934828, + "persist": 0.03779198825359344 + }, + "shuf|OOD_CommonPile|s2|d0": { + "min": 0.13895561844110488, + "persist": 0.021674325317144395 + }, + "shuf|OOD_CommonPile|s2|d1": { + "min": 0.15443600937724114, + "persist": 0.026810648664832116 + }, + "shuf|OOD_CommonPile|s2|d2": { + "min": 0.15316562503576278, + "persist": 0.020181251876056194 + }, + "shuf|OOD_CommonPile|s2|d3": { + "min": 0.13669089898467063, + "persist": 0.021304191276431083 + }, + "shuf|OOD_CommonPile|s2|d4": { + "min": 0.13880348578095436, + "persist": 0.026433794666081666 + }, + "rand|OOD_CommonPile": { + "min": 0.027811610139906406, + "persist": 0.0 + } + }, + "VERDICT": "NOT REACHED \u2014 (J^P - logit) includes or exceeds 0 on ALL three OOD rungs. Reported straight as a positive-framed negative: the bias term's practical magnitude is below the estimator worst case; the fitted operator tolerates shift the theory permits it to fail on. Publishable exactly as pre-registered and MUST NOT be re-described as a weaker crossover.", + "provenance": { + "schema_version": 1, + "experiment": "E48", + "utc": "2026-08-30T07:59:19+00:00", + "script": { + "path": "experiments/t48_crossover.py", + "sha256": "e1832c6c034dcb26f20309645c8465fed899ebff2ff69f621f7c6bcc7fbb8452", + "n_lines": 619 + }, + "git": { + "commit": "614809b36083400b4752d74e5be0e2670f9c6b26", + "commit_short": "614809b36083", + "git_repository_present": true, + "branch": "pythia", + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": "614809b" + }, + "argv": [ + "experiments/t48_crossover.py", + "--device", + "cpu", + "--n-derangements", + "5", + "--n-boot", + "10000" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_competence_gate_410m.json", + "sha256": "b705ef7305eea4a68a1cbbfa592b2dc88cf504e7bdb5cc524995e39aba481de8", + "bytes": 21580, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s0.pt", + "sha256": "efcf8b669825016233690a19162ff645dc58ed2e2628a2e113f26713e27cf998", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s1.pt", + "sha256": "bfa5aee55a338b05ccb9fc7d66a631da83b97c6e48cfa387105689bd30805033", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s2.pt", + "sha256": "33d97ade0445171e2329dc86da6ef93a218f178e3390f182e511c237ca2f1ed6", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "6f8f06ad3ce7ef6c3a0dc44afa63838df505ac8cd1617b25fda0aa9747a31524", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "2a2a3f4524ac4db804501e412d4ebc188935058687f1a468e2b5bfe0cdfbbfcd", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "a1e5dfd9f19145467500b542c74c1103bd90a44d430e4e3528aca22cdb589922", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s0.pt", + "sha256": "666e119f8cdcc6b218c020862e3de834c6920dcbed894121a108defa90877325", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s1.pt", + "sha256": "6c8e7e8900cdc01e54107cd6d4d786d12d199a7c50110f29606ea3ddbe64e0bc", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s2.pt", + "sha256": "fb4fc80ba2990f27e9343260491cfafbdb1d3bcaf9abc79cfc66b51a5fd38037", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s0.pt", + "sha256": "fba4f9a51cbd39b523043bea1286f8efc61ec72df542dc224a419c0e2e0a3a9f", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s1.pt", + "sha256": "7552cfc76417a4b50971284df5944f731e579c196137b8c32a3ae1c2d949d582", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s2.pt", + "sha256": "20e1867f9f0c7a5528f7b115ea67479c8875ff496ca35ee2323576e2aae81a63", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s0.pt", + "sha256": "3895668dce09dae91c6300f6adbba2de8359c63bf45fd5676d554f6ed90e9dc8", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s1.pt", + "sha256": "ca08e40ea0973004e8c21790e59334ea3fa6839be08e4753570c8ccfa3784283", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s2.pt", + "sha256": "fabb7cc8e515078c224375f23949ba336c33a4f86f35bf3e96df6cfbdba97bfc", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "e479cd0c039e83849725aa19cdb40072dff06c7f03e50a3e16cf455c2d77db75" + } +} \ No newline at end of file diff --git a/results/e48_crossover_410m_rstrip.json b/results/e48_crossover_410m_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..7017248ccd2c6251aca0cf67ea88a1fe1452d264 --- /dev/null +++ b/results/e48_crossover_410m_rstrip.json @@ -0,0 +1,3674 @@ +{ + "experiment": "E48 \u2014 the S3 crossover: does the unfitted logit lens overtake J^P under shift?", + "prereg": "specs/archive/prereg/PREREG_E48_CROSSOVER.md + Amendment 1, commits 408fa5c / 5869b63", + "status": "PRE-REGISTERED \u2014 rule fixed before any OOD lens existed", + "rule_verbatim": "CONFIRMED iff all three hold:\n (a) (J^P - logit) CI strictly > 0 for the in-stream English rungs\n (Wikipedia, USPTO, Pile-CC, StackExchange); AND\n (b) (J^P - logit) CI strictly < 0 for >=2 of 3 genuinely-OOD rungs\n (News_2024, arXiv_2023, CommonPile); AND\n (c) those OOD operators read strictly ABOVE the J_shuf floor\n (structured operator, wrong distribution - not a collapsed one).\nNOT REACHED iff (J^P - logit) CI includes or exceeds 0 for all OOD rungs.\n -> positive-framed negative; publishable as registered; NOT to be re-described as a\n weaker crossover, and no hunting for a rung that dips.\nDEGENERACY (excluded either way) iff OOD operators read BELOW J_shuf.\nAMENDMENT 1, clause (d), required for CONFIRMED:\n OOD_News_2024 must be among the rungs going strictly < 0, AND Github must sit AT the\n crossover (CI includes 0) or above - NOT strictly below. If Github goes strictly\n negative -> REGISTER-CONFOUNDED, not CONFIRMED.", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "n_items": 541, + "n_pairs": 893, + "smoke": false, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "excluded_sets": { + "association": "floored \u2014 196/205 ladder cells exactly zero (E28 crit 3)" + }, + "N_per_fit": 200, + "seeds": [ + 0, + 1, + 2 + ], + "n_derangements_per_cell": 5, + "adjudication_aggregation": "persist", + "ci_primary": "three-level: eval set > seed block > item pair (bootstrap.hierarchical_bootstrap_seeded). Chosen and written into this script BEFORE any OOD read was scored, because the pre-registration names both units and the repo's two-level resampler has no seed level.", + "ci_secondary": "two-level on the seed-averaged difference (bootstrap.hierarchical_bootstrap) \u2014 the repo's existing resampler, reported so the seed variance component is visible", + "ci_clause_disagreement": [], + "sensitivity_drop_contested_rung": { + "why": "OOD_CommonPile is 100% arXiv abstracts and arXiv is a Pile component; its OOD status is contested. Declared BEFORE any read was scored.", + "ood_set_used": [ + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "b_ok_needs_both": false, + "b_strictly_negative": [], + "verdict_would_change": false + }, + "contested_rungs": { + "OOD_CommonPile": "100% arXiv abstracts (2400/2400 start with two spaces, 79-col hard wrap, LaTeX escapes, 98.4% scientific vocabulary). arXiv is a Pile component at ~9% weight, so the manifest's 'domains not deliberately represented in the Pile' is false as sampled. k=32 containment 0.0002 IS below the 0.0014 single-line-reflow floor, so absence is not refuted \u2014 but the rung is not register-independent of OOD_arXiv_2023, which is what clause (b)'s '>=2 of 3' was meant to guarantee.", + "OOD_News_2024": "33.4% exact duplicates (2400 -> 1598 distinct), seed-asymmetric: the fitted 200 contain 199/166/161 distinct documents at seeds 0/1/2. This is the clause-(d) load-bearing rung. A deduplicated refit is a separate disclosed robustness arm." + }, + "rstrip_arm": true, + "scoring_path_forks_disclosed": { + "rstrip": "PRIMARY is UNSTRIPPED (matches trainval/t33/E28/E33, and is the only path that can reproduce C0). anchor_evals.py:32-34 calls .rstrip() load-bearing; it changes the readout token on 157/551 items. The stripped path is run separately via --rstrip.", + "passk_pooling": "both this scorer and t22 flat-pool over (item, intermediate) pairs; the anchor README defines a mean over items of the per-item fraction. Inherited unchanged so C0 can fire." + }, + "competence_gate": { + "file": "e48_competence_gate_410m.json", + "verdict": "ALL THREE OOD RUNGS PASS \u2014 a negative read on them cannot be attributed to model incompetence; E48 may be scored as registered", + "failed_rungs": [], + "ce_by_corpus": { + "Wikipedia_en": 2.417942479737886, + "USPTO_Backgrounds": 2.4130186644545546, + "Pile-CC": 2.9277699113536526, + "StackExchange": 2.412771591553101, + "Github": 1.0569410117018083, + "OOD_News_2024": 2.783762465468398, + "OOD_arXiv_2023": 2.826561468313406, + "OOD_CommonPile": 2.8191345634116782, + "OOD_Wikipedia_2023": 2.49233609070649, + "TRAP_FineWeb": 3.1196862617698873, + "CONTROL_PubMed_2023": 2.687820088383672 + } + }, + "controls": { + "C0_reproduces_e33_v2": { + "target": "e33_logit_baseline_410m_v2.json", + "max_abs_diff": 0.0, + "per_set": {}, + "skipped": true, + "why_skipped": "the rstrip sensitivity arm reads a DIFFERENT token on 157/551 items, so it cannot reproduce E33 by construction; C0 is a control on the PRIMARY (unstripped) path only" + }, + "C1_identity_equals_logit": { + "max_abs_per_pair_diff": 0.0, + "fires": true + }, + "C2_shuf_floor": { + "Wikipedia_en": { + "jp": 0.10422163642942905, + "shuf_mean_over_draws": 0.07876514978706836, + "ci_vs_shuf": { + "point": 0.025456484407186508, + "ci_lo": 0.00951878447085619, + "ci_hi": 0.04215948283672333, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "USPTO_Backgrounds": { + "jp": 0.1200522749684751, + "shuf_mean_over_draws": 0.08645985705661588, + "ci_vs_shuf": { + "point": 0.03359242528676987, + "ci_lo": 0.015270986594259739, + "ci_hi": 0.05336502194404602, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "Pile-CC": { + "jp": 0.11109574835281819, + "shuf_mean_over_draws": 0.08255375627195463, + "ci_vs_shuf": { + "point": 0.028541991487145424, + "ci_lo": 0.011218912899494171, + "ci_hi": 0.048610564321279526, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "StackExchange": { + "jp": 0.11614818507805466, + "shuf_mean_over_draws": 0.08374283523880877, + "ci_vs_shuf": { + "point": 0.032405368983745575, + "ci_lo": 0.014691319316625595, + "ci_hi": 0.05171893909573555, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "Github": { + "jp": 0.1007510763593018, + "shuf_mean_over_draws": 0.08166873238515109, + "ci_vs_shuf": { + "point": 0.01908234879374504, + "ci_lo": 0.009095155633985996, + "ci_hi": 0.030041687190532684, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "OOD_News_2024": { + "jp": 0.09791990842204541, + "shuf_mean_over_draws": 0.07102687416481786, + "ci_vs_shuf": { + "point": 0.02689303457736969, + "ci_lo": 0.0093351686373353, + "ci_hi": 0.045420676469802856, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "OOD_arXiv_2023": { + "jp": 0.10589559227228165, + "shuf_mean_over_draws": 0.0791645945981145, + "ci_vs_shuf": { + "point": 0.026730993762612343, + "ci_lo": 0.010988326743245125, + "ci_hi": 0.042101744562387466, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + }, + "OOD_CommonPile": { + "jp": 0.10663878545165062, + "shuf_mean_over_draws": 0.07937348280102015, + "ci_vs_shuf": { + "point": 0.027265304699540138, + "ci_lo": 0.010847318917512894, + "ci_hi": 0.042417705059051514, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "fires": true + } + }, + "C3_norm_matched_random": { + "Wikipedia_en": 0.0, + "USPTO_Backgrounds": 0.0, + "Pile-CC": 0.0, + "StackExchange": 0.0, + "Github": 0.0, + "OOD_News_2024": 0.0, + "OOD_arXiv_2023": 0.0, + "OOD_CommonPile": 0.0 + } + }, + "clauses": { + "a_instream_ci_positive": { + "Wikipedia_en": false, + "USPTO_Backgrounds": true, + "Pile-CC": false, + "StackExchange": true + }, + "a_ok": false, + "b_ood_ci_strictly_negative": [], + "b_ok": false, + "c_above_shuf_floor": { + "OOD_News_2024": true, + "OOD_arXiv_2023": true, + "OOD_CommonPile": true + }, + "c_ok": false, + "d_news_among_negatives": false, + "d_github_strictly_negative": false, + "d_ok": false + }, + "rungs": { + "Wikipedia_en": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.02104249782860279, + "ci_lo": -0.0029250604566186666, + "ci_hi": 0.05247287079691887, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.10422163642942905, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.07876514978706836, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.02104249969124794, + "ci_lo": -0.0029999352991580963, + "ci_hi": 0.052592530846595764, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.025456484407186508, + "ci_lo": 0.00951878447085619, + "ci_hi": 0.04215948283672333, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.10448124296963215, + "s1": 0.10463058389723301, + "s2": 0.1035530712455511 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.0813209842890501, + "s0|d1": 0.07815886037424206, + "s0|d2": 0.07532001119107008, + "s0|d3": 0.07884126789867878, + "s0|d4": 0.07891678363084793, + "s1|d0": 0.08280326202511787, + "s1|d1": 0.07869168482720852, + "s1|d2": 0.07610932774841786, + "s1|d3": 0.07892063967883586, + "s1|d4": 0.07768305167555808, + "s2|d0": 0.08325023017823696, + "s2|d1": 0.07850877046585084, + "s2|d2": 0.0764667907729745, + "s2|d3": 0.07845090553164483, + "s2|d4": 0.07803469803184271 + }, + "per_set": { + "multihop": { + "jp": 0.04484512656927109, + "logit": 0.0527045801281929, + "shuf": 0.013222375884652138, + "paired_item_ci": { + "point": -0.007859454490244389, + "ci_lo": -0.022191403433680534, + "ci_hi": 0.005547849927097559, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.23000000417232513, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.029248246923089027, + "logit": 0.023205220699310303, + "shuf": 0.01672709733247757, + "paired_item_ci": { + "point": 0.006043026223778725, + "ci_lo": -0.001208605826832354, + "ci_hi": 0.014019820839166641, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.11500000208616257, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.32251083850860596, + "logit": 0.290909081697464, + "shuf": 0.29056277871131897, + "paired_item_ci": { + "point": 0.03160172700881958, + "ci_lo": 0.00303029827773571, + "ci_hi": 0.05757575109601021, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.03200000151991844, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0, + "logit": 0.0014577260008081794, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.0014577260008081794, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7540000081062317, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1245039701461792, + "logit": 0.0476190485060215, + "shuf": 0.07331349700689316, + "paired_item_ci": { + "point": 0.0768849179148674, + "ci_lo": 0.0436507947742939, + "ci_hi": 0.11360368877649307, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0010351967066526413, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0010351967066526413, + "ci_lo": 0.0, + "ci_hi": 0.003105590119957924, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7039999961853027, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.04349992796778679, + "ci_lo": -9.144063369603828e-05, + "ci_hi": 0.09397823363542557, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.24160844944417476, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.25838803350925443, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.04349992796778679, + "ci_lo": -0.0010979481739923358, + "ci_hi": 0.09405913203954697, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.016779569908976555, + "ci_lo": -0.04266706481575966, + "ci_hi": 0.0034214784391224384, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.24112006239593028, + "s1": 0.24326577521860598, + "s2": 0.2404395245015621 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.24828002080321313, + "s0|d1": 0.25063798800110815, + "s0|d2": 0.27249130234122276, + "s0|d3": 0.2572912372648716, + "s0|d4": 0.27122137174010275, + "s1|d0": 0.24736156240105628, + "s1|d1": 0.2511155292391777, + "s1|d2": 0.2717252686619759, + "s1|d3": 0.2539973951876163, + "s1|d4": 0.26881202794611453, + "s2|d0": 0.24522592797875403, + "s2|d1": 0.24893060624599456, + "s2|d2": 0.2693440858274698, + "s2|d3": 0.2519726946949959, + "s2|d4": 0.26741342917084693 + }, + "per_set": { + "multihop": { + "jp": 0.24456772208213806, + "logit": 0.18446601927280426, + "shuf": 0.23800277709960938, + "paired_item_ci": { + "point": 0.0601017139852047, + "ci_lo": 0.025427646934986115, + "ci_hi": 0.09477577358484268, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.1429780125617981, + "logit": 0.10877447575330734, + "shuf": 0.1635001301765442, + "paired_item_ci": { + "point": 0.03420352563261986, + "ci_lo": 0.018129076808691025, + "ci_hi": 0.049431949853897095, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.48181816935539246, + "logit": 0.501298725605011, + "shuf": 0.5444155931472778, + "paired_item_ci": { + "point": -0.019480528309941292, + "ci_lo": -0.049783553928136826, + "ci_hi": 0.009101738221943378, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.1850000023841858, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.029154522344470024, + "logit": 0.02040816657245159, + "shuf": 0.03381924331188202, + "paired_item_ci": { + "point": 0.008746356703341007, + "ci_lo": -0.007774539291858673, + "ci_hi": 0.025279410183429718, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.33000001311302185, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3095238208770752, + "logit": 0.1755952388048172, + "shuf": 0.31220242381095886, + "paired_item_ci": { + "point": 0.1339285671710968, + "ci_lo": 0.0887896791100502, + "ci_hi": 0.1855158805847168, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.006211180239915848, + "logit": 0.006211180239915848, + "shuf": 0.017080746591091156, + "paired_item_ci": { + "point": -8.098457321414898e-11, + "ci_lo": -0.009834369644522667, + "ci_hi": 0.009834368713200092, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.9760000109672546, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "USPTO_Backgrounds": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.036873143166303635, + "ci_lo": 0.005331988446414471, + "ci_hi": 0.08567231148481369, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.1200522749684751, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.08645985705661588, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.03687314689159393, + "ci_lo": 0.005264620762318373, + "ci_hi": 0.08677493780851364, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.03359242528676987, + "ci_lo": 0.015270986594259739, + "ci_hi": 0.05336502194404602, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.12008049613796175, + "s1": 0.11979121798649431, + "s2": 0.12028511771932245 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.0909794308245182, + "s0|d1": 0.08494102142285556, + "s0|d2": 0.08603019118309022, + "s0|d3": 0.08238022662699222, + "s0|d4": 0.08888753782957792, + "s1|d0": 0.08966481909155846, + "s1|d1": 0.08335184752941131, + "s1|d2": 0.08593744933605194, + "s1|d3": 0.08251192029565572, + "s1|d4": 0.08784533105790615, + "s2|d0": 0.09051615260541439, + "s2|d1": 0.08496731789782644, + "s2|d2": 0.08608829639852048, + "s2|d3": 0.0844848508015275, + "s2|d4": 0.08831145130097866 + }, + "per_set": { + "multihop": { + "jp": 0.058252427726984024, + "logit": 0.0527045801281929, + "shuf": 0.01442441064864397, + "paired_item_ci": { + "point": 0.005547849927097559, + "ci_lo": -0.009708739817142487, + "ci_hi": 0.021729080006480217, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5210000276565552, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.03976311534643173, + "logit": 0.023205220699310303, + "shuf": 0.017597293481230736, + "paired_item_ci": { + "point": 0.01655789278447628, + "ci_lo": 0.007732051890343428, + "ci_hi": 0.026710176840424538, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.3207792043685913, + "logit": 0.290909081697464, + "shuf": 0.2878788113594055, + "paired_item_ci": { + "point": 0.02987012267112732, + "ci_lo": 0.0012986974325031042, + "ci_hi": 0.05670994892716408, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.03999999910593033, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0038872691802680492, + "logit": 0.0014577260008081794, + "shuf": 9.718173532746732e-05, + "paired_item_ci": { + "point": 0.0024295432958751917, + "ci_lo": -0.0029154520016163588, + "ci_hi": 0.009718173183500767, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5590000152587891, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1775793582201004, + "logit": 0.0476190485060215, + "shuf": 0.11230158805847168, + "paired_item_ci": { + "point": 0.1299603134393692, + "ci_lo": 0.0897817388176918, + "ci_hi": 0.1750991940498352, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.06301739066839218, + "ci_lo": 0.013715270906686783, + "ci_hi": 0.12858940660953522, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.26112590581178663, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.26514187678694723, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.06301739066839218, + "ci_lo": 0.014018489047884941, + "ci_hi": 0.12813681364059448, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.004015959799289703, + "ci_lo": -0.02807529829442501, + "ci_hi": 0.02401810511946678, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.259354492649436, + "s1": 0.26035019345581534, + "s2": 0.2636730432510376 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.25584742724895476, + "s0|d1": 0.26351498067379, + "s0|d2": 0.26983229219913485, + "s0|d3": 0.2610111027956009, + "s0|d4": 0.2718806996941566, + "s1|d0": 0.25631540194153785, + "s1|d1": 0.26299020983278754, + "s1|d2": 0.26882662773132326, + "s1|d3": 0.26178502663969994, + "s1|d4": 0.2712178058922291, + "s2|d0": 0.25852382332086565, + "s2|d1": 0.2640672240406275, + "s2|d2": 0.27396334931254385, + "s2|d3": 0.265429225564003, + "s2|d4": 0.27192303650081157 + }, + "per_set": { + "multihop": { + "jp": 0.2552011013031006, + "logit": 0.18446601927280426, + "shuf": 0.2064725011587143, + "paired_item_ci": { + "point": 0.07073508948087692, + "ci_lo": 0.03791030868887901, + "ci_hi": 0.10355987399816513, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.15155911445617676, + "logit": 0.10877447575330734, + "shuf": 0.15651439130306244, + "paired_item_ci": { + "point": 0.04278462007641792, + "ci_lo": 0.026951896026730537, + "ci_hi": 0.058375630527734756, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.5129870176315308, + "logit": 0.501298725605011, + "shuf": 0.5516883134841919, + "paired_item_ci": { + "point": 0.011688304133713245, + "ci_lo": -0.022088753059506416, + "ci_hi": 0.046320341527462006, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5210000276565552, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.02526725083589554, + "logit": 0.02040816657245159, + "shuf": 0.037026237696409225, + "paired_item_ci": { + "point": 0.004859085660427809, + "ci_lo": -0.009730319492518902, + "ci_hi": 0.019934413954615593, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5699999928474426, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3606150448322296, + "logit": 0.1755952388048172, + "shuf": 0.3740079402923584, + "paired_item_ci": { + "point": 0.185019850730896, + "ci_lo": 0.1289682537317276, + "ci_hi": 0.24505209922790527, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.020703934133052826, + "logit": 0.006211180239915848, + "shuf": 0.03322981670498848, + "paired_item_ci": { + "point": 0.014492753893136978, + "ci_lo": 0.0005175983533263206, + "ci_hi": 0.030538301914930344, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.039000000804662704, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "Pile-CC": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.027916613966226578, + "ci_lo": -0.002787945559248328, + "ci_hi": 0.07546353340148926, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.11109574835281819, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.08255375627195463, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.027916615828871727, + "ci_lo": -0.0028672225307673216, + "ci_hi": 0.07635460793972015, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.028541991487145424, + "ci_lo": 0.011218912899494171, + "ci_hi": 0.048610564321279526, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.11054239969234914, + "s1": 0.1118273551343009, + "s2": 0.11091751407366246 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.08584446720778942, + "s0|d1": 0.08377785235643387, + "s0|d2": 0.08186221458017826, + "s0|d3": 0.08023232966661453, + "s0|d4": 0.08126350324600935, + "s1|d0": 0.08509123313706368, + "s1|d1": 0.08387383986264467, + "s1|d2": 0.08191019371151924, + "s1|d3": 0.08150973357260227, + "s1|d4": 0.08268851265311242, + "s2|d0": 0.0852287447778508, + "s2|d1": 0.0828045567497611, + "s2|d2": 0.08076190613210202, + "s2|d3": 0.07991705778986216, + "s2|d4": 0.0815402265638113 + }, + "per_set": { + "multihop": { + "jp": 0.04715672880411148, + "logit": 0.0527045801281929, + "shuf": 0.015441518276929855, + "paired_item_ci": { + "point": -0.005547849927097559, + "ci_lo": -0.02450300194323063, + "ci_hi": 0.012944984249770641, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5389999747276306, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.037104181945323944, + "logit": 0.023205220699310303, + "shuf": 0.02100555971264839, + "paired_item_ci": { + "point": 0.013898961246013641, + "ci_lo": 0.005559583660215139, + "ci_hi": 0.023446939885616302, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0020000000949949026, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.30259740352630615, + "logit": 0.290909081697464, + "shuf": 0.2718614637851715, + "paired_item_ci": { + "point": 0.011688310652971268, + "ci_lo": -0.021212123334407806, + "ci_hi": 0.041136372834444046, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.46700000762939453, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0014577260008081794, + "logit": 0.0014577260008081794, + "shuf": 0.00019436347065493464, + "paired_item_ci": { + "point": 0.0, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.004373178351670504, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 1.281999945640564, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1671627014875412, + "logit": 0.0476190485060215, + "shuf": 0.10426587611436844, + "paired_item_ci": { + "point": 0.1195436492562294, + "ci_lo": 0.0798611119389534, + "ci_hi": 0.16520339250564575, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.06429049372673035, + "ci_lo": 0.009692131541669369, + "ci_hi": 0.12436782568693161, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.26239902824163436, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.27480331212282183, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.06429050117731094, + "ci_lo": 0.010017775930464268, + "ci_hi": 0.12449903041124344, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.012404273264110088, + "ci_lo": -0.03559108078479767, + "ci_hi": 0.006447996944189072, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.26216798909008504, + "s1": 0.2650879543274641, + "s2": 0.25994115322828293 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.2740033358335495, + "s0|d1": 0.2677078492939472, + "s0|d2": 0.2843741521239281, + "s0|d3": 0.26529838889837265, + "s0|d4": 0.27600362077355384, + "s1|d0": 0.2786212250590324, + "s1|d1": 0.2705193042755127, + "s1|d2": 0.2866306558251381, + "s1|d3": 0.2700627721846104, + "s1|d4": 0.27726985663175585, + "s2|d0": 0.2740327961742878, + "s2|d1": 0.2686062276363373, + "s2|d2": 0.2827077329158783, + "s2|d3": 0.2685593143105507, + "s2|d4": 0.27765234634280206 + }, + "per_set": { + "multihop": { + "jp": 0.2658345103263855, + "logit": 0.18446601927280426, + "shuf": 0.2555709779262543, + "paired_item_ci": { + "point": 0.08136846125125885, + "ci_lo": 0.047145165503025055, + "ci_hi": 0.11604253202676773, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.18491661548614502, + "logit": 0.10877447575330734, + "shuf": 0.17921200394630432, + "paired_item_ci": { + "point": 0.07614212483167648, + "ci_lo": 0.055837564170360565, + "ci_hi": 0.0968153327703476, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.4961039125919342, + "logit": 0.501298725605011, + "shuf": 0.5490043759346008, + "paired_item_ci": { + "point": -0.005194814410060644, + "ci_lo": -0.0350649431347847, + "ci_hi": 0.025541119277477264, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.671999990940094, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.020894072949886322, + "logit": 0.02040816657245159, + "shuf": 0.03566569834947586, + "paired_item_ci": { + "point": 0.0004859073960687965, + "ci_lo": -0.014577260240912437, + "ci_hi": 0.016034983098506927, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.9869999885559082, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3442460298538208, + "logit": 0.1755952388048172, + "shuf": 0.35456350445747375, + "paired_item_ci": { + "point": 0.1686507910490036, + "ci_lo": 0.12101934105157852, + "ci_hi": 0.2251983880996704, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.009834368713200092, + "logit": 0.006211180239915848, + "shuf": 0.02857142873108387, + "paired_item_ci": { + "point": 0.0036231884732842445, + "ci_lo": -0.00776397529989481, + "ci_hi": 0.01552795059978962, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5950000286102295, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "StackExchange": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.03296906128525734, + "ci_lo": 0.0009167171083390713, + "ci_hi": 0.079094797372818, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.11614818507805466, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.08374283523880877, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.03296906128525734, + "ci_lo": 0.0010298901470378041, + "ci_hi": 0.08060507476329803, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.032405368983745575, + "ci_lo": 0.014691319316625595, + "ci_hi": 0.05171893909573555, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.11692521655932069, + "s1": 0.11572976196184755, + "s2": 0.11578959980979561 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.08735615264158696, + "s0|d1": 0.08621515452396125, + "s0|d2": 0.08668109960854053, + "s0|d3": 0.07744382806122303, + "s0|d4": 0.08563328310847282, + "s1|d0": 0.08486844457220286, + "s1|d1": 0.08523496259003878, + "s1|d2": 0.08461906053125859, + "s1|d3": 0.0757306307554245, + "s1|d4": 0.08559647388756275, + "s2|d0": 0.08450331743806601, + "s2|d1": 0.08529306538403034, + "s2|d2": 0.0856543831527233, + "s2|d3": 0.07592109628021718, + "s2|d4": 0.08539159186184406 + }, + "per_set": { + "multihop": { + "jp": 0.04854368790984154, + "logit": 0.0527045801281929, + "shuf": 0.012852520681917667, + "paired_item_ci": { + "point": -0.004160887096077204, + "ci_lo": -0.022653721272945404, + "ci_hi": 0.013407304883003235, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.628000020980835, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.04012569785118103, + "logit": 0.023205220699310303, + "shuf": 0.020497946068644524, + "paired_item_ci": { + "point": 0.01692047342658043, + "ci_lo": 0.00773507310077548, + "ci_hi": 0.026951896026730537, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0010000000474974513, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.31904757022857666, + "logit": 0.290909081697464, + "shuf": 0.27991342544555664, + "paired_item_ci": { + "point": 0.028138522058725357, + "ci_lo": -0.009101739153265953, + "ci_hi": 0.06103895977139473, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.15700000524520874, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.004373178351670504, + "logit": 0.0014577260008081794, + "shuf": 0.00029154520598240197, + "paired_item_ci": { + "point": 0.0029154520016163588, + "ci_lo": -0.0029154520016163588, + "ci_hi": 0.011661808006465435, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5550000071525574, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1686507910490036, + "logit": 0.0476190485060215, + "shuf": 0.1051587387919426, + "paired_item_ci": { + "point": 0.1210317388176918, + "ci_lo": 0.07935267686843872, + "ci_hi": 0.1671627014875412, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.0661974549293518, + "ci_lo": 0.00616540526971221, + "ci_hi": 0.13472989201545715, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.26430597975850106, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.2649270880967379, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.0661974623799324, + "ci_lo": 0.006034563295543194, + "ci_hi": 0.1341061145067215, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.0006211016443558037, + "ci_lo": -0.018577732145786285, + "ci_hi": 0.01690845936536789, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.2665700364857912, + "s1": 0.2635776612907648, + "s2": 0.26277029849588873 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.26500423550605773, + "s0|d1": 0.27110425978899, + "s0|d2": 0.2754390150308609, + "s0|d3": 0.25646587610244753, + "s0|d4": 0.26446273662149905, + "s1|d0": 0.26482513546943665, + "s1|d1": 0.2690747335553169, + "s1|d2": 0.2727171786129475, + "s1|d3": 0.2517006419599056, + "s1|d4": 0.26243913993239404, + "s2|d0": 0.26502151116728784, + "s2|d1": 0.26919378712773323, + "s2|d2": 0.27218492031097413, + "s2|d3": 0.25335856787860395, + "s2|d4": 0.26091468334198 + }, + "per_set": { + "multihop": { + "jp": 0.2709200084209442, + "logit": 0.18446601927280426, + "shuf": 0.2463245391845703, + "paired_item_ci": { + "point": 0.08645399659872055, + "ci_lo": 0.051317617297172546, + "ci_hi": 0.12576283514499664, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.17959873378276825, + "logit": 0.10877447575330734, + "shuf": 0.17536862194538116, + "paired_item_ci": { + "point": 0.0708242729306221, + "ci_lo": 0.051365721970796585, + "ci_hi": 0.09161530435085297, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.4909090995788574, + "logit": 0.501298725605011, + "shuf": 0.5189610719680786, + "paired_item_ci": { + "point": -0.010389617644250393, + "ci_lo": -0.04199134558439255, + "ci_hi": 0.021645013242959976, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.48899999260902405, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.017006803303956985, + "logit": 0.02040816657245159, + "shuf": 0.030806610360741615, + "paired_item_ci": { + "point": -0.003401360707357526, + "ci_lo": -0.017492713406682014, + "ci_hi": 0.011661808006465435, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.625, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3630952537059784, + "logit": 0.1755952388048172, + "shuf": 0.3531745970249176, + "paired_item_ci": { + "point": 0.1875, + "ci_lo": 0.136408731341362, + "ci_hi": 0.2460317462682724, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.011904762126505375, + "logit": 0.006211180239915848, + "shuf": 0.013768116943538189, + "paired_item_ci": { + "point": 0.005693581886589527, + "ci_lo": -0.003623188938945532, + "ci_hi": 0.01552795059978962, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.2770000100135803, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "Github": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.01757194846868515, + "ci_lo": -0.010051947087049484, + "ci_hi": 0.04796682298183441, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.1007510763593018, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.08166873238515109, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.01757194846868515, + "ci_lo": -0.009690867736935616, + "ci_hi": 0.04795319586992264, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.01908234879374504, + "ci_lo": 0.009095155633985996, + "ci_hi": 0.030041687190532684, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.09901561820879579, + "s1": 0.10285943346098066, + "s2": 0.10037821242585779 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.08236480634659529, + "s0|d1": 0.07923254787456244, + "s0|d2": 0.08386189788579941, + "s0|d3": 0.0770264295861125, + "s0|d4": 0.08045144844800234, + "s1|d0": 0.08247843962162733, + "s1|d1": 0.08251860772725195, + "s1|d2": 0.08669814679306001, + "s1|d3": 0.07803292572498322, + "s1|d4": 0.08245038855820894, + "s2|d0": 0.08199466541409492, + "s2|d1": 0.0824049681192264, + "s2|d2": 0.08569017052650452, + "s2|d3": 0.07784030102193355, + "s2|d4": 0.0819853462278843 + }, + "per_set": { + "multihop": { + "jp": 0.02727692946791649, + "logit": 0.0527045801281929, + "shuf": 0.007951918989419937, + "paired_item_ci": { + "point": -0.025427646934986115, + "ci_lo": -0.045769765973091125, + "ci_hi": -0.008772526867687702, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0020000000949949026, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.02803964354097843, + "logit": 0.023205220699310303, + "shuf": 0.01619531214237213, + "paired_item_ci": { + "point": 0.004834420513361692, + "ci_lo": -0.0033871165942400694, + "ci_hi": 0.013536379672586918, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.2630000114440918, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.3597402274608612, + "logit": 0.290909081697464, + "shuf": 0.33212119340896606, + "paired_item_ci": { + "point": 0.06883116811513901, + "ci_lo": 0.04588743671774864, + "ci_hi": 0.09177488833665848, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.004373178351670504, + "logit": 0.0014577260008081794, + "shuf": 0.0003887269413098693, + "paired_item_ci": { + "point": 0.0029154520016163588, + "ci_lo": -0.0029154520016163588, + "ci_hi": 0.010204081423580647, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5929999947547913, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.0843254029750824, + "logit": 0.0476190485060215, + "shuf": 0.05168651044368744, + "paired_item_ci": { + "point": 0.0367063544690609, + "ci_lo": 0.0084325410425663, + "ci_hi": 0.06796876341104507, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.006000000052154064, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.04238881543278694, + "ci_lo": 0.0006703432300128043, + "ci_hi": 0.08835918456315994, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.24049734957516195, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.23240704722702504, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.04238881915807724, + "ci_lo": 0.0011467754375189543, + "ci_hi": 0.08760838955640793, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.008090298622846603, + "ci_lo": -0.009845133870840073, + "ci_hi": 0.0272099357098341, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.23628932386636733, + "s1": 0.2431370198726654, + "s2": 0.2420657031238079 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.22602029740810395, + "s0|d1": 0.23050989732146263, + "s0|d2": 0.24628960341215134, + "s0|d3": 0.20614932514727116, + "s0|d4": 0.23921805135905744, + "s1|d0": 0.23540843054652213, + "s1|d1": 0.2322789803147316, + "s1|d2": 0.25017106980085374, + "s1|d3": 0.21141563914716244, + "s1|d4": 0.2424768466502428, + "s2|d0": 0.22873966172337531, + "s2|d1": 0.23314449191093445, + "s2|d2": 0.250272324681282, + "s2|d3": 0.2102797579020262, + "s2|d4": 0.24373132549226284 + }, + "per_set": { + "multihop": { + "jp": 0.18908922374248505, + "logit": 0.18446601927280426, + "shuf": 0.1577438861131668, + "paired_item_ci": { + "point": 0.004623205866664648, + "ci_lo": -0.030513178557157516, + "ci_hi": 0.038372624665498734, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.800000011920929, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.1186850294470787, + "logit": 0.10877447575330734, + "shuf": 0.11438240855932236, + "paired_item_ci": { + "point": 0.00991056114435196, + "ci_lo": -0.006892072968184948, + "ci_hi": 0.026963993906974792, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.25099998712539673, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.6038961410522461, + "logit": 0.501298725605011, + "shuf": 0.6147186160087585, + "paired_item_ci": { + "point": 0.10259739309549332, + "ci_lo": 0.0627705529332161, + "ci_hi": 0.1441558301448822, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.017006801441311836, + "logit": 0.02040816657245159, + "shuf": 0.028960155323147774, + "paired_item_ci": { + "point": -0.0034013616386801004, + "ci_lo": -0.020408162847161293, + "ci_hi": 0.015549074858427048, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.6449999809265137, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.273809552192688, + "logit": 0.1755952388048172, + "shuf": 0.24623017013072968, + "paired_item_ci": { + "point": 0.0982142761349678, + "ci_lo": 0.0525793619453907, + "ci_hi": 0.14981399476528168, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.005175983998924494, + "logit": 0.006211180239915848, + "shuf": 0.01366459671407938, + "paired_item_ci": { + "point": -0.0010351968230679631, + "ci_lo": -0.010351967997848988, + "ci_hi": 0.00828157365322113, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.8450000286102295, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": true + } + }, + "OOD_News_2024": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.014740774407982826, + "ci_lo": -0.008469389751553535, + "ci_hi": 0.047540221363306046, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.09791990842204541, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.07102687416481786, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.014740774407982826, + "ci_lo": -0.008640674874186516, + "ci_hi": 0.048259831964969635, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.02689303457736969, + "ci_lo": 0.0093351686373353, + "ci_hi": 0.045420676469802856, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.0986472788034007, + "s1": 0.0974247383652255, + "s2": 0.0976877189008519 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.07370346884708852, + "s0|d1": 0.072022794932127, + "s0|d2": 0.07044114768505097, + "s0|d3": 0.07144842091947794, + "s0|d4": 0.06905782185494899, + "s1|d0": 0.07335275076329709, + "s1|d1": 0.07231609672307968, + "s1|d2": 0.07153821419924497, + "s1|d3": 0.070631312020123, + "s1|d4": 0.06975763961672783, + "s2|d0": 0.07303814720362425, + "s2|d1": 0.07085146754980087, + "s2|d2": 0.06865867599844933, + "s2|d3": 0.07009160798043013, + "s2|d4": 0.06849358156323433 + }, + "per_set": { + "multihop": { + "jp": 0.039759594947099686, + "logit": 0.0527045801281929, + "shuf": 0.013869625516235828, + "paired_item_ci": { + "point": -0.012944984249770641, + "ci_lo": -0.028663894161581993, + "ci_hi": 0.0009246416739188135, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.07000000029802322, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.02610587328672409, + "logit": 0.023205220699310303, + "shuf": 0.01653371937572956, + "paired_item_ci": { + "point": 0.002900651888921857, + "ci_lo": -0.004719604272395372, + "ci_hi": 0.011360889300704002, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.49000000953674316, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.2982684075832367, + "logit": 0.290909081697464, + "shuf": 0.2548917829990387, + "paired_item_ci": { + "point": 0.0073593007400631905, + "ci_lo": -0.03030303679406643, + "ci_hi": 0.040703464299440384, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.671999990940094, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0014577260008081794, + "logit": 0.0014577260008081794, + "shuf": 9.718173532746732e-05, + "paired_item_ci": { + "point": 0.0, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.004373178351670504, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 1.281999945640564, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1240079402923584, + "logit": 0.0476190485060215, + "shuf": 0.06974206119775772, + "paired_item_ci": { + "point": 0.0763888955116272, + "ci_lo": 0.0401785746216774, + "ci_hi": 0.11806797236204147, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0010351967066526413, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0010351967066526413, + "ci_lo": 0.0, + "ci_hi": 0.003105590119957924, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7039999961853027, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.05195476859807968, + "ci_lo": 0.010904176160693169, + "ci_hi": 0.10209808498620987, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.25006329715251924, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.25717504099011423, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.05195476859807968, + "ci_lo": 0.01106357667595148, + "ci_hi": 0.10248954594135284, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.00711175287142396, + "ci_lo": -0.023081177845597267, + "ci_hi": 0.00806505884975195, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.24892510399222373, + "s1": 0.2498748254030943, + "s2": 0.2513899695128202 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.2541537590324879, + "s0|d1": 0.24952763319015503, + "s0|d2": 0.27084239572286606, + "s0|d3": 0.2475816510617733, + "s0|d4": 0.26790230087935923, + "s1|d0": 0.2539294570684433, + "s1|d1": 0.2506876736879349, + "s1|d2": 0.26873930990695954, + "s1|d3": 0.24649405032396315, + "s1|d4": 0.2676916144788265, + "s2|d0": 0.25065327659249304, + "s2|d1": 0.24929407984018326, + "s2|d2": 0.2683473452925682, + "s2|d3": 0.2447837095707655, + "s2|d4": 0.2669975996017456 + }, + "per_set": { + "multihop": { + "jp": 0.23994453251361847, + "logit": 0.18446601927280426, + "shuf": 0.24308830499649048, + "paired_item_ci": { + "point": 0.055478498339653015, + "ci_lo": 0.024965323507785797, + "ci_hi": 0.08552934229373932, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.15977762639522552, + "logit": 0.10877447575330734, + "shuf": 0.16531305015087128, + "paired_item_ci": { + "point": 0.051003143191337585, + "ci_lo": 0.03250845894217491, + "ci_hi": 0.06852792203426361, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.5034632086753845, + "logit": 0.501298725605011, + "shuf": 0.5359306931495667, + "paired_item_ci": { + "point": 0.0021644949447363615, + "ci_lo": -0.030735939741134644, + "ci_hi": 0.03463202714920044, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.9350000023841858, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.028182700276374817, + "logit": 0.02040816657245159, + "shuf": 0.03449951484799385, + "paired_item_ci": { + "point": 0.007774537894874811, + "ci_lo": -0.006316813174635172, + "ci_hi": 0.02283770777285099, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.3269999921321869, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3189484179019928, + "logit": 0.1755952388048172, + "shuf": 0.3070436418056488, + "paired_item_ci": { + "point": 0.1433531790971756, + "ci_lo": 0.09621775150299072, + "ci_hi": 0.1969246119260788, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.009834368713200092, + "logit": 0.006211180239915848, + "shuf": 0.022877847775816917, + "paired_item_ci": { + "point": 0.0036231884732842445, + "ci_lo": -0.008281574584543705, + "ci_hi": 0.017080744728446007, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.5720000267028809, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "OOD_arXiv_2023": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.022716455161571503, + "ci_lo": -0.005232769064605236, + "ci_hi": 0.06359197944402695, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.10589559227228165, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.0791645945981145, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.022716455161571503, + "ci_lo": -0.005232771392911673, + "ci_hi": 0.06467151641845703, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.026730993762612343, + "ci_lo": 0.010988326743245125, + "ci_hi": 0.042101744562387466, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.10620390176773072, + "s1": 0.10661298111081123, + "s2": 0.10486989989876747 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.08164319396018982, + "s0|d1": 0.08086159788072109, + "s0|d2": 0.07931843660771846, + "s0|d3": 0.07451336067169904, + "s0|d4": 0.08041031993925571, + "s1|d0": 0.08155045211315155, + "s1|d1": 0.08128509055823088, + "s1|d2": 0.07937072664499283, + "s1|d3": 0.0759934613481164, + "s1|d4": 0.07981508262455464, + "s2|d0": 0.08120271135121584, + "s2|d1": 0.07898571649566292, + "s2|d2": 0.07789169233292341, + "s2|d3": 0.0748466819524765, + "s2|d4": 0.07978044673800469 + }, + "per_set": { + "multihop": { + "jp": 0.043458160012960434, + "logit": 0.0527045801281929, + "shuf": 0.01192787941545248, + "paired_item_ci": { + "point": -0.009246418252587318, + "ci_lo": -0.028201572597026825, + "ci_hi": 0.00924641452729702, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.31700000166893005, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.03251148387789726, + "logit": 0.023205220699310303, + "shuf": 0.015397632494568825, + "paired_item_ci": { + "point": 0.009306260384619236, + "ci_lo": 0.001329465419985354, + "ci_hi": 0.018491661176085472, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.02500000037252903, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.30519482493400574, + "logit": 0.290909081697464, + "shuf": 0.268398255109787, + "paired_item_ci": { + "point": 0.014285709708929062, + "ci_lo": -0.01991342380642891, + "ci_hi": 0.04588744044303894, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.4259999990463257, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0, + "logit": 0.0014577260008081794, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.0014577260008081794, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7540000081062317, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1483134925365448, + "logit": 0.0476190485060215, + "shuf": 0.10009920597076416, + "paired_item_ci": { + "point": 0.1006944477558136, + "ci_lo": 0.0625000074505806, + "ci_hi": 0.1428571492433548, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.07116450369358063, + "ci_lo": 0.021222492679953575, + "ci_hi": 0.12708593904972076, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.2692730328068137, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.27033296674489976, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.07116451114416122, + "ci_lo": 0.02090761810541153, + "ci_hi": 0.12630343437194824, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.0010599348461255431, + "ci_lo": -0.015796346589922905, + "ci_hi": 0.01297970674932003, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.2689389446750283, + "s1": 0.26879445035010574, + "s2": 0.27008571829646827 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.2580361794680357, + "s0|d1": 0.2720124736428261, + "s0|d2": 0.29188896268606185, + "s0|d3": 0.2627900928258896, + "s0|d4": 0.27306791618466375, + "s1|d0": 0.25658972561359406, + "s1|d1": 0.26868619695305823, + "s1|d2": 0.29177426397800443, + "s1|d3": 0.2612253025174141, + "s1|d4": 0.2692541994154453, + "s2|d0": 0.2571215949952602, + "s2|d1": 0.27173766642808916, + "s2|d2": 0.29140356555581093, + "s2|d3": 0.25893412753939626, + "s2|d4": 0.2704723007977009 + }, + "per_set": { + "multihop": { + "jp": 0.28155338764190674, + "logit": 0.18446601927280426, + "shuf": 0.27027276158332825, + "paired_item_ci": { + "point": 0.09708737581968307, + "ci_lo": 0.06380026787519455, + "ci_hi": 0.13177300989627838, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.15047135949134827, + "logit": 0.10877447575330734, + "shuf": 0.14500845968723297, + "paired_item_ci": { + "point": 0.041696883738040924, + "ci_lo": 0.026465430855751038, + "ci_hi": 0.058496490120887756, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.561904788017273, + "logit": 0.501298725605011, + "shuf": 0.5624242424964905, + "paired_item_ci": { + "point": 0.060606054961681366, + "ci_lo": 0.012987012043595314, + "ci_hi": 0.11428570002317429, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.014000000432133675, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.011661808006465435, + "logit": 0.02040816657245159, + "shuf": 0.03566569834947586, + "paired_item_ci": { + "point": -0.008746356703341007, + "ci_lo": -0.02235179767012596, + "ci_hi": 0.004373177420347929, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.17000000178813934, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3407738208770752, + "logit": 0.1755952388048172, + "shuf": 0.3382936716079712, + "paired_item_ci": { + "point": 0.165178582072258, + "ci_lo": 0.11060267686843872, + "ci_hi": 0.22671134769916534, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.011387163773179054, + "logit": 0.006211180239915848, + "shuf": 0.028778469190001488, + "paired_item_ci": { + "point": 0.005175983998924494, + "ci_lo": -0.0020703936461359262, + "ci_hi": 0.013975156471133232, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.22300000488758087, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + }, + "OOD_CommonPile": { + "persist": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.023459650576114655, + "ci_lo": -0.002050225157290697, + "ci_hi": 0.05795805901288986, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": false, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.10663878545165062, + "logit_admitted_mean": 0.08317913140635938, + "shuf_admitted_mean": 0.07937348280102015, + "rand_admitted_mean": 0.0, + "hierarchical_ci_vs_logit": { + "point": 0.023459652438759804, + "ci_lo": -0.0020511040929704905, + "ci_hi": 0.058900464326143265, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": 0.027265304699540138, + "ci_lo": 0.010847318917512894, + "ci_hi": 0.042417705059051514, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.10742316469550132, + "s1": 0.10653829500079155, + "s2": 0.1059548944234848 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.07998228296637536, + "s0|d1": 0.08328658873215318, + "s0|d2": 0.07980556171387435, + "s0|d3": 0.07883550319820642, + "s0|d4": 0.08045977549627423, + "s1|d0": 0.07729471735656261, + "s1|d1": 0.07961883107200265, + "s1|d2": 0.0780245989561081, + "s1|d3": 0.07607217859476804, + "s1|d4": 0.07973866332322359, + "s2|d0": 0.07946023121476173, + "s2|d1": 0.08081897310912609, + "s2|d2": 0.07980298958718776, + "s2|d3": 0.07772078756242991, + "s2|d4": 0.07968054311349988 + }, + "per_set": { + "multihop": { + "jp": 0.04669440910220146, + "logit": 0.0527045801281929, + "shuf": 0.011280630715191364, + "paired_item_ci": { + "point": -0.0060101719573140144, + "ci_lo": -0.022202961146831512, + "ci_hi": 0.010633381083607674, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.45899999141693115, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.031665459275245667, + "logit": 0.023205220699310303, + "shuf": 0.01677544414997101, + "paired_item_ci": { + "point": 0.00846023578196764, + "ci_lo": 0.00012086005881428719, + "ci_hi": 0.017769522964954376, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.04600000008940697, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.31991341710090637, + "logit": 0.290909081697464, + "shuf": 0.2764502167701721, + "paired_item_ci": { + "point": 0.029004326090216637, + "ci_lo": -0.002175329253077507, + "ci_hi": 0.05930735915899277, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.07900000363588333, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.0, + "logit": 0.0014577260008081794, + "shuf": 0.0, + "paired_item_ci": { + "point": -0.0014577260008081794, + "ci_lo": -0.004373178351670504, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.7540000081062317, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.1349206417798996, + "logit": 0.0476190485060215, + "shuf": 0.09236112236976624, + "paired_item_ci": { + "point": 0.0873015895485878, + "ci_lo": 0.0525793693959713, + "ci_hi": 0.1264880895614624, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.0, + "logit": 0.0, + "shuf": 0.0, + "paired_item_ci": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": false, + "ci_strictly_negative": false, + "above_shuf_floor": true, + "above_shuf_point": true + }, + "min": { + "hierarchical_ci_vs_logit_2level_seedavg": { + "point": 0.05477261543273926, + "ci_lo": 0.009714179672300816, + "ci_hi": 0.10627473145723343, + "n_sets": 5, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "ci_2level_strictly_positive": true, + "ci_2level_strictly_negative": false, + "jp_admitted_mean": 0.2528811445459723, + "logit_admitted_mean": 0.19810852520167826, + "shuf_admitted_mean": 0.26076692417263986, + "rand_admitted_mean": 0.04613274168223143, + "hierarchical_ci_vs_logit": { + "point": 0.054772622883319855, + "ci_lo": 0.009154736064374447, + "ci_hi": 0.10660968720912933, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "hierarchical_ci_vs_shuf": { + "point": -0.007885778322815895, + "ci_lo": -0.03249100223183632, + "ci_hi": 0.01627395488321781, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "per_seed_admitted_mean": { + "s0": 0.2531457645818591, + "s1": 0.25511594451963904, + "s2": 0.2503817303106189 + }, + "per_derangement_admitted_mean": { + "s0|d0": 0.2543313570320606, + "s0|d1": 0.26050583124160764, + "s0|d2": 0.2809336222708225, + "s0|d3": 0.24981799349188805, + "s0|d4": 0.2634584128856659, + "s1|d0": 0.2539062112569809, + "s1|d1": 0.26217192634940145, + "s1|d2": 0.2828520558774471, + "s1|d3": 0.24786126241087914, + "s1|d4": 0.26146751791238787, + "s2|d0": 0.2520745679736137, + "s2|d1": 0.2588454566895962, + "s2|d2": 0.277788944542408, + "s2|d3": 0.24596437588334083, + "s2|d4": 0.2595244117081165 + }, + "per_set": { + "multihop": { + "jp": 0.2662968039512634, + "logit": 0.18446601927280426, + "shuf": 0.23513638973236084, + "paired_item_ci": { + "point": 0.08183079212903976, + "ci_lo": 0.05039297044277191, + "ci_hi": 0.11512947082519531, + "n_items": 103, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "multilingual": { + "jp": 0.15035049617290497, + "logit": 0.10877447575330734, + "shuf": 0.14375151693820953, + "paired_item_ci": { + "point": 0.041576024144887924, + "ci_lo": 0.02549552544951439, + "ci_hi": 0.058013055473566055, + "n_items": 394, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "order-ops": { + "jp": 0.5147186517715454, + "logit": 0.501298725605011, + "shuf": 0.5598268508911133, + "paired_item_ci": { + "point": 0.013419907540082932, + "ci_lo": -0.02554113231599331, + "ci_hi": 0.052824683487415314, + "n_items": 110, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.550000011920929, + "unit": "item (paired)" + } + }, + "poetry": { + "jp": 0.014091352932155132, + "logit": 0.02040816657245159, + "shuf": 0.03615160658955574, + "paired_item_ci": { + "point": -0.0063168141059577465, + "ci_lo": -0.020420312881469727, + "ci_hi": 0.007774537429213524, + "n_items": 98, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.3700000047683716, + "unit": "item (paired)" + } + }, + "typo": { + "jp": 0.3189484179019928, + "logit": 0.1755952388048172, + "shuf": 0.32896825671195984, + "paired_item_ci": { + "point": 0.1433531790971756, + "ci_lo": 0.091269850730896, + "ci_hi": 0.1994047611951828, + "n_items": 96, + "n_boot": 2000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + } + }, + "association": { + "jp": 0.013975156471133232, + "logit": 0.006211180239915848, + "shuf": 0.030745340511202812, + "paired_item_ci": { + "point": 0.00776397529989481, + "ci_lo": -0.0015527952928096056, + "ci_hi": 0.018633542582392693, + "n_items": 92, + "n_boot": 2000, + "excludes_zero": false, + "p_two_sided": 0.13300000131130219, + "unit": "item (paired)" + } + } + }, + "ci_strictly_positive": true, + "ci_strictly_negative": false, + "above_shuf_floor": false, + "above_shuf_point": false + } + } + }, + "arms_admitted_mean": { + "logit_I": { + "min": 0.19810852520167826, + "persist": 0.08317913140635938 + }, + "C0_ref_Github_n400_s0": { + "min": 0.24063372761011123, + "persist": 0.10069348337128758 + }, + "C1_identity": { + "min": 0.19810852520167826, + "persist": 0.08317913140635938 + }, + "J|Wikipedia_en|s0": { + "min": 0.24112006239593028, + "persist": 0.10448124296963215 + }, + "shuf|Wikipedia_en|s0|d0": { + "min": 0.24828002080321313, + "persist": 0.0813209842890501 + }, + "shuf|Wikipedia_en|s0|d1": { + "min": 0.25063798800110815, + "persist": 0.07815886037424206 + }, + "shuf|Wikipedia_en|s0|d2": { + "min": 0.27249130234122276, + "persist": 0.07532001119107008 + }, + "shuf|Wikipedia_en|s0|d3": { + "min": 0.2572912372648716, + "persist": 0.07884126789867878 + }, + "shuf|Wikipedia_en|s0|d4": { + "min": 0.27122137174010275, + "persist": 0.07891678363084793 + }, + "J|Wikipedia_en|s1": { + "min": 0.24326577521860598, + "persist": 0.10463058389723301 + }, + "shuf|Wikipedia_en|s1|d0": { + "min": 0.24736156240105628, + "persist": 0.08280326202511787 + }, + "shuf|Wikipedia_en|s1|d1": { + "min": 0.2511155292391777, + "persist": 0.07869168482720852 + }, + "shuf|Wikipedia_en|s1|d2": { + "min": 0.2717252686619759, + "persist": 0.07610932774841786 + }, + "shuf|Wikipedia_en|s1|d3": { + "min": 0.2539973951876163, + "persist": 0.07892063967883586 + }, + "shuf|Wikipedia_en|s1|d4": { + "min": 0.26881202794611453, + "persist": 0.07768305167555808 + }, + "J|Wikipedia_en|s2": { + "min": 0.2404395245015621, + "persist": 0.1035530712455511 + }, + "shuf|Wikipedia_en|s2|d0": { + "min": 0.24522592797875403, + "persist": 0.08325023017823696 + }, + "shuf|Wikipedia_en|s2|d1": { + "min": 0.24893060624599456, + "persist": 0.07850877046585084 + }, + "shuf|Wikipedia_en|s2|d2": { + "min": 0.2693440858274698, + "persist": 0.0764667907729745 + }, + "shuf|Wikipedia_en|s2|d3": { + "min": 0.2519726946949959, + "persist": 0.07845090553164483 + }, + "shuf|Wikipedia_en|s2|d4": { + "min": 0.26741342917084693, + "persist": 0.07803469803184271 + }, + "rand|Wikipedia_en": { + "min": 0.04613274168223143, + "persist": 0.0 + }, + "J|USPTO_Backgrounds|s0": { + "min": 0.259354492649436, + "persist": 0.12008049613796175 + }, + "shuf|USPTO_Backgrounds|s0|d0": { + "min": 0.25584742724895476, + "persist": 0.0909794308245182 + }, + "shuf|USPTO_Backgrounds|s0|d1": { + "min": 0.26351498067379, + "persist": 0.08494102142285556 + }, + "shuf|USPTO_Backgrounds|s0|d2": { + "min": 0.26983229219913485, + "persist": 0.08603019118309022 + }, + "shuf|USPTO_Backgrounds|s0|d3": { + "min": 0.2610111027956009, + "persist": 0.08238022662699222 + }, + "shuf|USPTO_Backgrounds|s0|d4": { + "min": 0.2718806996941566, + "persist": 0.08888753782957792 + }, + "J|USPTO_Backgrounds|s1": { + "min": 0.26035019345581534, + "persist": 0.11979121798649431 + }, + "shuf|USPTO_Backgrounds|s1|d0": { + "min": 0.25631540194153785, + "persist": 0.08966481909155846 + }, + "shuf|USPTO_Backgrounds|s1|d1": { + "min": 0.26299020983278754, + "persist": 0.08335184752941131 + }, + "shuf|USPTO_Backgrounds|s1|d2": { + "min": 0.26882662773132326, + "persist": 0.08593744933605194 + }, + "shuf|USPTO_Backgrounds|s1|d3": { + "min": 0.26178502663969994, + "persist": 0.08251192029565572 + }, + "shuf|USPTO_Backgrounds|s1|d4": { + "min": 0.2712178058922291, + "persist": 0.08784533105790615 + }, + "J|USPTO_Backgrounds|s2": { + "min": 0.2636730432510376, + "persist": 0.12028511771932245 + }, + "shuf|USPTO_Backgrounds|s2|d0": { + "min": 0.25852382332086565, + "persist": 0.09051615260541439 + }, + "shuf|USPTO_Backgrounds|s2|d1": { + "min": 0.2640672240406275, + "persist": 0.08496731789782644 + }, + "shuf|USPTO_Backgrounds|s2|d2": { + "min": 0.27396334931254385, + "persist": 0.08608829639852048 + }, + "shuf|USPTO_Backgrounds|s2|d3": { + "min": 0.265429225564003, + "persist": 0.0844848508015275 + }, + "shuf|USPTO_Backgrounds|s2|d4": { + "min": 0.27192303650081157, + "persist": 0.08831145130097866 + }, + "rand|USPTO_Backgrounds": { + "min": 0.04613274168223143, + "persist": 0.0 + }, + "J|Pile-CC|s0": { + "min": 0.26216798909008504, + "persist": 0.11054239969234914 + }, + "shuf|Pile-CC|s0|d0": { + "min": 0.2740033358335495, + "persist": 0.08584446720778942 + }, + "shuf|Pile-CC|s0|d1": { + "min": 0.2677078492939472, + "persist": 0.08377785235643387 + }, + "shuf|Pile-CC|s0|d2": { + "min": 0.2843741521239281, + "persist": 0.08186221458017826 + }, + "shuf|Pile-CC|s0|d3": { + "min": 0.26529838889837265, + "persist": 0.08023232966661453 + }, + "shuf|Pile-CC|s0|d4": { + "min": 0.27600362077355384, + "persist": 0.08126350324600935 + }, + "J|Pile-CC|s1": { + "min": 0.2650879543274641, + "persist": 0.1118273551343009 + }, + "shuf|Pile-CC|s1|d0": { + "min": 0.2786212250590324, + "persist": 0.08509123313706368 + }, + "shuf|Pile-CC|s1|d1": { + "min": 0.2705193042755127, + "persist": 0.08387383986264467 + }, + "shuf|Pile-CC|s1|d2": { + "min": 0.2866306558251381, + "persist": 0.08191019371151924 + }, + "shuf|Pile-CC|s1|d3": { + "min": 0.2700627721846104, + "persist": 0.08150973357260227 + }, + "shuf|Pile-CC|s1|d4": { + "min": 0.27726985663175585, + "persist": 0.08268851265311242 + }, + "J|Pile-CC|s2": { + "min": 0.25994115322828293, + "persist": 0.11091751407366246 + }, + "shuf|Pile-CC|s2|d0": { + "min": 0.2740327961742878, + "persist": 0.0852287447778508 + }, + "shuf|Pile-CC|s2|d1": { + "min": 0.2686062276363373, + "persist": 0.0828045567497611 + }, + "shuf|Pile-CC|s2|d2": { + "min": 0.2827077329158783, + "persist": 0.08076190613210202 + }, + "shuf|Pile-CC|s2|d3": { + "min": 0.2685593143105507, + "persist": 0.07991705778986216 + }, + "shuf|Pile-CC|s2|d4": { + "min": 0.27765234634280206, + "persist": 0.0815402265638113 + }, + "rand|Pile-CC": { + "min": 0.04613274168223143, + "persist": 0.0 + }, + "J|StackExchange|s0": { + "min": 0.2665700364857912, + "persist": 0.11692521655932069 + }, + "shuf|StackExchange|s0|d0": { + "min": 0.26500423550605773, + "persist": 0.08735615264158696 + }, + "shuf|StackExchange|s0|d1": { + "min": 0.27110425978899, + "persist": 0.08621515452396125 + }, + "shuf|StackExchange|s0|d2": { + "min": 0.2754390150308609, + "persist": 0.08668109960854053 + }, + "shuf|StackExchange|s0|d3": { + "min": 0.25646587610244753, + "persist": 0.07744382806122303 + }, + "shuf|StackExchange|s0|d4": { + "min": 0.26446273662149905, + "persist": 0.08563328310847282 + }, + "J|StackExchange|s1": { + "min": 0.2635776612907648, + "persist": 0.11572976196184755 + }, + "shuf|StackExchange|s1|d0": { + "min": 0.26482513546943665, + "persist": 0.08486844457220286 + }, + "shuf|StackExchange|s1|d1": { + "min": 0.2690747335553169, + "persist": 0.08523496259003878 + }, + "shuf|StackExchange|s1|d2": { + "min": 0.2727171786129475, + "persist": 0.08461906053125859 + }, + "shuf|StackExchange|s1|d3": { + "min": 0.2517006419599056, + "persist": 0.0757306307554245 + }, + "shuf|StackExchange|s1|d4": { + "min": 0.26243913993239404, + "persist": 0.08559647388756275 + }, + "J|StackExchange|s2": { + "min": 0.26277029849588873, + "persist": 0.11578959980979561 + }, + "shuf|StackExchange|s2|d0": { + "min": 0.26502151116728784, + "persist": 0.08450331743806601 + }, + "shuf|StackExchange|s2|d1": { + "min": 0.26919378712773323, + "persist": 0.08529306538403034 + }, + "shuf|StackExchange|s2|d2": { + "min": 0.27218492031097413, + "persist": 0.0856543831527233 + }, + "shuf|StackExchange|s2|d3": { + "min": 0.25335856787860395, + "persist": 0.07592109628021718 + }, + "shuf|StackExchange|s2|d4": { + "min": 0.26091468334198, + "persist": 0.08539159186184406 + }, + "rand|StackExchange": { + "min": 0.04613274168223143, + "persist": 0.0 + }, + "J|Github|s0": { + "min": 0.23628932386636733, + "persist": 0.09901561820879579 + }, + "shuf|Github|s0|d0": { + "min": 0.22602029740810395, + "persist": 0.08236480634659529 + }, + "shuf|Github|s0|d1": { + "min": 0.23050989732146263, + "persist": 0.07923254787456244 + }, + "shuf|Github|s0|d2": { + "min": 0.24628960341215134, + "persist": 0.08386189788579941 + }, + "shuf|Github|s0|d3": { + "min": 0.20614932514727116, + "persist": 0.0770264295861125 + }, + "shuf|Github|s0|d4": { + "min": 0.23921805135905744, + "persist": 0.08045144844800234 + }, + "J|Github|s1": { + "min": 0.2431370198726654, + "persist": 0.10285943346098066 + }, + "shuf|Github|s1|d0": { + "min": 0.23540843054652213, + "persist": 0.08247843962162733 + }, + "shuf|Github|s1|d1": { + "min": 0.2322789803147316, + "persist": 0.08251860772725195 + }, + "shuf|Github|s1|d2": { + "min": 0.25017106980085374, + "persist": 0.08669814679306001 + }, + "shuf|Github|s1|d3": { + "min": 0.21141563914716244, + "persist": 0.07803292572498322 + }, + "shuf|Github|s1|d4": { + "min": 0.2424768466502428, + "persist": 0.08245038855820894 + }, + "J|Github|s2": { + "min": 0.2420657031238079, + "persist": 0.10037821242585779 + }, + "shuf|Github|s2|d0": { + "min": 0.22873966172337531, + "persist": 0.08199466541409492 + }, + "shuf|Github|s2|d1": { + "min": 0.23314449191093445, + "persist": 0.0824049681192264 + }, + "shuf|Github|s2|d2": { + "min": 0.250272324681282, + "persist": 0.08569017052650452 + }, + "shuf|Github|s2|d3": { + "min": 0.2102797579020262, + "persist": 0.07784030102193355 + }, + "shuf|Github|s2|d4": { + "min": 0.24373132549226284, + "persist": 0.0819853462278843 + }, + "rand|Github": { + "min": 0.04613274168223143, + "persist": 0.0 + }, + "J|OOD_News_2024|s0": { + "min": 0.24892510399222373, + "persist": 0.0986472788034007 + }, + "shuf|OOD_News_2024|s0|d0": { + "min": 0.2541537590324879, + "persist": 0.07370346884708852 + }, + "shuf|OOD_News_2024|s0|d1": { + "min": 0.24952763319015503, + "persist": 0.072022794932127 + }, + "shuf|OOD_News_2024|s0|d2": { + "min": 0.27084239572286606, + "persist": 0.07044114768505097 + }, + "shuf|OOD_News_2024|s0|d3": { + "min": 0.2475816510617733, + "persist": 0.07144842091947794 + }, + "shuf|OOD_News_2024|s0|d4": { + "min": 0.26790230087935923, + "persist": 0.06905782185494899 + }, + "J|OOD_News_2024|s1": { + "min": 0.2498748254030943, + "persist": 0.0974247383652255 + }, + "shuf|OOD_News_2024|s1|d0": { + "min": 0.2539294570684433, + "persist": 0.07335275076329709 + }, + "shuf|OOD_News_2024|s1|d1": { + "min": 0.2506876736879349, + "persist": 0.07231609672307968 + }, + "shuf|OOD_News_2024|s1|d2": { + "min": 0.26873930990695954, + "persist": 0.07153821419924497 + }, + "shuf|OOD_News_2024|s1|d3": { + "min": 0.24649405032396315, + "persist": 0.070631312020123 + }, + "shuf|OOD_News_2024|s1|d4": { + "min": 0.2676916144788265, + "persist": 0.06975763961672783 + }, + "J|OOD_News_2024|s2": { + "min": 0.2513899695128202, + "persist": 0.0976877189008519 + }, + "shuf|OOD_News_2024|s2|d0": { + "min": 0.25065327659249304, + "persist": 0.07303814720362425 + }, + "shuf|OOD_News_2024|s2|d1": { + "min": 0.24929407984018326, + "persist": 0.07085146754980087 + }, + "shuf|OOD_News_2024|s2|d2": { + "min": 0.2683473452925682, + "persist": 0.06865867599844933 + }, + "shuf|OOD_News_2024|s2|d3": { + "min": 0.2447837095707655, + "persist": 0.07009160798043013 + }, + "shuf|OOD_News_2024|s2|d4": { + "min": 0.2669975996017456, + "persist": 0.06849358156323433 + }, + "rand|OOD_News_2024": { + "min": 0.04613274168223143, + "persist": 0.0 + }, + "J|OOD_arXiv_2023|s0": { + "min": 0.2689389446750283, + "persist": 0.10620390176773072 + }, + "shuf|OOD_arXiv_2023|s0|d0": { + "min": 0.2580361794680357, + "persist": 0.08164319396018982 + }, + "shuf|OOD_arXiv_2023|s0|d1": { + "min": 0.2720124736428261, + "persist": 0.08086159788072109 + }, + "shuf|OOD_arXiv_2023|s0|d2": { + "min": 0.29188896268606185, + "persist": 0.07931843660771846 + }, + "shuf|OOD_arXiv_2023|s0|d3": { + "min": 0.2627900928258896, + "persist": 0.07451336067169904 + }, + "shuf|OOD_arXiv_2023|s0|d4": { + "min": 0.27306791618466375, + "persist": 0.08041031993925571 + }, + "J|OOD_arXiv_2023|s1": { + "min": 0.26879445035010574, + "persist": 0.10661298111081123 + }, + "shuf|OOD_arXiv_2023|s1|d0": { + "min": 0.25658972561359406, + "persist": 0.08155045211315155 + }, + "shuf|OOD_arXiv_2023|s1|d1": { + "min": 0.26868619695305823, + "persist": 0.08128509055823088 + }, + "shuf|OOD_arXiv_2023|s1|d2": { + "min": 0.29177426397800443, + "persist": 0.07937072664499283 + }, + "shuf|OOD_arXiv_2023|s1|d3": { + "min": 0.2612253025174141, + "persist": 0.0759934613481164 + }, + "shuf|OOD_arXiv_2023|s1|d4": { + "min": 0.2692541994154453, + "persist": 0.07981508262455464 + }, + "J|OOD_arXiv_2023|s2": { + "min": 0.27008571829646827, + "persist": 0.10486989989876747 + }, + "shuf|OOD_arXiv_2023|s2|d0": { + "min": 0.2571215949952602, + "persist": 0.08120271135121584 + }, + "shuf|OOD_arXiv_2023|s2|d1": { + "min": 0.27173766642808916, + "persist": 0.07898571649566292 + }, + "shuf|OOD_arXiv_2023|s2|d2": { + "min": 0.29140356555581093, + "persist": 0.07789169233292341 + }, + "shuf|OOD_arXiv_2023|s2|d3": { + "min": 0.25893412753939626, + "persist": 0.0748466819524765 + }, + "shuf|OOD_arXiv_2023|s2|d4": { + "min": 0.2704723007977009, + "persist": 0.07978044673800469 + }, + "rand|OOD_arXiv_2023": { + "min": 0.04613274168223143, + "persist": 0.0 + }, + "J|OOD_CommonPile|s0": { + "min": 0.2531457645818591, + "persist": 0.10742316469550132 + }, + "shuf|OOD_CommonPile|s0|d0": { + "min": 0.2543313570320606, + "persist": 0.07998228296637536 + }, + "shuf|OOD_CommonPile|s0|d1": { + "min": 0.26050583124160764, + "persist": 0.08328658873215318 + }, + "shuf|OOD_CommonPile|s0|d2": { + "min": 0.2809336222708225, + "persist": 0.07980556171387435 + }, + "shuf|OOD_CommonPile|s0|d3": { + "min": 0.24981799349188805, + "persist": 0.07883550319820642 + }, + "shuf|OOD_CommonPile|s0|d4": { + "min": 0.2634584128856659, + "persist": 0.08045977549627423 + }, + "J|OOD_CommonPile|s1": { + "min": 0.25511594451963904, + "persist": 0.10653829500079155 + }, + "shuf|OOD_CommonPile|s1|d0": { + "min": 0.2539062112569809, + "persist": 0.07729471735656261 + }, + "shuf|OOD_CommonPile|s1|d1": { + "min": 0.26217192634940145, + "persist": 0.07961883107200265 + }, + "shuf|OOD_CommonPile|s1|d2": { + "min": 0.2828520558774471, + "persist": 0.0780245989561081 + }, + "shuf|OOD_CommonPile|s1|d3": { + "min": 0.24786126241087914, + "persist": 0.07607217859476804 + }, + "shuf|OOD_CommonPile|s1|d4": { + "min": 0.26146751791238787, + "persist": 0.07973866332322359 + }, + "J|OOD_CommonPile|s2": { + "min": 0.2503817303106189, + "persist": 0.1059548944234848 + }, + "shuf|OOD_CommonPile|s2|d0": { + "min": 0.2520745679736137, + "persist": 0.07946023121476173 + }, + "shuf|OOD_CommonPile|s2|d1": { + "min": 0.2588454566895962, + "persist": 0.08081897310912609 + }, + "shuf|OOD_CommonPile|s2|d2": { + "min": 0.277788944542408, + "persist": 0.07980298958718776 + }, + "shuf|OOD_CommonPile|s2|d3": { + "min": 0.24596437588334083, + "persist": 0.07772078756242991 + }, + "shuf|OOD_CommonPile|s2|d4": { + "min": 0.2595244117081165, + "persist": 0.07968054311349988 + }, + "rand|OOD_CommonPile": { + "min": 0.04613274168223143, + "persist": 0.0 + } + }, + "VERDICT": "NOT REACHED \u2014 (J^P - logit) includes or exceeds 0 on ALL three OOD rungs. Reported straight as a positive-framed negative: the bias term's practical magnitude is below the estimator worst case; the fitted operator tolerates shift the theory permits it to fail on. Publishable exactly as pre-registered and MUST NOT be re-described as a weaker crossover.", + "provenance": { + "schema_version": 1, + "experiment": "E48", + "utc": "2026-08-20T07:52:49+00:00", + "script": { + "path": "experiments/t48_crossover.py", + "sha256": "a76a103f93a2ba7867f349fba0f937863c9c55aba7474e8a60109f49e00a69db", + "n_lines": 619 + }, + "git": { + "commit": "3457e0c5f9f03539b32d8b757eed99708702e1c1", + "commit_short": "3457e0c5f9f0", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/e48_crossover_410m.json", + "results/ladder410_cpu/", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "dafacfd609a5e3e070141cf92b8b10a29b7ae3b348527e03ac70f5bbe3d10d00", + "describe": "3457e0c-dirty" + }, + "argv": [ + "experiments/t48_crossover.py", + "--device", + "cpu", + "--rstrip", + "--out", + "results/e48_crossover_410m_rstrip.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_competence_gate_410m.json", + "sha256": "37b9f18a33810e338c921e998af5eaab98164cc9b0a9848fc549a1d0b0950f8a", + "bytes": 19534, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s0.pt", + "sha256": "efcf8b669825016233690a19162ff645dc58ed2e2628a2e113f26713e27cf998", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s1.pt", + "sha256": "bfa5aee55a338b05ccb9fc7d66a631da83b97c6e48cfa387105689bd30805033", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n200_s2.pt", + "sha256": "33d97ade0445171e2329dc86da6ef93a218f178e3390f182e511c237ca2f1ed6", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "6f8f06ad3ce7ef6c3a0dc44afa63838df505ac8cd1617b25fda0aa9747a31524", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "2a2a3f4524ac4db804501e412d4ebc188935058687f1a468e2b5bfe0cdfbbfcd", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "a1e5dfd9f19145467500b542c74c1103bd90a44d430e4e3528aca22cdb589922", + "bytes": 46145173, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s0.pt", + "sha256": "666e119f8cdcc6b218c020862e3de834c6920dcbed894121a108defa90877325", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s1.pt", + "sha256": "6c8e7e8900cdc01e54107cd6d4d786d12d199a7c50110f29606ea3ddbe64e0bc", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n200_s2.pt", + "sha256": "fb4fc80ba2990f27e9343260491cfafbdb1d3bcaf9abc79cfc66b51a5fd38037", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s0.pt", + "sha256": "fba4f9a51cbd39b523043bea1286f8efc61ec72df542dc224a419c0e2e0a3a9f", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s1.pt", + "sha256": "7552cfc76417a4b50971284df5944f731e579c196137b8c32a3ae1c2d949d582", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n200_s2.pt", + "sha256": "20e1867f9f0c7a5528f7b115ea67479c8875ff496ca35ee2323576e2aae81a63", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s0.pt", + "sha256": "3895668dce09dae91c6300f6adbba2de8359c63bf45fd5676d554f6ed90e9dc8", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s1.pt", + "sha256": "ca08e40ea0973004e8c21790e59334ea3fa6839be08e4753570c8ccfa3784283", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n200_s2.pt", + "sha256": "fabb7cc8e515078c224375f23949ba336c33a4f86f35bf3e96df6cfbdba97bfc", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "4f871d2e6a046579e795f507d1af9cde1745f3c9eb7b3afa897041d226461aa4", + "payload_sha256_before_redaction": "da1cb4e3e01c5c63465e5654426a1233900124b3c19618a883384e71bb7b937f", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/e48b_exposure_growth.json b/results/e48b_exposure_growth.json new file mode 100644 index 0000000000000000000000000000000000000000..90afaa8f4d89f47cab5d049c406de35dc16164cc --- /dev/null +++ b/results/e48b_exposure_growth.json @@ -0,0 +1,1429 @@ +{ + "experiment": "E48b \u2014 containment vs stream coverage: does the OOD designation earn itself?", + "why": "the E48 pre-registration classifies rungs by the GROWTH RATE of containment with coverage, and no stored file contained a growth rate for the OOD corpora at all \u2014 results/e35_containment_shard0.json has only the five Pile components", + "shards_used": [ + "shard00.npz", + "shard01.npz", + "shard02.npz", + "shard03.npz", + "shard04.npz", + "shard05.npz", + "shard06.npz", + "shard07.npz", + "shard08.npz", + "shard09.npz", + "shard10.npz", + "shard11.npz", + "shard12.npz", + "shard13.npz", + "shard14.npz", + "shard15.npz", + "shard16.npz", + "shard17.npz", + "shard18.npz", + "shard19.npz" + ], + "n_shards_total_in_stream": 20, + "k_values": [ + 8, + 13, + 20, + 32 + ], + "window_tokens": 128, + "primary_k": 32, + "controls": { + "C1_random_floor": { + "max_over_all": 0.0, + "fires": true + }, + "C2_index_matches_bitmaps": { + "per_corpus_counts_vs_reference": { + "8": { + "n_checked": 12, + "mismatches": {}, + "fires": true + }, + "13": { + "n_checked": 12, + "mismatches": {}, + "fires": true + }, + "20": { + "n_checked": 12, + "mismatches": {}, + "fires": true + }, + "32": { + "n_checked": 12, + "mismatches": {}, + "fires": true + } + }, + "length_vs_bitmaps": { + "8": { + "index_len": 2999819, + "bitmap_lens": [ + 2999819 + ], + "fires": true + }, + "13": { + "index_len": 2920399, + "bitmap_lens": [ + 2920399 + ], + "fires": true + }, + "20": { + "index_len": 2760367, + "bitmap_lens": [ + 2760367 + ], + "fires": true + }, + "32": { + "index_len": 2465010, + "bitmap_lens": [ + 2465010 + ], + "fires": true + } + } + }, + "C3_reproduces_stored_5shard": { + "checked": true, + "matching_coverage_shards": 5, + "max_abs_diff": 0.0, + "fires": true + } + }, + "containment_by_coverage": { + "8": { + "CONTROL_PubMed_2023": [ + 0.10180322858492401, + 0.17104623378066738, + 0.2401190575421633, + 0.29884901781000806, + 0.34787169014475844, + 0.3962275291044983, + 0.4450834930954961, + 0.49331430079742156, + 0.5308723014086855, + 0.5657178183434748, + 0.6022408379872746, + 0.6359124503348059, + 0.6579769942485622, + 0.6892209163401961, + 0.7159532938790253, + 0.741612486454947, + 0.7640278125086827, + 0.7836820316190158, + 0.8058021449806896, + 0.8239386235447751 + ], + "Github": [ + 0.5724903725754247, + 0.6574337893070616, + 0.7083374678101448, + 0.7499468424409951, + 0.7798426536253455, + 0.8057504666052402, + 0.8265031776407494, + 0.8478370779880454, + 0.8659200982824202, + 0.8820091194745671, + 0.8951685685259999, + 0.906338743591561, + 0.9166111465494838, + 0.9256172183240012, + 0.9340232003213079, + 0.9435679353604083, + 0.9540246178562147, + 0.9604129751695135, + 0.9665650766650129, + 0.9736669265480662 + ], + "OOD_CommonPile": [ + 0.0357996791277796, + 0.052184863801818276, + 0.06627563958804147, + 0.08005934410957959, + 0.09085169147963497, + 0.10326910138527093, + 0.1142719133299981, + 0.12280435420152845, + 0.1310883778701675, + 0.1386685527972812, + 0.14462366518881434, + 0.15104455983576862, + 0.15655114116652577, + 0.16218193109872858, + 0.1679196784377318, + 0.17307088515879723, + 0.17806683112805563, + 0.18184829299428987, + 0.18519157451653526, + 0.18745838149291838 + ], + "OOD_News_2024": [ + 0.029309548949175205, + 0.03802707047768622, + 0.04525364644958641, + 0.050097893044553275, + 0.053965862122024075, + 0.05695836494739244, + 0.05950517586259955, + 0.06192995208811966, + 0.06405760037353227, + 0.06572363918056359, + 0.06733131357078807, + 0.06880634155917886, + 0.07010097044107581, + 0.07163436284627343, + 0.07265839306842962, + 0.07348080075979858, + 0.07420239718577394, + 0.07500358145284951, + 0.07556600219662442, + 0.07613372879647266 + ], + "OOD_Wikipedia_2023": [ + 0.11954121794227443, + 0.18290970977771462, + 0.2335608295234365, + 0.2792137343459167, + 0.32380599161880463, + 0.3554555758518976, + 0.3813582527738013, + 0.40651223730775765, + 0.4282728429167114, + 0.4489381392914458, + 0.465811447367965, + 0.483551297560164, + 0.4999774699049923, + 0.5158663861380991, + 0.5293497814580784, + 0.5400498435024939, + 0.5506355219876397, + 0.5611726741143073, + 0.5679213041112224, + 0.5736300835693217 + ], + "OOD_arXiv_2023": [ + 0.02355505480735819, + 0.030044891841899287, + 0.034386659506435435, + 0.03776319992137785, + 0.0401850428385403, + 0.0423506617949015, + 0.044116150182340205, + 0.045692103037131414, + 0.047075010442003885, + 0.04821924347243135, + 0.04925817898472133, + 0.05028307482792631, + 0.05119916323572253, + 0.051950285531770016, + 0.0527014078278175, + 0.05336127227481248, + 0.05388074003095747, + 0.05438265820074621, + 0.054845967280551196, + 0.05521801851130369 + ], + "Pile-CC": [ + 0.15484516883353286, + 0.23412656738988888, + 0.30552813425468905, + 0.37363214383230764, + 0.433747803378771, + 0.4894457163260591, + 0.5307176912898279, + 0.572861313561992, + 0.613566192686564, + 0.6568755119615215, + 0.6940833000777131, + 0.7317636680599021, + 0.7671651509105039, + 0.8026226432267053, + 0.8249249123101804, + 0.8552190320164108, + 0.8781549081794824, + 0.8997290542101615, + 0.9232460285788299, + 0.9434829486183165 + ], + "StackExchange": [ + 0.17365869379405047, + 0.25994857567222945, + 0.3301490885142993, + 0.3869225272184623, + 0.44562262670421904, + 0.497705522160102, + 0.546635699973377, + 0.5924692084576906, + 0.6379489119621113, + 0.6779078565723654, + 0.7121954124455281, + 0.7454320624378215, + 0.7758733027870024, + 0.8026048453767147, + 0.8255496237756946, + 0.8579665671809099, + 0.8842742443986716, + 0.9041608866843219, + 0.924152619558059, + 0.9371418162456037 + ], + "TRAP_FineWeb": [ + 0.04889778589110622, + 0.06828333955662504, + 0.08378625067418993, + 0.094880996833036, + 0.10673636753377864, + 0.11854333485458242, + 0.12580384184541343, + 0.13302977499343097, + 0.14049426765686152, + 0.1474989282108728, + 0.15262968648439337, + 0.1580646945746726, + 0.16252817768189298, + 0.16650071222116195, + 0.17046287460758688, + 0.17528592568006748, + 0.1786499605858192, + 0.1824876571381156, + 0.18435118726576222, + 0.1872242736035625 + ], + "USPTO_Backgrounds": [ + 0.20597797096434697, + 0.3047734382884247, + 0.38791214336173463, + 0.4523304932024278, + 0.5111694995891182, + 0.5624612545593473, + 0.6120121678704785, + 0.6505773971714026, + 0.6942245865951588, + 0.7280034023903234, + 0.7587222294306762, + 0.7949086688868705, + 0.8208627077836887, + 0.8461751942678373, + 0.8691989965831928, + 0.8879626025402593, + 0.9108746738174531, + 0.9253240200106685, + 0.9389660193474907, + 0.9508455516629903 + ], + "Wikipedia_en": [ + 0.16650006594611855, + 0.25691219447857444, + 0.3255759872828117, + 0.39258765627494674, + 0.44922148871627204, + 0.503252184899033, + 0.5536315486231145, + 0.6009045030786425, + 0.6396773499378718, + 0.6797482940780386, + 0.719010530553878, + 0.7561416661460394, + 0.7907078448114289, + 0.82202530942613, + 0.8490215678516143, + 0.8685520314875362, + 0.8842194409157487, + 0.9079149225653734, + 0.9278653588510104, + 0.9438000236017687 + ], + "_CONTROL_random": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "13": { + "CONTROL_PubMed_2023": [ + 0.06655420230635867, + 0.1278280949880222, + 0.19475637917514946, + 0.2521887386966124, + 0.3011107354305898, + 0.35009388016430826, + 0.3997280712482105, + 0.44926154797959816, + 0.4881624666383707, + 0.5246534347191149, + 0.5637593790240779, + 0.5998151172243125, + 0.6235729135942795, + 0.6567906652182983, + 0.6855158373319329, + 0.7130288402742307, + 0.7371175552310315, + 0.7585193551403886, + 0.7834749329170474, + 0.8032293337745581 + ], + "Github": [ + 0.48856291521883943, + 0.5773274760095781, + 0.6349855067245197, + 0.681494517761014, + 0.7156167293808401, + 0.7456385143042328, + 0.7714293430315251, + 0.7976027582233585, + 0.8204274165961507, + 0.8407540104063518, + 0.8576103199323047, + 0.8716669967412636, + 0.88539960030967, + 0.8968456871252903, + 0.9080352159588067, + 0.9212727076319248, + 0.9346857389770088, + 0.9427965720252777, + 0.9515240444340421, + 0.9606656104279567 + ], + "OOD_CommonPile": [ + 0.00485002640569932, + 0.008216303992469885, + 0.011891546224344258, + 0.015333268666314591, + 0.0183007663041721, + 0.02209097212492231, + 0.025349471347121778, + 0.027637965288181385, + 0.030073756327488155, + 0.032470028633118854, + 0.03418729724195165, + 0.03639316110350675, + 0.03782661335230233, + 0.03952591890037327, + 0.04170663447686178, + 0.04352449622596093, + 0.045371098872279046, + 0.04649199386381844, + 0.04765600020118628, + 0.048381707855964996 + ], + "OOD_News_2024": [ + 0.0008914215713082569, + 0.001175801459087578, + 0.002220350662277008, + 0.0026359828059544775, + 0.002778172749844138, + 0.00290395616174653, + 0.0029422380697168233, + 0.0030406772616404345, + 0.003215680269504632, + 0.0033086506174324873, + 0.0033469325254027803, + 0.0034399028733306356, + 0.0036969385411311758, + 0.0040578822448510835, + 0.004068819932842596, + 0.004123508372800158, + 0.004150852592778939, + 0.004221947564723769, + 0.00427663600468133, + 0.004347730976626161 + ], + "OOD_Wikipedia_2023": [ + 0.055348043872527855, + 0.09854046117972191, + 0.13488139447850994, + 0.17124391858824883, + 0.20917897342890865, + 0.2360631315312203, + 0.25875507384057345, + 0.280795693352909, + 0.3000942798744854, + 0.3195655928836687, + 0.33522612776002536, + 0.3522144975098598, + 0.3681845006765121, + 0.3839278003281803, + 0.3978574718599764, + 0.4083865906670121, + 0.41947347209027835, + 0.4297435011659038, + 0.4364438494976538, + 0.44222298989550046 + ], + "OOD_arXiv_2023": [ + 0.0010665744223922976, + 0.001531399086205157, + 0.0017656130640953576, + 0.0019818105821478504, + 0.002104322509044263, + 0.0022412476038108416, + 0.0024250154941554605, + 0.002536717545149248, + 0.0026231965523702454, + 0.002731295311396492, + 0.002785344690909615, + 0.0028429973623902796, + 0.0029799224571568587, + 0.0029871290410919417, + 0.003070004756345397, + 0.003174500223404102, + 0.003221343018982142, + 0.003232152894884767, + 0.0032898055663654314, + 0.0033186319021057636 + ], + "Pile-CC": [ + 0.11586139686764775, + 0.18759931782066247, + 0.25664997581826976, + 0.32429336620594107, + 0.3854749619092294, + 0.4433038425314817, + 0.4859582328791532, + 0.530118072297919, + 0.5741288213497405, + 0.6206886520412653, + 0.660972141919484, + 0.7019065385216781, + 0.7402445809287966, + 0.7790008036334414, + 0.803142534027149, + 0.8367496845466017, + 0.8622368645932197, + 0.8867422300282545, + 0.9131530430798434, + 0.9354002349082366 + ], + "StackExchange": [ + 0.1010960349112768, + 0.1818392000462401, + 0.25172316629096586, + 0.31005071961158315, + 0.3721316686896711, + 0.427796803652968, + 0.48174599734119417, + 0.5322127333680134, + 0.5831960580313277, + 0.6275793306745275, + 0.6669737298422056, + 0.7045221374487024, + 0.7400041905092192, + 0.7708766834287035, + 0.7970782035720478, + 0.8345399109878042, + 0.864263626379978, + 0.887914715912375, + 0.9111756545864401, + 0.9260953124096873 + ], + "TRAP_FineWeb": [ + 0.018117363372982313, + 0.028842381817854635, + 0.03806301848086232, + 0.044534019542567166, + 0.052700149358478346, + 0.06161847006532184, + 0.06641953536916737, + 0.07176045059473467, + 0.0773604937827284, + 0.08284176998794335, + 0.08668190243112414, + 0.09097550880855122, + 0.09444854330496122, + 0.0974573068686905, + 0.10073239639380252, + 0.10492523078584154, + 0.10761728239549405, + 0.11101473790286301, + 0.1123319717118641, + 0.11472170736535243 + ], + "USPTO_Backgrounds": [ + 0.15446205415137354, + 0.24780163281138504, + 0.3295958956493737, + 0.393949595822462, + 0.4553867772284655, + 0.5097138005862313, + 0.5630381562624648, + 0.6053295080707429, + 0.6530184322522331, + 0.6904318423835679, + 0.7246798965129853, + 0.7651049690600173, + 0.7937505260345957, + 0.8213751184721394, + 0.8477957778176243, + 0.8697263888228169, + 0.8953163709284923, + 0.9116920566031522, + 0.9278372018984817, + 0.9416000966073971 + ], + "Wikipedia_en": [ + 0.099935954635733, + 0.18004663078207306, + 0.2451282346507009, + 0.31316024294061773, + 0.37257131343369504, + 0.42934860827264615, + 0.48454995538412826, + 0.5363763276819896, + 0.5800639014421001, + 0.6254965314757779, + 0.6700044615871736, + 0.7122887942200857, + 0.7522343916410006, + 0.7886538959730578, + 0.8198993984053424, + 0.8421965401111079, + 0.8606941366109209, + 0.8883775654126248, + 0.911973604674592, + 0.9313131458507239 + ], + "_CONTROL_random": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "20": { + "CONTROL_PubMed_2023": [ + 0.062416592420378024, + 0.12189647326942417, + 0.18819044268632631, + 0.24494774718278353, + 0.29342643116891065, + 0.3421804318648501, + 0.39172979194469193, + 0.4411300221400521, + 0.4799993881850893, + 0.5167388735722726, + 0.556028862368412, + 0.592206242806395, + 0.6163691079356218, + 0.6498468550801668, + 0.6790992554977305, + 0.7065353303991709, + 0.7307593770194671, + 0.7523258526214357, + 0.7777467621607773, + 0.7977225189949411 + ], + "Github": [ + 0.4280769845437319, + 0.516079541562978, + 0.5773521152485103, + 0.6259446131366864, + 0.6629392023013173, + 0.696367662838748, + 0.7261888997968083, + 0.7555169973288282, + 0.7820278075843017, + 0.8059450697472661, + 0.8259126503961097, + 0.8425378416017899, + 0.8589118969886532, + 0.8717061254309262, + 0.8852172324832767, + 0.901340152051323, + 0.9176776785918129, + 0.9270975548503458, + 0.9376726558754367, + 0.9485445537773111 + ], + "OOD_CommonPile": [ + 0.00040138535287506595, + 0.0007339617881144063, + 0.0012882558468466402, + 0.0016361231526716973, + 0.0018845997996895954, + 0.002553575387814705, + 0.003027592375971926, + 0.0032875371759291117, + 0.003623936328814881, + 0.003922108305236358, + 0.004052080705214951, + 0.004373188987515004, + 0.004449643340443588, + 0.004606374763947185, + 0.0052868185050115825, + 0.005466486234393755, + 0.005623217657897353, + 0.005695849293179507, + 0.005757012775522374, + 0.005806708104925954 + ], + "OOD_News_2024": [ + 0.00012684794391014552, + 0.00012684794391014552, + 0.0007553218478285939, + 0.0009455937636938122, + 0.0009513595793260915, + 0.0009513595793260915, + 0.0009513595793260915, + 0.0009513595793260915, + 0.0009686570262229295, + 0.0009801886574874882, + 0.0009801886574874882, + 0.0010205493669134436, + 0.0012857768859982934, + 0.0014933462487603496, + 0.0014933462487603496, + 0.0014933462487603496, + 0.0015048778800249084, + 0.0015048778800249084, + 0.0015106436956571878, + 0.0015279411425540258 + ], + "OOD_Wikipedia_2023": [ + 0.03568873637619119, + 0.06802244844088921, + 0.09550151301267411, + 0.12488953668529719, + 0.15561650962704524, + 0.1778507186332006, + 0.19670312434248027, + 0.21456853316194782, + 0.230329878844219, + 0.24726949988332014, + 0.2608043642095035, + 0.27564757325008893, + 0.28978305196270837, + 0.30401799547817704, + 0.3171894307170265, + 0.32649321535277487, + 0.33700205433073577, + 0.34561341091587955, + 0.3514053228971802, + 0.35669225972555363 + ], + "OOD_arXiv_2023": [ + 6.891666826195992e-05, + 0.00015314815169324426, + 0.00016080555927790646, + 0.00016463426307023756, + 0.00016463426307023756, + 0.00016463426307023756, + 0.0001952638934088864, + 0.00020675000478587974, + 0.00020675000478587974, + 0.00021823611616287306, + 0.00021823611616287306, + 0.00021823611616287306, + 0.0003062963033864885, + 0.0003062963033864885, + 0.0003139537109711507, + 0.0003445833413097996, + 0.00035224074889446177, + 0.00035224074889446177, + 0.00035224074889446177, + 0.00035224074889446177 + ], + "Pile-CC": [ + 0.10712866013916923, + 0.17629260272703765, + 0.24451099730229364, + 0.3113477339652424, + 0.37259608890466944, + 0.4307144869533671, + 0.47357289789239276, + 0.5177627948083625, + 0.5628210521440784, + 0.6100637183579112, + 0.6511236583831514, + 0.6928165366698957, + 0.7315415520109297, + 0.7709496775488308, + 0.7955223477378246, + 0.8299480141562046, + 0.8564773898243596, + 0.8820959518966921, + 0.9090537187052553, + 0.9320556213794194 + ], + "StackExchange": [ + 0.0840036465036465, + 0.1630125022982166, + 0.23242630385487528, + 0.29077036219893365, + 0.3528183796040939, + 0.4093085432371147, + 0.4645078752221609, + 0.5159151498437212, + 0.5680693448550591, + 0.6135357296071582, + 0.6540494576208862, + 0.6925292639578354, + 0.7292356131641846, + 0.7613914935343506, + 0.7883304222589936, + 0.827005577005577, + 0.857334375191518, + 0.8820670159955875, + 0.9063629956487099, + 0.9220444934730649 + ], + "TRAP_FineWeb": [ + 0.0120624399652516, + 0.020232906121917898, + 0.027106047675722427, + 0.03218436551642321, + 0.03867481544696468, + 0.04598039853505646, + 0.04941696931195873, + 0.05336634686403349, + 0.058016050086679656, + 0.06232133084838906, + 0.06545174832668088, + 0.06925570306191568, + 0.0721488517337844, + 0.07475880860443845, + 0.07755628437049142, + 0.08126839311614308, + 0.08341146620641621, + 0.08632757637568071, + 0.08733788226109519, + 0.08935849403192414 + ], + "USPTO_Backgrounds": [ + 0.13277103572426208, + 0.22183726993388994, + 0.3017706943108104, + 0.36564604736335704, + 0.4278531301406003, + 0.4830767559514572, + 0.5374856451162358, + 0.5810701759831155, + 0.6305246904000745, + 0.669674726093299, + 0.7055347776156926, + 0.7477264967876098, + 0.7775846550172258, + 0.806542723237841, + 0.8345502653713647, + 0.8580845153480865, + 0.885036779540023, + 0.9024527452745275, + 0.9200238989416183, + 0.9348792637884478 + ], + "Wikipedia_en": [ + 0.08057673400840848, + 0.1566355397604465, + 0.21928210464302245, + 0.28759425102236774, + 0.34699295723461476, + 0.4041078334984679, + 0.4606833127393335, + 0.5141333496555127, + 0.5596186730833235, + 0.6067068855368913, + 0.652261068159126, + 0.6961473280719808, + 0.7377076774176272, + 0.7758480201068848, + 0.8084911037746315, + 0.8316622226982858, + 0.851386556389942, + 0.8804451364368427, + 0.9050661239542009, + 0.9261944201099451 + ], + "_CONTROL_random": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "32": { + "CONTROL_PubMed_2023": [ + 0.061166716783366985, + 0.11960565316379569, + 0.1853258301473431, + 0.24163838652863096, + 0.2897504188324241, + 0.338141672752266, + 0.38725031143949484, + 0.43627303578332405, + 0.4747282958889987, + 0.5114910434297006, + 0.5506550968684222, + 0.5867906697023068, + 0.6109755573693028, + 0.644417715537609, + 0.673950771081232, + 0.7012242793934448, + 0.7253146612827012, + 0.7469478929507282, + 0.7725804373040079, + 0.7927273508312213 + ], + "Github": [ + 0.3742096158434741, + 0.4577368169887855, + 0.5195011134971765, + 0.5686640022269943, + 0.6077905034597948, + 0.6443569553805775, + 0.6786467032530025, + 0.7105752405949256, + 0.7410721387099339, + 0.7680744452397996, + 0.7919400699912511, + 0.8114014157321244, + 0.8298983933826454, + 0.844239640499483, + 0.8595303427980593, + 0.8783255786208543, + 0.8980553567167741, + 0.9088274477054005, + 0.9214288554839736, + 0.9340004374453194 + ], + "OOD_CommonPile": [ + 0.0, + 2.577319587628866e-05, + 0.00012457044673539519, + 0.00018900343642611685, + 0.00019329896907216494, + 0.0006443298969072165, + 0.0007259450171821306, + 0.0007259450171821306, + 0.000743127147766323, + 0.0007860824742268041, + 0.0007860824742268041, + 0.0007989690721649485, + 0.0007989690721649485, + 0.0007989690721649485, + 0.00125, + 0.00125, + 0.00125, + 0.00125, + 0.00125, + 0.00125 + ], + "OOD_News_2024": [ + 1.936533347104237e-05, + 1.936533347104237e-05, + 0.0004324924475199463, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0005486844483462006, + 0.0007423377830566242, + 0.0007487928942136383, + 0.0007487928942136383, + 0.0007487928942136383, + 0.0007487928942136383, + 0.0007487928942136383, + 0.0007487928942136383, + 0.0007487928942136383 + ], + "OOD_Wikipedia_2023": [ + 0.022133038856352236, + 0.04430904614379815, + 0.06429366344546661, + 0.08671029394703711, + 0.11009801099132467, + 0.12712640023374827, + 0.14158527729677006, + 0.15459182138882563, + 0.1662061883134461, + 0.17953069879559488, + 0.1900794486290922, + 0.2019688135127122, + 0.21365622677019194, + 0.226095587571811, + 0.23774432924130642, + 0.24528528889824647, + 0.25429147205548086, + 0.2606894714453291, + 0.2651152198479777, + 0.26957534299550123 + ], + "OOD_arXiv_2023": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 8.598932872430531e-05, + 8.598932872430531e-05, + 8.598932872430531e-05, + 8.598932872430531e-05, + 8.598932872430531e-05, + 8.598932872430531e-05, + 8.598932872430531e-05, + 8.598932872430531e-05 + ], + "Pile-CC": [ + 0.09955354052744514, + 0.16657437530283103, + 0.23348705613622206, + 0.2993787637571814, + 0.36025645462725825, + 0.41824859832491174, + 0.4614063473385478, + 0.5054509586765419, + 0.5512173807710944, + 0.5990603585519485, + 0.6408813248425279, + 0.6832819962621998, + 0.7223428739530698, + 0.7621478507648647, + 0.7870665190004845, + 0.8223117256177753, + 0.8498606977227106, + 0.876626635287603, + 0.9038122101474354, + 0.9276060773863086 + ], + "StackExchange": [ + 0.07686719226900293, + 0.15416426649242118, + 0.2230184591311226, + 0.2813061533302955, + 0.34294262795436375, + 0.3998418034407752, + 0.4551891050717473, + 0.5066674691129815, + 0.559272123874784, + 0.6051534248695308, + 0.6458245565767641, + 0.6847718616467918, + 0.7220297306359673, + 0.7550102741787105, + 0.7822218018932001, + 0.8213711514818031, + 0.8520174360120711, + 0.8771655303453672, + 0.9017547781379234, + 0.9179914195561899 + ], + "TRAP_FineWeb": [ + 0.007237811980212063, + 0.013104565752771127, + 0.01772490297291838, + 0.02130942505813029, + 0.02606299990974225, + 0.0317105562886013, + 0.033653246915119034, + 0.036180463924836787, + 0.039562980568795744, + 0.04249850644913116, + 0.04496125363717244, + 0.04796554732729609, + 0.04993402588248441, + 0.05192399437823155, + 0.05381081115929633, + 0.056815104849419984, + 0.05843114837944358, + 0.06064891024511426, + 0.061435442069567234, + 0.06309446548070848 + ], + "USPTO_Backgrounds": [ + 0.12033439046490556, + 0.2053173137241235, + 0.2827638182198369, + 0.3454772774796125, + 0.4081210357566042, + 0.4635638112497386, + 0.5176909806928278, + 0.5611234055900188, + 0.6115782045026835, + 0.651882797797449, + 0.6885629399874538, + 0.731903882344741, + 0.762703004112358, + 0.7927179898236565, + 0.8213345995678539, + 0.8459216212448596, + 0.8741113124695058, + 0.8923729699588764, + 0.9112096605562139, + 0.9267616923398619 + ], + "Wikipedia_en": [ + 0.07066448338254318, + 0.14398071145837363, + 0.2045797392952462, + 0.2726569450353924, + 0.3318978670001762, + 0.3885042354852435, + 0.4455060018824379, + 0.49941334983689833, + 0.5458253286745145, + 0.593750134306356, + 0.6392423832179374, + 0.6840942594240084, + 0.7268702965914121, + 0.7660533701225304, + 0.799593427799052, + 0.8232872179029298, + 0.8439811412387129, + 0.8737821099635976, + 0.8991993192279426, + 0.9215951727072293 + ], + "_CONTROL_random": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } + }, + "growth_ratio_first_to_last_k32": { + "CONTROL_PubMed_2023": 12.960109558255494, + "Github": 2.4959284793900025, + "OOD_CommonPile": Infinity, + "OOD_News_2024": 38.666666666666664, + "OOD_Wikipedia_2023": 12.1797709182683, + "OOD_arXiv_2023": Infinity, + "Pile-CC": 9.317660351121155, + "StackExchange": 11.942564733516024, + "TRAP_FineWeb": 8.71733966745843, + "USPTO_Backgrounds": 7.701553053614741, + "Wikipedia_en": 13.0418440579005, + "_CONTROL_random": NaN + }, + "formatting_sensitivity_measured": { + "note": "fraction of OOD_CommonPile's own k=32 k-grams that survive re-wrapping the IDENTICAL text \u2014 the band in which a containment value cannot be read as absence", + "80col_its_own_wrap": 0.2877, + "72col": 0.0243, + "100col": 0.0132, + "single_line_reflow": 0.0014 + }, + "reading": "A corpus is 'not found' only if its containment is BELOW the formatting floor AND flat in coverage. Containment inside 0.001-0.03 at k=32 is consistent with present-but-reformatted and must not be called absence.", + "provenance": { + "schema_version": 1, + "experiment": "E48b", + "utc": "2026-08-30T06:34:31+00:00", + "script": { + "path": "experiments/t48b_exposure_growth.py", + "sha256": "215eb1b7e9d35f89f5fb2d301d8400a77af4830e073404ce48ad3c43e593b8a1", + "n_lines": 246 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/t48b_exposure_growth.py", + "--exclude", + "OOD_News_2024_dedup" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/m1/merged/shard00.npz", + "sha256": "271dacb8f530278fba62a1a0599a0294781f4cb2b81f0c8d5383ef341903abb6", + "bytes": 1039631, + "exists": true + }, + { + "path": "results/m1/merged/shard01.npz", + "sha256": "bc499f9d5558cfd18fb25daefc6041313953e788557f8bfda08f7be5ff9baf87", + "bytes": 1052626, + "exists": true + }, + { + "path": "results/m1/merged/shard02.npz", + "sha256": "d1bb6cbeb70b055f16dc6ad1f6d9653eb747b55c6ecec451b7861b3e422f2f23", + "bytes": 1048923, + "exists": true + }, + { + "path": "results/m1/merged/shard03.npz", + "sha256": "f5303578e3918fa2cab600d501238fd844e8fa26832007dfdbbe0483fa051834", + "bytes": 1046557, + "exists": true + }, + { + "path": "results/m1/merged/shard04.npz", + "sha256": "0f3c825f329c831d753ef2636640e6270a09205632ddf544b49020cb08c30e29", + "bytes": 1047396, + "exists": true + }, + { + "path": "results/m1/merged/shard05.npz", + "sha256": "a573dffbbc1225482769693b643325350b7270ddd36aececf883949022762208", + "bytes": 1050274, + "exists": true + }, + { + "path": "results/m1/merged/shard06.npz", + "sha256": "efd6470fd6ce2369bf099a1b140f2f086c2e845a3600dd97822cb4a8c48400cc", + "bytes": 1049424, + "exists": true + }, + { + "path": "results/m1/merged/shard07.npz", + "sha256": "37db3648354e22ed6b5619794399de0fccd29611e5a9ad92ff0ca694c14f05d0", + "bytes": 1037538, + "exists": true + }, + { + "path": "results/m1/merged/shard08.npz", + "sha256": "33a51b8473d3ce54db3d1a317bf34c7360114c6204462b7b91f62ebaa4a9cad7", + "bytes": 1042641, + "exists": true + }, + { + "path": "results/m1/merged/shard09.npz", + "sha256": "3f1527d8af10e27a1e12767ee503213f00652fa99d4c22ec31a2e9e9caf63020", + "bytes": 1048836, + "exists": true + }, + { + "path": "results/m1/merged/shard10.npz", + "sha256": "1a931f59851c54b1cb23c9b2025b60b16c7a09a2c32946d725560b4d8ba54e2f", + "bytes": 1031060, + "exists": true + }, + { + "path": "results/m1/merged/shard11.npz", + "sha256": "2b7c65ef6da707ba0dbe1fb592479c13fd0d1534dda5ba3d558053f280a76250", + "bytes": 1050414, + "exists": true + }, + { + "path": "results/m1/merged/shard12.npz", + "sha256": "3f6583e9d941f58bd395ce589cdedf5602b071915bda715db14a800d99d66a85", + "bytes": 1028856, + "exists": true + }, + { + "path": "results/m1/merged/shard13.npz", + "sha256": "e4c1165f15b052a3a762158f05e2ca1db3eee20b7e9c04af1ee48131d0958ff0", + "bytes": 1049628, + "exists": true + }, + { + "path": "results/m1/merged/shard14.npz", + "sha256": "0c275d4ccc84c7d4aea84681e16f1338c5f91e81b5c60cb0e6304c8c9c5c4ea4", + "bytes": 1041328, + "exists": true + }, + { + "path": "results/m1/merged/shard15.npz", + "sha256": "debd79b3c5163291183a84b26f3392ef71d762c2f048656d50fbd73d6704c8c9", + "bytes": 1057596, + "exists": true + }, + { + "path": "results/m1/merged/shard16.npz", + "sha256": "889d11318360d394fac55174b9a39c9a8ecb199d66ea10ee9bfb04bad10e1ffe", + "bytes": 1051198, + "exists": true + }, + { + "path": "results/m1/merged/shard17.npz", + "sha256": "8605129b53db7ddc2ad5b38e154d50a2fb669089393b150f0c45296df469daca", + "bytes": 1049840, + "exists": true + }, + { + "path": "results/m1/merged/shard18.npz", + "sha256": "7ec716f947bcc7479cec5e0f84b7e800f28441db2026ba92f513e2bb2d628e31", + "bytes": 1052755, + "exists": true + }, + { + "path": "results/m1/merged/shard19.npz", + "sha256": "13678ac5fb4cf617ef89b98d59e920285d0845b98645cb457f9a9e684fb48633", + "bytes": 1044727, + "exists": true + }, + { + "path": "results/m1/m1_5shard.json", + "sha256": "e14a43851c804a97227c8348b8b78f885e2c37317eefb4139e7f3733d56d57e5", + "bytes": 6675, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "a3904599356aee3c0265c5c1b7b54240e9814b735dd84c0812fd7ad8b6777eef" + } +} \ No newline at end of file diff --git a/results/e48c_exposure_vs_read.json b/results/e48c_exposure_vs_read.json new file mode 100644 index 0000000000000000000000000000000000000000..dbb2908a77b7af6da1ad11caa289e97593240aa7 --- /dev/null +++ b/results/e48c_exposure_vs_read.json @@ -0,0 +1,239 @@ +{ + "experiment": "E48c \u2014 does exposure of the FITTING corpus govern the read?", + "why": "E48 returned NOT REACHED on a CI that could not resolve the in-stream advantage either. This measures the positive content: whether the read orders with measured containment across four orders of magnitude of exposure.", + "not_a_new_predictor": "tests ONE pre-existing externally-measured variable that E48's own rung classification is already built on; proposes no new scalar and does not target the corpus x concept interaction (HANDOFF sec0.5)", + "sources": [ + "e48_crossover_410m.json", + "e48b_exposure_growth.json" + ], + "containment_k": 32, + "containment_coverage_shards": 20, + "aggregation": "persist", + "logit_baseline": 0.028439503419212996, + "by_rung": { + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "read_persist": 0.03820955517391364, + "seed_sd": 0.0008041814500509604, + "delta_vs_logit": 0.009770051754700644, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "read_persist": 0.04961537096338967, + "seed_sd": 0.0002657278869433335, + "delta_vs_logit": 0.021175867544176677, + "tier": "in-stream" + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "read_persist": 0.04546591031830758, + "seed_sd": 0.0005658129573954557, + "delta_vs_logit": 0.017026406899094584, + "tier": "in-stream" + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "read_persist": 0.045729962767412265, + "seed_sd": 0.0005801408785964227, + "delta_vs_logit": 0.01729045934819927, + "tier": "in-stream" + }, + "Github": { + "containment_k32": 0.9340004374453194, + "read_persist": 0.02789688172439734, + "seed_sd": 0.0010255852488031789, + "delta_vs_logit": -0.0005426216948156565, + "tier": "github" + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "read_persist": 0.0337636109519129, + "seed_sd": 0.0003581168111076974, + "delta_vs_logit": 0.0053241075326999066, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "read_persist": 0.04105739652489623, + "seed_sd": 0.00029864155760697485, + "delta_vs_logit": 0.012617893105683231, + "tier": "OOD" + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "read_persist": 0.03823967572922508, + "seed_sd": 0.0006203803993649463, + "delta_vs_logit": 0.009800172310012081, + "tier": "OOD" + } + }, + "correlations": { + "all8_pearson": 0.2663554061916339, + "all8_spearman": 0.0, + "without_github_pearson": 0.6900549924615283, + "without_github_spearman": 0.5 + }, + "tier_contrast_corpus_as_unit": { + "note": "n=4 in-stream vs n=3 OOD; Github excluded (it is neither a clean in-stream English rung nor OOD, and it is the leverage point in seven prior analyses)", + "mean_a": 0.04475519980575579, + "mean_b": 0.0376868944020114, + "diff": 0.007068305403744389, + "t": 2.2164522110159894, + "df": 4.961001162329945, + "se": 0.003189017732308508, + "significant_at_5pct": false + }, + "headline_pair_seed_axis": { + "note": "OOD_arXiv_2023 (containment 0.0001) vs Wikipedia_en (containment 0.7996), SEED as the replication unit (3 disjoint prompt blocks)", + "mean_a": 0.04105739652489623, + "mean_b": 0.03820955517391364, + "diff": 0.002847841350982587, + "t": 5.750010207015007, + "df": 2.5413400341640955, + "se": 0.0004952758775120474, + "arxiv_above_wikipedia": true, + "significant_at_5pct": true + }, + "controls": { + "C1_containment_spans_orders": { + "max_over_min": 10861.817987204726, + "fires": true + }, + "C2_seed_sd_small_vs_spread": { + "max_seed_sd": 0.0010255852488031789, + "between_corpus_range": 0.021718489238992333, + "ratio": 0.04722175826861347, + "fires": true + }, + "C3_github_leverage": { + "spearman_with": 0.0, + "spearman_without": 0.5, + "note": "claim is stated only in the form that survives both" + } + }, + "VERDICT": "EXPOSURE DOES NOT ORDER THE READ. Containment spans 10862x across the eight fitting corpora and the rank correlation with read quality is +0.000 (all eight) / +0.500 (without Github). The most-contained corpus (Github, 0.934) is the WORST reader; the least-contained (arXiv_2023, 0.0001) reads ABOVE in-stream Wikipedia by t=+5.75 on the seed axis. The in-stream/OOD tier gap is +0.00707 with the corpus as the replication unit (t=2.22, df=5.0) \u2014 suggestive, NOT significant, and far smaller than the within-tier spread.", + "provenance": { + "schema_version": 1, + "experiment": "E48c", + "utc": "2026-08-31T23:19:48+00:00", + "script": { + "path": "experiments/t48c_exposure_vs_read.py", + "sha256": "44398e8faf4c2437de63858ffec1ba24ecc4cad46e4802009f21945d8c90c4e3", + "n_lines": 200 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/t48c_exposure_vs_read.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "3d15cf8111aab635e1894aa66b3dfd9b4a6449f7d79558d3a0923a87683f4daf", + "bytes": 114484, + "exists": true + }, + { + "path": "results/e48b_exposure_growth.json", + "sha256": "37807735102faf3e37ad0bc59fbc9a622a65c9d4768ac581358c9fdeaa80bc9e", + "bytes": 34930, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "77af14c08520331c43a36332f4d18bebfe49b906d5c484b3ef60afdd2b91f1b6" + } +} \ No newline at end of file diff --git a/results/e49_derangement_stability.json b/results/e49_derangement_stability.json new file mode 100644 index 0000000000000000000000000000000000000000..b56e69d6173efc81a26e59d612a222c79d4ea814 --- /dev/null +++ b/results/e49_derangement_stability.json @@ -0,0 +1,1063 @@ +{ + "experiment": "E49 \u2014 stability of the layer-derangement control across random draws", + "why": "every standing derangement claim rests on ONE draw; this reports the distribution", + "n_seeds": 20, + "cells": { + "70m|wikitext": { + "lens": "lens_70m_n200_db128_pen.pt", + "band": [ + 2, + 3 + ], + "baseline": { + "multihop": { + "min": 0.004160887694784573, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.015591007923441274, + "persist": 0.015591007923441274 + }, + "order-ops": { + "min": 0.11168830894998141, + "persist": 0.11168830894998141 + }, + "poetry": { + "min": 0.011661807341235024, + "persist": 0.011661807341235024 + }, + "typo": { + "min": 0.06696428491600923, + "persist": 0.06696428491600923 + }, + "association": { + "min": 0.00776397529989481, + "persist": 0.00776397529989481 + } + }, + "min": { + "n_win_mean": 6, + "n_win_sd": 0.0, + "n_win_min": 6, + "n_win_max": 6, + "n_win_all": [ + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 20 + }, + "per_set_win_rate": { + "multihop": 1.0, + "multilingual": 1.0, + "order-ops": 1.0, + "poetry": 1.0, + "typo": 1.0, + "association": 1.0 + } + }, + "persist": { + "n_win_mean": 6, + "n_win_sd": 0.0, + "n_win_min": 6, + "n_win_max": 6, + "n_win_all": [ + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 20 + }, + "per_set_win_rate": { + "multihop": 1.0, + "multilingual": 1.0, + "order-ops": 1.0, + "poetry": 1.0, + "typo": 1.0, + "association": 1.0 + } + } + }, + "160m|wikitext": { + "lens": "lens_160m_n200_db128_pen.pt", + "band": [ + 4, + 5, + 6, + 7, + 8, + 9 + ], + "baseline": { + "multihop": { + "min": 0.04299583951277392, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.021029730859611715, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.015584415756165981, + "persist": 0.0 + }, + "poetry": { + "min": 0.002915451835308756, + "persist": 0.0 + }, + "typo": { + "min": 0.19494047654526575, + "persist": 0.12202380863683564 + }, + "association": { + "min": 0.0, + "persist": 0.0 + } + }, + "min": { + "n_win_mean": 3.5, + "n_win_sd": 0.6708203932499369, + "n_win_min": 2, + "n_win_max": 4, + "n_win_all": [ + 3, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 3, + 4, + 3, + 2, + 3, + 3, + 4, + 3, + 4, + 2, + 4 + ], + "hist": { + "0": 0, + "1": 0, + "2": 2, + "3": 6, + "4": 12, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.85, + "multilingual": 0.95, + "order-ops": 1.0, + "poetry": 0.0, + "typo": 0.7, + "association": 0.0 + } + }, + "persist": { + "n_win_mean": 2.35, + "n_win_sd": 0.5722761571129799, + "n_win_min": 1, + "n_win_max": 3, + "n_win_all": [ + 2, + 2, + 3, + 3, + 2, + 3, + 2, + 2, + 2, + 2, + 2, + 3, + 2, + 2, + 3, + 3, + 2, + 3, + 1, + 3 + ], + "hist": { + "0": 0, + "1": 1, + "2": 11, + "3": 8, + "4": 0, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.55, + "multilingual": 0.8, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 1.0, + "association": 0.0 + } + } + }, + "410m|wikitext": { + "lens": "lens_410m_n200_db128_pen.pt", + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "baseline": { + "multihop": { + "min": 0.09292649238237313, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.14720812014171056, + "persist": 0.021029732057026455 + }, + "order-ops": { + "min": 0.04415584368897336, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408162980207374, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34226190618106295, + "persist": 0.14583333022892475 + }, + "association": { + "min": 0.009316770093781608, + "persist": 0.001552795059978962 + } + }, + "min": { + "n_win_mean": 2.2, + "n_win_sd": 0.812403840463596, + "n_win_min": 0, + "n_win_max": 3, + "n_win_all": [ + 3, + 3, + 2, + 2, + 2, + 2, + 1, + 2, + 3, + 2, + 2, + 3, + 3, + 2, + 3, + 3, + 1, + 3, + 0, + 2 + ], + "hist": { + "0": 1, + "1": 2, + "2": 9, + "3": 8, + "4": 0, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.6, + "multilingual": 0.9, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 0.7, + "association": 0.0 + } + }, + "persist": { + "n_win_mean": 4.9, + "n_win_sd": 0.3, + "n_win_min": 4, + "n_win_max": 5, + "n_win_all": [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 4, + 4, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 0, + "4": 2, + "5": 18, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.9, + "multilingual": 1.0, + "order-ops": 0.0, + "poetry": 1.0, + "typo": 1.0, + "association": 1.0 + } + } + }, + "410m|Github": { + "lens": "e28_Github_410m_n400_s0.pt", + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "baseline": { + "multihop": { + "min": 0.06241331435739994, + "persist": 0.0 + }, + "multilingual": { + "min": 0.10986221821180411, + "persist": 0.02284263925892966 + }, + "order-ops": { + "min": 0.07922077950622354, + "persist": 0.009090908936091832 + }, + "poetry": { + "min": 0.017492711410990784, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.25744047335215975, + "persist": 0.07886904863906759 + }, + "association": { + "min": 0.006211180106869766, + "persist": 0.0 + } + }, + "min": { + "n_win_mean": 2.2, + "n_win_sd": 1.0295630140987, + "n_win_min": 0, + "n_win_max": 3, + "n_win_all": [ + 3, + 3, + 3, + 0, + 1, + 1, + 1, + 1, + 3, + 1, + 3, + 3, + 3, + 3, + 1, + 3, + 3, + 3, + 2, + 3 + ], + "hist": { + "0": 1, + "1": 6, + "2": 1, + "3": 12, + "4": 0, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.65, + "multilingual": 0.75, + "order-ops": 0.0, + "poetry": 0.0, + "typo": 0.8, + "association": 0.0 + } + }, + "persist": { + "n_win_mean": 3.45, + "n_win_sd": 0.49749371855331, + "n_win_min": 3, + "n_win_max": 4, + "n_win_all": [ + 4, + 4, + 3, + 3, + 4, + 3, + 4, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 3, + 4, + 3, + 4, + 3, + 4 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 11, + "4": 9, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.0, + "multilingual": 1.0, + "order-ops": 0.45, + "poetry": 1.0, + "typo": 1.0, + "association": 0.0 + } + } + }, + "410m|Wikipedia_en": { + "lens": "e28_Wikipedia_en_410m_n400_s0.pt", + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "baseline": { + "multihop": { + "min": 0.08460471486406666, + "persist": 0.006934812558548791 + }, + "multilingual": { + "min": 0.13089195026883058, + "persist": 0.021392312832176685 + }, + "order-ops": { + "min": 0.06233766182724919, + "persist": 0.0 + }, + "poetry": { + "min": 0.0291545195505023, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952381023338865, + "persist": 0.12053571601531335 + }, + "association": { + "min": 0.004658385046890804, + "persist": 0.0 + } + }, + "min": { + "n_win_mean": 1.5, + "n_win_sd": 0.6708203932499369, + "n_win_min": 0, + "n_win_max": 3, + "n_win_all": [ + 2, + 2, + 2, + 0, + 1, + 2, + 1, + 1, + 2, + 3, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1 + ], + "hist": { + "0": 1, + "1": 9, + "2": 9, + "3": 1, + "4": 0, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.0, + "multilingual": 0.65, + "order-ops": 0.05, + "poetry": 0.0, + "typo": 0.8, + "association": 0.0 + } + }, + "persist": { + "n_win_mean": 3.85, + "n_win_sd": 0.3570714214271425, + "n_win_min": 3, + "n_win_max": 4, + "n_win_all": [ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 3, + 4, + 4, + 4, + 3, + 4, + 4, + 3, + 4, + 4, + 4 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 3, + "4": 17, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.85, + "multilingual": 1.0, + "order-ops": 0.0, + "poetry": 1.0, + "typo": 1.0, + "association": 0.0 + } + } + }, + "410m|StackExchange": { + "lens": "e28_StackExchange_410m_n400_s0.pt", + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "baseline": { + "multihop": { + "min": 0.10124826670757361, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.17077592547450746, + "persist": 0.026831037364900112 + }, + "order-ops": { + "min": 0.0467532468693597, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985227244242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309524199792315, + "persist": 0.1592261902987957 + }, + "association": { + "min": 0.015527949934559209, + "persist": 0.0 + } + }, + "min": { + "n_win_mean": 3.3, + "n_win_sd": 0.714142842854285, + "n_win_min": 2, + "n_win_max": 4, + "n_win_all": [ + 3, + 4, + 4, + 4, + 4, + 3, + 3, + 3, + 3, + 2, + 2, + 4, + 4, + 4, + 4, + 3, + 2, + 3, + 3, + 4 + ], + "hist": { + "0": 0, + "1": 0, + "2": 3, + "3": 8, + "4": 9, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.95, + "multilingual": 0.9, + "order-ops": 0.1, + "poetry": 0.0, + "typo": 0.8, + "association": 0.55 + } + }, + "persist": { + "n_win_mean": 3.05, + "n_win_sd": 0.21794494717703367, + "n_win_min": 3, + "n_win_max": 4, + "n_win_all": [ + 3, + 3, + 3, + 4, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 19, + "4": 1, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.05, + "multilingual": 1.0, + "order-ops": 0.0, + "poetry": 1.0, + "typo": 1.0, + "association": 0.0 + } + } + }, + "410m|Pile-CC": { + "lens": "e28_Pile-CC_410m_n400_s0.pt", + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "baseline": { + "multihop": { + "min": 0.09986130334436893, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.16896301774041994, + "persist": 0.024292966350913048 + }, + "order-ops": { + "min": 0.04415584462029593, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.024781341132308756, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3377976215311459, + "persist": 0.15922619216144085 + }, + "association": { + "min": 0.01086956528680665, + "persist": 0.0 + } + }, + "min": { + "n_win_mean": 1.75, + "n_win_sd": 0.82915619758885, + "n_win_min": 0, + "n_win_max": 3, + "n_win_all": [ + 2, + 3, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 3, + 1, + 3, + 1, + 0, + 2, + 3, + 1 + ], + "hist": { + "0": 1, + "1": 7, + "2": 8, + "3": 4, + "4": 0, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.55, + "multilingual": 0.95, + "order-ops": 0.05, + "poetry": 0.0, + "typo": 0.2, + "association": 0.0 + } + }, + "persist": { + "n_win_mean": 4.15, + "n_win_sd": 0.7262919523166975, + "n_win_min": 3, + "n_win_max": 5, + "n_win_all": [ + 4, + 4, + 5, + 3, + 5, + 4, + 5, + 5, + 4, + 3, + 3, + 5, + 4, + 4, + 5, + 4, + 4, + 5, + 4, + 3 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 4, + "4": 9, + "5": 7, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.65, + "multilingual": 1.0, + "order-ops": 0.95, + "poetry": 0.55, + "typo": 1.0, + "association": 0.0 + } + } + }, + "410m|USPTO_Backgrounds": { + "lens": "e28_USPTO_Backgrounds_410m_n400_s0.pt", + "band": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "baseline": { + "multihop": { + "min": 0.10402219183743, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.14358230094824517, + "persist": 0.024655547259109362 + }, + "order-ops": { + "min": 0.1285714294229235, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.02332361481551613, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190447807312, + "persist": 0.18005952265645778 + }, + "association": { + "min": 0.026397515487458025, + "persist": 0.0 + } + }, + "min": { + "n_win_mean": 2.9, + "n_win_sd": 0.7, + "n_win_min": 2, + "n_win_max": 4, + "n_win_all": [ + 3, + 3, + 4, + 2, + 2, + 2, + 2, + 3, + 2, + 2, + 3, + 4, + 4, + 3, + 3, + 4, + 3, + 3, + 3, + 3 + ], + "hist": { + "0": 0, + "1": 0, + "2": 6, + "3": 10, + "4": 4, + "5": 0, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.7, + "multilingual": 0.8, + "order-ops": 0.9, + "poetry": 0.0, + "typo": 0.4, + "association": 0.1 + } + }, + "persist": { + "n_win_mean": 4.8, + "n_win_sd": 0.4, + "n_win_min": 4, + "n_win_max": 5, + "n_win_all": [ + 5, + 5, + 5, + 4, + 5, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 5, + 4, + 5, + 5, + 4, + 5, + 5, + 5 + ], + "hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 0, + "4": 4, + "5": 16, + "6": 0 + }, + "per_set_win_rate": { + "multihop": 0.8, + "multilingual": 1.0, + "order-ops": 1.0, + "poetry": 1.0, + "typo": 1.0, + "association": 0.0 + } + } + } + }, + "max_n_win_range_persist": 2, + "VERDICT": "UNSTABLE \u2014 a single-draw n_win carries no information; every standing derangement claim must be restated as a distribution" +} \ No newline at end of file diff --git a/results/e50_concept_energy.json b/results/e50_concept_energy.json new file mode 100644 index 0000000000000000000000000000000000000000..767def594eb77d23e9f9a4be91b2470c9c463327 --- /dev/null +++ b/results/e50_concept_energy.json @@ -0,0 +1,37 @@ +{ + "experiment": "E50 \u2014 concept-conditional activation energy, targeting the 6.8% interaction", + "prereg": "in-file docstring, fixed before running", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_cells": 5815, + "n_concept_tokens": 1163, + "r_primary": 0.09872587496579752, + "control_C1_random_direction_r": 0.0711147247683827, + "control_C2_dose_r": 0.022596516886992272, + "loo_concept_worst": 0.09800211043223954, + "loo_concept_n": 200, + "required_pattern_github_zA_by_set": { + "multihop": -0.8963096102458632, + "multilingual": -0.979721459984356, + "order-ops": -0.8224535183798235, + "poetry": -0.8918440922870096, + "typo": -1.081617467413437 + }, + "required_pattern_sign_reversal_ok": false, + "controls_fire": true, + "VERDICT": "NULL \u2014 the strongest object-level account also fails; reported as such" +} \ No newline at end of file diff --git a/results/e51_interaction_variance.json b/results/e51_interaction_variance.json new file mode 100644 index 0000000000000000000000000000000000000000..288790625261412f29a92bb59a6b8000596885d1 --- /dev/null +++ b/results/e51_interaction_variance.json @@ -0,0 +1,790 @@ +{ + "experiment": "E51 \u2014 the corpus x concept-set interaction decomposition, STORED", + "why": "the programme's headline interaction numbers (6.8% / 1.9% / 4614x at 410M, 9.2% / 2.2% at 1B) appear in six documents including the paper abstract and in NO results JSON; no committed script computed them. This stores the decomposition and reports whether it reproduces them.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json" + ], + "aggregation": "persist", + "n_min": 75, + "decomposition": "balanced two-factor (eval set, corpus) with seed as the replication unit, after averaging over the NESTED N axis (SPINE A1.1)", + "scales": { + "410m": { + "admitted5": { + "grand_mean": 0.041563127900002834, + "SS_total": 0.22925593623758608, + "frac_set_main": 0.9127829399393488, + "frac_corpus_main": 0.01906450114831279, + "frac_interaction": 0.06780626428099203, + "frac_residual_seed": 0.00034629463134648467, + "SS_set": 0.2092609074774917, + "SS_corpus": 0.004370650059658984, + "SS_interaction": 0.015544988600512018, + "SS_residual_seed": 7.939009992338807e-05, + "interaction_over_noise_floor": 195.8051270311163, + "candidate_noise_floors": { + "seed_x_set": 2768.1445461585845, + "seed_x_corpus": 2547.878926111827, + "seed_main": 10268.19304676178, + "three_way_residual": 234.96272012579493, + "full_two_factor_residual": 195.8051270311163, + "mean_squares_F": 611.8910219722385 + }, + "SS_seed_main": 1.5138971900625057e-06, + "SS_seed_x_set": 5.615670837017576e-06, + "SS_seed_x_corpus": 6.10114885805596e-06, + "SS_three_way_residual": 6.615938303825178e-05, + "interaction_over_corpus_main": 3.5566765557352586, + "per_corpus_main": { + "Github": -0.01347907410647357, + "Pile-CC": 0.004152620384147217, + "StackExchange": 0.004112475200761065, + "USPTO_Backgrounds": 0.00817213694672702, + "Wikipedia_en": -0.0029581584251617415 + }, + "per_set_main": { + "multihop": -0.026766659081874818, + "multilingual": -0.007186749365197628, + "order-ops": -0.03100104450900741, + "poetry": -0.038628393975836214, + "typo": 0.10358284693191608 + }, + "interaction_cells": { + "Github|multihop": 0.0023481492099414816, + "Github|multilingual": 0.007358160156824216, + "Github|order-ops": 0.02190276691766114, + "Github|poetry": 0.014917517935270082, + "Github|typo": -0.046526594219696946, + "Pile-CC|multihop": -0.0020486935037057966, + "Pile-CC|multilingual": -0.002015691274651746, + "Pile-CC|order-ops": -0.009327498430404486, + "Pile-CC|poetry": -0.005629628390659461, + "Pile-CC|typo": 0.01902151159942149, + "StackExchange|multihop": -0.005006295451394596, + "StackExchange|multilingual": 0.0005318301469256631, + "StackExchange|order-ops": -0.012138998990460316, + "StackExchange|poetry": -0.002743446891852852, + "StackExchange|typo": 0.0193569111867821, + "USPTO_Backgrounds|multihop": -0.003962081805792314, + "USPTO_Backgrounds|multilingual": -0.0031076970647595306, + "USPTO_Backgrounds|order-ops": -0.010028111813667362, + "USPTO_Backgrounds|poetry": -0.007273591605950647, + "USPTO_Backgrounds|typo": 0.024371482290169853, + "Wikipedia_en|multihop": 0.008668921550951246, + "Wikipedia_en|multilingual": -0.0027666019643385886, + "Wikipedia_en|order-ops": 0.009591842316871033, + "Wikipedia_en|poetry": 0.0007291489531928777, + "Wikipedia_en|typo": -0.016223310856676575 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06780626428099203, + "null2_within_cell_mean": 0.023463539326378492, + "null2_max": 0.069952084379219, + "n_perm": 2000, + "n_null_ge_observed": 3, + "p_value": 0.001999000499750125, + "observed": 0.06780626428099203, + "fires": null + }, + "C4_F_test": { + "F": 611.8910219722385, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03464607279448547, + "SS_total": 0.25078694938130414, + "frac_set_main": 0.9202689818852297, + "frac_corpus_main": 0.014505336726722001, + "frac_interaction": 0.06490812752157134, + "frac_residual_seed": 0.00031755386647712333, + "SS_set": 0.23079145057723538, + "SS_corpus": 0.003637749147443202, + "SS_interaction": 0.016278111291187544, + "SS_residual_seed": 7.963836543803573e-05, + "interaction_over_noise_floor": 204.40036911421873, + "candidate_noise_floors": { + "seed_x_set": 2664.38297317969, + "seed_x_corpus": 3357.1772819678595, + "seed_main": 15217.493238347859, + "three_way_residual": 240.763440450699, + "full_two_factor_residual": 204.40036911421873, + "mean_squares_F": 613.2011073426562 + }, + "SS_seed_main": 1.0696972908893393e-06, + "SS_seed_x_set": 6.109523839120302e-06, + "SS_seed_x_corpus": 4.848749387951859e-06, + "SS_three_way_residual": 6.761039492007427e-05, + "interaction_over_corpus_main": 4.474775645986651, + "per_corpus_main": { + "Github": -0.011242694633211085, + "Pile-CC": 0.003450384108972908, + "StackExchange": 0.003416929789484449, + "USPTO_Backgrounds": 0.006799981244456077, + "Wikipedia_en": -0.0024246005097023557 + }, + "per_set_main": { + "multihop": -0.019849603976357454, + "multilingual": -0.0002696942596802643, + "order-ops": -0.024083989403490047, + "poetry": -0.03171133887031885, + "typo": 0.11049990203743343, + "association": -0.03458527552758682 + }, + "interaction_cells": { + "Github|multihop": 0.0001117697366790002, + "Github|multilingual": 0.005121780683561734, + "Github|order-ops": 0.01966638744439865, + "Github|poetry": 0.0126811384620076, + "Github|typo": -0.04876297369295941, + "Github|association": 0.011181897366312435, + "Pile-CC|multihop": -0.0013464572285314877, + "Pile-CC|multilingual": -0.001313454999477437, + "Pile-CC|order-ops": -0.008625262155230178, + "Pile-CC|poetry": -0.004927392115485152, + "Pile-CC|typo": 0.019723747874595812, + "Pile-CC|association": -0.003511181375871558, + "StackExchange|multihop": -0.0043107500401179805, + "StackExchange|multilingual": 0.0012273755582022788, + "StackExchange|order-ops": -0.011443453579183704, + "StackExchange|poetry": -0.0020479014805762363, + "StackExchange|typo": 0.02005245659805873, + "StackExchange|association": -0.0034777270563830995, + "USPTO_Backgrounds|multihop": -0.0025899261035213714, + "USPTO_Backgrounds|multilingual": -0.001735541362488588, + "USPTO_Backgrounds|order-ops": -0.00865595611139642, + "USPTO_Backgrounds|poetry": -0.005901435903679704, + "USPTO_Backgrounds|typo": 0.025743637992440817, + "USPTO_Backgrounds|association": -0.006860778511354727, + "Wikipedia_en|multihop": 0.00813536363549186, + "Wikipedia_en|multilingual": -0.0033001598797979745, + "Wikipedia_en|order-ops": 0.009058284401411647, + "Wikipedia_en|poetry": 0.00019559103773349185, + "Wikipedia_en|typo": -0.016756868772135947, + "Wikipedia_en|association": 0.002667789577296943 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06490812752157134, + "null2_within_cell_mean": 0.022123878177645398, + "null2_max": 0.06493063958182166, + "n_perm": 2000, + "n_null_ge_observed": 1, + "p_value": 0.0009995002498750624, + "observed": 0.06490812752157134, + "fires": null + }, + "C4_F_test": { + "F": 613.2011073426562, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + }, + "1b": { + "admitted5": { + "grand_mean": 0.04294701039968502, + "SS_total": 0.5877733083330603, + "frac_set_main": 0.8857950497345805, + "frac_corpus_main": 0.022117202251311613, + "frac_interaction": 0.09158325214755474, + "frac_residual_seed": 0.0005044958665530746, + "SS_set": 0.520646686887542, + "SS_corpus": 0.012999901138324835, + "SS_interaction": 0.053830191102669095, + "SS_residual_seed": 0.00029652920452425476, + "interaction_over_noise_floor": 181.5341972438537, + "candidate_noise_floors": { + "seed_x_set": 701.2113462326924, + "seed_x_corpus": 1300.55822013104, + "seed_main": 3666.99058665031, + "three_way_residual": 328.85037669047756, + "full_two_factor_residual": 181.5341972438537, + "mean_squares_F": 567.2943663870428 + }, + "SS_seed_main": 1.4679664381642555e-05, + "SS_seed_x_set": 7.676742738387734e-05, + "SS_seed_x_corpus": 4.139006641105643e-05, + "SS_three_way_residual": 0.00016369204634767822, + "interaction_over_corpus_main": 4.1408154208167804, + "per_corpus_main": { + "Github": -0.015690614938115084, + "Pile-CC": 0.0030669355334802736, + "StackExchange": 0.015805140123480846, + "USPTO_Backgrounds": 0.011754592504973217, + "Wikipedia_en": -0.014936053223819253 + }, + "per_set_main": { + "multihop": -0.04065852216755351, + "multilingual": -0.04060835923999548, + "order-ops": -0.04294701039968502, + "poetry": -0.04241251089654508, + "typo": 0.1666264027037791 + }, + "interaction_cells": { + "Github|multihop": 0.01617605183583995, + "Github|multilingual": 0.01740079140379315, + "Github|order-ops": 0.015690614938115084, + "Github|poetry": 0.015399069754584209, + "Github|typo": -0.06466652793233241, + "Pile-CC|multihop": -0.005239843551867775, + "Pile-CC|multilingual": -0.003502033432679519, + "Pile-CC|order-ops": -0.0030669355334802736, + "Pile-CC|poetry": -0.003601435036620214, + "Pile-CC|typo": 0.01541024755464776, + "StackExchange|multihop": -0.01531970322575598, + "StackExchange|multilingual": -0.015847441318134474, + "StackExchange|order-ops": -0.015805140123480846, + "StackExchange|poetry": -0.016339639626620786, + "StackExchange|typo": 0.06331192429399204, + "USPTO_Backgrounds|multihop": -0.009882193042320159, + "USPTO_Backgrounds|multilingual": -0.012975283813263697, + "USPTO_Backgrounds|order-ops": -0.011754592504973217, + "USPTO_Backgrounds|poetry": -0.012289092008113157, + "USPTO_Backgrounds|typo": 0.046901161368670195, + "Wikipedia_en|multihop": 0.014265687984103964, + "Wikipedia_en|multilingual": 0.014923967160284518, + "Wikipedia_en|order-ops": 0.014936053223819253, + "Wikipedia_en|poetry": 0.01683109691676994, + "Wikipedia_en|typo": -0.06095680528497768 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.09158325214755474, + "null2_within_cell_mean": 0.030773442912407283, + "null2_max": 0.09038351687329153, + "n_perm": 2000, + "n_null_ge_observed": 0, + "p_value": 0.0004997501249375312, + "observed": 0.09158325214755474, + "fires": null + }, + "C4_F_test": { + "F": 567.2943663870428, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03645773986613171, + "SS_total": 0.6067938204847383, + "frac_set_main": 0.8892583495369545, + "frac_corpus_main": 0.018617391676965456, + "frac_interaction": 0.0916251460361479, + "frac_residual_seed": 0.000499112749932075, + "SS_set": 0.5395964713134814, + "SS_corpus": 0.011296918223126638, + "SS_interaction": 0.055597572415746255, + "SS_residual_seed": 0.0003028585323839276, + "interaction_over_noise_floor": 183.5760477940451, + "candidate_noise_floors": { + "seed_x_set": 696.3305263109522, + "seed_x_corpus": 1715.3652591502535, + "seed_main": 4592.651780597178, + "three_way_residual": 311.4751704961482, + "full_two_factor_residual": 183.5760477940451, + "mean_squares_F": 550.7281433821353 + }, + "SS_seed_main": 1.2105767010386526e-05, + "SS_seed_x_set": 7.984365228147229e-05, + "SS_seed_x_corpus": 3.241150659847677e-05, + "SS_three_way_residual": 0.00017849760649359299, + "interaction_over_corpus_main": 4.921481356032917, + "per_corpus_main": { + "Github": -0.013744076981490095, + "Pile-CC": 0.002771445592894911, + "StackExchange": 0.013257216481447578, + "USPTO_Backgrounds": 0.01009742612987461, + "Wikipedia_en": -0.012382011222727005 + }, + "per_set_main": { + "multihop": -0.0341692516340002, + "multilingual": -0.034119088706442174, + "order-ops": -0.03645773986613171, + "poetry": -0.03592324036299177, + "typo": 0.1731156732373324, + "association": -0.03244635266776655 + }, + "interaction_cells": { + "Github|multihop": 0.014229513879214964, + "Github|multilingual": 0.015454253447168165, + "Github|order-ops": 0.013744076981490091, + "Github|poetry": 0.013452531797959216, + "Github|typo": -0.06661306588895739, + "Github|association": 0.009732689783124929, + "Pile-CC|multihop": -0.004944353611282412, + "Pile-CC|multilingual": -0.0032065434920941494, + "Pile-CC|order-ops": -0.002771445592894911, + "Pile-CC|poetry": -0.0033059450960348513, + "Pile-CC|typo": 0.01570573749523313, + "Pile-CC|association": -0.0014774497029268058, + "StackExchange|multihop": -0.012771779583722713, + "StackExchange|multilingual": -0.0132995176761012, + "StackExchange|order-ops": -0.013257216481447578, + "StackExchange|poetry": -0.013791715984587519, + "StackExchange|typo": 0.06585984793602533, + "StackExchange|association": -0.012739618210166344, + "USPTO_Backgrounds|multihop": -0.008225026667221552, + "USPTO_Backgrounds|multilingual": -0.011318117438165083, + "USPTO_Backgrounds|order-ops": -0.01009742612987461, + "USPTO_Backgrounds|poetry": -0.010631925633014551, + "USPTO_Backgrounds|typo": 0.04855832774376881, + "USPTO_Backgrounds|association": -0.008285831875493012, + "Wikipedia_en|multihop": 0.01171164598301172, + "Wikipedia_en|multilingual": 0.012369925159192274, + "Wikipedia_en|order-ops": 0.012382011222727002, + "Wikipedia_en|poetry": 0.014277054915677698, + "Wikipedia_en|typo": -0.06351084728606993, + "Wikipedia_en|association": 0.012770210005461219 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.0916251460361479, + "null2_within_cell_mean": 0.0305718474037837, + "null2_max": 0.09044355107395313, + "n_perm": 2000, + "n_null_ge_observed": 0, + "p_value": 0.0004997501249375312, + "observed": 0.0916251460361479, + "fires": null + }, + "C4_F_test": { + "F": 550.7281433821353, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + } + }, + "ladder1b_dir": "results/ladder1b", + "ladder410_dir": "results/ladder410", + "reproduces_the_prose": { + "410m": { + "claimed": { + "interaction": 6.8, + "corpus_main": 1.9, + "set_main": 91.3, + "noise_floor": 4614 + }, + "recomputed": { + "interaction": 6.780626428099203, + "corpus_main": 1.9064501148312791, + "set_main": 91.27829399393488, + "noise_floor": 195.8051270311163 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + }, + "1b": { + "claimed": { + "interaction": 9.2, + "corpus_main": 2.2, + "set_main": 88.6 + }, + "recomputed": { + "interaction": 9.158325214755473, + "corpus_main": 2.2117202251311614, + "set_main": 88.57950497345804, + "noise_floor": 181.5341972438537 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + } + }, + "noise_floor_4614_audit": { + "claimed": 4614, + "claimed_where": "PLAN.tex:52 and :270, HANDOFF.md:63 \u2014 described as 'the seed x set noise floor'. In no results file.", + "candidates_computed": { + "seed_x_set": 2768.1445461585845, + "seed_x_corpus": 2547.878926111827, + "seed_main": 10268.19304676178, + "three_way_residual": 234.96272012579493, + "full_two_factor_residual": 195.8051270311163, + "mean_squares_F": 611.8910219722385 + }, + "any_candidate_within_2pct": {}, + "reproduces": false, + "reading": "The six VARIANCE FRACTIONS reproduce exactly at both scales, so the decomposition specified in this file is the one that was used. The 4614x figure does NOT follow from it under any of six natural denominators. Cite a named, stored ratio from candidate_noise_floors instead, or drop the multiplier \u2014 do not restate 4614x." + }, + "VERDICT": "REPRODUCES \u2014 the headline interaction numbers now have a results file and match the prose to within 0.1 percentage points. || BUT the '4614x noise floor' does NOT reproduce under any natural denominator (nearest candidates: seed x set 2768x, seed x corpus 2548x, seed main 10268x, three-way residual 235x). The multiplier is unsupported and must not be restated until the operator names the intended denominator.", + "provenance": { + "schema_version": 1, + "experiment": "E51", + "utc": "2026-08-31T23:20:05+00:00", + "script": { + "path": "experiments/t51_interaction_variance.py", + "sha256": "715aa2606a7552e80ddde57bcfd08173971070d0b74f903a5da83176a07e6ee2", + "n_lines": 327 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/t51_interaction_variance.py", + "--agg", + "persist", + "--n-min", + "75", + "--n-perm", + "2000" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "147651d805a99a12b4309de6b719b3b347cdf90930e956eda45c026235a5cfe6" + } +} \ No newline at end of file diff --git a/results/e52_factorial_410m.json b/results/e52_factorial_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..d5a44626de6dbad90988a833aa8ca79c07f3ca29 --- /dev/null +++ b/results/e52_factorial_410m.json @@ -0,0 +1,1023 @@ +{ + "experiment": "E52 \u2014 the fit x read factorial: J^Q read on Q, the missing cell", + "prereg": "pythia/specs/PREREG_E52_FACTORIAL.md, written before this ran", + "status": "PRE-REGISTERED", + "question": "can a J-lens trained on an out-of-distribution sample read latent structure on an out-of-distribution corpus?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_operators": 24, + "n_items": 541, + "n_pairs": 893, + "prefix_tokens": 128, + "smoke": false, + "all_operators_share_one_fitter": "trainval.py N=200 --band 9,21 dim_batch=128 max_seq_len=128 skip_first=16 target_layer=-2 fp32", + "adjudication": { + "aggregation": "persist", + "min_does_not_vote": true, + "D": 0.0010325341964960988, + "D_without_degenerate": 0.0009875729428136123, + "D_hierarchical_ci": { + "point": 0.001032533822581172, + "ci_lo": 1.4873118743707892e-05, + "ci_hi": 0.002772060688585043, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "degenerate_excluded_from_C2": [ + "Github" + ] + }, + "by_aggregation": { + "persist": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.040705389040522275, + "Wikipedia_en|USPTO_Backgrounds": 0.04152529486972425, + "Wikipedia_en|Pile-CC": 0.040249800131035345, + "Wikipedia_en|StackExchange": 0.0339988582327755, + "Wikipedia_en|Github": 0.03484094970497406, + "Wikipedia_en|OOD_News_2024": 0.04145839762253066, + "Wikipedia_en|OOD_arXiv_2023": 0.03625677386040075, + "Wikipedia_en|OOD_CommonPile": 0.03785195941002004, + "USPTO_Backgrounds|Wikipedia_en": 0.04909250016013781, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.049914510855968625, + "USPTO_Backgrounds|Pile-CC": 0.0489167712835802, + "USPTO_Backgrounds|StackExchange": 0.04372346260740111, + "USPTO_Backgrounds|Github": 0.045737664468793404, + "USPTO_Backgrounds|OOD_News_2024": 0.048868775621263516, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.046633168247838815, + "USPTO_Backgrounds|OOD_CommonPile": 0.04696154247762428, + "Pile-CC|Wikipedia_en": 0.049671283695432875, + "Pile-CC|USPTO_Backgrounds": 0.04903185960008866, + "Pile-CC|Pile-CC": 0.049649708304140304, + "Pile-CC|StackExchange": 0.04309877515511794, + "Pile-CC|Github": 0.04413961774876548, + "Pile-CC|OOD_News_2024": 0.049862991304447254, + "Pile-CC|OOD_arXiv_2023": 0.04598900407242278, + "Pile-CC|OOD_CommonPile": 0.04587176217594081, + "StackExchange|Wikipedia_en": 0.04540587743330333, + "StackExchange|USPTO_Backgrounds": 0.048255354476471744, + "StackExchange|Pile-CC": 0.04563351409613258, + "StackExchange|StackExchange": 0.042693324909648965, + "StackExchange|Github": 0.04448954296919207, + "StackExchange|OOD_News_2024": 0.045542550169759326, + "StackExchange|OOD_arXiv_2023": 0.04429536027730339, + "StackExchange|OOD_CommonPile": 0.04517444061736266, + "Github|Wikipedia_en": 0.026004515438237123, + "Github|USPTO_Backgrounds": 0.026875987550657655, + "Github|Pile-CC": 0.026034934611784088, + "Github|StackExchange": 0.021615857266200085, + "Github|Github": 0.02419525204329855, + "Github|OOD_News_2024": 0.0259296004091286, + "Github|OOD_arXiv_2023": 0.02626178347402149, + "Github|OOD_CommonPile": 0.02639343175623152, + "OOD_News_2024|Wikipedia_en": 0.038795182419319944, + "OOD_News_2024|USPTO_Backgrounds": 0.04006493637700462, + "OOD_News_2024|Pile-CC": 0.03854981670673523, + "OOD_News_2024|StackExchange": 0.03337902032718476, + "OOD_News_2024|Github": 0.033795293430901235, + "OOD_News_2024|OOD_News_2024": 0.040212060573200384, + "OOD_News_2024|OOD_arXiv_2023": 0.035892593482923174, + "OOD_News_2024|OOD_CommonPile": 0.034603384209589826, + "OOD_arXiv_2023|Wikipedia_en": 0.04051736176220907, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.04081738338702255, + "OOD_arXiv_2023|Pile-CC": 0.04089874563376523, + "OOD_arXiv_2023|StackExchange": 0.037503251353175275, + "OOD_arXiv_2023|Github": 0.03651138133265906, + "OOD_arXiv_2023|OOD_News_2024": 0.0401454847254273, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.03884629919969787, + "OOD_arXiv_2023|OOD_CommonPile": 0.040131539209849305, + "OOD_CommonPile|Wikipedia_en": 0.037925809619223906, + "OOD_CommonPile|USPTO_Backgrounds": 0.039744688337668775, + "OOD_CommonPile|Pile-CC": 0.039942382951267064, + "OOD_CommonPile|StackExchange": 0.03600302289447023, + "OOD_CommonPile|Github": 0.035526463063433766, + "OOD_CommonPile|OOD_News_2024": 0.03911558420966483, + "OOD_CommonPile|OOD_arXiv_2023": 0.037926333162209225, + "OOD_CommonPile|OOD_CommonPile": 0.039112817444321185 + }, + "all8": { + "D": 0.0010325341964960988, + "diag_mean": 0.0009034674219340851, + "offdiag_mean": -0.00012906677456201365, + "a_fit": { + "Wikipedia_en": -0.0014017750154178207, + "USPTO_Backgrounds": 0.007718346590910287, + "Pile-CC": 0.007401672382628829, + "StackExchange": 0.005423542744231076, + "Github": -0.014348782555720793, + "OOD_News_2024": -0.0028511669335582884, + "OOD_arXiv_2023": -0.0003412720489399737, + "OOD_CommonPile": -0.0016005651641333093 + }, + "b_read": { + "Wikipedia_en": 0.001252037071632607, + "USPTO_Backgrounds": 0.00226604905741018, + "Pile-CC": 0.0014717563403893186, + "StackExchange": -0.003260756281168946, + "Github": -0.0023581822791634774, + "OOD_News_2024": 0.0016292277050120527, + "OOD_arXiv_2023": -0.0007500384023134965, + "OOD_CommonPile": -0.0002500932117982316 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0010924241098918056, + "Wikipedia_en|USPTO_Backgrounds": 0.0008983179533162064, + "Wikipedia_en|Pile-CC": 0.000417115931648164, + "Wikipedia_en|StackExchange": -0.001101313345053416, + "Wikipedia_en|Github": -0.0011617958748603233, + "Wikipedia_en|OOD_News_2024": 0.0014682420585207454, + "Wikipedia_en|OOD_arXiv_2023": -0.0013541155962836157, + "Wikipedia_en|OOD_CommonPile": -0.00025887523717958716, + "USPTO_Backgrounds|Wikipedia_en": 0.0003594136231792333, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.00016741233323247506, + "USPTO_Backgrounds|Pile-CC": -3.60345221350894e-05, + "USPTO_Backgrounds|StackExchange": -0.0004968305767559106, + "USPTO_Backgrounds|Github": 0.000614797282630912, + "USPTO_Backgrounds|OOD_News_2024": -0.00024150154907450622, + "USPTO_Backgrounds|OOD_arXiv_2023": -9.78428151736585e-05, + "USPTO_Backgrounds|OOD_CommonPile": -0.00026941377590345567, + "Pile-CC|Wikipedia_en": 0.0012548713667557554, + "Pile-CC|USPTO_Backgrounds": -0.0003985647143660287, + "Pile-CC|Pile-CC": 0.0010135767067064733, + "Pile-CC|StackExchange": -0.0008048438207576281, + "Pile-CC|Github": -0.0006665752291155555, + "Pile-CC|OOD_News_2024": 0.0010693883423906897, + "Pile-CC|OOD_arXiv_2023": -0.00042533278230823257, + "Pile-CC|OOD_CommonPile": -0.0010425198693054735, + "StackExchange|Wikipedia_en": -0.0010324052569760378, + "StackExchange|USPTO_Backgrounds": 0.0008030598004148057, + "StackExchange|Pile-CC": -0.0010244878629034962, + "StackExchange|StackExchange": 0.0007678355721711522, + "StackExchange|Github": 0.0016614796297087855, + "StackExchange|OOD_News_2024": -0.0012729231538994856, + "StackExchange|OOD_arXiv_2023": -0.00014084693902987472, + "StackExchange|OOD_CommonPile": 0.00023828821051413013, + "Github|Wikipedia_en": -0.000661441952090374, + "Github|USPTO_Backgrounds": -0.000803981825447414, + "Github|Pile-CC": -0.0008507420473001202, + "Github|StackExchange": -0.0005373067713258586, + "Github|Github": 0.001139514003767137, + "Github|OOD_News_2024": -0.0011135476145783425, + "Github|OOD_arXiv_2023": 0.001597901557640096, + "Github|OOD_CommonPile": 0.0012296046493348624, + "OOD_News_2024|Wikipedia_en": 0.000631609406829943, + "OOD_News_2024|USPTO_Backgrounds": 0.0008873513787370432, + "OOD_News_2024|Pile-CC": 0.0001665244254885137, + "OOD_News_2024|StackExchange": -0.00027175933250368767, + "OOD_News_2024|Github": -0.0007580602307926815, + "OOD_News_2024|OOD_News_2024": 0.001671296927330937, + "OOD_News_2024|OOD_arXiv_2023": -0.00026890405562072367, + "OOD_News_2024|OOD_CommonPile": -0.002058058519469337, + "OOD_arXiv_2023|Wikipedia_en": -0.00015610613489924763, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0008700964958633373, + "OOD_arXiv_2023|Pile-CC": 5.558467900200481e-06, + "OOD_arXiv_2023|StackExchange": 0.001342576808868512, + "OOD_arXiv_2023|Github": -0.0005518672136531691, + "OOD_arXiv_2023|OOD_News_2024": -0.0009051738050604649, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00017490677653565767, + "OOD_arXiv_2023|OOD_CommonPile": 0.000960201596171828, + "OOD_CommonPile|Wikipedia_en": -0.0014883651626910743, + "OOD_CommonPile|USPTO_Backgrounds": -0.0006834984300237781, + "OOD_CommonPile|Pile-CC": 0.0003084889005953717, + "OOD_CommonPile|StackExchange": 0.0011016414653568021, + "OOD_CommonPile|Github": -0.0002774923676851296, + "OOD_CommonPile|OOD_News_2024": -0.000675781205629597, + "OOD_CommonPile|OOD_arXiv_2023": 0.000514233854240348, + "OOD_CommonPile|OOD_CommonPile": 0.0012007729458370434 + } + }, + "without_degenerate": { + "D": 0.0009875729428136123, + "diag_mean": 0.0008464910938402409, + "offdiag_mean": -0.00014108184897337135, + "a_fit": { + "Wikipedia_en": -0.0033088856433727346, + "USPTO_Backgrounds": 0.005557436940456628, + "Pile-CC": 0.005423815948138955, + "StackExchange": 0.00311310704419486, + "OOD_News_2024": -0.004815954082094287, + "OOD_arXiv_2023": -0.002335515342781619, + "OOD_CommonPile": -0.0036340048645418163 + }, + "b_read": { + "Wikipedia_en": 0.0009863902085044673, + "USPTO_Backgrounds": 0.0020207650333330385, + "Pile-CC": 0.0012331523480054313, + "StackExchange": -0.0035441367415491656, + "OOD_News_2024": 0.001428167365096475, + "OOD_arXiv_2023": -0.0013384486239745655, + "OOD_CommonPile": -0.0007858895894156948 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0008552169510165492, + "Wikipedia_en|USPTO_Backgrounds": 0.0006407479553899517, + "Wikipedia_en|Pile-CC": 0.0001528659020286552, + "Wikipedia_en|StackExchange": -0.0013207869066765923, + "Wikipedia_en|OOD_News_2024": 0.001166448376432927, + "Wikipedia_en|OOD_arXiv_2023": -0.0012685593966259429, + "Wikipedia_en|OOD_CommonPile": -0.0002259328815655201, + "USPTO_Backgrounds|Wikipedia_en": 0.00037600548680272183, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0001636413578049653, + "USPTO_Backgrounds|Pile-CC": -4.648552925585331e-05, + "USPTO_Backgrounds|StackExchange": -0.000462505115880342, + "USPTO_Backgrounds|OOD_News_2024": -0.00028949620866357967, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.00024151240698275928, + "USPTO_Backgrounds|OOD_CommonPile": 1.7327602209356308e-05, + "Pile-CC|Wikipedia_en": 0.0010884100144154599, + "Pile-CC|USPTO_Backgrounds": -0.0005853889057573225, + "Pile-CC|Pile-CC": 0.0008200724836219253, + "Pile-CC|StackExchange": -0.0009535715758458435, + "Pile-CC|OOD_News_2024": 0.0008383404668378322, + "Pile-CC|OOD_arXiv_2023": -0.00026903077611559884, + "Pile-CC|OOD_CommonPile": -0.0009388317071564456, + "StackExchange|Wikipedia_en": -0.000866287343769992, + "StackExchange|USPTO_Backgrounds": 0.0009488148745698533, + "StackExchange|Pile-CC": -0.0008854128204417028, + "StackExchange|StackExchange": 0.0009516870826292781, + "StackExchange|OOD_News_2024": -0.0011713917639060017, + "StackExchange|OOD_arXiv_2023": 0.0003480343327091004, + "StackExchange|OOD_CommonPile": 0.0006745556382094994, + "OOD_News_2024|Wikipedia_en": 0.00045207876853577134, + "OOD_News_2024|USPTO_Backgrounds": 0.0006874579013918733, + "OOD_News_2024|Pile-CC": -4.004908354991038e-05, + "OOD_News_2024|StackExchange": -0.00043355637354577925, + "OOD_News_2024|OOD_News_2024": 0.0014271797658242033, + "OOD_News_2024|OOD_arXiv_2023": -0.00012567133538196607, + "OOD_News_2024|OOD_CommonPile": -0.0019674396432741853, + "OOD_arXiv_2023|Wikipedia_en": -0.00030618062788777267, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0010405338279028606, + "OOD_arXiv_2023|Pile-CC": -0.000171558895832577, + "OOD_arXiv_2023|StackExchange": 0.001210235913132067, + "OOD_arXiv_2023|OOD_News_2024": -0.001119834821261552, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0003475956420800619, + "OOD_arXiv_2023|OOD_CommonPile": 0.0010802766176726264, + "OOD_CommonPile|Wikipedia_en": -0.0015992432491127376, + "OOD_CommonPile|USPTO_Backgrounds": -0.0008147393554964397, + "OOD_CommonPile|Pile-CC": 0.00017056794342945603, + "OOD_CommonPile|StackExchange": 0.001008496976187219, + "OOD_CommonPile|OOD_News_2024": -0.0008512458152638222, + "OOD_CommonPile|OOD_arXiv_2023": 0.000726119126351614, + "OOD_CommonPile|OOD_CommonPile": 0.0013600443739047036 + } + }, + "C4_permutation": { + "null_mean": -1.342009317004193e-05, + "null_sd": 0.0003901576568318512, + "n_abs_ge_observed": 10, + "n_perm": 2000, + "p_two_sided": 0.005497251374312844 + }, + "C4_permutation_without_degenerate": { + "null_mean": -1.0247287295497412e-06, + "null_sd": 0.0004117535650963638, + "n_abs_ge_observed": 25, + "n_perm": 2000, + "p_two_sided": 0.012993503248375811 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 0.00013936889285791815, + "surrogate_D_without_degenerate": 1.0770963581868456e-05, + "D_minus_surrogate_all8": 0.0008931653036381806, + "D_minus_surrogate_without_degenerate": 0.0009768019792317439, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + }, + "min": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.13328435383737086, + "Wikipedia_en|USPTO_Backgrounds": 0.13784370236098767, + "Wikipedia_en|Pile-CC": 0.1383333519515064, + "Wikipedia_en|StackExchange": 0.11953544283492698, + "Wikipedia_en|Github": 0.11716976339618365, + "Wikipedia_en|OOD_News_2024": 0.13719095596008832, + "Wikipedia_en|OOD_arXiv_2023": 0.13225573628313012, + "Wikipedia_en|OOD_CommonPile": 0.12354723012281789, + "USPTO_Backgrounds|Wikipedia_en": 0.1589013874116871, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.16444444126553007, + "USPTO_Backgrounds|Pile-CC": 0.16392662384443812, + "USPTO_Backgrounds|StackExchange": 0.1540478135355645, + "USPTO_Backgrounds|Github": 0.14482734395811955, + "USPTO_Backgrounds|OOD_News_2024": 0.16169015301598444, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.15767217957311205, + "USPTO_Backgrounds|OOD_CommonPile": 0.1438613244228893, + "Pile-CC|Wikipedia_en": 0.1456771372920937, + "Pile-CC|USPTO_Backgrounds": 0.15375699135992263, + "Pile-CC|Pile-CC": 0.14843652219408088, + "Pile-CC|StackExchange": 0.13440058796356122, + "Pile-CC|Github": 0.1325610138475895, + "Pile-CC|OOD_News_2024": 0.1505866065621376, + "Pile-CC|OOD_arXiv_2023": 0.14537321800986927, + "Pile-CC|OOD_CommonPile": 0.14141270303063921, + "StackExchange|Wikipedia_en": 0.14468121946685844, + "StackExchange|USPTO_Backgrounds": 0.15529588140133355, + "StackExchange|Pile-CC": 0.14998613929169047, + "StackExchange|StackExchange": 0.13777427354620564, + "StackExchange|Github": 0.13621440918909178, + "StackExchange|OOD_News_2024": 0.14825708121061326, + "StackExchange|OOD_arXiv_2023": 0.15157426219019626, + "StackExchange|OOD_CommonPile": 0.14583224952220916, + "Github|Wikipedia_en": 0.108150950488117, + "Github|USPTO_Backgrounds": 0.12000470521549383, + "Github|Pile-CC": 0.11796630918979645, + "Github|StackExchange": 0.10981142164932357, + "Github|Github": 0.10700858065651522, + "Github|OOD_News_2024": 0.11542985724906127, + "Github|OOD_arXiv_2023": 0.11210602997905678, + "Github|OOD_CommonPile": 0.11318731150693363, + "OOD_News_2024|Wikipedia_en": 0.1364180642697546, + "OOD_News_2024|USPTO_Backgrounds": 0.14418143530686697, + "OOD_News_2024|Pile-CC": 0.14283755603763792, + "OOD_News_2024|StackExchange": 0.12079984980324904, + "OOD_News_2024|Github": 0.12478081385294597, + "OOD_News_2024|OOD_News_2024": 0.14264462573660744, + "OOD_News_2024|OOD_arXiv_2023": 0.13951912265684868, + "OOD_News_2024|OOD_CommonPile": 0.13029280362857712, + "OOD_arXiv_2023|Wikipedia_en": 0.14334331928855842, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.149317937820322, + "OOD_arXiv_2023|Pile-CC": 0.14782937183562253, + "OOD_arXiv_2023|StackExchange": 0.13626646215303076, + "OOD_arXiv_2023|Github": 0.1268347776805361, + "OOD_arXiv_2023|OOD_News_2024": 0.14620044955776798, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.14460420372585456, + "OOD_arXiv_2023|OOD_CommonPile": 0.12925454620271923, + "OOD_CommonPile|Wikipedia_en": 0.14085149384207196, + "OOD_CommonPile|USPTO_Backgrounds": 0.14806802579098277, + "OOD_CommonPile|Pile-CC": 0.1476166748131315, + "OOD_CommonPile|StackExchange": 0.13746774678842888, + "OOD_CommonPile|Github": 0.1275541218618552, + "OOD_CommonPile|OOD_News_2024": 0.14699387763523392, + "OOD_CommonPile|OOD_arXiv_2023": 0.14019710266341764, + "OOD_CommonPile|OOD_CommonPile": 0.13042868367499774 + }, + "all8": { + "D": 0.0005479490122063032, + "diag_mean": 0.0004794553856805085, + "offdiag_mean": -6.849362652579462e-05, + "a_fit": { + "Wikipedia_en": -0.008203688100588302, + "USPTO_Backgrounds": 0.018072653184450865, + "Pile-CC": 0.005926842338521976, + "StackExchange": 0.008103184283310055, + "Github": -0.025140609452177554, + "OOD_News_2024": -0.0029144712824037966, + "OOD_arXiv_2023": 0.002357628339086676, + "OOD_CommonPile": 0.0017984606898001632 + }, + "b_read": { + "Wikipedia_en": 0.0008147355430992431, + "USPTO_Backgrounds": 0.008515384871215143, + "Pile-CC": 0.006517813450773263, + "StackExchange": -0.006835805409678458, + "Github": -0.010979902138610165, + "OOD_News_2024": 0.005525445671971996, + "OOD_arXiv_2023": 0.0023139766912208815, + "OOD_CommonPile": -0.0058716486799918755 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002574551200895142, + "Wikipedia_en|USPTO_Backgrounds": -0.0005667496036039499, + "Wikipedia_en|Pile-CC": 0.0019204714073566553, + "Wikipedia_en|StackExchange": -0.0035238188487710376, + "Wikipedia_en|Github": -0.0017454015585826599, + "Wikipedia_en|OOD_News_2024": 0.0017704431947398458, + "Wikipedia_en|OOD_arXiv_2023": 4.66924985327577e-05, + "Wikipedia_en|OOD_CommonPile": -0.00047618829056671197, + "USPTO_Backgrounds|Wikipedia_en": 0.0019152434901722193, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.00024235198410071934, + "USPTO_Backgrounds|Pile-CC": 0.0012374020152492171, + "USPTO_Backgrounds|StackExchange": 0.004712210566827302, + "USPTO_Backgrounds|Github": -0.0003641622816859258, + "USPTO_Backgrounds|OOD_News_2024": -6.701034403200357e-06, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0008132054965244717, + "USPTO_Backgrounds|OOD_CommonPile": -0.0064384352755344765, + "Pile-CC|Wikipedia_en": 0.0008368042165076972, + "Pile-CC|USPTO_Backgrounds": 0.0012160089562207321, + "Pile-CC|Pile-CC": -0.00210688878917914, + "Pile-CC|StackExchange": -0.0027892041592470773, + "Pile-CC|Github": -0.0004846815462870857, + "Pile-CC|OOD_News_2024": 0.001035563357678848, + "Pile-CC|OOD_arXiv_2023": -0.0009663562138383652, + "Pile-CC|OOD_CommonPile": 0.003258754178144335, + "StackExchange|Wikipedia_en": -0.0023354555535156363, + "StackExchange|USPTO_Backgrounds": 0.0005785570528435702, + "StackExchange|Pile-CC": -0.002733613636357629, + "StackExchange|StackExchange": -0.0015918605213907366, + "StackExchange|Github": 0.0009923718504271095, + "StackExchange|OOD_News_2024": -0.0034703039386335677, + "StackExchange|OOD_arXiv_2023": 0.0030583460217005476, + "StackExchange|OOD_CommonPile": 0.005501958724926204, + "Github|Wikipedia_en": -0.005621930796769467, + "Github|USPTO_Backgrounds": -0.001468825397508533, + "Github|Pile-CC": -0.0015096500027640403, + "Github|StackExchange": 0.003689081317214807, + "Github|Github": 0.005030337053338163, + "Github|OOD_News_2024": -0.003053734164697952, + "Github|OOD_arXiv_2023": -0.00316609245395133, + "Github|OOD_CommonPile": 0.006100814445138283, + "OOD_News_2024|Wikipedia_en": 0.0004190448150943882, + "OOD_News_2024|USPTO_Backgrounds": 0.00048176652409084864, + "OOD_News_2024|Pile-CC": 0.00113545867530368, + "OOD_News_2024|StackExchange": -0.007548628698633486, + "OOD_News_2024|Github": 0.0005764320799951511, + "OOD_News_2024|OOD_News_2024": 0.0019348961530744613, + "OOD_News_2024|OOD_arXiv_2023": 0.0020208620540668187, + "OOD_News_2024|OOD_CommonPile": 0.0009801683970080133, + "OOD_arXiv_2023|Wikipedia_en": 0.0020722002124077266, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.00034616941605539187, + "OOD_arXiv_2023|Pile-CC": 0.000855174851797813, + "OOD_arXiv_2023|StackExchange": 0.0026458840296577635, + "OOD_arXiv_2023|Github": -0.0026417037139051802, + "OOD_arXiv_2023|OOD_News_2024": 0.00021862035274453073, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0018338435015822219, + "OOD_arXiv_2023|OOD_CommonPile": -0.005330188650340351, + "OOD_CommonPile|Wikipedia_en": 0.00013954241520777733, + "OOD_CommonPile|USPTO_Backgrounds": -0.0003445749639973128, + "OOD_CommonPile|Pile-CC": 0.001201645478593305, + "OOD_CommonPile|StackExchange": 0.004406336314342396, + "OOD_CommonPile|Github": -0.001363191883299586, + "OOD_CommonPile|OOD_News_2024": 0.0015712160794969787, + "OOD_CommonPile|OOD_arXiv_2023": -0.0020140899115681787, + "OOD_CommonPile|OOD_CommonPile": -0.0035968835287753242 + } + }, + "without_degenerate": { + "D": -0.00031888559115705905, + "diag_mean": -0.00027333050670604243, + "offdiag_mean": 4.555508445101665e-05, + "a_fit": { + "Wikipedia_en": -0.0116485205966802, + "USPTO_Backgrounds": 0.014430500791659417, + "Pile-CC": 0.002301906983530694, + "StackExchange": 0.004267241300216745, + "OOD_News_2024": -0.006690994298292535, + "OOD_arXiv_2023": -0.0009591609919591682, + "OOD_CommonPile": -0.0017009731884750356 + }, + "b_read": { + "Wikipedia_en": -5.334887417063339e-05, + "USPTO_Backgrounds": 0.007053999682622297, + "Pile-CC": 0.005062260348645459, + "StackExchange": -0.009034034414660358, + "OOD_News_2024": 0.004290476021549056, + "OOD_arXiv_2023": 0.0010950582249769913, + "OOD_CommonPile": -0.00841441098896284 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0016247350899946111, + "Wikipedia_en|USPTO_Backgrounds": -0.0009232649431815121, + "Wikipedia_en|Pile-CC": 0.0015581239813140524, + "Wikipedia_en|StackExchange": -0.0031434903719595453, + "Wikipedia_en|OOD_News_2024": 0.0011875123169923785, + "Wikipedia_en|OOD_arXiv_2023": -0.0005522895633937597, + "Wikipedia_en|OOD_CommonPile": 0.00024867349023384444, + "USPTO_Backgrounds|Wikipedia_en": 0.0011627472759712376, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.00040154742697873225, + "USPTO_Backgrounds|Pile-CC": 0.0010723744859061635, + "USPTO_Backgrounds|StackExchange": 0.005289858940338343, + "USPTO_Backgrounds|OOD_News_2024": -0.0003923120154511184, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0012148676617514398, + "USPTO_Backgrounds|OOD_CommonPile": -0.005516253598034371, + "Pile-CC|Wikipedia_en": 6.709096450654983e-05, + "Pile-CC|USPTO_Backgrounds": 0.0010395964755425535, + "Pile-CC|Pile-CC": -0.0022891333563223593, + "Pile-CC|StackExchange": -0.0022287728235362014, + "Pile-CC|OOD_News_2024": 0.0006327353388307644, + "Pile-CC|OOD_arXiv_2023": -0.001385235416865499, + "Pile-CC|OOD_CommonPile": 0.004163718817844275, + "StackExchange|Wikipedia_en": -0.0028941611774147558, + "StackExchange|USPTO_Backgrounds": 0.0006131522002674195, + "StackExchange|Pile-CC": -0.0027048505753988206, + "StackExchange|StackExchange": -0.0008204215575778329, + "StackExchange|OOD_News_2024": -0.0036621243293796235, + "StackExchange|OOD_arXiv_2023": 0.0028504744467754417, + "StackExchange|OOD_CommonPile": 0.0066179309927281715, + "OOD_News_2024|Wikipedia_en": -0.00019908077600930274, + "OOD_News_2024|USPTO_Backgrounds": 0.0004569417043101265, + "OOD_News_2024|Pile-CC": 0.001104801769057917, + "OOD_News_2024|StackExchange": -0.006836609702025154, + "OOD_News_2024|OOD_News_2024": 0.001683655795123834, + "OOD_News_2024|OOD_arXiv_2023": 0.0017535705119371414, + "OOD_News_2024|OOD_CommonPile": 0.0020367206976054097, + "OOD_arXiv_2023|Wikipedia_en": 0.000994340936461141, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.00013838908856822485, + "OOD_arXiv_2023|Pile-CC": 0.0003647842607091556, + "OOD_arXiv_2023|StackExchange": 0.0028981693414232013, + "OOD_arXiv_2023|OOD_News_2024": -0.0004923536900489911, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00110681827460965, + "OOD_arXiv_2023|OOD_CommonPile": -0.004733370034585849, + "OOD_CommonPile|Wikipedia_en": -0.0007556723135094534, + "OOD_CommonPile|USPTO_Backgrounds": -0.0006464889213915748, + "OOD_CommonPile|Pile-CC": 0.0008938994347340024, + "OOD_CommonPile|StackExchange": 0.004841266173337189, + "OOD_CommonPile|OOD_News_2024": 0.0010428865839328116, + "OOD_CommonPile|OOD_arXiv_2023": -0.002558470591311396, + "OOD_CommonPile|OOD_CommonPile": -0.0028174203657914676 + } + }, + "C4_permutation": { + "null_mean": -5.2312281163084104e-05, + "null_sd": 0.0011953980452625355, + "n_abs_ge_observed": 1327, + "n_perm": 2000, + "p_two_sided": 0.6636681659170415 + }, + "C4_permutation_without_degenerate": { + "null_mean": 7.965028577449722e-06, + "null_sd": 0.001228504235812649, + "n_abs_ge_observed": 1587, + "n_perm": 2000, + "p_two_sided": 0.7936031984007996 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 0.0003985587486577635, + "surrogate_D_without_degenerate": 6.983678305843262e-05, + "D_minus_surrogate_all8": 0.00014939026354853966, + "D_minus_surrogate_without_degenerate": -0.0003887223742154917, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + } + }, + "per_set_D": { + "multihop": { + "per_prefix_seed": [ + 0.00017336982585090612, + -0.00012383584648237696, + 0.000759527349146083 + ], + "mean": 0.00026968710950487073, + "sd": 0.00036700626533760965 + }, + "multilingual": { + "per_prefix_seed": [ + -0.00013165215828589031, + 0.0006280423341585058, + 0.0011913387237914971 + ], + "mean": 0.0005625762998880375, + "sd": 0.0005420889039922649 + }, + "order-ops": { + "per_prefix_seed": [ + -0.0002164501150088784, + 3.8651784832057145e-05, + 0.000139146571330327 + ], + "mean": -1.2883919615498084e-05, + "sd": 0.00014967564366218737 + }, + "poetry": { + "per_prefix_seed": [ + 5.206154725913491e-05, + -0.0002689851472886013, + 0.00020824664118533422 + ], + "mean": -2.892319614710725e-06, + "sd": 0.0001986663712455282 + }, + "typo": { + "per_prefix_seed": [ + 0.005978955621165889, + 0.0031976310004081044, + 0.0038619598905955043 + ], + "mean": 0.004346182170723166, + "sd": 0.0011859721958341132 + } + }, + "per_set_sign_split": { + "n_positive": 3, + "n_negative": 2, + "note": "if the sets split in sign, a null on the set-averaged D is cancellation, not absence of matching" + }, + "PARTIAL_UNBLINDING_DISCLOSED": "E36's stored ladder already contains a 5x5 in-stream fit x read matrix (E28 N=400 operators, same aggregation, same admitted sets). Applying this script's diag_excess to it gives D = +0.001214 with all five diagonal residuals positive and a permutation p of 0.134 \u2014 i.e. positive but NOT significant. That estimate was known before E52 ran and is recorded here so E52's primary is not presented as blind for the in-stream half. It also carries ~17% fit/prefix leakage (E28 fitted N=400 from the same 800-doc block), which is the confound Amendment 1 removes, so its sign is exactly what leakage predicts.", + "secondary_ood_rungs": { + "OOD_News_2024": { + "matched_J^q": 0.040212060573200384, + "logit": 0.03683846357744187, + "shuf_floor": 0.023494280191759268, + "best_instream_J^P": 0.049862991304447254, + "best_instream_corpus": "Pile-CC", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.003373596995758517, + "delta_vs_best_instream": -0.00965093073124687 + }, + "OOD_arXiv_2023": { + "matched_J^q": 0.03884629919969787, + "logit": 0.026718863868154585, + "shuf_floor": 0.02148574317495028, + "best_instream_J^P": 0.046633168247838815, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.012127435331543285, + "delta_vs_best_instream": -0.0077868690481409444 + }, + "OOD_CommonPile": { + "matched_J^q": 0.039112817444321185, + "logit": 0.03460996786598116, + "shuf_floor": 0.0209646623271207, + "best_instream_J^P": 0.04696154247762428, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.004502849578340025, + "delta_vs_best_instream": -0.007848725033303097 + } + }, + "controls": { + "C1_Q0_vs_e48": { + "per_corpus": { + "Wikipedia_en": { + "here": 0.03820955517391364, + "e48": 0.038209554925560954, + "abs_diff": 2.4835268563894175e-10 + }, + "USPTO_Backgrounds": { + "here": 0.04961537096338967, + "e48": 0.04961536889895797, + "abs_diff": 2.0644317019757885e-09 + }, + "Pile-CC": { + "here": 0.04546591031830758, + "e48": 0.04546590892132372, + "abs_diff": 1.3969838619232178e-09 + }, + "StackExchange": { + "here": 0.045729962767412265, + "e48": 0.04572996227070689, + "abs_diff": 4.967053712778835e-10 + }, + "Github": { + "here": 0.027860941054920357, + "e48": 0.027896882360801102, + "abs_diff": 3.594130588074548e-05 + }, + "OOD_News_2024": { + "here": 0.0337636109519129, + "e48": 0.03376360982656479, + "abs_diff": 1.1253481133066678e-09 + }, + "OOD_arXiv_2023": { + "here": 0.04105739652489623, + "e48": 0.04105739537626505, + "abs_diff": 1.148631176284276e-09 + }, + "OOD_CommonPile": { + "here": 0.03823967572922508, + "e48": 0.038239676505327225, + "abs_diff": 7.761021478258634e-10 + } + }, + "max_abs_diff": 3.594130588074548e-05, + "tolerance": 1e-06, + "note": "in-stream rungs are EXPECTED to differ: E48 scored the fastfit E28 lenses, this scores the matched-provenance INSTREAM refits. The OOD rungs use the SAME .pt files in both and must agree to fp32.", + "ood_max_abs_diff": 1.148631176284276e-09 + }, + "C1_fires": true, + "C2_shuf_floor_excl_degenerate": { + "below_floor": { + "Q0": [], + "Wikipedia_en": [], + "USPTO_Backgrounds": [], + "Pile-CC": [], + "StackExchange": [], + "Github": [], + "OOD_News_2024": [], + "OOD_arXiv_2023": [], + "OOD_CommonPile": [], + "SHUFFLED_Pile-CC": [] + }, + "fires": true + }, + "C3_capability": { + "q0_mean_rank": 4224.63045912654, + "per_rung": { + "Q0": 4224.63045912654, + "Wikipedia_en": 4229.167599850691, + "USPTO_Backgrounds": 4182.112355356477, + "Pile-CC": 4321.102276969018, + "StackExchange": 4709.87420679358, + "Github": 4838.026129152669, + "OOD_News_2024": 4429.988428518104, + "OOD_arXiv_2023": 4422.400522583053, + "OOD_CommonPile": 4499.4897349757375, + "SHUFFLED_Pile-CC": 4654.458753266144 + }, + "collapsed": [], + "fires": true + }, + "C5_self_distance": { + "value": 0.0, + "fires": true + } + }, + "VERDICT": "MATCHING CONFIRMED \u2014 the diagonal excess D = +0.00103 has a CI strictly above zero [+0.00001,+0.00277] AND exceeds the link-artifact null (+0.00014; +0.00001 without the degenerate operator). Fitting on the corpus you then read on buys something beyond operator quality, read difficulty, and the curvature of a bounded rate.", + "provenance": { + "schema_version": 1, + "experiment": "E52", + "utc": "2026-08-15T05:09:32+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "5420e3e62ccfdc0a1977c1aa8fd3e3741ae415b689e98f5870285f79d2c32ec1", + "n_lines": 540 + }, + "git": { + "commit": "e103aff3bc51d490d32e99927e5f952f7b3c76fa", + "commit_short": "e103aff3bc51", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "LAUDE.md", + "pythia/specs/HANDOFF.md", + "pythia/specs/PLAN.tex", + "pythia/specs/RESULTS.tex", + "pythia/specs/SPINE.tex" + ], + "dirty_diff_sha256": "905d4d636e37ceb5c1674d10af8e85ef27e601ab0620f20a5afea0cdd2f9eb32", + "describe": "e103aff-dirty" + }, + "argv": [ + "pythia/t52_factorial.py", + "--device", + "cpu", + "--prefix-seeds", + "3", + "--n-boot", + "4000", + "--n-perm", + "2000" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", + "bytes": 116036, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "de0fd855cefa636a23c70be9d4d9d62cf2ab53e550e92e61762bdcd52115ec9d", + "path_migration": { + "date": "2026-08-15", + "reason": "the programme directory `pythia/` was dissolved; its contents moved to the repo root", + "rule": "re-point provenance.script.path and provenance.inputs[].path at where each file lives now, resolving by unique basename; refuse if the basename is ambiguous", + "not_touched": [ + "provenance.argv", + "provenance.git.dirty_files", + "every field outside the provenance block (payload_sha256 must not move)" + ], + "script_path_before": "t52_factorial.py", + "script_path_after": "experiments/t52_factorial.py", + "script_sha256_before": "e049966955c3d77838771ab8c243523d31b6c1549dca749d033942b3e71418b2", + "script_content_changed": true, + "script_sha256_on_disk_now": "5420e3e62ccfdc0a1977c1aa8fd3e3741ae415b689e98f5870285f79d2c32ec1", + "restamp_required": "content differs; run --restamp-scripts , which proves the diff is path-only before recording it", + "script_restamps": [ + { + "sha256_before": "e049966955c3d77838771ab8c243523d31b6c1549dca749d033942b3e71418b2", + "sha256_after": "5420e3e62ccfdc0a1977c1aa8fd3e3741ae415b689e98f5870285f79d2c32ec1", + "verified": "every differing line vs the pre-move blob is a path line (contains a separator, a path-construction call, or a moved directory name)", + "baseline_commit": "88d3697" + } + ], + "inputs_refreshed": [ + { + "path": "results/e48_crossover_410m.json", + "sha256_before": "c0f9a47b8e2e1f8806e187f403caa108df4d2299656af2b2b9be05e976d29f8c", + "sha256_after": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", + "payload_sha256_of_input": "b316c1440c6b634cef4b1751e8d2d88dc256760208dd02fcd1d7289afd12b799", + "why": "file bytes moved (provenance re-pointed); the input's own payload hash still matches its content, so the science it carries is unchanged", + "pre_existing": false + } + ] + } + } +} \ No newline at end of file diff --git a/results/e52_factorial_410m_pooled.json b/results/e52_factorial_410m_pooled.json new file mode 100644 index 0000000000000000000000000000000000000000..4e3d5b7cf110fb568738d8f199cce53560d0d9f6 --- /dev/null +++ b/results/e52_factorial_410m_pooled.json @@ -0,0 +1,1137 @@ +{ + "experiment": "E52 \u2014 the fit x read factorial: J^Q read on Q, the missing cell", + "prereg": "specs/archive/prereg/PREREG_E52_FACTORIAL.md, written before this ran", + "prereg_files": [ + { + "path": "specs/archive/prereg/PREREG_E52_FACTORIAL.md", + "role": "E52, the original design", + "sha256": "740ddb1d6aeb35d8c0d93b626043e62c53d35d149ec9d6a2e7bb8817875902ce" + }, + { + "path": "specs/experiments/P0_t52_parallel_port.md", + "role": "P0, the pool port", + "sha256": "4aaa3dcf09c376b64f46f0e0edcb21f34174e92ac890c36a346f36cf812348a5" + }, + { + "path": "specs/experiments/R1_grid_rstrip.md", + "role": "R1, the readout correction", + "sha256": "5fdf52dfe227b329333bd3d385e0ecbdc0f9c0ee0eafe9fcfd8a23e297372789" + } + ], + "decision_rule_verbatim": { + "R1": "ACCEPT (headline robust): stripped fit_pct exceeds stripped read_pct under BOTH aggregations, and stripped fit_pct lies inside the published bootstrap interval [87.609, 92.171] under persist. | QUALIFIED (ordering survives, magnitude does not): stripped fit_pct still exceeds read_pct under both aggregations but falls outside the published interval. | REJECT (ordering does not survive): stripped fit_pct does not exceed read_pct under both aggregations. STOP. Alert the operator.", + "P0": "ACCEPT: the pooled run reproduces the serial run's stored matrix at max_abs_diff = 0.0 across all 128 cell values, and wallclock falls by at least 3x. | REJECT: any non-zero difference. STOP. Do not \"investigate the tolerance\" -- there is no tolerance." + }, + "readout_convention": "UNSTRIPPED (LEGACY) \u2014 readout at the trailing-space token, which does not occur in the tokenisation of prompt+target. Retained only to reproduce the published run; superseded by the stripped arm.", + "rstrip": false, + "workers": 3, + "wallclock_seconds": 3059.0, + "cell_completion_seconds": { + "Q0|p1": 143.6, + "Q0|p0": 143.7, + "Q0|p2": 143.7, + "Wikipedia_en|p0": 378.8, + "Wikipedia_en|p1": 382.3, + "Wikipedia_en|p2": 383.7, + "USPTO_Backgrounds|p0": 647.9, + "USPTO_Backgrounds|p1": 652.2, + "USPTO_Backgrounds|p2": 652.7, + "Pile-CC|p0": 879.0, + "Pile-CC|p1": 883.7, + "Pile-CC|p2": 884.9, + "StackExchange|p0": 1200.6, + "StackExchange|p1": 1207.8, + "StackExchange|p2": 1210.2, + "Github|p0": 1574.1, + "Github|p1": 1581.6, + "Github|p2": 1583.3, + "OOD_News_2024|p0": 1895.3, + "OOD_News_2024|p2": 1903.2, + "OOD_News_2024|p1": 1903.3, + "OOD_arXiv_2023|p0": 2155.5, + "OOD_arXiv_2023|p1": 2180.0, + "OOD_arXiv_2023|p2": 2202.4, + "OOD_CommonPile|p0": 2466.2, + "OOD_CommonPile|p1": 2503.9, + "OOD_CommonPile|p2": 2529.6, + "SHUFFLED_Pile-CC|p0": 3020.3, + "SHUFFLED_Pile-CC|p1": 3049.1, + "SHUFFLED_Pile-CC|p2": 3058.3 + }, + "status": "PRE-REGISTERED", + "question": "can a J-lens trained on an out-of-distribution sample read latent structure on an out-of-distribution corpus?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_operators": 24, + "n_items": 541, + "n_pairs": 893, + "prefix_tokens": 128, + "smoke": false, + "all_operators_share_one_fitter": "trainval.py N=200 --band 9,21 dim_batch=128 max_seq_len=128 skip_first=16 target_layer=-2 fp32", + "adjudication": { + "aggregation": "persist", + "min_does_not_vote": true, + "D": 0.0010325341964960988, + "D_without_degenerate": 0.0009875729428136123, + "D_hierarchical_ci": { + "point": 0.001032533822581172, + "ci_lo": 1.4873118743707892e-05, + "ci_hi": 0.002772060688585043, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "degenerate_excluded_from_C2": [ + "Github" + ] + }, + "by_aggregation": { + "persist": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.040705389040522275, + "Wikipedia_en|USPTO_Backgrounds": 0.04152529486972425, + "Wikipedia_en|Pile-CC": 0.040249800131035345, + "Wikipedia_en|StackExchange": 0.0339988582327755, + "Wikipedia_en|Github": 0.03484094970497406, + "Wikipedia_en|OOD_News_2024": 0.04145839762253066, + "Wikipedia_en|OOD_arXiv_2023": 0.03625677386040075, + "Wikipedia_en|OOD_CommonPile": 0.03785195941002004, + "USPTO_Backgrounds|Wikipedia_en": 0.04909250016013781, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.049914510855968625, + "USPTO_Backgrounds|Pile-CC": 0.0489167712835802, + "USPTO_Backgrounds|StackExchange": 0.04372346260740111, + "USPTO_Backgrounds|Github": 0.045737664468793404, + "USPTO_Backgrounds|OOD_News_2024": 0.048868775621263516, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.046633168247838815, + "USPTO_Backgrounds|OOD_CommonPile": 0.04696154247762428, + "Pile-CC|Wikipedia_en": 0.049671283695432875, + "Pile-CC|USPTO_Backgrounds": 0.04903185960008866, + "Pile-CC|Pile-CC": 0.049649708304140304, + "Pile-CC|StackExchange": 0.04309877515511794, + "Pile-CC|Github": 0.04413961774876548, + "Pile-CC|OOD_News_2024": 0.049862991304447254, + "Pile-CC|OOD_arXiv_2023": 0.04598900407242278, + "Pile-CC|OOD_CommonPile": 0.04587176217594081, + "StackExchange|Wikipedia_en": 0.04540587743330333, + "StackExchange|USPTO_Backgrounds": 0.048255354476471744, + "StackExchange|Pile-CC": 0.04563351409613258, + "StackExchange|StackExchange": 0.042693324909648965, + "StackExchange|Github": 0.04448954296919207, + "StackExchange|OOD_News_2024": 0.045542550169759326, + "StackExchange|OOD_arXiv_2023": 0.04429536027730339, + "StackExchange|OOD_CommonPile": 0.04517444061736266, + "Github|Wikipedia_en": 0.026004515438237123, + "Github|USPTO_Backgrounds": 0.026875987550657655, + "Github|Pile-CC": 0.026034934611784088, + "Github|StackExchange": 0.021615857266200085, + "Github|Github": 0.02419525204329855, + "Github|OOD_News_2024": 0.0259296004091286, + "Github|OOD_arXiv_2023": 0.02626178347402149, + "Github|OOD_CommonPile": 0.02639343175623152, + "OOD_News_2024|Wikipedia_en": 0.038795182419319944, + "OOD_News_2024|USPTO_Backgrounds": 0.04006493637700462, + "OOD_News_2024|Pile-CC": 0.03854981670673523, + "OOD_News_2024|StackExchange": 0.03337902032718476, + "OOD_News_2024|Github": 0.033795293430901235, + "OOD_News_2024|OOD_News_2024": 0.040212060573200384, + "OOD_News_2024|OOD_arXiv_2023": 0.035892593482923174, + "OOD_News_2024|OOD_CommonPile": 0.034603384209589826, + "OOD_arXiv_2023|Wikipedia_en": 0.04051736176220907, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.04081738338702255, + "OOD_arXiv_2023|Pile-CC": 0.04089874563376523, + "OOD_arXiv_2023|StackExchange": 0.037503251353175275, + "OOD_arXiv_2023|Github": 0.03651138133265906, + "OOD_arXiv_2023|OOD_News_2024": 0.0401454847254273, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.03884629919969787, + "OOD_arXiv_2023|OOD_CommonPile": 0.040131539209849305, + "OOD_CommonPile|Wikipedia_en": 0.037925809619223906, + "OOD_CommonPile|USPTO_Backgrounds": 0.039744688337668775, + "OOD_CommonPile|Pile-CC": 0.039942382951267064, + "OOD_CommonPile|StackExchange": 0.03600302289447023, + "OOD_CommonPile|Github": 0.035526463063433766, + "OOD_CommonPile|OOD_News_2024": 0.03911558420966483, + "OOD_CommonPile|OOD_arXiv_2023": 0.037926333162209225, + "OOD_CommonPile|OOD_CommonPile": 0.039112817444321185 + }, + "all8": { + "D": 0.0010325341964960988, + "diag_mean": 0.0009034674219340851, + "offdiag_mean": -0.00012906677456201365, + "a_fit": { + "Wikipedia_en": -0.0014017750154178207, + "USPTO_Backgrounds": 0.007718346590910287, + "Pile-CC": 0.007401672382628829, + "StackExchange": 0.005423542744231076, + "Github": -0.014348782555720793, + "OOD_News_2024": -0.0028511669335582884, + "OOD_arXiv_2023": -0.0003412720489399737, + "OOD_CommonPile": -0.0016005651641333093 + }, + "b_read": { + "Wikipedia_en": 0.001252037071632607, + "USPTO_Backgrounds": 0.00226604905741018, + "Pile-CC": 0.0014717563403893186, + "StackExchange": -0.003260756281168946, + "Github": -0.0023581822791634774, + "OOD_News_2024": 0.0016292277050120527, + "OOD_arXiv_2023": -0.0007500384023134965, + "OOD_CommonPile": -0.0002500932117982316 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0010924241098918056, + "Wikipedia_en|USPTO_Backgrounds": 0.0008983179533162064, + "Wikipedia_en|Pile-CC": 0.000417115931648164, + "Wikipedia_en|StackExchange": -0.001101313345053416, + "Wikipedia_en|Github": -0.0011617958748603233, + "Wikipedia_en|OOD_News_2024": 0.0014682420585207454, + "Wikipedia_en|OOD_arXiv_2023": -0.0013541155962836157, + "Wikipedia_en|OOD_CommonPile": -0.00025887523717958716, + "USPTO_Backgrounds|Wikipedia_en": 0.0003594136231792333, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.00016741233323247506, + "USPTO_Backgrounds|Pile-CC": -3.60345221350894e-05, + "USPTO_Backgrounds|StackExchange": -0.0004968305767559106, + "USPTO_Backgrounds|Github": 0.000614797282630912, + "USPTO_Backgrounds|OOD_News_2024": -0.00024150154907450622, + "USPTO_Backgrounds|OOD_arXiv_2023": -9.78428151736585e-05, + "USPTO_Backgrounds|OOD_CommonPile": -0.00026941377590345567, + "Pile-CC|Wikipedia_en": 0.0012548713667557554, + "Pile-CC|USPTO_Backgrounds": -0.0003985647143660287, + "Pile-CC|Pile-CC": 0.0010135767067064733, + "Pile-CC|StackExchange": -0.0008048438207576281, + "Pile-CC|Github": -0.0006665752291155555, + "Pile-CC|OOD_News_2024": 0.0010693883423906897, + "Pile-CC|OOD_arXiv_2023": -0.00042533278230823257, + "Pile-CC|OOD_CommonPile": -0.0010425198693054735, + "StackExchange|Wikipedia_en": -0.0010324052569760378, + "StackExchange|USPTO_Backgrounds": 0.0008030598004148057, + "StackExchange|Pile-CC": -0.0010244878629034962, + "StackExchange|StackExchange": 0.0007678355721711522, + "StackExchange|Github": 0.0016614796297087855, + "StackExchange|OOD_News_2024": -0.0012729231538994856, + "StackExchange|OOD_arXiv_2023": -0.00014084693902987472, + "StackExchange|OOD_CommonPile": 0.00023828821051413013, + "Github|Wikipedia_en": -0.000661441952090374, + "Github|USPTO_Backgrounds": -0.000803981825447414, + "Github|Pile-CC": -0.0008507420473001202, + "Github|StackExchange": -0.0005373067713258586, + "Github|Github": 0.001139514003767137, + "Github|OOD_News_2024": -0.0011135476145783425, + "Github|OOD_arXiv_2023": 0.001597901557640096, + "Github|OOD_CommonPile": 0.0012296046493348624, + "OOD_News_2024|Wikipedia_en": 0.000631609406829943, + "OOD_News_2024|USPTO_Backgrounds": 0.0008873513787370432, + "OOD_News_2024|Pile-CC": 0.0001665244254885137, + "OOD_News_2024|StackExchange": -0.00027175933250368767, + "OOD_News_2024|Github": -0.0007580602307926815, + "OOD_News_2024|OOD_News_2024": 0.001671296927330937, + "OOD_News_2024|OOD_arXiv_2023": -0.00026890405562072367, + "OOD_News_2024|OOD_CommonPile": -0.002058058519469337, + "OOD_arXiv_2023|Wikipedia_en": -0.00015610613489924763, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0008700964958633373, + "OOD_arXiv_2023|Pile-CC": 5.558467900200481e-06, + "OOD_arXiv_2023|StackExchange": 0.001342576808868512, + "OOD_arXiv_2023|Github": -0.0005518672136531691, + "OOD_arXiv_2023|OOD_News_2024": -0.0009051738050604649, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00017490677653565767, + "OOD_arXiv_2023|OOD_CommonPile": 0.000960201596171828, + "OOD_CommonPile|Wikipedia_en": -0.0014883651626910743, + "OOD_CommonPile|USPTO_Backgrounds": -0.0006834984300237781, + "OOD_CommonPile|Pile-CC": 0.0003084889005953717, + "OOD_CommonPile|StackExchange": 0.0011016414653568021, + "OOD_CommonPile|Github": -0.0002774923676851296, + "OOD_CommonPile|OOD_News_2024": -0.000675781205629597, + "OOD_CommonPile|OOD_arXiv_2023": 0.000514233854240348, + "OOD_CommonPile|OOD_CommonPile": 0.0012007729458370434 + } + }, + "without_degenerate": { + "D": 0.0009875729428136123, + "diag_mean": 0.0008464910938402409, + "offdiag_mean": -0.00014108184897337135, + "a_fit": { + "Wikipedia_en": -0.0033088856433727346, + "USPTO_Backgrounds": 0.005557436940456628, + "Pile-CC": 0.005423815948138955, + "StackExchange": 0.00311310704419486, + "OOD_News_2024": -0.004815954082094287, + "OOD_arXiv_2023": -0.002335515342781619, + "OOD_CommonPile": -0.0036340048645418163 + }, + "b_read": { + "Wikipedia_en": 0.0009863902085044673, + "USPTO_Backgrounds": 0.0020207650333330385, + "Pile-CC": 0.0012331523480054313, + "StackExchange": -0.0035441367415491656, + "OOD_News_2024": 0.001428167365096475, + "OOD_arXiv_2023": -0.0013384486239745655, + "OOD_CommonPile": -0.0007858895894156948 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0008552169510165492, + "Wikipedia_en|USPTO_Backgrounds": 0.0006407479553899517, + "Wikipedia_en|Pile-CC": 0.0001528659020286552, + "Wikipedia_en|StackExchange": -0.0013207869066765923, + "Wikipedia_en|OOD_News_2024": 0.001166448376432927, + "Wikipedia_en|OOD_arXiv_2023": -0.0012685593966259429, + "Wikipedia_en|OOD_CommonPile": -0.0002259328815655201, + "USPTO_Backgrounds|Wikipedia_en": 0.00037600548680272183, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0001636413578049653, + "USPTO_Backgrounds|Pile-CC": -4.648552925585331e-05, + "USPTO_Backgrounds|StackExchange": -0.000462505115880342, + "USPTO_Backgrounds|OOD_News_2024": -0.00028949620866357967, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.00024151240698275928, + "USPTO_Backgrounds|OOD_CommonPile": 1.7327602209356308e-05, + "Pile-CC|Wikipedia_en": 0.0010884100144154599, + "Pile-CC|USPTO_Backgrounds": -0.0005853889057573225, + "Pile-CC|Pile-CC": 0.0008200724836219253, + "Pile-CC|StackExchange": -0.0009535715758458435, + "Pile-CC|OOD_News_2024": 0.0008383404668378322, + "Pile-CC|OOD_arXiv_2023": -0.00026903077611559884, + "Pile-CC|OOD_CommonPile": -0.0009388317071564456, + "StackExchange|Wikipedia_en": -0.000866287343769992, + "StackExchange|USPTO_Backgrounds": 0.0009488148745698533, + "StackExchange|Pile-CC": -0.0008854128204417028, + "StackExchange|StackExchange": 0.0009516870826292781, + "StackExchange|OOD_News_2024": -0.0011713917639060017, + "StackExchange|OOD_arXiv_2023": 0.0003480343327091004, + "StackExchange|OOD_CommonPile": 0.0006745556382094994, + "OOD_News_2024|Wikipedia_en": 0.00045207876853577134, + "OOD_News_2024|USPTO_Backgrounds": 0.0006874579013918733, + "OOD_News_2024|Pile-CC": -4.004908354991038e-05, + "OOD_News_2024|StackExchange": -0.00043355637354577925, + "OOD_News_2024|OOD_News_2024": 0.0014271797658242033, + "OOD_News_2024|OOD_arXiv_2023": -0.00012567133538196607, + "OOD_News_2024|OOD_CommonPile": -0.0019674396432741853, + "OOD_arXiv_2023|Wikipedia_en": -0.00030618062788777267, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0010405338279028606, + "OOD_arXiv_2023|Pile-CC": -0.000171558895832577, + "OOD_arXiv_2023|StackExchange": 0.001210235913132067, + "OOD_arXiv_2023|OOD_News_2024": -0.001119834821261552, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0003475956420800619, + "OOD_arXiv_2023|OOD_CommonPile": 0.0010802766176726264, + "OOD_CommonPile|Wikipedia_en": -0.0015992432491127376, + "OOD_CommonPile|USPTO_Backgrounds": -0.0008147393554964397, + "OOD_CommonPile|Pile-CC": 0.00017056794342945603, + "OOD_CommonPile|StackExchange": 0.001008496976187219, + "OOD_CommonPile|OOD_News_2024": -0.0008512458152638222, + "OOD_CommonPile|OOD_arXiv_2023": 0.000726119126351614, + "OOD_CommonPile|OOD_CommonPile": 0.0013600443739047036 + } + }, + "C4_permutation": { + "null_mean": -1.342009317004193e-05, + "null_sd": 0.0003901576568318512, + "n_abs_ge_observed": 10, + "n_perm": 2000, + "p_two_sided": 0.005497251374312844 + }, + "C4_permutation_without_degenerate": { + "null_mean": -1.0247287295497412e-06, + "null_sd": 0.0004117535650963638, + "n_abs_ge_observed": 25, + "n_perm": 2000, + "p_two_sided": 0.012993503248375811 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 0.00013936889285791815, + "surrogate_D_without_degenerate": 1.0770963581868456e-05, + "D_minus_surrogate_all8": 0.0008931653036381806, + "D_minus_surrogate_without_degenerate": 0.0009768019792317439, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + }, + "min": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.13328435383737086, + "Wikipedia_en|USPTO_Backgrounds": 0.13784370236098767, + "Wikipedia_en|Pile-CC": 0.1383333519515064, + "Wikipedia_en|StackExchange": 0.11953544283492698, + "Wikipedia_en|Github": 0.11716976339618365, + "Wikipedia_en|OOD_News_2024": 0.13719095596008832, + "Wikipedia_en|OOD_arXiv_2023": 0.13225573628313012, + "Wikipedia_en|OOD_CommonPile": 0.12354723012281789, + "USPTO_Backgrounds|Wikipedia_en": 0.1589013874116871, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.16444444126553007, + "USPTO_Backgrounds|Pile-CC": 0.16392662384443812, + "USPTO_Backgrounds|StackExchange": 0.1540478135355645, + "USPTO_Backgrounds|Github": 0.14482734395811955, + "USPTO_Backgrounds|OOD_News_2024": 0.16169015301598444, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.15767217957311205, + "USPTO_Backgrounds|OOD_CommonPile": 0.1438613244228893, + "Pile-CC|Wikipedia_en": 0.1456771372920937, + "Pile-CC|USPTO_Backgrounds": 0.15375699135992263, + "Pile-CC|Pile-CC": 0.14843652219408088, + "Pile-CC|StackExchange": 0.13440058796356122, + "Pile-CC|Github": 0.1325610138475895, + "Pile-CC|OOD_News_2024": 0.1505866065621376, + "Pile-CC|OOD_arXiv_2023": 0.14537321800986927, + "Pile-CC|OOD_CommonPile": 0.14141270303063921, + "StackExchange|Wikipedia_en": 0.14468121946685844, + "StackExchange|USPTO_Backgrounds": 0.15529588140133355, + "StackExchange|Pile-CC": 0.14998613929169047, + "StackExchange|StackExchange": 0.13777427354620564, + "StackExchange|Github": 0.13621440918909178, + "StackExchange|OOD_News_2024": 0.14825708121061326, + "StackExchange|OOD_arXiv_2023": 0.15157426219019626, + "StackExchange|OOD_CommonPile": 0.14583224952220916, + "Github|Wikipedia_en": 0.108150950488117, + "Github|USPTO_Backgrounds": 0.12000470521549383, + "Github|Pile-CC": 0.11796630918979645, + "Github|StackExchange": 0.10981142164932357, + "Github|Github": 0.10700858065651522, + "Github|OOD_News_2024": 0.11542985724906127, + "Github|OOD_arXiv_2023": 0.11210602997905678, + "Github|OOD_CommonPile": 0.11318731150693363, + "OOD_News_2024|Wikipedia_en": 0.1364180642697546, + "OOD_News_2024|USPTO_Backgrounds": 0.14418143530686697, + "OOD_News_2024|Pile-CC": 0.14283755603763792, + "OOD_News_2024|StackExchange": 0.12079984980324904, + "OOD_News_2024|Github": 0.12478081385294597, + "OOD_News_2024|OOD_News_2024": 0.14264462573660744, + "OOD_News_2024|OOD_arXiv_2023": 0.13951912265684868, + "OOD_News_2024|OOD_CommonPile": 0.13029280362857712, + "OOD_arXiv_2023|Wikipedia_en": 0.14334331928855842, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.149317937820322, + "OOD_arXiv_2023|Pile-CC": 0.14782937183562253, + "OOD_arXiv_2023|StackExchange": 0.13626646215303076, + "OOD_arXiv_2023|Github": 0.1268347776805361, + "OOD_arXiv_2023|OOD_News_2024": 0.14620044955776798, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.14460420372585456, + "OOD_arXiv_2023|OOD_CommonPile": 0.12925454620271923, + "OOD_CommonPile|Wikipedia_en": 0.14085149384207196, + "OOD_CommonPile|USPTO_Backgrounds": 0.14806802579098277, + "OOD_CommonPile|Pile-CC": 0.1476166748131315, + "OOD_CommonPile|StackExchange": 0.13746774678842888, + "OOD_CommonPile|Github": 0.1275541218618552, + "OOD_CommonPile|OOD_News_2024": 0.14699387763523392, + "OOD_CommonPile|OOD_arXiv_2023": 0.14019710266341764, + "OOD_CommonPile|OOD_CommonPile": 0.13042868367499774 + }, + "all8": { + "D": 0.0005479490122063032, + "diag_mean": 0.0004794553856805085, + "offdiag_mean": -6.849362652579462e-05, + "a_fit": { + "Wikipedia_en": -0.008203688100588302, + "USPTO_Backgrounds": 0.018072653184450865, + "Pile-CC": 0.005926842338521976, + "StackExchange": 0.008103184283310055, + "Github": -0.025140609452177554, + "OOD_News_2024": -0.0029144712824037966, + "OOD_arXiv_2023": 0.002357628339086676, + "OOD_CommonPile": 0.0017984606898001632 + }, + "b_read": { + "Wikipedia_en": 0.0008147355430992431, + "USPTO_Backgrounds": 0.008515384871215143, + "Pile-CC": 0.006517813450773263, + "StackExchange": -0.006835805409678458, + "Github": -0.010979902138610165, + "OOD_News_2024": 0.005525445671971996, + "OOD_arXiv_2023": 0.0023139766912208815, + "OOD_CommonPile": -0.0058716486799918755 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002574551200895142, + "Wikipedia_en|USPTO_Backgrounds": -0.0005667496036039499, + "Wikipedia_en|Pile-CC": 0.0019204714073566553, + "Wikipedia_en|StackExchange": -0.0035238188487710376, + "Wikipedia_en|Github": -0.0017454015585826599, + "Wikipedia_en|OOD_News_2024": 0.0017704431947398458, + "Wikipedia_en|OOD_arXiv_2023": 4.66924985327577e-05, + "Wikipedia_en|OOD_CommonPile": -0.00047618829056671197, + "USPTO_Backgrounds|Wikipedia_en": 0.0019152434901722193, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.00024235198410071934, + "USPTO_Backgrounds|Pile-CC": 0.0012374020152492171, + "USPTO_Backgrounds|StackExchange": 0.004712210566827302, + "USPTO_Backgrounds|Github": -0.0003641622816859258, + "USPTO_Backgrounds|OOD_News_2024": -6.701034403200357e-06, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0008132054965244717, + "USPTO_Backgrounds|OOD_CommonPile": -0.0064384352755344765, + "Pile-CC|Wikipedia_en": 0.0008368042165076972, + "Pile-CC|USPTO_Backgrounds": 0.0012160089562207321, + "Pile-CC|Pile-CC": -0.00210688878917914, + "Pile-CC|StackExchange": -0.0027892041592470773, + "Pile-CC|Github": -0.0004846815462870857, + "Pile-CC|OOD_News_2024": 0.001035563357678848, + "Pile-CC|OOD_arXiv_2023": -0.0009663562138383652, + "Pile-CC|OOD_CommonPile": 0.003258754178144335, + "StackExchange|Wikipedia_en": -0.0023354555535156363, + "StackExchange|USPTO_Backgrounds": 0.0005785570528435702, + "StackExchange|Pile-CC": -0.002733613636357629, + "StackExchange|StackExchange": -0.0015918605213907366, + "StackExchange|Github": 0.0009923718504271095, + "StackExchange|OOD_News_2024": -0.0034703039386335677, + "StackExchange|OOD_arXiv_2023": 0.0030583460217005476, + "StackExchange|OOD_CommonPile": 0.005501958724926204, + "Github|Wikipedia_en": -0.005621930796769467, + "Github|USPTO_Backgrounds": -0.001468825397508533, + "Github|Pile-CC": -0.0015096500027640403, + "Github|StackExchange": 0.003689081317214807, + "Github|Github": 0.005030337053338163, + "Github|OOD_News_2024": -0.003053734164697952, + "Github|OOD_arXiv_2023": -0.00316609245395133, + "Github|OOD_CommonPile": 0.006100814445138283, + "OOD_News_2024|Wikipedia_en": 0.0004190448150943882, + "OOD_News_2024|USPTO_Backgrounds": 0.00048176652409084864, + "OOD_News_2024|Pile-CC": 0.00113545867530368, + "OOD_News_2024|StackExchange": -0.007548628698633486, + "OOD_News_2024|Github": 0.0005764320799951511, + "OOD_News_2024|OOD_News_2024": 0.0019348961530744613, + "OOD_News_2024|OOD_arXiv_2023": 0.0020208620540668187, + "OOD_News_2024|OOD_CommonPile": 0.0009801683970080133, + "OOD_arXiv_2023|Wikipedia_en": 0.0020722002124077266, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.00034616941605539187, + "OOD_arXiv_2023|Pile-CC": 0.000855174851797813, + "OOD_arXiv_2023|StackExchange": 0.0026458840296577635, + "OOD_arXiv_2023|Github": -0.0026417037139051802, + "OOD_arXiv_2023|OOD_News_2024": 0.00021862035274453073, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0018338435015822219, + "OOD_arXiv_2023|OOD_CommonPile": -0.005330188650340351, + "OOD_CommonPile|Wikipedia_en": 0.00013954241520777733, + "OOD_CommonPile|USPTO_Backgrounds": -0.0003445749639973128, + "OOD_CommonPile|Pile-CC": 0.001201645478593305, + "OOD_CommonPile|StackExchange": 0.004406336314342396, + "OOD_CommonPile|Github": -0.001363191883299586, + "OOD_CommonPile|OOD_News_2024": 0.0015712160794969787, + "OOD_CommonPile|OOD_arXiv_2023": -0.0020140899115681787, + "OOD_CommonPile|OOD_CommonPile": -0.0035968835287753242 + } + }, + "without_degenerate": { + "D": -0.00031888559115705905, + "diag_mean": -0.00027333050670604243, + "offdiag_mean": 4.555508445101665e-05, + "a_fit": { + "Wikipedia_en": -0.0116485205966802, + "USPTO_Backgrounds": 0.014430500791659417, + "Pile-CC": 0.002301906983530694, + "StackExchange": 0.004267241300216745, + "OOD_News_2024": -0.006690994298292535, + "OOD_arXiv_2023": -0.0009591609919591682, + "OOD_CommonPile": -0.0017009731884750356 + }, + "b_read": { + "Wikipedia_en": -5.334887417063339e-05, + "USPTO_Backgrounds": 0.007053999682622297, + "Pile-CC": 0.005062260348645459, + "StackExchange": -0.009034034414660358, + "OOD_News_2024": 0.004290476021549056, + "OOD_arXiv_2023": 0.0010950582249769913, + "OOD_CommonPile": -0.00841441098896284 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0016247350899946111, + "Wikipedia_en|USPTO_Backgrounds": -0.0009232649431815121, + "Wikipedia_en|Pile-CC": 0.0015581239813140524, + "Wikipedia_en|StackExchange": -0.0031434903719595453, + "Wikipedia_en|OOD_News_2024": 0.0011875123169923785, + "Wikipedia_en|OOD_arXiv_2023": -0.0005522895633937597, + "Wikipedia_en|OOD_CommonPile": 0.00024867349023384444, + "USPTO_Backgrounds|Wikipedia_en": 0.0011627472759712376, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.00040154742697873225, + "USPTO_Backgrounds|Pile-CC": 0.0010723744859061635, + "USPTO_Backgrounds|StackExchange": 0.005289858940338343, + "USPTO_Backgrounds|OOD_News_2024": -0.0003923120154511184, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0012148676617514398, + "USPTO_Backgrounds|OOD_CommonPile": -0.005516253598034371, + "Pile-CC|Wikipedia_en": 6.709096450654983e-05, + "Pile-CC|USPTO_Backgrounds": 0.0010395964755425535, + "Pile-CC|Pile-CC": -0.0022891333563223593, + "Pile-CC|StackExchange": -0.0022287728235362014, + "Pile-CC|OOD_News_2024": 0.0006327353388307644, + "Pile-CC|OOD_arXiv_2023": -0.001385235416865499, + "Pile-CC|OOD_CommonPile": 0.004163718817844275, + "StackExchange|Wikipedia_en": -0.0028941611774147558, + "StackExchange|USPTO_Backgrounds": 0.0006131522002674195, + "StackExchange|Pile-CC": -0.0027048505753988206, + "StackExchange|StackExchange": -0.0008204215575778329, + "StackExchange|OOD_News_2024": -0.0036621243293796235, + "StackExchange|OOD_arXiv_2023": 0.0028504744467754417, + "StackExchange|OOD_CommonPile": 0.0066179309927281715, + "OOD_News_2024|Wikipedia_en": -0.00019908077600930274, + "OOD_News_2024|USPTO_Backgrounds": 0.0004569417043101265, + "OOD_News_2024|Pile-CC": 0.001104801769057917, + "OOD_News_2024|StackExchange": -0.006836609702025154, + "OOD_News_2024|OOD_News_2024": 0.001683655795123834, + "OOD_News_2024|OOD_arXiv_2023": 0.0017535705119371414, + "OOD_News_2024|OOD_CommonPile": 0.0020367206976054097, + "OOD_arXiv_2023|Wikipedia_en": 0.000994340936461141, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.00013838908856822485, + "OOD_arXiv_2023|Pile-CC": 0.0003647842607091556, + "OOD_arXiv_2023|StackExchange": 0.0028981693414232013, + "OOD_arXiv_2023|OOD_News_2024": -0.0004923536900489911, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00110681827460965, + "OOD_arXiv_2023|OOD_CommonPile": -0.004733370034585849, + "OOD_CommonPile|Wikipedia_en": -0.0007556723135094534, + "OOD_CommonPile|USPTO_Backgrounds": -0.0006464889213915748, + "OOD_CommonPile|Pile-CC": 0.0008938994347340024, + "OOD_CommonPile|StackExchange": 0.004841266173337189, + "OOD_CommonPile|OOD_News_2024": 0.0010428865839328116, + "OOD_CommonPile|OOD_arXiv_2023": -0.002558470591311396, + "OOD_CommonPile|OOD_CommonPile": -0.0028174203657914676 + } + }, + "C4_permutation": { + "null_mean": -5.2312281163084104e-05, + "null_sd": 0.0011953980452625355, + "n_abs_ge_observed": 1327, + "n_perm": 2000, + "p_two_sided": 0.6636681659170415 + }, + "C4_permutation_without_degenerate": { + "null_mean": 7.965028577449722e-06, + "null_sd": 0.001228504235812649, + "n_abs_ge_observed": 1587, + "n_perm": 2000, + "p_two_sided": 0.7936031984007996 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 0.0003985587486577635, + "surrogate_D_without_degenerate": 6.983678305843262e-05, + "D_minus_surrogate_all8": 0.00014939026354853966, + "D_minus_surrogate_without_degenerate": -0.0003887223742154917, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + } + }, + "per_set_D": { + "multihop": { + "per_prefix_seed": [ + 0.00017336982585090612, + -0.00012383584648237696, + 0.000759527349146083 + ], + "mean": 0.00026968710950487073, + "sd": 0.00036700626533760965 + }, + "multilingual": { + "per_prefix_seed": [ + -0.00013165215828589031, + 0.0006280423341585058, + 0.0011913387237914971 + ], + "mean": 0.0005625762998880375, + "sd": 0.0005420889039922649 + }, + "order-ops": { + "per_prefix_seed": [ + -0.0002164501150088784, + 3.8651784832057145e-05, + 0.000139146571330327 + ], + "mean": -1.2883919615498084e-05, + "sd": 0.00014967564366218737 + }, + "poetry": { + "per_prefix_seed": [ + 5.206154725913491e-05, + -0.0002689851472886013, + 0.00020824664118533422 + ], + "mean": -2.892319614710725e-06, + "sd": 0.0001986663712455282 + }, + "typo": { + "per_prefix_seed": [ + 0.005978955621165889, + 0.0031976310004081044, + 0.0038619598905955043 + ], + "mean": 0.004346182170723166, + "sd": 0.0011859721958341132 + } + }, + "per_set_sign_split": { + "n_positive": 3, + "n_negative": 2, + "note": "if the sets split in sign, a null on the set-averaged D is cancellation, not absence of matching" + }, + "PARTIAL_UNBLINDING_DISCLOSED": "E36's stored ladder already contains a 5x5 in-stream fit x read matrix (E28 N=400 operators, same aggregation, same admitted sets). Applying this script's diag_excess to it gives D = +0.001214 with all five diagonal residuals positive and a permutation p of 0.134 \u2014 i.e. positive but NOT significant. That estimate was known before E52 ran and is recorded here so E52's primary is not presented as blind for the in-stream half. It also carries ~17% fit/prefix leakage (E28 fitted N=400 from the same 800-doc block), which is the confound Amendment 1 removes, so its sign is exactly what leakage predicts.", + "secondary_ood_rungs": { + "OOD_News_2024": { + "matched_J^q": 0.040212060573200384, + "logit": 0.03683846357744187, + "shuf_floor": 0.023494280191759268, + "best_instream_J^P": 0.049862991304447254, + "best_instream_corpus": "Pile-CC", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.003373596995758517, + "delta_vs_best_instream": -0.00965093073124687 + }, + "OOD_arXiv_2023": { + "matched_J^q": 0.03884629919969787, + "logit": 0.026718863868154585, + "shuf_floor": 0.02148574317495028, + "best_instream_J^P": 0.046633168247838815, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.012127435331543285, + "delta_vs_best_instream": -0.0077868690481409444 + }, + "OOD_CommonPile": { + "matched_J^q": 0.039112817444321185, + "logit": 0.03460996786598116, + "shuf_floor": 0.0209646623271207, + "best_instream_J^P": 0.04696154247762428, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.004502849578340025, + "delta_vs_best_instream": -0.007848725033303097 + } + }, + "controls": { + "C1_Q0_vs_e48": { + "per_corpus": { + "Wikipedia_en": { + "here": 0.03820955517391364, + "e48": 0.038209554925560954, + "abs_diff": 2.4835268563894175e-10 + }, + "USPTO_Backgrounds": { + "here": 0.04961537096338967, + "e48": 0.04961536889895797, + "abs_diff": 2.0644317019757885e-09 + }, + "Pile-CC": { + "here": 0.04546591031830758, + "e48": 0.04546590892132372, + "abs_diff": 1.3969838619232178e-09 + }, + "StackExchange": { + "here": 0.045729962767412265, + "e48": 0.04572996227070689, + "abs_diff": 4.967053712778835e-10 + }, + "Github": { + "here": 0.027860941054920357, + "e48": 0.027896882360801102, + "abs_diff": 3.594130588074548e-05 + }, + "OOD_News_2024": { + "here": 0.0337636109519129, + "e48": 0.03376360982656479, + "abs_diff": 1.1253481133066678e-09 + }, + "OOD_arXiv_2023": { + "here": 0.04105739652489623, + "e48": 0.04105739537626505, + "abs_diff": 1.148631176284276e-09 + }, + "OOD_CommonPile": { + "here": 0.03823967572922508, + "e48": 0.038239676505327225, + "abs_diff": 7.761021478258634e-10 + } + }, + "max_abs_diff": 3.594130588074548e-05, + "tolerance": 1e-06, + "note": "in-stream rungs are EXPECTED to differ: E48 scored the fastfit E28 lenses, this scores the matched-provenance INSTREAM refits. The OOD rungs use the SAME .pt files in both and must agree to fp32.", + "ood_max_abs_diff": 1.148631176284276e-09 + }, + "C1_fires": true, + "C2_shuf_floor_excl_degenerate": { + "below_floor": { + "Q0": [], + "Wikipedia_en": [], + "USPTO_Backgrounds": [], + "Pile-CC": [], + "StackExchange": [], + "Github": [], + "OOD_News_2024": [], + "OOD_arXiv_2023": [], + "OOD_CommonPile": [], + "SHUFFLED_Pile-CC": [] + }, + "fires": true + }, + "C3_capability": { + "q0_mean_rank": 4224.63045912654, + "per_rung": { + "Q0": 4224.63045912654, + "Wikipedia_en": 4229.167599850691, + "USPTO_Backgrounds": 4182.112355356477, + "Pile-CC": 4321.102276969018, + "StackExchange": 4709.87420679358, + "Github": 4838.026129152669, + "OOD_News_2024": 4429.988428518104, + "OOD_arXiv_2023": 4422.400522583053, + "OOD_CommonPile": 4499.4897349757375, + "SHUFFLED_Pile-CC": 4654.458753266144 + }, + "collapsed": [], + "fires": true + }, + "C5_self_distance": { + "value": 0.0, + "fires": true + }, + "R1_C2_readout_divergence": { + "required_total": 157, + "required_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "observed_total": 157, + "observed_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "n_released_items": 551, + "examples": { + "multihop": { + "unstripped_token_id": 209, + "stripped_token_id": 253, + "unstripped": "' '", + "stripped": "' the'" + }, + "multilingual": { + "unstripped_token_id": 209, + "stripped_token_id": 10863, + "unstripped": "' '", + "stripped": "' ist'" + }, + "order-ops": { + "unstripped_token_id": 209, + "stripped_token_id": 426, + "unstripped": "' '", + "stripped": "' ='" + } + }, + "note": "computed over ALL released items in all six sets, independently of --rstrip: it describes the difference between the two conventions, not this run's arm.", + "fires": true + }, + "R1_C3_logit_constant_moves": { + "arm": "UNSTRIPPED", + "observed_logit_Q0_persist": 0.028439503419212996, + "unstripped_reference": 0.0284395, + "reference_source": "E33/E36 agree to 2.6e-10 on the unstripped Q0 logit constant", + "abs_diff_vs_unstripped_reference": 3.4192129960697315e-09, + "fires": true + }, + "R1_C4_derangement_floor": { + "alias_of": "C2_shuf_floor_excl_degenerate", + "required": "no non-degenerate operator at or below its rung's shuf floor under persist", + "fires": true + }, + "P0_C3_worker_threads": { + "workers": 3, + "observed_threads_in_cell": 1, + "required": "1 when --workers > 1", + "fires": true + } + }, + "VERDICT": "MATCHING CONFIRMED \u2014 the diagonal excess D = +0.00103 has a CI strictly above zero [+0.00001,+0.00277] AND exceeds the link-artifact null (+0.00014; +0.00001 without the degenerate operator). Fitting on the corpus you then read on buys something beyond operator quality, read difficulty, and the curvature of a bounded rate.", + "provenance": { + "schema_version": 1, + "experiment": "E52", + "utc": "2026-08-20T06:18:58+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "6d74ea7f4ef2a968b73923e3858cbafe48c5a8d735f3118ad31422b7d43a272d", + "n_lines": 851 + }, + "git": { + "commit": "64b28ecb5a5600eda58cf20e64c1d8ae680943e1", + "commit_short": "64b28ecb5a56", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/RESULTS_TAXONOMY.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "46c97419e7eecf171879be40caaf59516ff52e2d5c24a0721ce5c7f1fe41418c", + "describe": "64b28ec-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--workers", + "3", + "--out", + "results/e52_factorial_410m_pooled.json", + "--cells-out", + "results/e57_factorial_cells_410m_pooled.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", + "bytes": 116036, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "466570432a7847b1db9678026018fec3d444d820a3d977b2e552a4ad3a72c742" + } +} \ No newline at end of file diff --git a/results/e52_factorial_410m_rerun.json b/results/e52_factorial_410m_rerun.json new file mode 100644 index 0000000000000000000000000000000000000000..9b0196fc95e2c1d68b3a5c4ceacf2b0c800e470c --- /dev/null +++ b/results/e52_factorial_410m_rerun.json @@ -0,0 +1,999 @@ +{ + "experiment": "E52 \u2014 the fit x read factorial: J^Q read on Q, the missing cell", + "prereg": "specs/archive/prereg/PREREG_E52_FACTORIAL.md, written before this ran", + "status": "PRE-REGISTERED", + "question": "can a J-lens trained on an out-of-distribution sample read latent structure on an out-of-distribution corpus?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_operators": 24, + "n_items": 541, + "n_pairs": 893, + "prefix_tokens": 128, + "smoke": false, + "all_operators_share_one_fitter": "trainval.py N=200 --band 9,21 dim_batch=128 max_seq_len=128 skip_first=16 target_layer=-2 fp32", + "adjudication": { + "aggregation": "persist", + "min_does_not_vote": true, + "D": 0.0010325341964960988, + "D_without_degenerate": 0.0009875729428136123, + "D_hierarchical_ci": { + "point": 0.001032533822581172, + "ci_lo": 1.4873118743707892e-05, + "ci_hi": 0.002772060688585043, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": true, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "degenerate_excluded_from_C2": [ + "Github" + ] + }, + "by_aggregation": { + "persist": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.040705389040522275, + "Wikipedia_en|USPTO_Backgrounds": 0.04152529486972425, + "Wikipedia_en|Pile-CC": 0.040249800131035345, + "Wikipedia_en|StackExchange": 0.0339988582327755, + "Wikipedia_en|Github": 0.03484094970497406, + "Wikipedia_en|OOD_News_2024": 0.04145839762253066, + "Wikipedia_en|OOD_arXiv_2023": 0.03625677386040075, + "Wikipedia_en|OOD_CommonPile": 0.03785195941002004, + "USPTO_Backgrounds|Wikipedia_en": 0.04909250016013781, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.049914510855968625, + "USPTO_Backgrounds|Pile-CC": 0.0489167712835802, + "USPTO_Backgrounds|StackExchange": 0.04372346260740111, + "USPTO_Backgrounds|Github": 0.045737664468793404, + "USPTO_Backgrounds|OOD_News_2024": 0.048868775621263516, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.046633168247838815, + "USPTO_Backgrounds|OOD_CommonPile": 0.04696154247762428, + "Pile-CC|Wikipedia_en": 0.049671283695432875, + "Pile-CC|USPTO_Backgrounds": 0.04903185960008866, + "Pile-CC|Pile-CC": 0.049649708304140304, + "Pile-CC|StackExchange": 0.04309877515511794, + "Pile-CC|Github": 0.04413961774876548, + "Pile-CC|OOD_News_2024": 0.049862991304447254, + "Pile-CC|OOD_arXiv_2023": 0.04598900407242278, + "Pile-CC|OOD_CommonPile": 0.04587176217594081, + "StackExchange|Wikipedia_en": 0.04540587743330333, + "StackExchange|USPTO_Backgrounds": 0.048255354476471744, + "StackExchange|Pile-CC": 0.04563351409613258, + "StackExchange|StackExchange": 0.042693324909648965, + "StackExchange|Github": 0.04448954296919207, + "StackExchange|OOD_News_2024": 0.045542550169759326, + "StackExchange|OOD_arXiv_2023": 0.04429536027730339, + "StackExchange|OOD_CommonPile": 0.04517444061736266, + "Github|Wikipedia_en": 0.026004515438237123, + "Github|USPTO_Backgrounds": 0.026875987550657655, + "Github|Pile-CC": 0.026034934611784088, + "Github|StackExchange": 0.021615857266200085, + "Github|Github": 0.02419525204329855, + "Github|OOD_News_2024": 0.0259296004091286, + "Github|OOD_arXiv_2023": 0.02626178347402149, + "Github|OOD_CommonPile": 0.02639343175623152, + "OOD_News_2024|Wikipedia_en": 0.038795182419319944, + "OOD_News_2024|USPTO_Backgrounds": 0.04006493637700462, + "OOD_News_2024|Pile-CC": 0.03854981670673523, + "OOD_News_2024|StackExchange": 0.03337902032718476, + "OOD_News_2024|Github": 0.033795293430901235, + "OOD_News_2024|OOD_News_2024": 0.040212060573200384, + "OOD_News_2024|OOD_arXiv_2023": 0.035892593482923174, + "OOD_News_2024|OOD_CommonPile": 0.034603384209589826, + "OOD_arXiv_2023|Wikipedia_en": 0.04051736176220907, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.04081738338702255, + "OOD_arXiv_2023|Pile-CC": 0.04089874563376523, + "OOD_arXiv_2023|StackExchange": 0.037503251353175275, + "OOD_arXiv_2023|Github": 0.03651138133265906, + "OOD_arXiv_2023|OOD_News_2024": 0.0401454847254273, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.03884629919969787, + "OOD_arXiv_2023|OOD_CommonPile": 0.040131539209849305, + "OOD_CommonPile|Wikipedia_en": 0.037925809619223906, + "OOD_CommonPile|USPTO_Backgrounds": 0.039744688337668775, + "OOD_CommonPile|Pile-CC": 0.039942382951267064, + "OOD_CommonPile|StackExchange": 0.03600302289447023, + "OOD_CommonPile|Github": 0.035526463063433766, + "OOD_CommonPile|OOD_News_2024": 0.03911558420966483, + "OOD_CommonPile|OOD_arXiv_2023": 0.037926333162209225, + "OOD_CommonPile|OOD_CommonPile": 0.039112817444321185 + }, + "all8": { + "D": 0.0010325341964960988, + "diag_mean": 0.0009034674219340851, + "offdiag_mean": -0.00012906677456201365, + "a_fit": { + "Wikipedia_en": -0.0014017750154178207, + "USPTO_Backgrounds": 0.007718346590910287, + "Pile-CC": 0.007401672382628829, + "StackExchange": 0.005423542744231076, + "Github": -0.014348782555720793, + "OOD_News_2024": -0.0028511669335582884, + "OOD_arXiv_2023": -0.0003412720489399737, + "OOD_CommonPile": -0.0016005651641333093 + }, + "b_read": { + "Wikipedia_en": 0.001252037071632607, + "USPTO_Backgrounds": 0.00226604905741018, + "Pile-CC": 0.0014717563403893186, + "StackExchange": -0.003260756281168946, + "Github": -0.0023581822791634774, + "OOD_News_2024": 0.0016292277050120527, + "OOD_arXiv_2023": -0.0007500384023134965, + "OOD_CommonPile": -0.0002500932117982316 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0010924241098918056, + "Wikipedia_en|USPTO_Backgrounds": 0.0008983179533162064, + "Wikipedia_en|Pile-CC": 0.000417115931648164, + "Wikipedia_en|StackExchange": -0.001101313345053416, + "Wikipedia_en|Github": -0.0011617958748603233, + "Wikipedia_en|OOD_News_2024": 0.0014682420585207454, + "Wikipedia_en|OOD_arXiv_2023": -0.0013541155962836157, + "Wikipedia_en|OOD_CommonPile": -0.00025887523717958716, + "USPTO_Backgrounds|Wikipedia_en": 0.0003594136231792333, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.00016741233323247506, + "USPTO_Backgrounds|Pile-CC": -3.60345221350894e-05, + "USPTO_Backgrounds|StackExchange": -0.0004968305767559106, + "USPTO_Backgrounds|Github": 0.000614797282630912, + "USPTO_Backgrounds|OOD_News_2024": -0.00024150154907450622, + "USPTO_Backgrounds|OOD_arXiv_2023": -9.78428151736585e-05, + "USPTO_Backgrounds|OOD_CommonPile": -0.00026941377590345567, + "Pile-CC|Wikipedia_en": 0.0012548713667557554, + "Pile-CC|USPTO_Backgrounds": -0.0003985647143660287, + "Pile-CC|Pile-CC": 0.0010135767067064733, + "Pile-CC|StackExchange": -0.0008048438207576281, + "Pile-CC|Github": -0.0006665752291155555, + "Pile-CC|OOD_News_2024": 0.0010693883423906897, + "Pile-CC|OOD_arXiv_2023": -0.00042533278230823257, + "Pile-CC|OOD_CommonPile": -0.0010425198693054735, + "StackExchange|Wikipedia_en": -0.0010324052569760378, + "StackExchange|USPTO_Backgrounds": 0.0008030598004148057, + "StackExchange|Pile-CC": -0.0010244878629034962, + "StackExchange|StackExchange": 0.0007678355721711522, + "StackExchange|Github": 0.0016614796297087855, + "StackExchange|OOD_News_2024": -0.0012729231538994856, + "StackExchange|OOD_arXiv_2023": -0.00014084693902987472, + "StackExchange|OOD_CommonPile": 0.00023828821051413013, + "Github|Wikipedia_en": -0.000661441952090374, + "Github|USPTO_Backgrounds": -0.000803981825447414, + "Github|Pile-CC": -0.0008507420473001202, + "Github|StackExchange": -0.0005373067713258586, + "Github|Github": 0.001139514003767137, + "Github|OOD_News_2024": -0.0011135476145783425, + "Github|OOD_arXiv_2023": 0.001597901557640096, + "Github|OOD_CommonPile": 0.0012296046493348624, + "OOD_News_2024|Wikipedia_en": 0.000631609406829943, + "OOD_News_2024|USPTO_Backgrounds": 0.0008873513787370432, + "OOD_News_2024|Pile-CC": 0.0001665244254885137, + "OOD_News_2024|StackExchange": -0.00027175933250368767, + "OOD_News_2024|Github": -0.0007580602307926815, + "OOD_News_2024|OOD_News_2024": 0.001671296927330937, + "OOD_News_2024|OOD_arXiv_2023": -0.00026890405562072367, + "OOD_News_2024|OOD_CommonPile": -0.002058058519469337, + "OOD_arXiv_2023|Wikipedia_en": -0.00015610613489924763, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0008700964958633373, + "OOD_arXiv_2023|Pile-CC": 5.558467900200481e-06, + "OOD_arXiv_2023|StackExchange": 0.001342576808868512, + "OOD_arXiv_2023|Github": -0.0005518672136531691, + "OOD_arXiv_2023|OOD_News_2024": -0.0009051738050604649, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00017490677653565767, + "OOD_arXiv_2023|OOD_CommonPile": 0.000960201596171828, + "OOD_CommonPile|Wikipedia_en": -0.0014883651626910743, + "OOD_CommonPile|USPTO_Backgrounds": -0.0006834984300237781, + "OOD_CommonPile|Pile-CC": 0.0003084889005953717, + "OOD_CommonPile|StackExchange": 0.0011016414653568021, + "OOD_CommonPile|Github": -0.0002774923676851296, + "OOD_CommonPile|OOD_News_2024": -0.000675781205629597, + "OOD_CommonPile|OOD_arXiv_2023": 0.000514233854240348, + "OOD_CommonPile|OOD_CommonPile": 0.0012007729458370434 + } + }, + "without_degenerate": { + "D": 0.0009875729428136123, + "diag_mean": 0.0008464910938402409, + "offdiag_mean": -0.00014108184897337135, + "a_fit": { + "Wikipedia_en": -0.0033088856433727346, + "USPTO_Backgrounds": 0.005557436940456628, + "Pile-CC": 0.005423815948138955, + "StackExchange": 0.00311310704419486, + "OOD_News_2024": -0.004815954082094287, + "OOD_arXiv_2023": -0.002335515342781619, + "OOD_CommonPile": -0.0036340048645418163 + }, + "b_read": { + "Wikipedia_en": 0.0009863902085044673, + "USPTO_Backgrounds": 0.0020207650333330385, + "Pile-CC": 0.0012331523480054313, + "StackExchange": -0.0035441367415491656, + "OOD_News_2024": 0.001428167365096475, + "OOD_arXiv_2023": -0.0013384486239745655, + "OOD_CommonPile": -0.0007858895894156948 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0008552169510165492, + "Wikipedia_en|USPTO_Backgrounds": 0.0006407479553899517, + "Wikipedia_en|Pile-CC": 0.0001528659020286552, + "Wikipedia_en|StackExchange": -0.0013207869066765923, + "Wikipedia_en|OOD_News_2024": 0.001166448376432927, + "Wikipedia_en|OOD_arXiv_2023": -0.0012685593966259429, + "Wikipedia_en|OOD_CommonPile": -0.0002259328815655201, + "USPTO_Backgrounds|Wikipedia_en": 0.00037600548680272183, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0001636413578049653, + "USPTO_Backgrounds|Pile-CC": -4.648552925585331e-05, + "USPTO_Backgrounds|StackExchange": -0.000462505115880342, + "USPTO_Backgrounds|OOD_News_2024": -0.00028949620866357967, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.00024151240698275928, + "USPTO_Backgrounds|OOD_CommonPile": 1.7327602209356308e-05, + "Pile-CC|Wikipedia_en": 0.0010884100144154599, + "Pile-CC|USPTO_Backgrounds": -0.0005853889057573225, + "Pile-CC|Pile-CC": 0.0008200724836219253, + "Pile-CC|StackExchange": -0.0009535715758458435, + "Pile-CC|OOD_News_2024": 0.0008383404668378322, + "Pile-CC|OOD_arXiv_2023": -0.00026903077611559884, + "Pile-CC|OOD_CommonPile": -0.0009388317071564456, + "StackExchange|Wikipedia_en": -0.000866287343769992, + "StackExchange|USPTO_Backgrounds": 0.0009488148745698533, + "StackExchange|Pile-CC": -0.0008854128204417028, + "StackExchange|StackExchange": 0.0009516870826292781, + "StackExchange|OOD_News_2024": -0.0011713917639060017, + "StackExchange|OOD_arXiv_2023": 0.0003480343327091004, + "StackExchange|OOD_CommonPile": 0.0006745556382094994, + "OOD_News_2024|Wikipedia_en": 0.00045207876853577134, + "OOD_News_2024|USPTO_Backgrounds": 0.0006874579013918733, + "OOD_News_2024|Pile-CC": -4.004908354991038e-05, + "OOD_News_2024|StackExchange": -0.00043355637354577925, + "OOD_News_2024|OOD_News_2024": 0.0014271797658242033, + "OOD_News_2024|OOD_arXiv_2023": -0.00012567133538196607, + "OOD_News_2024|OOD_CommonPile": -0.0019674396432741853, + "OOD_arXiv_2023|Wikipedia_en": -0.00030618062788777267, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0010405338279028606, + "OOD_arXiv_2023|Pile-CC": -0.000171558895832577, + "OOD_arXiv_2023|StackExchange": 0.001210235913132067, + "OOD_arXiv_2023|OOD_News_2024": -0.001119834821261552, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0003475956420800619, + "OOD_arXiv_2023|OOD_CommonPile": 0.0010802766176726264, + "OOD_CommonPile|Wikipedia_en": -0.0015992432491127376, + "OOD_CommonPile|USPTO_Backgrounds": -0.0008147393554964397, + "OOD_CommonPile|Pile-CC": 0.00017056794342945603, + "OOD_CommonPile|StackExchange": 0.001008496976187219, + "OOD_CommonPile|OOD_News_2024": -0.0008512458152638222, + "OOD_CommonPile|OOD_arXiv_2023": 0.000726119126351614, + "OOD_CommonPile|OOD_CommonPile": 0.0013600443739047036 + } + }, + "C4_permutation": { + "null_mean": -1.342009317004193e-05, + "null_sd": 0.0003901576568318512, + "n_abs_ge_observed": 10, + "n_perm": 2000, + "p_two_sided": 0.005497251374312844 + }, + "C4_permutation_without_degenerate": { + "null_mean": -1.0247287295497412e-06, + "null_sd": 0.0004117535650963638, + "n_abs_ge_observed": 25, + "n_perm": 2000, + "p_two_sided": 0.012993503248375811 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 0.00013936889285791815, + "surrogate_D_without_degenerate": 1.0770963581868456e-05, + "D_minus_surrogate_all8": 0.0008931653036381806, + "D_minus_surrogate_without_degenerate": 0.0009768019792317439, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + }, + "min": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.13328435383737086, + "Wikipedia_en|USPTO_Backgrounds": 0.13784370236098767, + "Wikipedia_en|Pile-CC": 0.1383333519515064, + "Wikipedia_en|StackExchange": 0.11953544283492698, + "Wikipedia_en|Github": 0.11716976339618365, + "Wikipedia_en|OOD_News_2024": 0.13719095596008832, + "Wikipedia_en|OOD_arXiv_2023": 0.13225573628313012, + "Wikipedia_en|OOD_CommonPile": 0.12354723012281789, + "USPTO_Backgrounds|Wikipedia_en": 0.1589013874116871, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.16444444126553007, + "USPTO_Backgrounds|Pile-CC": 0.16392662384443812, + "USPTO_Backgrounds|StackExchange": 0.1540478135355645, + "USPTO_Backgrounds|Github": 0.14482734395811955, + "USPTO_Backgrounds|OOD_News_2024": 0.16169015301598444, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.15767217957311205, + "USPTO_Backgrounds|OOD_CommonPile": 0.1438613244228893, + "Pile-CC|Wikipedia_en": 0.1456771372920937, + "Pile-CC|USPTO_Backgrounds": 0.15375699135992263, + "Pile-CC|Pile-CC": 0.14843652219408088, + "Pile-CC|StackExchange": 0.13440058796356122, + "Pile-CC|Github": 0.1325610138475895, + "Pile-CC|OOD_News_2024": 0.1505866065621376, + "Pile-CC|OOD_arXiv_2023": 0.14537321800986927, + "Pile-CC|OOD_CommonPile": 0.14141270303063921, + "StackExchange|Wikipedia_en": 0.14468121946685844, + "StackExchange|USPTO_Backgrounds": 0.15529588140133355, + "StackExchange|Pile-CC": 0.14998613929169047, + "StackExchange|StackExchange": 0.13777427354620564, + "StackExchange|Github": 0.13621440918909178, + "StackExchange|OOD_News_2024": 0.14825708121061326, + "StackExchange|OOD_arXiv_2023": 0.15157426219019626, + "StackExchange|OOD_CommonPile": 0.14583224952220916, + "Github|Wikipedia_en": 0.108150950488117, + "Github|USPTO_Backgrounds": 0.12000470521549383, + "Github|Pile-CC": 0.11796630918979645, + "Github|StackExchange": 0.10981142164932357, + "Github|Github": 0.10700858065651522, + "Github|OOD_News_2024": 0.11542985724906127, + "Github|OOD_arXiv_2023": 0.11210602997905678, + "Github|OOD_CommonPile": 0.11318731150693363, + "OOD_News_2024|Wikipedia_en": 0.1364180642697546, + "OOD_News_2024|USPTO_Backgrounds": 0.14418143530686697, + "OOD_News_2024|Pile-CC": 0.14283755603763792, + "OOD_News_2024|StackExchange": 0.12079984980324904, + "OOD_News_2024|Github": 0.12478081385294597, + "OOD_News_2024|OOD_News_2024": 0.14264462573660744, + "OOD_News_2024|OOD_arXiv_2023": 0.13951912265684868, + "OOD_News_2024|OOD_CommonPile": 0.13029280362857712, + "OOD_arXiv_2023|Wikipedia_en": 0.14334331928855842, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.149317937820322, + "OOD_arXiv_2023|Pile-CC": 0.14782937183562253, + "OOD_arXiv_2023|StackExchange": 0.13626646215303076, + "OOD_arXiv_2023|Github": 0.1268347776805361, + "OOD_arXiv_2023|OOD_News_2024": 0.14620044955776798, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.14460420372585456, + "OOD_arXiv_2023|OOD_CommonPile": 0.12925454620271923, + "OOD_CommonPile|Wikipedia_en": 0.14085149384207196, + "OOD_CommonPile|USPTO_Backgrounds": 0.14806802579098277, + "OOD_CommonPile|Pile-CC": 0.1476166748131315, + "OOD_CommonPile|StackExchange": 0.13746774678842888, + "OOD_CommonPile|Github": 0.1275541218618552, + "OOD_CommonPile|OOD_News_2024": 0.14699387763523392, + "OOD_CommonPile|OOD_arXiv_2023": 0.14019710266341764, + "OOD_CommonPile|OOD_CommonPile": 0.13042868367499774 + }, + "all8": { + "D": 0.0005479490122063032, + "diag_mean": 0.0004794553856805085, + "offdiag_mean": -6.849362652579462e-05, + "a_fit": { + "Wikipedia_en": -0.008203688100588302, + "USPTO_Backgrounds": 0.018072653184450865, + "Pile-CC": 0.005926842338521976, + "StackExchange": 0.008103184283310055, + "Github": -0.025140609452177554, + "OOD_News_2024": -0.0029144712824037966, + "OOD_arXiv_2023": 0.002357628339086676, + "OOD_CommonPile": 0.0017984606898001632 + }, + "b_read": { + "Wikipedia_en": 0.0008147355430992431, + "USPTO_Backgrounds": 0.008515384871215143, + "Pile-CC": 0.006517813450773263, + "StackExchange": -0.006835805409678458, + "Github": -0.010979902138610165, + "OOD_News_2024": 0.005525445671971996, + "OOD_arXiv_2023": 0.0023139766912208815, + "OOD_CommonPile": -0.0058716486799918755 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002574551200895142, + "Wikipedia_en|USPTO_Backgrounds": -0.0005667496036039499, + "Wikipedia_en|Pile-CC": 0.0019204714073566553, + "Wikipedia_en|StackExchange": -0.0035238188487710376, + "Wikipedia_en|Github": -0.0017454015585826599, + "Wikipedia_en|OOD_News_2024": 0.0017704431947398458, + "Wikipedia_en|OOD_arXiv_2023": 4.66924985327577e-05, + "Wikipedia_en|OOD_CommonPile": -0.00047618829056671197, + "USPTO_Backgrounds|Wikipedia_en": 0.0019152434901722193, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.00024235198410071934, + "USPTO_Backgrounds|Pile-CC": 0.0012374020152492171, + "USPTO_Backgrounds|StackExchange": 0.004712210566827302, + "USPTO_Backgrounds|Github": -0.0003641622816859258, + "USPTO_Backgrounds|OOD_News_2024": -6.701034403200357e-06, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0008132054965244717, + "USPTO_Backgrounds|OOD_CommonPile": -0.0064384352755344765, + "Pile-CC|Wikipedia_en": 0.0008368042165076972, + "Pile-CC|USPTO_Backgrounds": 0.0012160089562207321, + "Pile-CC|Pile-CC": -0.00210688878917914, + "Pile-CC|StackExchange": -0.0027892041592470773, + "Pile-CC|Github": -0.0004846815462870857, + "Pile-CC|OOD_News_2024": 0.001035563357678848, + "Pile-CC|OOD_arXiv_2023": -0.0009663562138383652, + "Pile-CC|OOD_CommonPile": 0.003258754178144335, + "StackExchange|Wikipedia_en": -0.0023354555535156363, + "StackExchange|USPTO_Backgrounds": 0.0005785570528435702, + "StackExchange|Pile-CC": -0.002733613636357629, + "StackExchange|StackExchange": -0.0015918605213907366, + "StackExchange|Github": 0.0009923718504271095, + "StackExchange|OOD_News_2024": -0.0034703039386335677, + "StackExchange|OOD_arXiv_2023": 0.0030583460217005476, + "StackExchange|OOD_CommonPile": 0.005501958724926204, + "Github|Wikipedia_en": -0.005621930796769467, + "Github|USPTO_Backgrounds": -0.001468825397508533, + "Github|Pile-CC": -0.0015096500027640403, + "Github|StackExchange": 0.003689081317214807, + "Github|Github": 0.005030337053338163, + "Github|OOD_News_2024": -0.003053734164697952, + "Github|OOD_arXiv_2023": -0.00316609245395133, + "Github|OOD_CommonPile": 0.006100814445138283, + "OOD_News_2024|Wikipedia_en": 0.0004190448150943882, + "OOD_News_2024|USPTO_Backgrounds": 0.00048176652409084864, + "OOD_News_2024|Pile-CC": 0.00113545867530368, + "OOD_News_2024|StackExchange": -0.007548628698633486, + "OOD_News_2024|Github": 0.0005764320799951511, + "OOD_News_2024|OOD_News_2024": 0.0019348961530744613, + "OOD_News_2024|OOD_arXiv_2023": 0.0020208620540668187, + "OOD_News_2024|OOD_CommonPile": 0.0009801683970080133, + "OOD_arXiv_2023|Wikipedia_en": 0.0020722002124077266, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.00034616941605539187, + "OOD_arXiv_2023|Pile-CC": 0.000855174851797813, + "OOD_arXiv_2023|StackExchange": 0.0026458840296577635, + "OOD_arXiv_2023|Github": -0.0026417037139051802, + "OOD_arXiv_2023|OOD_News_2024": 0.00021862035274453073, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0018338435015822219, + "OOD_arXiv_2023|OOD_CommonPile": -0.005330188650340351, + "OOD_CommonPile|Wikipedia_en": 0.00013954241520777733, + "OOD_CommonPile|USPTO_Backgrounds": -0.0003445749639973128, + "OOD_CommonPile|Pile-CC": 0.001201645478593305, + "OOD_CommonPile|StackExchange": 0.004406336314342396, + "OOD_CommonPile|Github": -0.001363191883299586, + "OOD_CommonPile|OOD_News_2024": 0.0015712160794969787, + "OOD_CommonPile|OOD_arXiv_2023": -0.0020140899115681787, + "OOD_CommonPile|OOD_CommonPile": -0.0035968835287753242 + } + }, + "without_degenerate": { + "D": -0.00031888559115705905, + "diag_mean": -0.00027333050670604243, + "offdiag_mean": 4.555508445101665e-05, + "a_fit": { + "Wikipedia_en": -0.0116485205966802, + "USPTO_Backgrounds": 0.014430500791659417, + "Pile-CC": 0.002301906983530694, + "StackExchange": 0.004267241300216745, + "OOD_News_2024": -0.006690994298292535, + "OOD_arXiv_2023": -0.0009591609919591682, + "OOD_CommonPile": -0.0017009731884750356 + }, + "b_read": { + "Wikipedia_en": -5.334887417063339e-05, + "USPTO_Backgrounds": 0.007053999682622297, + "Pile-CC": 0.005062260348645459, + "StackExchange": -0.009034034414660358, + "OOD_News_2024": 0.004290476021549056, + "OOD_arXiv_2023": 0.0010950582249769913, + "OOD_CommonPile": -0.00841441098896284 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0016247350899946111, + "Wikipedia_en|USPTO_Backgrounds": -0.0009232649431815121, + "Wikipedia_en|Pile-CC": 0.0015581239813140524, + "Wikipedia_en|StackExchange": -0.0031434903719595453, + "Wikipedia_en|OOD_News_2024": 0.0011875123169923785, + "Wikipedia_en|OOD_arXiv_2023": -0.0005522895633937597, + "Wikipedia_en|OOD_CommonPile": 0.00024867349023384444, + "USPTO_Backgrounds|Wikipedia_en": 0.0011627472759712376, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.00040154742697873225, + "USPTO_Backgrounds|Pile-CC": 0.0010723744859061635, + "USPTO_Backgrounds|StackExchange": 0.005289858940338343, + "USPTO_Backgrounds|OOD_News_2024": -0.0003923120154511184, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0012148676617514398, + "USPTO_Backgrounds|OOD_CommonPile": -0.005516253598034371, + "Pile-CC|Wikipedia_en": 6.709096450654983e-05, + "Pile-CC|USPTO_Backgrounds": 0.0010395964755425535, + "Pile-CC|Pile-CC": -0.0022891333563223593, + "Pile-CC|StackExchange": -0.0022287728235362014, + "Pile-CC|OOD_News_2024": 0.0006327353388307644, + "Pile-CC|OOD_arXiv_2023": -0.001385235416865499, + "Pile-CC|OOD_CommonPile": 0.004163718817844275, + "StackExchange|Wikipedia_en": -0.0028941611774147558, + "StackExchange|USPTO_Backgrounds": 0.0006131522002674195, + "StackExchange|Pile-CC": -0.0027048505753988206, + "StackExchange|StackExchange": -0.0008204215575778329, + "StackExchange|OOD_News_2024": -0.0036621243293796235, + "StackExchange|OOD_arXiv_2023": 0.0028504744467754417, + "StackExchange|OOD_CommonPile": 0.0066179309927281715, + "OOD_News_2024|Wikipedia_en": -0.00019908077600930274, + "OOD_News_2024|USPTO_Backgrounds": 0.0004569417043101265, + "OOD_News_2024|Pile-CC": 0.001104801769057917, + "OOD_News_2024|StackExchange": -0.006836609702025154, + "OOD_News_2024|OOD_News_2024": 0.001683655795123834, + "OOD_News_2024|OOD_arXiv_2023": 0.0017535705119371414, + "OOD_News_2024|OOD_CommonPile": 0.0020367206976054097, + "OOD_arXiv_2023|Wikipedia_en": 0.000994340936461141, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.00013838908856822485, + "OOD_arXiv_2023|Pile-CC": 0.0003647842607091556, + "OOD_arXiv_2023|StackExchange": 0.0028981693414232013, + "OOD_arXiv_2023|OOD_News_2024": -0.0004923536900489911, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00110681827460965, + "OOD_arXiv_2023|OOD_CommonPile": -0.004733370034585849, + "OOD_CommonPile|Wikipedia_en": -0.0007556723135094534, + "OOD_CommonPile|USPTO_Backgrounds": -0.0006464889213915748, + "OOD_CommonPile|Pile-CC": 0.0008938994347340024, + "OOD_CommonPile|StackExchange": 0.004841266173337189, + "OOD_CommonPile|OOD_News_2024": 0.0010428865839328116, + "OOD_CommonPile|OOD_arXiv_2023": -0.002558470591311396, + "OOD_CommonPile|OOD_CommonPile": -0.0028174203657914676 + } + }, + "C4_permutation": { + "null_mean": -5.2312281163084104e-05, + "null_sd": 0.0011953980452625355, + "n_abs_ge_observed": 1327, + "n_perm": 2000, + "p_two_sided": 0.6636681659170415 + }, + "C4_permutation_without_degenerate": { + "null_mean": 7.965028577449722e-06, + "null_sd": 0.001228504235812649, + "n_abs_ge_observed": 1587, + "n_perm": 2000, + "p_two_sided": 0.7936031984007996 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 0.0003985587486577635, + "surrogate_D_without_degenerate": 6.983678305843262e-05, + "D_minus_surrogate_all8": 0.00014939026354853966, + "D_minus_surrogate_without_degenerate": -0.0003887223742154917, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + } + }, + "per_set_D": { + "multihop": { + "per_prefix_seed": [ + 0.00017336982585090612, + -0.00012383584648237696, + 0.000759527349146083 + ], + "mean": 0.00026968710950487073, + "sd": 0.00036700626533760965 + }, + "multilingual": { + "per_prefix_seed": [ + -0.00013165215828589031, + 0.0006280423341585058, + 0.0011913387237914971 + ], + "mean": 0.0005625762998880375, + "sd": 0.0005420889039922649 + }, + "order-ops": { + "per_prefix_seed": [ + -0.0002164501150088784, + 3.8651784832057145e-05, + 0.000139146571330327 + ], + "mean": -1.2883919615498084e-05, + "sd": 0.00014967564366218737 + }, + "poetry": { + "per_prefix_seed": [ + 5.206154725913491e-05, + -0.0002689851472886013, + 0.00020824664118533422 + ], + "mean": -2.892319614710725e-06, + "sd": 0.0001986663712455282 + }, + "typo": { + "per_prefix_seed": [ + 0.005978955621165889, + 0.0031976310004081044, + 0.0038619598905955043 + ], + "mean": 0.004346182170723166, + "sd": 0.0011859721958341132 + } + }, + "per_set_sign_split": { + "n_positive": 3, + "n_negative": 2, + "note": "if the sets split in sign, a null on the set-averaged D is cancellation, not absence of matching" + }, + "PARTIAL_UNBLINDING_DISCLOSED": "E36's stored ladder already contains a 5x5 in-stream fit x read matrix (E28 N=400 operators, same aggregation, same admitted sets). Applying this script's diag_excess to it gives D = +0.001214 with all five diagonal residuals positive and a permutation p of 0.134 \u2014 i.e. positive but NOT significant. That estimate was known before E52 ran and is recorded here so E52's primary is not presented as blind for the in-stream half. It also carries ~17% fit/prefix leakage (E28 fitted N=400 from the same 800-doc block), which is the confound Amendment 1 removes, so its sign is exactly what leakage predicts.", + "secondary_ood_rungs": { + "OOD_News_2024": { + "matched_J^q": 0.040212060573200384, + "logit": 0.03683846357744187, + "shuf_floor": 0.023494280191759268, + "best_instream_J^P": 0.049862991304447254, + "best_instream_corpus": "Pile-CC", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.003373596995758517, + "delta_vs_best_instream": -0.00965093073124687 + }, + "OOD_arXiv_2023": { + "matched_J^q": 0.03884629919969787, + "logit": 0.026718863868154585, + "shuf_floor": 0.02148574317495028, + "best_instream_J^P": 0.046633168247838815, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.012127435331543285, + "delta_vs_best_instream": -0.0077868690481409444 + }, + "OOD_CommonPile": { + "matched_J^q": 0.039112817444321185, + "logit": 0.03460996786598116, + "shuf_floor": 0.0209646623271207, + "best_instream_J^P": 0.04696154247762428, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.004502849578340025, + "delta_vs_best_instream": -0.007848725033303097 + } + }, + "controls": { + "C1_Q0_vs_e48": { + "per_corpus": { + "Wikipedia_en": { + "here": 0.03820955517391364, + "e48": 0.038209554925560954, + "abs_diff": 2.4835268563894175e-10 + }, + "USPTO_Backgrounds": { + "here": 0.04961537096338967, + "e48": 0.04961536889895797, + "abs_diff": 2.0644317019757885e-09 + }, + "Pile-CC": { + "here": 0.04546591031830758, + "e48": 0.04546590892132372, + "abs_diff": 1.3969838619232178e-09 + }, + "StackExchange": { + "here": 0.045729962767412265, + "e48": 0.04572996227070689, + "abs_diff": 4.967053712778835e-10 + }, + "Github": { + "here": 0.027860941054920357, + "e48": 0.027896882360801102, + "abs_diff": 3.594130588074548e-05 + }, + "OOD_News_2024": { + "here": 0.0337636109519129, + "e48": 0.03376360982656479, + "abs_diff": 1.1253481133066678e-09 + }, + "OOD_arXiv_2023": { + "here": 0.04105739652489623, + "e48": 0.04105739537626505, + "abs_diff": 1.148631176284276e-09 + }, + "OOD_CommonPile": { + "here": 0.03823967572922508, + "e48": 0.038239676505327225, + "abs_diff": 7.761021478258634e-10 + } + }, + "max_abs_diff": 3.594130588074548e-05, + "tolerance": 1e-06, + "note": "in-stream rungs are EXPECTED to differ: E48 scored the fastfit E28 lenses, this scores the matched-provenance INSTREAM refits. The OOD rungs use the SAME .pt files in both and must agree to fp32.", + "ood_max_abs_diff": 1.148631176284276e-09 + }, + "C1_fires": true, + "C2_shuf_floor_excl_degenerate": { + "below_floor": { + "Q0": [], + "Wikipedia_en": [], + "USPTO_Backgrounds": [], + "Pile-CC": [], + "StackExchange": [], + "Github": [], + "OOD_News_2024": [], + "OOD_arXiv_2023": [], + "OOD_CommonPile": [], + "SHUFFLED_Pile-CC": [] + }, + "fires": true + }, + "C3_capability": { + "q0_mean_rank": 4224.63045912654, + "per_rung": { + "Q0": 4224.63045912654, + "Wikipedia_en": 4229.167599850691, + "USPTO_Backgrounds": 4182.112355356477, + "Pile-CC": 4321.102276969018, + "StackExchange": 4709.87420679358, + "Github": 4838.026129152669, + "OOD_News_2024": 4429.988428518104, + "OOD_arXiv_2023": 4422.400522583053, + "OOD_CommonPile": 4499.4897349757375, + "SHUFFLED_Pile-CC": 4654.458753266144 + }, + "collapsed": [], + "fires": true + }, + "C5_self_distance": { + "value": 0.0, + "fires": true + } + }, + "VERDICT": "MATCHING CONFIRMED \u2014 the diagonal excess D = +0.00103 has a CI strictly above zero [+0.00001,+0.00277] AND exceeds the link-artifact null (+0.00014; +0.00001 without the degenerate operator). Fitting on the corpus you then read on buys something beyond operator quality, read difficulty, and the curvature of a bounded rate.", + "provenance": { + "schema_version": 1, + "experiment": "E52", + "utc": "2026-08-17T01:31:15+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "38b436c06a057ed96391f6a9c386ceecf34d7a29a51780bd470e740aa39d923f", + "n_lines": 599 + }, + "git": { + "commit": "f8ef435b29365012c0492e023fb6b4dfece5f0f3", + "commit_short": "f8ef435b2936", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t53_ladder_summary.py", + "experiments/t55_matrix_robustness.py", + "experiments/t56_predictor_registry.py", + "experiments/t58_algebra_audit.py", + "experiments/t59_read_dose.py", + "experiments/t60_fitter_determinism.py", + "paper/paper.tex", + "results/e35_containment.json", + "results/e53_ladder_summary.json", + "results/e55_matrix_robustness.json", + "results/e56_predictor_registry.json", + "results/e58_algebra_audit.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/experiments/E57_grid_variance_ci.md", + "specs/experiments/E59_read_dose_response.md", + "tests/test_anchor_fidelity.py", + "xperiments/t52_factorial.py" + ], + "dirty_diff_sha256": "4a72c81a812589b5e476dff564e05d42f643a7dd4b402e3ce4f3d7b37238d070", + "describe": "f8ef435-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--out", + "results/e52_factorial_410m_rerun.json", + "--cells-out", + "results/e57_factorial_cells_410m.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", + "bytes": 116036, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "13c41c0ff994fefdda17939c19a77c7fa564d9e744f0f21f47e8b59b84e9119b" + } +} \ No newline at end of file diff --git a/results/e52_factorial_410m_rstrip.json b/results/e52_factorial_410m_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..7961a1154070b4fe2de980bb05a8c77f2a1b1130 --- /dev/null +++ b/results/e52_factorial_410m_rstrip.json @@ -0,0 +1,1132 @@ +{ + "experiment": "E52 \u2014 the fit x read factorial: J^Q read on Q, the missing cell", + "prereg": "docs/experiments/preregs/superseded/PREREG_E52_FACTORIAL.md, written before this ran", + "prereg_files": [ + { + "path": "docs/experiments/preregs/superseded/PREREG_E52_FACTORIAL.md", + "role": "E52, the original design", + "sha256": "740ddb1d6aeb35d8c0d93b626043e62c53d35d149ec9d6a2e7bb8817875902ce" + }, + { + "path": "docs/experiments/preregs/P0_t52_parallel_port.md", + "role": "P0, the pool port", + "sha256": "d8041a76bb5e305c04b435dd1d5a7402694b447ca920dcf0d4402ac75c54345e" + }, + { + "path": "docs/experiments/preregs/R1_grid_rstrip.md", + "role": "R1, the readout correction", + "sha256": "b0f874a0c007a9c31b5995db83729a42aeb3b6150bf578999bb2de77c766ab57" + } + ], + "decision_rule_verbatim": { + "R1": "ACCEPT (headline robust): stripped fit_pct exceeds stripped read_pct under BOTH aggregations, and stripped fit_pct lies inside the published bootstrap interval [87.609, 92.171] under persist. | QUALIFIED (ordering survives, magnitude does not): stripped fit_pct still exceeds read_pct under both aggregations but falls outside the published interval. | REJECT (ordering does not survive): stripped fit_pct does not exceed read_pct under both aggregations. STOP. Alert the operator.", + "P0": "ACCEPT: the pooled run reproduces the serial run's stored matrix at max_abs_diff = 0.0 across all 128 cell values, and wallclock falls by at least 3x. | REJECT: any non-zero difference. STOP. Do not \"investigate the tolerance\" -- there is no tolerance." + }, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`, i.e. the final prompt token after .rstrip(). jacobian-lens/data/evaluations/README.md; src/anchor_evals.py:228", + "rstrip": true, + "workers": 6, + "wallclock_seconds": 2105.5, + "cell_completion_seconds": { + "Q0|p0": 235.9, + "Q0|p1": 236.9, + "Q0|p2": 238.6, + "Wikipedia_en|p1": 423.5, + "Wikipedia_en|p2": 423.5, + "Wikipedia_en|p0": 424.8, + "USPTO_Backgrounds|p0": 643.4, + "USPTO_Backgrounds|p1": 647.9, + "USPTO_Backgrounds|p2": 652.7, + "Pile-CC|p0": 858.8, + "Pile-CC|p1": 859.0, + "Pile-CC|p2": 861.5, + "StackExchange|p0": 1085.9, + "StackExchange|p1": 1090.0, + "StackExchange|p2": 1094.9, + "Github|p0": 1312.9, + "Github|p1": 1314.1, + "Github|p2": 1321.0, + "OOD_News_2024|p0": 1531.8, + "OOD_News_2024|p1": 1536.2, + "OOD_News_2024|p2": 1538.9, + "OOD_arXiv_2023|p0": 1750.8, + "OOD_arXiv_2023|p1": 1775.3, + "OOD_arXiv_2023|p2": 1795.8, + "OOD_CommonPile|p2": 1989.4, + "OOD_CommonPile|p0": 1989.5, + "OOD_CommonPile|p1": 1991.4, + "SHUFFLED_Pile-CC|p0": 2090.8, + "SHUFFLED_Pile-CC|p1": 2096.6, + "SHUFFLED_Pile-CC|p2": 2104.9 + }, + "status": "PRE-REGISTERED", + "question": "can a J-lens trained on an out-of-distribution sample read latent structure on an out-of-distribution corpus?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_operators": 24, + "n_items": 541, + "n_pairs": 893, + "prefix_tokens": 128, + "smoke": false, + "all_operators_share_one_fitter": "trainval.py N=200 --band 9,21 dim_batch=128 max_seq_len=128 skip_first=16 target_layer=-2 fp32", + "adjudication": { + "aggregation": "persist", + "min_does_not_vote": true, + "D": 0.000845372218579525, + "D_without_degenerate": 0.001007020774004221, + "D_hierarchical_ci": { + "point": 0.000845370814204216, + "ci_lo": -0.0008957392419688404, + "ci_hi": 0.0027882857248187065, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "degenerate_excluded_from_C2": [ + "Github" + ] + }, + "by_aggregation": { + "persist": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.1057175321996005, + "Wikipedia_en|USPTO_Backgrounds": 0.10812408523634076, + "Wikipedia_en|Pile-CC": 0.10244530709201677, + "Wikipedia_en|StackExchange": 0.08896334908850904, + "Wikipedia_en|Github": 0.09078200098075387, + "Wikipedia_en|OOD_News_2024": 0.10550682633183897, + "Wikipedia_en|OOD_arXiv_2023": 0.09642561516569306, + "Wikipedia_en|OOD_CommonPile": 0.09897492753031353, + "USPTO_Backgrounds|Wikipedia_en": 0.1134465583651844, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.11788387473206967, + "USPTO_Backgrounds|Pile-CC": 0.11393354688253668, + "USPTO_Backgrounds|StackExchange": 0.10075250616969747, + "USPTO_Backgrounds|Github": 0.0997736249739925, + "USPTO_Backgrounds|OOD_News_2024": 0.11525068599213328, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.10726413512602448, + "USPTO_Backgrounds|OOD_CommonPile": 0.10878850278548069, + "Pile-CC|Wikipedia_en": 0.11220522388401959, + "Pile-CC|USPTO_Backgrounds": 0.11493656668915517, + "Pile-CC|Pile-CC": 0.11208727546036243, + "Pile-CC|StackExchange": 0.0982377641963669, + "Pile-CC|Github": 0.09963205248220927, + "Pile-CC|OOD_News_2024": 0.11498919603311353, + "Pile-CC|OOD_arXiv_2023": 0.10682292872936361, + "Pile-CC|OOD_CommonPile": 0.107277470589098, + "StackExchange|Wikipedia_en": 0.1126410995092657, + "StackExchange|USPTO_Backgrounds": 0.11810599462025695, + "StackExchange|Pile-CC": 0.11142965133508874, + "StackExchange|StackExchange": 0.10109850274812844, + "StackExchange|Github": 0.10198042160417471, + "StackExchange|OOD_News_2024": 0.11420692674194773, + "StackExchange|OOD_arXiv_2023": 0.10534035389622053, + "StackExchange|OOD_CommonPile": 0.10708697298541665, + "Github|Wikipedia_en": 0.09720011671694616, + "Github|USPTO_Backgrounds": 0.09792679025170704, + "Github|Pile-CC": 0.09611689302449425, + "Github|StackExchange": 0.08193752315774974, + "Github|Github": 0.0831806702642805, + "Github|OOD_News_2024": 0.09784760768007901, + "Github|OOD_arXiv_2023": 0.09211890187321438, + "Github|OOD_CommonPile": 0.09339318144031697, + "OOD_News_2024|Wikipedia_en": 0.09874760847952631, + "OOD_News_2024|USPTO_Backgrounds": 0.10250586529728026, + "OOD_News_2024|Pile-CC": 0.09953238663470579, + "OOD_News_2024|StackExchange": 0.08627744833680077, + "OOD_News_2024|Github": 0.08632135616822376, + "OOD_News_2024|OOD_News_2024": 0.10254994873992271, + "OOD_News_2024|OOD_arXiv_2023": 0.09456200205927921, + "OOD_News_2024|OOD_CommonPile": 0.09503316304956873, + "OOD_arXiv_2023|Wikipedia_en": 0.10122454878356722, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.10528044994506572, + "OOD_arXiv_2023|Pile-CC": 0.10172094841642926, + "OOD_arXiv_2023|StackExchange": 0.08987056811909296, + "OOD_arXiv_2023|Github": 0.08733742833137512, + "OOD_arXiv_2023|OOD_News_2024": 0.10298278572348257, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.09594758972784298, + "OOD_arXiv_2023|OOD_CommonPile": 0.09871495900054773, + "OOD_CommonPile|Wikipedia_en": 0.1013734284038138, + "OOD_CommonPile|USPTO_Backgrounds": 0.10645713993969062, + "OOD_CommonPile|Pile-CC": 0.10359419542106076, + "OOD_CommonPile|StackExchange": 0.09089663302422397, + "OOD_CommonPile|Github": 0.08911381827977796, + "OOD_CommonPile|OOD_News_2024": 0.10473065648693591, + "OOD_CommonPile|OOD_arXiv_2023": 0.09662212647963316, + "OOD_CommonPile|OOD_CommonPile": 0.09897827323940064 + }, + "all8": { + "D": 0.000845372218579525, + "diag_mean": 0.0007397006912570839, + "offdiag_mean": -0.00010567152732244106, + "a_fit": { + "Wikipedia_en": -0.001823302244560579, + "USPTO_Backgrounds": 0.008195921680695994, + "Pile-CC": 0.006832802060267165, + "StackExchange": 0.007545482732368533, + "Github": -0.008975547146595389, + "OOD_News_2024": -0.00574953535203046, + "OOD_arXiv_2023": -0.0035558479417684558, + "OOD_CommonPile": -0.0024699737883767947 + }, + "b_read": { + "Wikipedia_en": 0.0038787568450465715, + "USPTO_Backgrounds": 0.007461838141251884, + "Pile-CC": 0.003666767835642931, + "StackExchange": -0.009186470842622738, + "Github": -0.00917558606209544, + "OOD_News_2024": 0.005817321518487822, + "OOD_arXiv_2023": -0.002052801065534973, + "OOD_CommonPile": -0.00040982637017603074 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002221319901420618, + "Wikipedia_en|USPTO_Backgrounds": 0.0010447916419555614, + "Wikipedia_en|Pile-CC": -0.0008389161967594788, + "Wikipedia_en|StackExchange": -0.0014676355220015341, + "Wikipedia_en|Github": 0.0003401315897159918, + "Wikipedia_en|OOD_News_2024": 7.204936021783237e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0011390392219052803, + "Wikipedia_en|OOD_CommonPile": -0.00023270155264375203, + "USPTO_Backgrounds|Wikipedia_en": -6.887785825206161e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0007853572124278951, + "USPTO_Backgrounds|Pile-CC": 0.0006300996685038585, + "USPTO_Backgrounds|StackExchange": 0.00030229763393031794, + "USPTO_Backgrounds|Github": -0.0006874683423019473, + "USPTO_Backgrounds|OOD_News_2024": -0.00020331490474442826, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00031974318683043734, + "USPTO_Backgrounds|OOD_CommonPile": -0.00043835022273316926, + "Pile-CC|Wikipedia_en": 5.290728101195463e-05, + "Pile-CC|USPTO_Backgrounds": -0.0007988312100577766, + "Pile-CC|Pile-CC": 0.0001469478667584373, + "Pile-CC|StackExchange": -0.0008493247189714226, + "Pile-CC|Github": 0.0005340787863436502, + "Pile-CC|OOD_News_2024": 0.0008983147566646449, + "Pile-CC|OOD_arXiv_2023": 0.0006021700369375216, + "Pile-CC|OOD_CommonPile": -0.0005862627986870234, + "StackExchange|Wikipedia_en": -0.00022389776584330612, + "StackExchange|USPTO_Backgrounds": 0.001657916048942637, + "StackExchange|Pile-CC": -0.0012233569306166203, + "StackExchange|StackExchange": 0.0012987331606887503, + "StackExchange|Github": 0.002169767236207723, + "StackExchange|OOD_News_2024": -0.0005966352066025249, + "StackExchange|OOD_arXiv_2023": -0.001593085468306929, + "StackExchange|OOD_CommonPile": -0.001489441074469744, + "Github|Wikipedia_en": 0.0008561493208010806, + "Github|USPTO_Backgrounds": -0.002000258440643346, + "Github|Pile-CC": -1.5085362247188594e-05, + "Github|StackExchange": -0.0013412165507260265, + "Github|Github": -0.00010895422472256833, + "Github|OOD_News_2024": -0.0004349243895073185, + "Github|OOD_arXiv_2023": 0.0017064923876508459, + "Github|OOD_CommonPile": 0.0013377972593944937, + "OOD_News_2024|Wikipedia_en": -0.0008223707111836981, + "OOD_News_2024|USPTO_Backgrounds": -0.0006471951896350608, + "OOD_News_2024|Pile-CC": 0.00017439645339942245, + "OOD_News_2024|StackExchange": -0.00022730316623992275, + "OOD_News_2024|Github": -0.00019428011534423917, + "OOD_News_2024|OOD_News_2024": 0.0010414048757714511, + "OOD_News_2024|OOD_arXiv_2023": 0.0009235807791507511, + "OOD_News_2024|OOD_CommonPile": -0.00024823292591867607, + "OOD_arXiv_2023|Wikipedia_en": -0.0005391178174047884, + "OOD_arXiv_2023|USPTO_Backgrounds": -6.629795211160472e-05, + "OOD_arXiv_2023|Pile-CC": 0.00016927082486088707, + "OOD_arXiv_2023|StackExchange": 0.0011721292057902538, + "OOD_arXiv_2023|Github": -0.0013718953624548835, + "OOD_arXiv_2023|OOD_News_2024": -0.0007194455509306891, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00011548103745251392, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012398756147983248, + "OOD_CommonPile|Wikipedia_en": -0.0014761123505498686, + "OOD_CommonPile|USPTO_Backgrounds": 2.4517889121639103e-05, + "OOD_CommonPile|Pile-CC": 0.0009566436761007241, + "OOD_CommonPile|StackExchange": 0.0011123199575296117, + "OOD_CommonPile|Github": -0.000681379567443699, + "OOD_CommonPile|OOD_News_2024": -5.744894086900931e-05, + "OOD_CommonPile|OOD_arXiv_2023": -0.000295856364148972, + "OOD_CommonPile|OOD_CommonPile": 0.000417315700259574 + } + }, + "without_degenerate": { + "D": 0.001007020774004221, + "diag_mean": 0.0008631606634321978, + "offdiag_mean": -0.00014386011057202305, + "a_fit": { + "Wikipedia_en": -0.0031518899369984976, + "USPTO_Backgrounds": 0.007014133978546375, + "Pile-CC": 0.005476507625453875, + "StackExchange": 0.005955518519003236, + "OOD_News_2024": -0.0070017785151740425, + "OOD_arXiv_2023": -0.004639860355324807, + "OOD_CommonPile": -0.00365263131550618 + }, + "b_read": { + "Wikipedia_en": 0.002447875345953629, + "USPTO_Backgrounds": 0.006439014893793865, + "Pile-CC": 0.0023603484341283287, + "StackExchange": -0.0103034429314975, + "OOD_News_2024": 0.00457087912086751, + "OOD_arXiv_2023": -0.0036051601456064397, + "OOD_CommonPile": -0.001909514717639449 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002389993618745076, + "Wikipedia_en|USPTO_Backgrounds": 0.0008054071076450953, + "Wikipedia_en|Pile-CC": -0.000794704577013361, + "Wikipedia_en|StackExchange": -0.0016128712148952562, + "Wikipedia_en|OOD_News_2024": 5.628397606965985e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0008488879236022984, + "Wikipedia_en|OOD_CommonPile": 4.779013051181558e-06, + "USPTO_Backgrounds|Wikipedia_en": -4.700413121590252e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0003991726878291302, + "USPTO_Backgrounds|Pile-CC": 0.0005275112979616775, + "USPTO_Backgrounds|StackExchange": 1.0261950748297055e-05, + "USPTO_Backgrounds|OOD_News_2024": -0.0003658802791808996, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00017639187881575424, + "USPTO_Backgrounds|OOD_CommonPile": -0.0003476696473265345, + "Pile-CC|Wikipedia_en": 0.0002492877407117844, + "Pile-CC|USPTO_Backgrounds": -0.0010105090019928709, + "Pile-CC|Pile-CC": 0.00021886622887992702, + "Pile-CC|StackExchange": -0.0009668536694897728, + "Pile-CC|OOD_News_2024": 0.0009102561148918442, + "Pile-CC|OOD_arXiv_2023": 0.0009200280776158754, + "Pile-CC|OOD_CommonPile": -0.0003210754906167179, + "StackExchange|Wikipedia_en": 0.00020615247240853007, + "StackExchange|USPTO_Backgrounds": 0.0016799080355595492, + "StackExchange|Pile-CC": -0.0009177687899431242, + "StackExchange|StackExchange": 0.0014148739887224066, + "StackExchange|OOD_News_2024": -0.00035102406982331913, + "StackExchange|OOD_arXiv_2023": -0.0010415576490765688, + "StackExchange|OOD_CommonPile": -0.000990583987847432, + "OOD_News_2024|Wikipedia_en": -0.0007300415231535767, + "OOD_News_2024|USPTO_Backgrounds": -0.0009629242532398635, + "OOD_News_2024|Pile-CC": 0.00014226354385120377, + "OOD_News_2024|StackExchange": -0.00044888338842798137, + "OOD_News_2024|OOD_News_2024": 0.0009492949623289421, + "OOD_News_2024|OOD_arXiv_2023": 0.0011373875481593965, + "OOD_News_2024|OOD_CommonPile": -8.709688951807903e-05, + "OOD_arXiv_2023|Wikipedia_en": -0.000615019378961898, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0005502577653036383, + "OOD_arXiv_2023|Pile-CC": -3.109283427456255e-05, + "OOD_arXiv_2023|StackExchange": 0.0007823182340149643, + "OOD_arXiv_2023|OOD_News_2024": -0.000979786213960429, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00016105705687392835, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012327809016116908, + "OOD_CommonPile|Wikipedia_en": -0.0014533687985339439, + "OOD_CommonPile|USPTO_Backgrounds": -0.00036079681049736023, + "OOD_CommonPile|Pile-CC": 0.0008549251305383088, + "OOD_CommonPile|StackExchange": 0.0008211540993273564, + "OOD_CommonPile|OOD_News_2024": -0.00021914449032571504, + "OOD_CommonPile|OOD_arXiv_2023": -0.00015163523115452326, + "OOD_CommonPile|OOD_CommonPile": 0.0005088661006459744 + } + }, + "C4_permutation": { + "null_mean": -9.644068875497913e-06, + "null_sd": 0.00040128992078327074, + "n_abs_ge_observed": 68, + "n_perm": 2000, + "p_two_sided": 0.034482758620689655 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.244750499708084e-06, + "null_sd": 0.00039985230956766146, + "n_abs_ge_observed": 24, + "n_perm": 2000, + "p_two_sided": 0.01249375312343828 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 9.441563525961398e-05, + "surrogate_D_without_degenerate": -3.09029476336121e-05, + "D_minus_surrogate_all8": 0.000750956583319911, + "D_minus_surrogate_without_degenerate": 0.001037923721637833, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + }, + "min": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.23895396097666688, + "Wikipedia_en|USPTO_Backgrounds": 0.2436402971131934, + "Wikipedia_en|Pile-CC": 0.24109223555359574, + "Wikipedia_en|StackExchange": 0.22112794599185387, + "Wikipedia_en|Github": 0.22357925524314246, + "Wikipedia_en|OOD_News_2024": 0.2387056555185053, + "Wikipedia_en|OOD_arXiv_2023": 0.2240878723975685, + "Wikipedia_en|OOD_CommonPile": 0.2209852614750465, + "USPTO_Backgrounds|Wikipedia_en": 0.262192660694321, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.2683731574151251, + "USPTO_Backgrounds|Pile-CC": 0.26694677480392986, + "USPTO_Backgrounds|StackExchange": 0.2489887709625893, + "USPTO_Backgrounds|Github": 0.24927170336660412, + "USPTO_Backgrounds|OOD_News_2024": 0.26407354954216217, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.2434683405690723, + "USPTO_Backgrounds|OOD_CommonPile": 0.24144293343027431, + "Pile-CC|Wikipedia_en": 0.26439758812387787, + "Pile-CC|USPTO_Backgrounds": 0.2711278670363956, + "Pile-CC|Pile-CC": 0.2672508102738195, + "Pile-CC|StackExchange": 0.23920876369294192, + "Pile-CC|Github": 0.24696667301985953, + "Pile-CC|OOD_News_2024": 0.2684913569026523, + "Pile-CC|OOD_arXiv_2023": 0.2459330331120226, + "Pile-CC|OOD_CommonPile": 0.2441944960090849, + "StackExchange|Wikipedia_en": 0.26712176754242845, + "StackExchange|USPTO_Backgrounds": 0.27237275782972575, + "StackExchange|Pile-CC": 0.2704890080831117, + "StackExchange|StackExchange": 0.24824307101468246, + "StackExchange|Github": 0.25470268204808233, + "StackExchange|OOD_News_2024": 0.27020158254437976, + "StackExchange|OOD_arXiv_2023": 0.24713207156293923, + "StackExchange|OOD_CommonPile": 0.24989844709634781, + "Github|Wikipedia_en": 0.2348206989467144, + "Github|USPTO_Backgrounds": 0.23774738440083132, + "Github|Pile-CC": 0.2397848286562496, + "Github|StackExchange": 0.22261133020122845, + "Github|Github": 0.22343710913426346, + "Github|OOD_News_2024": 0.2349901809874508, + "Github|OOD_arXiv_2023": 0.21322937744359174, + "Github|OOD_CommonPile": 0.2154477684862084, + "OOD_News_2024|Wikipedia_en": 0.251803253011571, + "OOD_News_2024|USPTO_Backgrounds": 0.2560335245397356, + "OOD_News_2024|Pile-CC": 0.25415840529733236, + "OOD_News_2024|StackExchange": 0.22414421873788037, + "OOD_News_2024|Github": 0.23448309062255754, + "OOD_News_2024|OOD_News_2024": 0.25150823344786966, + "OOD_News_2024|OOD_arXiv_2023": 0.23495631789167723, + "OOD_News_2024|OOD_CommonPile": 0.23010762880245844, + "OOD_arXiv_2023|Wikipedia_en": 0.25748952556815413, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.2689400637108419, + "OOD_arXiv_2023|Pile-CC": 0.2626505770410101, + "OOD_arXiv_2023|StackExchange": 0.2423444380155868, + "OOD_arXiv_2023|Github": 0.24505835103078022, + "OOD_arXiv_2023|OOD_News_2024": 0.2630884069121546, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.2443132063994805, + "OOD_arXiv_2023|OOD_CommonPile": 0.2444441412265102, + "OOD_CommonPile|Wikipedia_en": 0.2519655141565535, + "OOD_CommonPile|USPTO_Backgrounds": 0.26096289563510155, + "OOD_CommonPile|Pile-CC": 0.2566619581439429, + "OOD_CommonPile|StackExchange": 0.23913137020750178, + "OOD_CommonPile|Github": 0.2396458975970745, + "OOD_CommonPile|OOD_News_2024": 0.25631876453343366, + "OOD_CommonPile|OOD_arXiv_2023": 0.2370515153846807, + "OOD_CommonPile|OOD_CommonPile": 0.23404423404071067 + }, + "all8": { + "D": 0.0003741732225679442, + "diag_mean": 0.0003274015697469525, + "offdiag_mean": -4.6771652820991726e-05, + "a_fit": { + "Wikipedia_en": -0.015166510734383742, + "USPTO_Backgrounds": 0.00890666507992946, + "Pile-CC": 0.00925825225325147, + "StackExchange": 0.013332102197131851, + "Github": -0.01892948648601306, + "OOD_News_2024": -0.0045387372241950386, + "OOD_arXiv_2023": 0.006853017469984485, + "OOD_CommonPile": 0.0002846974442945738 + }, + "b_read": { + "Wikipedia_en": 0.0069050498594555865, + "USPTO_Backgrounds": 0.013211672192038476, + "Pile-CC": 0.010691253463543632, + "StackExchange": -0.010963082665047197, + "Github": -0.007044976010284809, + "OOD_News_2024": 0.009234145030495716, + "OOD_arXiv_2023": -0.010416604422951226, + "OOD_CommonPile": -0.011617457447250151 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0005273505835147185, + "Wikipedia_en|USPTO_Backgrounds": -0.0010929356125416456, + "Wikipedia_en|Pile-CC": -0.0011205784436444655, + "Wikipedia_en|StackExchange": 0.0005694681232044918, + "Wikipedia_en|Github": -0.00089732928026931, + "Wikipedia_en|OOD_News_2024": -0.0020500500456869797, + "Wikipedia_en|OOD_arXiv_2023": 0.0029829162868231507, + "Wikipedia_en|OOD_CommonPile": 0.0010811583886000675, + "USPTO_Backgrounds|Wikipedia_en": -0.00030712551314435177, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0004332511249231785, + "USPTO_Backgrounds|Pile-CC": 0.0006607849923764564, + "USPTO_Backgrounds|StackExchange": 0.004357117279626732, + "USPTO_Backgrounds|Github": 0.0007219430288791517, + "USPTO_Backgrounds|OOD_News_2024": -0.0007553318363433259, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0017097913559862565, + "USPTO_Backgrounds|OOD_CommonPile": -0.0025343454704853108, + "Pile-CC|Wikipedia_en": 0.001546214743090496, + "Pile-CC|USPTO_Backgrounds": 0.001969871323025313, + "Pile-CC|Pile-CC": 0.0006132332889440539, + "Pile-CC|StackExchange": -0.005774477163342673, + "Pile-CC|Github": -0.0019346744911874492, + "Pile-CC|OOD_News_2024": 0.0033108883508247977, + "Pile-CC|OOD_arXiv_2023": 0.00040331401364204766, + "Pile-CC|OOD_CommonPile": -0.000134370064996725, + "StackExchange|Wikipedia_en": 0.0001965442177606913, + "StackExchange|USPTO_Backgrounds": -0.0008590878275248903, + "StackExchange|Pile-CC": -0.00022241884564408454, + "StackExchange|StackExchange": -0.0008140197854825115, + "StackExchange|Github": 0.001727484593154971, + "StackExchange|OOD_News_2024": 0.0009472640486718753, + "StackExchange|OOD_arXiv_2023": -0.0024714974793217104, + "StackExchange|OOD_CommonPile": 0.001495731078385798, + "Github|Wikipedia_en": 0.000157064305191551, + "Github|USPTO_Backgrounds": -0.0032228725732744157, + "Github|Pile-CC": 0.0013349904106386978, + "Github|StackExchange": 0.005815828084208385, + "Github|Github": 0.0027235003624810072, + "Github|OOD_News_2024": -0.002002548825112166, + "Github|OOD_arXiv_2023": -0.004112602915524288, + "Github|OOD_CommonPile": -0.0006933588486087161, + "OOD_News_2024|Wikipedia_en": 0.002748869108230162, + "OOD_News_2024|USPTO_Backgrounds": 0.0006725183038118476, + "OOD_News_2024|Pile-CC": 0.0013178177899034471, + "OOD_News_2024|StackExchange": -0.007042032640957707, + "OOD_News_2024|Github": -0.0006212674110429306, + "OOD_News_2024|OOD_News_2024": 0.00012475437348866136, + "OOD_News_2024|OOD_arXiv_2023": 0.0032235882707431773, + "OOD_News_2024|OOD_CommonPile": -0.00042424779417668534, + "OOD_arXiv_2023|Wikipedia_en": -0.0029566130293662563, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0021873027807386147, + "OOD_arXiv_2023|Pile-CC": -0.001581765160598353, + "OOD_arXiv_2023|StackExchange": -0.00023356805743079967, + "OOD_arXiv_2023|Github": -0.001437761696999773, + "OOD_arXiv_2023|OOD_News_2024": 0.00031317314359410875, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0011887220843669333, + "OOD_arXiv_2023|OOD_CommonPile": 0.002520509935695553, + "OOD_CommonPile|Wikipedia_en": -0.001912304415276983, + "OOD_CommonPile|USPTO_Backgrounds": 0.0007784547306881884, + "OOD_CommonPile|Pile-CC": -0.0010020640319756413, + "OOD_CommonPile|StackExchange": 0.003121684160174082, + "OOD_CommonPile|Github": -0.0002818951050155838, + "OOD_CommonPile|OOD_News_2024": 0.00011185079056305613, + "OOD_CommonPile|OOD_arXiv_2023": 0.0004953510952570295, + "OOD_CommonPile|OOD_CommonPile": -0.0013110772244140645 + } + }, + "without_degenerate": { + "D": -8.222654747663485e-05, + "diag_mean": -7.047989783710653e-05, + "offdiag_mean": 1.1746649639528328e-05, + "a_fit": { + "Wikipedia_en": -0.017798114832397766, + "USPTO_Backgrounds": 0.006043736366322772, + "Pile-CC": 0.006774840328225729, + "StackExchange": 0.010325524688628629, + "OOD_News_2024": -0.0072097787320985485, + "OOD_arXiv_2023": 0.004298618002931964, + "OOD_CommonPile": -0.002434825821612835 + }, + "b_read": { + "Wikipedia_en": 0.0058206053171940275, + "USPTO_Backgrounds": 0.012610075775272012, + "Pile-CC": 0.009438533763361068, + "StackExchange": -0.012855922032882577, + "OOD_News_2024": 0.00845821664970614, + "OOD_arXiv_2023": -0.010891096505110498, + "OOD_CommonPile": -0.012580412967540255 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0004771800842685303, + "Wikipedia_en|USPTO_Backgrounds": -0.0016259542372829283, + "Wikipedia_en|Pile-CC": -0.0010024737849696486, + "Wikipedia_en|StackExchange": 0.0013276924495321252, + "Wikipedia_en|OOD_News_2024": -0.0024087367064051513, + "Wikipedia_en|OOD_arXiv_2023": 0.002322793327474676, + "Wikipedia_en|OOD_CommonPile": 0.0009094988673824245, + "USPTO_Backgrounds|Wikipedia_en": -0.00012597139679787484, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0007349451340717961, + "USPTO_Backgrounds|Pile-CC": 0.0010102142666439384, + "USPTO_Backgrounds|StackExchange": 0.005346666221547031, + "USPTO_Backgrounds|OOD_News_2024": -0.0008826938814688323, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.002138589699742066, + "USPTO_Backgrounds|OOD_CommonPile": -0.0024746803761102887, + "Pile-CC|Wikipedia_en": 0.001347852070856026, + "Pile-CC|USPTO_Backgrounds": 0.0012886605252957484, + "Pile-CC|Pile-CC": 0.000583145774630589, + "Pile-CC|StackExchange": -0.005164445010003321, + "Pile-CC|OOD_News_2024": 0.0028040095171183443, + "Pile-CC|OOD_arXiv_2023": -0.0004050011186947089, + "Pile-CC|OOD_CommonPile": -0.00045422175920264984, + "StackExchange|Wikipedia_en": 0.000521347129003702, + "StackExchange|USPTO_Backgrounds": -0.0010171330417769742, + "StackExchange|Pile-CC": 0.0002706592235199312, + "StackExchange|StackExchange": 0.0003191779513343207, + "StackExchange|OOD_News_2024": 0.0009635507984429026, + "StackExchange|OOD_arXiv_2023": -0.0027566470281809863, + "StackExchange|OOD_CommonPile": 0.0016990449676573538, + "OOD_News_2024|Wikipedia_en": 0.0027381360188734605, + "OOD_News_2024|USPTO_Backgrounds": 0.00017893708896005167, + "OOD_News_2024|Pile-CC": 0.0014753598584677508, + "OOD_News_2024|StackExchange": -0.006244370904740587, + "OOD_News_2024|OOD_News_2024": -0.00019449487734002346, + "OOD_News_2024|OOD_arXiv_2023": 0.0026029027212841893, + "OOD_News_2024|OOD_CommonPile": -0.0005564699055048417, + "OOD_arXiv_2023|Wikipedia_en": -0.003083988159573947, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0015770795250358294, + "OOD_arXiv_2023|Pile-CC": -0.0015408651328850387, + "OOD_arXiv_2023|StackExchange": 0.00044745163793533105, + "OOD_arXiv_2023|OOD_News_2024": -0.00012271814808556547, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00045139449405695586, + "OOD_arXiv_2023|OOD_CommonPile": 0.0022716457835164072, + "OOD_CommonPile|Wikipedia_en": -0.0018745557466297857, + "OOD_CommonPile|USPTO_Backgrounds": 0.0003333552738402912, + "OOD_CommonPile|Pile-CC": -0.0007960402054074389, + "OOD_CommonPile|StackExchange": 0.003967827654395101, + "OOD_CommonPile|OOD_News_2024": -0.00015891670226172994, + "OOD_CommonPile|OOD_arXiv_2023": -7.685269619805979e-05, + "OOD_CommonPile|OOD_CommonPile": -0.001394817577738322 + } + }, + "C4_permutation": { + "null_mean": -2.3054927495855173e-05, + "null_sd": 0.0009489086857509815, + "n_abs_ge_observed": 1426, + "n_perm": 2000, + "p_two_sided": 0.7131434282858571 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.452356661881763e-06, + "null_sd": 0.001010189712926274, + "n_abs_ge_observed": 1887, + "n_perm": 2000, + "p_two_sided": 0.9435282358820589 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": -1.0121748673588243e-05, + "surrogate_D_without_degenerate": -0.00011535759602394957, + "D_minus_surrogate_all8": 0.00038429497124153243, + "D_minus_surrogate_without_degenerate": 3.313104854731472e-05, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + } + }, + "per_set_D": { + "multihop": { + "per_prefix_seed": [ + 0.0011145231962603117, + 0.000726503469715161, + 0.0008255728420668415 + ], + "mean": 0.0008888665026807713, + "sd": 0.00016460942186982471 + }, + "multilingual": { + "per_prefix_seed": [ + -0.00024172088264354637, + 0.000781276895265494, + 0.0012064470377351557 + ], + "mean": 0.0005820010167857012, + "sd": 0.0006077722919051255 + }, + "order-ops": { + "per_prefix_seed": [ + -0.0012291302638394491, + -0.004769640043377876, + 0.0012368599751165935 + ], + "mean": -0.0015873034440335772, + "sd": 0.0024651878411595463 + }, + "poetry": { + "per_prefix_seed": [ + 5.206154725913491e-05, + -0.0002689851472886013, + 0.00020824664118533422 + ], + "mean": -2.892319614710725e-06, + "sd": 0.0001986663712455282 + }, + "typo": { + "per_prefix_seed": [ + 0.005978955621165889, + 0.0031976310004081044, + 0.0038619598905955043 + ], + "mean": 0.004346182170723166, + "sd": 0.0011859721958341132 + } + }, + "per_set_sign_split": { + "n_positive": 3, + "n_negative": 2, + "note": "if the sets split in sign, a null on the set-averaged D is cancellation, not absence of matching" + }, + "PARTIAL_UNBLINDING_DISCLOSED": "E36's stored ladder already contains a 5x5 in-stream fit x read matrix (E28 N=400 operators, same aggregation, same admitted sets). Applying this script's diag_excess to it gives D = +0.001214 with all five diagonal residuals positive and a permutation p of 0.134 \u2014 i.e. positive but NOT significant. That estimate was known before E52 ran and is recorded here so E52's primary is not presented as blind for the in-stream half. It also carries ~17% fit/prefix leakage (E28 fitted N=400 from the same 800-doc block), which is the confound Amendment 1 removes, so its sign is exactly what leakage predicts.", + "secondary_ood_rungs": { + "OOD_News_2024": { + "matched_J^q": 0.10254994873992271, + "logit": 0.07871548670809715, + "shuf_floor": 0.08219137775401274, + "best_instream_J^P": 0.11525068599213328, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.023834462031825557, + "delta_vs_best_instream": -0.012700737252210575 + }, + "OOD_arXiv_2023": { + "matched_J^q": 0.09594758972784298, + "logit": 0.06943058591180791, + "shuf_floor": 0.07357985358685255, + "best_instream_J^P": 0.10726413512602448, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.02651700381603507, + "delta_vs_best_instream": -0.011316545398181499 + }, + "OOD_CommonPile": { + "matched_J^q": 0.09897827323940064, + "logit": 0.07235888629996529, + "shuf_floor": 0.07529745303715268, + "best_instream_J^P": 0.10878850278548069, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.026619386939435358, + "delta_vs_best_instream": -0.009810229546080046 + } + }, + "controls": { + "C1_Q0_vs_e48": { + "applicable": false, + "why": "results/e48_crossover_410m.json was scored at the unstripped readout. Comparing a stripped Q0 rung against it would compare two conventions, not two runs. The stripped counterpart is R2 (results/e48_crossover_410m_rstrip.json).", + "fires": null + }, + "C1_fires": null, + "C2_shuf_floor_excl_degenerate": { + "below_floor": { + "Q0": [], + "Wikipedia_en": [], + "USPTO_Backgrounds": [], + "Pile-CC": [], + "StackExchange": [], + "Github": [], + "OOD_News_2024": [], + "OOD_arXiv_2023": [], + "OOD_CommonPile": [], + "SHUFFLED_Pile-CC": [] + }, + "fires": true + }, + "C3_capability": { + "q0_mean_rank": 3124.912653975364, + "per_rung": { + "Q0": 3124.912653975364, + "Wikipedia_en": 2912.431131019037, + "USPTO_Backgrounds": 2895.9477416946625, + "Pile-CC": 3041.0272489734975, + "StackExchange": 3170.777902202314, + "Github": 3232.4229189996267, + "OOD_News_2024": 3204.6147816349385, + "OOD_arXiv_2023": 3026.1086226203806, + "OOD_CommonPile": 3008.9858156028367, + "SHUFFLED_Pile-CC": 2797.349757372154 + }, + "collapsed": [], + "fires": true + }, + "C5_self_distance": { + "value": 0.0, + "fires": true + }, + "R1_C2_readout_divergence": { + "required_total": 157, + "required_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "observed_total": 157, + "observed_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "n_released_items": 551, + "examples": { + "multihop": { + "unstripped_token_id": 209, + "stripped_token_id": 253, + "unstripped": "' '", + "stripped": "' the'" + }, + "multilingual": { + "unstripped_token_id": 209, + "stripped_token_id": 10863, + "unstripped": "' '", + "stripped": "' ist'" + }, + "order-ops": { + "unstripped_token_id": 209, + "stripped_token_id": 426, + "unstripped": "' '", + "stripped": "' ='" + } + }, + "note": "computed over ALL released items in all six sets, independently of --rstrip: it describes the difference between the two conventions, not this run's arm.", + "fires": true + }, + "R1_C3_logit_constant_moves": { + "arm": "STRIPPED", + "observed_logit_Q0_persist": 0.08317913140635938, + "unstripped_reference": 0.0284395, + "reference_source": "E33/E36 agree to 2.6e-10 on the unstripped Q0 logit constant", + "abs_diff_vs_unstripped_reference": 0.05473963140635938, + "fires": true + }, + "R1_C4_derangement_floor": { + "alias_of": "C2_shuf_floor_excl_degenerate", + "required": "no non-degenerate operator at or below its rung's shuf floor under persist", + "fires": true + }, + "P0_C3_worker_threads": { + "workers": 6, + "observed_threads_in_cell": 1, + "required": "1 when --workers > 1", + "fires": true + } + }, + "VERDICT": "NO MATCHING (the publishable null) \u2014 D = +0.00085, CI [-0.00090,+0.00279] includes zero. The corpus effect established in E28/E33/E48 is a property of the FITTING CORPUS PER SE; the fit-read relationship is not the operative variable, and EQ2's ||J^P - J^Q|| gap framing is not what the data measures.", + "provenance": { + "schema_version": 1, + "experiment": "E52", + "utc": "2026-08-30T07:22:30+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "73ea8c493c2f36b26522e577752339a77fc84063610c786e9c199f6cfc390962", + "n_lines": 851 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "9b5e7c3d0df63ba700d0e85d1608e851e53fdc6f379ceb7d1e3414552b342ad8", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--workers", + "6", + "--rstrip", + "--out", + "results/e52_factorial_410m_rstrip.json", + "--cells-out", + "results/e57_factorial_cells_410m_rstrip.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "cc21cc60dc9df63dd2e49d5296467b2ec0a1976d3adf9dfcddff67b5a276b7be", + "bytes": 115713, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "2a992ddc89b768907bad92d80f8bfabcc2d02dc796f64117afa2143426a486ae" + } +} \ No newline at end of file diff --git a/results/e52_factorial_410m_rstrip_w1.json b/results/e52_factorial_410m_rstrip_w1.json new file mode 100644 index 0000000000000000000000000000000000000000..a4015ff11978a746b5452f1689fa6253f811f72a --- /dev/null +++ b/results/e52_factorial_410m_rstrip_w1.json @@ -0,0 +1,1094 @@ +{ + "experiment": "E52 \u2014 the fit x read factorial: J^Q read on Q, the missing cell", + "prereg": "specs/archive/prereg/PREREG_E52_FACTORIAL.md, written before this ran", + "prereg_files": [ + { + "path": "specs/archive/prereg/PREREG_E52_FACTORIAL.md", + "role": "E52, the original design", + "sha256": "740ddb1d6aeb35d8c0d93b626043e62c53d35d149ec9d6a2e7bb8817875902ce" + }, + { + "path": "specs/experiments/P0_t52_parallel_port.md", + "role": "P0, the pool port", + "sha256": "d2cd8cb4b3296249ace1ae995b238b2035b21d7884858596777202971869565e" + }, + { + "path": "specs/experiments/R1_grid_rstrip.md", + "role": "R1, the readout correction", + "sha256": "3b13a90cd3730455c6d1328a84d50653f3f146e39ed115f08dc2f77e872fcd07" + } + ], + "decision_rule_verbatim": { + "R1": "ACCEPT (headline robust): stripped fit_pct exceeds stripped read_pct under BOTH aggregations, and stripped fit_pct lies inside the published bootstrap interval [87.609, 92.171] under persist. | QUALIFIED (ordering survives, magnitude does not): stripped fit_pct still exceeds read_pct under both aggregations but falls outside the published interval. | REJECT (ordering does not survive): stripped fit_pct does not exceed read_pct under both aggregations. STOP. Alert the operator.", + "P0": "ACCEPT: the pooled run reproduces the serial run's stored matrix at max_abs_diff = 0.0 across all 128 cell values, and wallclock falls by at least 3x. | REJECT: any non-zero difference. STOP. Do not \"investigate the tolerance\" -- there is no tolerance." + }, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`, i.e. the final prompt token after .rstrip(). jacobian-lens/data/evaluations/README.md; src/anchor_evals.py:228", + "rstrip": true, + "workers": 1, + "wallclock_seconds": 4834.9, + "cell_completion_seconds": { + "Q0|p0": 744.1, + "Q0|p1": 1420.2, + "Q0|p2": 2020.9, + "Wikipedia_en|p0": 2272.1, + "Wikipedia_en|p1": 2511.4, + "Wikipedia_en|p2": 2668.1, + "USPTO_Backgrounds|p0": 2757.9, + "USPTO_Backgrounds|p1": 2844.2, + "USPTO_Backgrounds|p2": 2930.5, + "Pile-CC|p0": 3023.8, + "Pile-CC|p1": 3110.3, + "Pile-CC|p2": 3197.4, + "StackExchange|p0": 3285.9, + "StackExchange|p1": 3371.8, + "StackExchange|p2": 3457.7, + "Github|p0": 3551.8, + "Github|p1": 3639.2, + "Github|p2": 3726.9, + "OOD_News_2024|p0": 3815.9, + "OOD_News_2024|p1": 3902.0, + "OOD_News_2024|p2": 3988.1, + "OOD_arXiv_2023|p0": 4126.8, + "OOD_arXiv_2023|p1": 4222.2, + "OOD_arXiv_2023|p2": 4317.2, + "OOD_CommonPile|p0": 4403.8, + "OOD_CommonPile|p1": 4489.5, + "OOD_CommonPile|p2": 4575.6, + "SHUFFLED_Pile-CC|p0": 4662.2, + "SHUFFLED_Pile-CC|p1": 4748.5, + "SHUFFLED_Pile-CC|p2": 4834.9 + }, + "status": "PRE-REGISTERED", + "question": "can a J-lens trained on an out-of-distribution sample read latent structure on an out-of-distribution corpus?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_operators": 24, + "n_items": 541, + "n_pairs": 893, + "prefix_tokens": 128, + "smoke": false, + "all_operators_share_one_fitter": "trainval.py N=200 --band 9,21 dim_batch=128 max_seq_len=128 skip_first=16 target_layer=-2 fp32", + "adjudication": { + "aggregation": "persist", + "min_does_not_vote": true, + "D": 0.000845372218579525, + "D_without_degenerate": 0.001007020774004221, + "D_hierarchical_ci": { + "point": 0.000845370814204216, + "ci_lo": -0.0008957392419688404, + "ci_hi": 0.0027882857248187065, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "degenerate_excluded_from_C2": [ + "Github" + ] + }, + "by_aggregation": { + "persist": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.1057175321996005, + "Wikipedia_en|USPTO_Backgrounds": 0.10812408523634076, + "Wikipedia_en|Pile-CC": 0.10244530709201677, + "Wikipedia_en|StackExchange": 0.08896334908850904, + "Wikipedia_en|Github": 0.09078200098075387, + "Wikipedia_en|OOD_News_2024": 0.10550682633183897, + "Wikipedia_en|OOD_arXiv_2023": 0.09642561516569306, + "Wikipedia_en|OOD_CommonPile": 0.09897492753031353, + "USPTO_Backgrounds|Wikipedia_en": 0.1134465583651844, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.11788387473206967, + "USPTO_Backgrounds|Pile-CC": 0.11393354688253668, + "USPTO_Backgrounds|StackExchange": 0.10075250616969747, + "USPTO_Backgrounds|Github": 0.0997736249739925, + "USPTO_Backgrounds|OOD_News_2024": 0.11525068599213328, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.10726413512602448, + "USPTO_Backgrounds|OOD_CommonPile": 0.10878850278548069, + "Pile-CC|Wikipedia_en": 0.11220522388401959, + "Pile-CC|USPTO_Backgrounds": 0.11493656668915517, + "Pile-CC|Pile-CC": 0.11208727546036243, + "Pile-CC|StackExchange": 0.0982377641963669, + "Pile-CC|Github": 0.09963205248220927, + "Pile-CC|OOD_News_2024": 0.11498919603311353, + "Pile-CC|OOD_arXiv_2023": 0.10682292872936361, + "Pile-CC|OOD_CommonPile": 0.107277470589098, + "StackExchange|Wikipedia_en": 0.1126410995092657, + "StackExchange|USPTO_Backgrounds": 0.11810599462025695, + "StackExchange|Pile-CC": 0.11142965133508874, + "StackExchange|StackExchange": 0.10109850274812844, + "StackExchange|Github": 0.10198042160417471, + "StackExchange|OOD_News_2024": 0.11420692674194773, + "StackExchange|OOD_arXiv_2023": 0.10534035389622053, + "StackExchange|OOD_CommonPile": 0.10708697298541665, + "Github|Wikipedia_en": 0.09720011671694616, + "Github|USPTO_Backgrounds": 0.09792679025170704, + "Github|Pile-CC": 0.09611689302449425, + "Github|StackExchange": 0.08193752315774974, + "Github|Github": 0.0831806702642805, + "Github|OOD_News_2024": 0.09784760768007901, + "Github|OOD_arXiv_2023": 0.09211890187321438, + "Github|OOD_CommonPile": 0.09339318144031697, + "OOD_News_2024|Wikipedia_en": 0.09874760847952631, + "OOD_News_2024|USPTO_Backgrounds": 0.10250586529728026, + "OOD_News_2024|Pile-CC": 0.09953238663470579, + "OOD_News_2024|StackExchange": 0.08627744833680077, + "OOD_News_2024|Github": 0.08632135616822376, + "OOD_News_2024|OOD_News_2024": 0.10254994873992271, + "OOD_News_2024|OOD_arXiv_2023": 0.09456200205927921, + "OOD_News_2024|OOD_CommonPile": 0.09503316304956873, + "OOD_arXiv_2023|Wikipedia_en": 0.10122454878356722, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.10528044994506572, + "OOD_arXiv_2023|Pile-CC": 0.10172094841642926, + "OOD_arXiv_2023|StackExchange": 0.08987056811909296, + "OOD_arXiv_2023|Github": 0.08733742833137512, + "OOD_arXiv_2023|OOD_News_2024": 0.10298278572348257, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.09594758972784298, + "OOD_arXiv_2023|OOD_CommonPile": 0.09871495900054773, + "OOD_CommonPile|Wikipedia_en": 0.1013734284038138, + "OOD_CommonPile|USPTO_Backgrounds": 0.10645713993969062, + "OOD_CommonPile|Pile-CC": 0.10359419542106076, + "OOD_CommonPile|StackExchange": 0.09089663302422397, + "OOD_CommonPile|Github": 0.08911381827977796, + "OOD_CommonPile|OOD_News_2024": 0.10473065648693591, + "OOD_CommonPile|OOD_arXiv_2023": 0.09662212647963316, + "OOD_CommonPile|OOD_CommonPile": 0.09897827323940064 + }, + "all8": { + "D": 0.000845372218579525, + "diag_mean": 0.0007397006912570839, + "offdiag_mean": -0.00010567152732244106, + "a_fit": { + "Wikipedia_en": -0.001823302244560579, + "USPTO_Backgrounds": 0.008195921680695994, + "Pile-CC": 0.006832802060267165, + "StackExchange": 0.007545482732368533, + "Github": -0.008975547146595389, + "OOD_News_2024": -0.00574953535203046, + "OOD_arXiv_2023": -0.0035558479417684558, + "OOD_CommonPile": -0.0024699737883767947 + }, + "b_read": { + "Wikipedia_en": 0.0038787568450465715, + "USPTO_Backgrounds": 0.007461838141251884, + "Pile-CC": 0.003666767835642931, + "StackExchange": -0.009186470842622738, + "Github": -0.00917558606209544, + "OOD_News_2024": 0.005817321518487822, + "OOD_arXiv_2023": -0.002052801065534973, + "OOD_CommonPile": -0.00040982637017603074 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002221319901420618, + "Wikipedia_en|USPTO_Backgrounds": 0.0010447916419555614, + "Wikipedia_en|Pile-CC": -0.0008389161967594788, + "Wikipedia_en|StackExchange": -0.0014676355220015341, + "Wikipedia_en|Github": 0.0003401315897159918, + "Wikipedia_en|OOD_News_2024": 7.204936021783237e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0011390392219052803, + "Wikipedia_en|OOD_CommonPile": -0.00023270155264375203, + "USPTO_Backgrounds|Wikipedia_en": -6.887785825206161e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0007853572124278951, + "USPTO_Backgrounds|Pile-CC": 0.0006300996685038585, + "USPTO_Backgrounds|StackExchange": 0.00030229763393031794, + "USPTO_Backgrounds|Github": -0.0006874683423019473, + "USPTO_Backgrounds|OOD_News_2024": -0.00020331490474442826, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00031974318683043734, + "USPTO_Backgrounds|OOD_CommonPile": -0.00043835022273316926, + "Pile-CC|Wikipedia_en": 5.290728101195463e-05, + "Pile-CC|USPTO_Backgrounds": -0.0007988312100577766, + "Pile-CC|Pile-CC": 0.0001469478667584373, + "Pile-CC|StackExchange": -0.0008493247189714226, + "Pile-CC|Github": 0.0005340787863436502, + "Pile-CC|OOD_News_2024": 0.0008983147566646449, + "Pile-CC|OOD_arXiv_2023": 0.0006021700369375216, + "Pile-CC|OOD_CommonPile": -0.0005862627986870234, + "StackExchange|Wikipedia_en": -0.00022389776584330612, + "StackExchange|USPTO_Backgrounds": 0.001657916048942637, + "StackExchange|Pile-CC": -0.0012233569306166203, + "StackExchange|StackExchange": 0.0012987331606887503, + "StackExchange|Github": 0.002169767236207723, + "StackExchange|OOD_News_2024": -0.0005966352066025249, + "StackExchange|OOD_arXiv_2023": -0.001593085468306929, + "StackExchange|OOD_CommonPile": -0.001489441074469744, + "Github|Wikipedia_en": 0.0008561493208010806, + "Github|USPTO_Backgrounds": -0.002000258440643346, + "Github|Pile-CC": -1.5085362247188594e-05, + "Github|StackExchange": -0.0013412165507260265, + "Github|Github": -0.00010895422472256833, + "Github|OOD_News_2024": -0.0004349243895073185, + "Github|OOD_arXiv_2023": 0.0017064923876508459, + "Github|OOD_CommonPile": 0.0013377972593944937, + "OOD_News_2024|Wikipedia_en": -0.0008223707111836981, + "OOD_News_2024|USPTO_Backgrounds": -0.0006471951896350608, + "OOD_News_2024|Pile-CC": 0.00017439645339942245, + "OOD_News_2024|StackExchange": -0.00022730316623992275, + "OOD_News_2024|Github": -0.00019428011534423917, + "OOD_News_2024|OOD_News_2024": 0.0010414048757714511, + "OOD_News_2024|OOD_arXiv_2023": 0.0009235807791507511, + "OOD_News_2024|OOD_CommonPile": -0.00024823292591867607, + "OOD_arXiv_2023|Wikipedia_en": -0.0005391178174047884, + "OOD_arXiv_2023|USPTO_Backgrounds": -6.629795211160472e-05, + "OOD_arXiv_2023|Pile-CC": 0.00016927082486088707, + "OOD_arXiv_2023|StackExchange": 0.0011721292057902538, + "OOD_arXiv_2023|Github": -0.0013718953624548835, + "OOD_arXiv_2023|OOD_News_2024": -0.0007194455509306891, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00011548103745251392, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012398756147983248, + "OOD_CommonPile|Wikipedia_en": -0.0014761123505498686, + "OOD_CommonPile|USPTO_Backgrounds": 2.4517889121639103e-05, + "OOD_CommonPile|Pile-CC": 0.0009566436761007241, + "OOD_CommonPile|StackExchange": 0.0011123199575296117, + "OOD_CommonPile|Github": -0.000681379567443699, + "OOD_CommonPile|OOD_News_2024": -5.744894086900931e-05, + "OOD_CommonPile|OOD_arXiv_2023": -0.000295856364148972, + "OOD_CommonPile|OOD_CommonPile": 0.000417315700259574 + } + }, + "without_degenerate": { + "D": 0.001007020774004221, + "diag_mean": 0.0008631606634321978, + "offdiag_mean": -0.00014386011057202305, + "a_fit": { + "Wikipedia_en": -0.0031518899369984976, + "USPTO_Backgrounds": 0.007014133978546375, + "Pile-CC": 0.005476507625453875, + "StackExchange": 0.005955518519003236, + "OOD_News_2024": -0.0070017785151740425, + "OOD_arXiv_2023": -0.004639860355324807, + "OOD_CommonPile": -0.00365263131550618 + }, + "b_read": { + "Wikipedia_en": 0.002447875345953629, + "USPTO_Backgrounds": 0.006439014893793865, + "Pile-CC": 0.0023603484341283287, + "StackExchange": -0.0103034429314975, + "OOD_News_2024": 0.00457087912086751, + "OOD_arXiv_2023": -0.0036051601456064397, + "OOD_CommonPile": -0.001909514717639449 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002389993618745076, + "Wikipedia_en|USPTO_Backgrounds": 0.0008054071076450953, + "Wikipedia_en|Pile-CC": -0.000794704577013361, + "Wikipedia_en|StackExchange": -0.0016128712148952562, + "Wikipedia_en|OOD_News_2024": 5.628397606965985e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0008488879236022984, + "Wikipedia_en|OOD_CommonPile": 4.779013051181558e-06, + "USPTO_Backgrounds|Wikipedia_en": -4.700413121590252e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0003991726878291302, + "USPTO_Backgrounds|Pile-CC": 0.0005275112979616775, + "USPTO_Backgrounds|StackExchange": 1.0261950748297055e-05, + "USPTO_Backgrounds|OOD_News_2024": -0.0003658802791808996, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00017639187881575424, + "USPTO_Backgrounds|OOD_CommonPile": -0.0003476696473265345, + "Pile-CC|Wikipedia_en": 0.0002492877407117844, + "Pile-CC|USPTO_Backgrounds": -0.0010105090019928709, + "Pile-CC|Pile-CC": 0.00021886622887992702, + "Pile-CC|StackExchange": -0.0009668536694897728, + "Pile-CC|OOD_News_2024": 0.0009102561148918442, + "Pile-CC|OOD_arXiv_2023": 0.0009200280776158754, + "Pile-CC|OOD_CommonPile": -0.0003210754906167179, + "StackExchange|Wikipedia_en": 0.00020615247240853007, + "StackExchange|USPTO_Backgrounds": 0.0016799080355595492, + "StackExchange|Pile-CC": -0.0009177687899431242, + "StackExchange|StackExchange": 0.0014148739887224066, + "StackExchange|OOD_News_2024": -0.00035102406982331913, + "StackExchange|OOD_arXiv_2023": -0.0010415576490765688, + "StackExchange|OOD_CommonPile": -0.000990583987847432, + "OOD_News_2024|Wikipedia_en": -0.0007300415231535767, + "OOD_News_2024|USPTO_Backgrounds": -0.0009629242532398635, + "OOD_News_2024|Pile-CC": 0.00014226354385120377, + "OOD_News_2024|StackExchange": -0.00044888338842798137, + "OOD_News_2024|OOD_News_2024": 0.0009492949623289421, + "OOD_News_2024|OOD_arXiv_2023": 0.0011373875481593965, + "OOD_News_2024|OOD_CommonPile": -8.709688951807903e-05, + "OOD_arXiv_2023|Wikipedia_en": -0.000615019378961898, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0005502577653036383, + "OOD_arXiv_2023|Pile-CC": -3.109283427456255e-05, + "OOD_arXiv_2023|StackExchange": 0.0007823182340149643, + "OOD_arXiv_2023|OOD_News_2024": -0.000979786213960429, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00016105705687392835, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012327809016116908, + "OOD_CommonPile|Wikipedia_en": -0.0014533687985339439, + "OOD_CommonPile|USPTO_Backgrounds": -0.00036079681049736023, + "OOD_CommonPile|Pile-CC": 0.0008549251305383088, + "OOD_CommonPile|StackExchange": 0.0008211540993273564, + "OOD_CommonPile|OOD_News_2024": -0.00021914449032571504, + "OOD_CommonPile|OOD_arXiv_2023": -0.00015163523115452326, + "OOD_CommonPile|OOD_CommonPile": 0.0005088661006459744 + } + }, + "C4_permutation": { + "null_mean": -9.644068875497913e-06, + "null_sd": 0.00040128992078327074, + "n_abs_ge_observed": 68, + "n_perm": 2000, + "p_two_sided": 0.034482758620689655 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.244750499708084e-06, + "null_sd": 0.00039985230956766146, + "n_abs_ge_observed": 24, + "n_perm": 2000, + "p_two_sided": 0.01249375312343828 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 9.441563525961398e-05, + "surrogate_D_without_degenerate": -3.09029476336121e-05, + "D_minus_surrogate_all8": 0.000750956583319911, + "D_minus_surrogate_without_degenerate": 0.001037923721637833, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + }, + "min": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.23895396097666688, + "Wikipedia_en|USPTO_Backgrounds": 0.2436402971131934, + "Wikipedia_en|Pile-CC": 0.24109223555359574, + "Wikipedia_en|StackExchange": 0.22112794599185387, + "Wikipedia_en|Github": 0.22357925524314246, + "Wikipedia_en|OOD_News_2024": 0.2387056555185053, + "Wikipedia_en|OOD_arXiv_2023": 0.2240878723975685, + "Wikipedia_en|OOD_CommonPile": 0.2209852614750465, + "USPTO_Backgrounds|Wikipedia_en": 0.262192660694321, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.2683731574151251, + "USPTO_Backgrounds|Pile-CC": 0.26694677480392986, + "USPTO_Backgrounds|StackExchange": 0.2489887709625893, + "USPTO_Backgrounds|Github": 0.24927170336660412, + "USPTO_Backgrounds|OOD_News_2024": 0.26407354954216217, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.2434683405690723, + "USPTO_Backgrounds|OOD_CommonPile": 0.24144293343027431, + "Pile-CC|Wikipedia_en": 0.26439758812387787, + "Pile-CC|USPTO_Backgrounds": 0.2711278670363956, + "Pile-CC|Pile-CC": 0.2672508102738195, + "Pile-CC|StackExchange": 0.23920876369294192, + "Pile-CC|Github": 0.24696667301985953, + "Pile-CC|OOD_News_2024": 0.2684913569026523, + "Pile-CC|OOD_arXiv_2023": 0.2459330331120226, + "Pile-CC|OOD_CommonPile": 0.2441944960090849, + "StackExchange|Wikipedia_en": 0.26712176754242845, + "StackExchange|USPTO_Backgrounds": 0.27237275782972575, + "StackExchange|Pile-CC": 0.2704890080831117, + "StackExchange|StackExchange": 0.24824307101468246, + "StackExchange|Github": 0.25470268204808233, + "StackExchange|OOD_News_2024": 0.27020158254437976, + "StackExchange|OOD_arXiv_2023": 0.24713207156293923, + "StackExchange|OOD_CommonPile": 0.24989844709634781, + "Github|Wikipedia_en": 0.2348206989467144, + "Github|USPTO_Backgrounds": 0.23774738440083132, + "Github|Pile-CC": 0.2397848286562496, + "Github|StackExchange": 0.22261133020122845, + "Github|Github": 0.22343710913426346, + "Github|OOD_News_2024": 0.2349901809874508, + "Github|OOD_arXiv_2023": 0.21322937744359174, + "Github|OOD_CommonPile": 0.2154477684862084, + "OOD_News_2024|Wikipedia_en": 0.251803253011571, + "OOD_News_2024|USPTO_Backgrounds": 0.2560335245397356, + "OOD_News_2024|Pile-CC": 0.25415840529733236, + "OOD_News_2024|StackExchange": 0.22414421873788037, + "OOD_News_2024|Github": 0.23448309062255754, + "OOD_News_2024|OOD_News_2024": 0.25150823344786966, + "OOD_News_2024|OOD_arXiv_2023": 0.23495631789167723, + "OOD_News_2024|OOD_CommonPile": 0.23010762880245844, + "OOD_arXiv_2023|Wikipedia_en": 0.25748952556815413, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.2689400637108419, + "OOD_arXiv_2023|Pile-CC": 0.2626505770410101, + "OOD_arXiv_2023|StackExchange": 0.2423444380155868, + "OOD_arXiv_2023|Github": 0.24505835103078022, + "OOD_arXiv_2023|OOD_News_2024": 0.2630884069121546, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.2443132063994805, + "OOD_arXiv_2023|OOD_CommonPile": 0.2444441412265102, + "OOD_CommonPile|Wikipedia_en": 0.2519655141565535, + "OOD_CommonPile|USPTO_Backgrounds": 0.26096289563510155, + "OOD_CommonPile|Pile-CC": 0.2566619581439429, + "OOD_CommonPile|StackExchange": 0.23913137020750178, + "OOD_CommonPile|Github": 0.2396458975970745, + "OOD_CommonPile|OOD_News_2024": 0.25631876453343366, + "OOD_CommonPile|OOD_arXiv_2023": 0.2370515153846807, + "OOD_CommonPile|OOD_CommonPile": 0.23404423404071067 + }, + "all8": { + "D": 0.0003741732225679442, + "diag_mean": 0.0003274015697469525, + "offdiag_mean": -4.6771652820991726e-05, + "a_fit": { + "Wikipedia_en": -0.015166510734383742, + "USPTO_Backgrounds": 0.00890666507992946, + "Pile-CC": 0.00925825225325147, + "StackExchange": 0.013332102197131851, + "Github": -0.01892948648601306, + "OOD_News_2024": -0.0045387372241950386, + "OOD_arXiv_2023": 0.006853017469984485, + "OOD_CommonPile": 0.0002846974442945738 + }, + "b_read": { + "Wikipedia_en": 0.0069050498594555865, + "USPTO_Backgrounds": 0.013211672192038476, + "Pile-CC": 0.010691253463543632, + "StackExchange": -0.010963082665047197, + "Github": -0.007044976010284809, + "OOD_News_2024": 0.009234145030495716, + "OOD_arXiv_2023": -0.010416604422951226, + "OOD_CommonPile": -0.011617457447250151 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0005273505835147185, + "Wikipedia_en|USPTO_Backgrounds": -0.0010929356125416456, + "Wikipedia_en|Pile-CC": -0.0011205784436444655, + "Wikipedia_en|StackExchange": 0.0005694681232044918, + "Wikipedia_en|Github": -0.00089732928026931, + "Wikipedia_en|OOD_News_2024": -0.0020500500456869797, + "Wikipedia_en|OOD_arXiv_2023": 0.0029829162868231507, + "Wikipedia_en|OOD_CommonPile": 0.0010811583886000675, + "USPTO_Backgrounds|Wikipedia_en": -0.00030712551314435177, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0004332511249231785, + "USPTO_Backgrounds|Pile-CC": 0.0006607849923764564, + "USPTO_Backgrounds|StackExchange": 0.004357117279626732, + "USPTO_Backgrounds|Github": 0.0007219430288791517, + "USPTO_Backgrounds|OOD_News_2024": -0.0007553318363433259, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0017097913559862565, + "USPTO_Backgrounds|OOD_CommonPile": -0.0025343454704853108, + "Pile-CC|Wikipedia_en": 0.001546214743090496, + "Pile-CC|USPTO_Backgrounds": 0.001969871323025313, + "Pile-CC|Pile-CC": 0.0006132332889440539, + "Pile-CC|StackExchange": -0.005774477163342673, + "Pile-CC|Github": -0.0019346744911874492, + "Pile-CC|OOD_News_2024": 0.0033108883508247977, + "Pile-CC|OOD_arXiv_2023": 0.00040331401364204766, + "Pile-CC|OOD_CommonPile": -0.000134370064996725, + "StackExchange|Wikipedia_en": 0.0001965442177606913, + "StackExchange|USPTO_Backgrounds": -0.0008590878275248903, + "StackExchange|Pile-CC": -0.00022241884564408454, + "StackExchange|StackExchange": -0.0008140197854825115, + "StackExchange|Github": 0.001727484593154971, + "StackExchange|OOD_News_2024": 0.0009472640486718753, + "StackExchange|OOD_arXiv_2023": -0.0024714974793217104, + "StackExchange|OOD_CommonPile": 0.001495731078385798, + "Github|Wikipedia_en": 0.000157064305191551, + "Github|USPTO_Backgrounds": -0.0032228725732744157, + "Github|Pile-CC": 0.0013349904106386978, + "Github|StackExchange": 0.005815828084208385, + "Github|Github": 0.0027235003624810072, + "Github|OOD_News_2024": -0.002002548825112166, + "Github|OOD_arXiv_2023": -0.004112602915524288, + "Github|OOD_CommonPile": -0.0006933588486087161, + "OOD_News_2024|Wikipedia_en": 0.002748869108230162, + "OOD_News_2024|USPTO_Backgrounds": 0.0006725183038118476, + "OOD_News_2024|Pile-CC": 0.0013178177899034471, + "OOD_News_2024|StackExchange": -0.007042032640957707, + "OOD_News_2024|Github": -0.0006212674110429306, + "OOD_News_2024|OOD_News_2024": 0.00012475437348866136, + "OOD_News_2024|OOD_arXiv_2023": 0.0032235882707431773, + "OOD_News_2024|OOD_CommonPile": -0.00042424779417668534, + "OOD_arXiv_2023|Wikipedia_en": -0.0029566130293662563, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0021873027807386147, + "OOD_arXiv_2023|Pile-CC": -0.001581765160598353, + "OOD_arXiv_2023|StackExchange": -0.00023356805743079967, + "OOD_arXiv_2023|Github": -0.001437761696999773, + "OOD_arXiv_2023|OOD_News_2024": 0.00031317314359410875, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0011887220843669333, + "OOD_arXiv_2023|OOD_CommonPile": 0.002520509935695553, + "OOD_CommonPile|Wikipedia_en": -0.001912304415276983, + "OOD_CommonPile|USPTO_Backgrounds": 0.0007784547306881884, + "OOD_CommonPile|Pile-CC": -0.0010020640319756413, + "OOD_CommonPile|StackExchange": 0.003121684160174082, + "OOD_CommonPile|Github": -0.0002818951050155838, + "OOD_CommonPile|OOD_News_2024": 0.00011185079056305613, + "OOD_CommonPile|OOD_arXiv_2023": 0.0004953510952570295, + "OOD_CommonPile|OOD_CommonPile": -0.0013110772244140645 + } + }, + "without_degenerate": { + "D": -8.222654747663485e-05, + "diag_mean": -7.047989783710653e-05, + "offdiag_mean": 1.1746649639528328e-05, + "a_fit": { + "Wikipedia_en": -0.017798114832397766, + "USPTO_Backgrounds": 0.006043736366322772, + "Pile-CC": 0.006774840328225729, + "StackExchange": 0.010325524688628629, + "OOD_News_2024": -0.0072097787320985485, + "OOD_arXiv_2023": 0.004298618002931964, + "OOD_CommonPile": -0.002434825821612835 + }, + "b_read": { + "Wikipedia_en": 0.0058206053171940275, + "USPTO_Backgrounds": 0.012610075775272012, + "Pile-CC": 0.009438533763361068, + "StackExchange": -0.012855922032882577, + "OOD_News_2024": 0.00845821664970614, + "OOD_arXiv_2023": -0.010891096505110498, + "OOD_CommonPile": -0.012580412967540255 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0004771800842685303, + "Wikipedia_en|USPTO_Backgrounds": -0.0016259542372829283, + "Wikipedia_en|Pile-CC": -0.0010024737849696486, + "Wikipedia_en|StackExchange": 0.0013276924495321252, + "Wikipedia_en|OOD_News_2024": -0.0024087367064051513, + "Wikipedia_en|OOD_arXiv_2023": 0.002322793327474676, + "Wikipedia_en|OOD_CommonPile": 0.0009094988673824245, + "USPTO_Backgrounds|Wikipedia_en": -0.00012597139679787484, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0007349451340717961, + "USPTO_Backgrounds|Pile-CC": 0.0010102142666439384, + "USPTO_Backgrounds|StackExchange": 0.005346666221547031, + "USPTO_Backgrounds|OOD_News_2024": -0.0008826938814688323, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.002138589699742066, + "USPTO_Backgrounds|OOD_CommonPile": -0.0024746803761102887, + "Pile-CC|Wikipedia_en": 0.001347852070856026, + "Pile-CC|USPTO_Backgrounds": 0.0012886605252957484, + "Pile-CC|Pile-CC": 0.000583145774630589, + "Pile-CC|StackExchange": -0.005164445010003321, + "Pile-CC|OOD_News_2024": 0.0028040095171183443, + "Pile-CC|OOD_arXiv_2023": -0.0004050011186947089, + "Pile-CC|OOD_CommonPile": -0.00045422175920264984, + "StackExchange|Wikipedia_en": 0.000521347129003702, + "StackExchange|USPTO_Backgrounds": -0.0010171330417769742, + "StackExchange|Pile-CC": 0.0002706592235199312, + "StackExchange|StackExchange": 0.0003191779513343207, + "StackExchange|OOD_News_2024": 0.0009635507984429026, + "StackExchange|OOD_arXiv_2023": -0.0027566470281809863, + "StackExchange|OOD_CommonPile": 0.0016990449676573538, + "OOD_News_2024|Wikipedia_en": 0.0027381360188734605, + "OOD_News_2024|USPTO_Backgrounds": 0.00017893708896005167, + "OOD_News_2024|Pile-CC": 0.0014753598584677508, + "OOD_News_2024|StackExchange": -0.006244370904740587, + "OOD_News_2024|OOD_News_2024": -0.00019449487734002346, + "OOD_News_2024|OOD_arXiv_2023": 0.0026029027212841893, + "OOD_News_2024|OOD_CommonPile": -0.0005564699055048417, + "OOD_arXiv_2023|Wikipedia_en": -0.003083988159573947, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0015770795250358294, + "OOD_arXiv_2023|Pile-CC": -0.0015408651328850387, + "OOD_arXiv_2023|StackExchange": 0.00044745163793533105, + "OOD_arXiv_2023|OOD_News_2024": -0.00012271814808556547, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00045139449405695586, + "OOD_arXiv_2023|OOD_CommonPile": 0.0022716457835164072, + "OOD_CommonPile|Wikipedia_en": -0.0018745557466297857, + "OOD_CommonPile|USPTO_Backgrounds": 0.0003333552738402912, + "OOD_CommonPile|Pile-CC": -0.0007960402054074389, + "OOD_CommonPile|StackExchange": 0.003967827654395101, + "OOD_CommonPile|OOD_News_2024": -0.00015891670226172994, + "OOD_CommonPile|OOD_arXiv_2023": -7.685269619805979e-05, + "OOD_CommonPile|OOD_CommonPile": -0.001394817577738322 + } + }, + "C4_permutation": { + "null_mean": -2.3054927495855173e-05, + "null_sd": 0.0009489086857509815, + "n_abs_ge_observed": 1426, + "n_perm": 2000, + "p_two_sided": 0.7131434282858571 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.452356661881763e-06, + "null_sd": 0.001010189712926274, + "n_abs_ge_observed": 1887, + "n_perm": 2000, + "p_two_sided": 0.9435282358820589 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": -1.0121748673588243e-05, + "surrogate_D_without_degenerate": -0.00011535759602394957, + "D_minus_surrogate_all8": 0.00038429497124153243, + "D_minus_surrogate_without_degenerate": 3.313104854731472e-05, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + } + }, + "per_set_D": { + "multihop": { + "per_prefix_seed": [ + 0.0011145231962603117, + 0.000726503469715161, + 0.0008255728420668415 + ], + "mean": 0.0008888665026807713, + "sd": 0.00016460942186982471 + }, + "multilingual": { + "per_prefix_seed": [ + -0.00024172088264354637, + 0.000781276895265494, + 0.0012064470377351557 + ], + "mean": 0.0005820010167857012, + "sd": 0.0006077722919051255 + }, + "order-ops": { + "per_prefix_seed": [ + -0.0012291302638394491, + -0.004769640043377876, + 0.0012368599751165935 + ], + "mean": -0.0015873034440335772, + "sd": 0.0024651878411595463 + }, + "poetry": { + "per_prefix_seed": [ + 5.206154725913491e-05, + -0.0002689851472886013, + 0.00020824664118533422 + ], + "mean": -2.892319614710725e-06, + "sd": 0.0001986663712455282 + }, + "typo": { + "per_prefix_seed": [ + 0.005978955621165889, + 0.0031976310004081044, + 0.0038619598905955043 + ], + "mean": 0.004346182170723166, + "sd": 0.0011859721958341132 + } + }, + "per_set_sign_split": { + "n_positive": 3, + "n_negative": 2, + "note": "if the sets split in sign, a null on the set-averaged D is cancellation, not absence of matching" + }, + "PARTIAL_UNBLINDING_DISCLOSED": "E36's stored ladder already contains a 5x5 in-stream fit x read matrix (E28 N=400 operators, same aggregation, same admitted sets). Applying this script's diag_excess to it gives D = +0.001214 with all five diagonal residuals positive and a permutation p of 0.134 \u2014 i.e. positive but NOT significant. That estimate was known before E52 ran and is recorded here so E52's primary is not presented as blind for the in-stream half. It also carries ~17% fit/prefix leakage (E28 fitted N=400 from the same 800-doc block), which is the confound Amendment 1 removes, so its sign is exactly what leakage predicts.", + "secondary_ood_rungs": { + "OOD_News_2024": { + "matched_J^q": 0.10254994873992271, + "logit": 0.07871548670809715, + "shuf_floor": 0.08219137775401274, + "best_instream_J^P": 0.11525068599213328, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.023834462031825557, + "delta_vs_best_instream": -0.012700737252210575 + }, + "OOD_arXiv_2023": { + "matched_J^q": 0.09594758972784298, + "logit": 0.06943058591180791, + "shuf_floor": 0.07357985358685255, + "best_instream_J^P": 0.10726413512602448, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.02651700381603507, + "delta_vs_best_instream": -0.011316545398181499 + }, + "OOD_CommonPile": { + "matched_J^q": 0.09897827323940064, + "logit": 0.07235888629996529, + "shuf_floor": 0.07529745303715268, + "best_instream_J^P": 0.10878850278548069, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.026619386939435358, + "delta_vs_best_instream": -0.009810229546080046 + } + }, + "controls": { + "C1_Q0_vs_e48": { + "applicable": false, + "why": "results/e48_crossover_410m.json was scored at the unstripped readout. Comparing a stripped Q0 rung against it would compare two conventions, not two runs. The stripped counterpart is R2 (results/e48_crossover_410m_rstrip.json).", + "fires": null + }, + "C1_fires": null, + "C2_shuf_floor_excl_degenerate": { + "below_floor": { + "Q0": [], + "Wikipedia_en": [], + "USPTO_Backgrounds": [], + "Pile-CC": [], + "StackExchange": [], + "Github": [], + "OOD_News_2024": [], + "OOD_arXiv_2023": [], + "OOD_CommonPile": [], + "SHUFFLED_Pile-CC": [] + }, + "fires": true + }, + "C3_capability": { + "q0_mean_rank": 3124.912653975364, + "per_rung": { + "Q0": 3124.912653975364, + "Wikipedia_en": 2912.431131019037, + "USPTO_Backgrounds": 2895.9477416946625, + "Pile-CC": 3041.0272489734975, + "StackExchange": 3170.777902202314, + "Github": 3232.4229189996267, + "OOD_News_2024": 3204.6147816349385, + "OOD_arXiv_2023": 3026.1086226203806, + "OOD_CommonPile": 3008.9858156028367, + "SHUFFLED_Pile-CC": 2797.349757372154 + }, + "collapsed": [], + "fires": true + }, + "C5_self_distance": { + "value": 0.0, + "fires": true + }, + "R1_C2_readout_divergence": { + "required_total": 157, + "required_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "observed_total": 157, + "observed_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "n_released_items": 551, + "examples": { + "multihop": { + "unstripped_token_id": 209, + "stripped_token_id": 253, + "unstripped": "' '", + "stripped": "' the'" + }, + "multilingual": { + "unstripped_token_id": 209, + "stripped_token_id": 10863, + "unstripped": "' '", + "stripped": "' ist'" + }, + "order-ops": { + "unstripped_token_id": 209, + "stripped_token_id": 426, + "unstripped": "' '", + "stripped": "' ='" + } + }, + "note": "computed over ALL released items in all six sets, independently of --rstrip: it describes the difference between the two conventions, not this run's arm.", + "fires": true + }, + "R1_C3_logit_constant_moves": { + "arm": "STRIPPED", + "observed_logit_Q0_persist": 0.08317913140635938, + "unstripped_reference": 0.0284395, + "reference_source": "E33/E36 agree to 2.6e-10 on the unstripped Q0 logit constant", + "abs_diff_vs_unstripped_reference": 0.05473963140635938, + "fires": true + }, + "R1_C4_derangement_floor": { + "alias_of": "C2_shuf_floor_excl_degenerate", + "required": "no non-degenerate operator at or below its rung's shuf floor under persist", + "fires": true + }, + "P0_C3_worker_threads": { + "workers": 1, + "observed_threads_in_cell": 8, + "required": "1 when --workers > 1", + "fires": null + } + }, + "VERDICT": "NO MATCHING (the publishable null) \u2014 D = +0.00085, CI [-0.00090,+0.00279] includes zero. The corpus effect established in E28/E33/E48 is a property of the FITTING CORPUS PER SE; the fit-read relationship is not the operative variable, and EQ2's ||J^P - J^Q|| gap framing is not what the data measures.", + "provenance": { + "schema_version": 1, + "experiment": "E52", + "utc": "2026-08-20T10:21:58+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "6d74ea7f4ef2a968b73923e3858cbafe48c5a8d735f3118ad31422b7d43a272d", + "n_lines": 851 + }, + "git": { + "commit": "0085188b142f807a95aed2394063dae7009cf1b7", + "commit_short": "0085188b142f", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/e36_qladder_410m_c1arm.json", + "results/e36_qladder_410m_rstrip.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "aee92baa8be53162e66b7c00eec04f9286b40c2afaf4dbb2dbe8c1782597cc28", + "describe": "0085188-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--workers", + "1", + "--rstrip", + "--out", + "results/e52_factorial_410m_rstrip_w1.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "cb00c6c92899012c350c180d05e5928ea183811de386004899e6812fbacf37aa", + "bytes": 115278, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "c4a7b87227a4b9b3b8e60777d952285592d29cf719c9970c493153b0ba551010" + } +} \ No newline at end of file diff --git a/results/e52_factorial_410m_rstrip_w2.json b/results/e52_factorial_410m_rstrip_w2.json new file mode 100644 index 0000000000000000000000000000000000000000..c7ada25c01de184094aed09ba063104fb4984c76 --- /dev/null +++ b/results/e52_factorial_410m_rstrip_w2.json @@ -0,0 +1,1096 @@ +{ + "experiment": "E52 \u2014 the fit x read factorial: J^Q read on Q, the missing cell", + "prereg": "specs/archive/prereg/PREREG_E52_FACTORIAL.md, written before this ran", + "prereg_files": [ + { + "path": "specs/archive/prereg/PREREG_E52_FACTORIAL.md", + "role": "E52, the original design", + "sha256": "740ddb1d6aeb35d8c0d93b626043e62c53d35d149ec9d6a2e7bb8817875902ce" + }, + { + "path": "specs/experiments/P0_t52_parallel_port.md", + "role": "P0, the pool port", + "sha256": "d2cd8cb4b3296249ace1ae995b238b2035b21d7884858596777202971869565e" + }, + { + "path": "specs/experiments/R1_grid_rstrip.md", + "role": "R1, the readout correction", + "sha256": "3b13a90cd3730455c6d1328a84d50653f3f146e39ed115f08dc2f77e872fcd07" + } + ], + "decision_rule_verbatim": { + "R1": "ACCEPT (headline robust): stripped fit_pct exceeds stripped read_pct under BOTH aggregations, and stripped fit_pct lies inside the published bootstrap interval [87.609, 92.171] under persist. | QUALIFIED (ordering survives, magnitude does not): stripped fit_pct still exceeds read_pct under both aggregations but falls outside the published interval. | REJECT (ordering does not survive): stripped fit_pct does not exceed read_pct under both aggregations. STOP. Alert the operator.", + "P0": "ACCEPT: the pooled run reproduces the serial run's stored matrix at max_abs_diff = 0.0 across all 128 cell values, and wallclock falls by at least 3x. | REJECT: any non-zero difference. STOP. Do not \"investigate the tolerance\" -- there is no tolerance." + }, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`, i.e. the final prompt token after .rstrip(). jacobian-lens/data/evaluations/README.md; src/anchor_evals.py:228", + "rstrip": true, + "workers": 2, + "wallclock_seconds": 5246.2, + "cell_completion_seconds": { + "Q0|p1": 119.7, + "Q0|p0": 119.7, + "Q0|p2": 257.4, + "Wikipedia_en|p0": 376.1, + "Wikipedia_en|p1": 507.3, + "Wikipedia_en|p2": 604.5, + "USPTO_Backgrounds|p0": 742.8, + "USPTO_Backgrounds|p1": 916.2, + "USPTO_Backgrounds|p2": 1101.3, + "Pile-CC|p0": 1255.0, + "Pile-CC|p1": 1458.7, + "Pile-CC|p2": 1614.2, + "StackExchange|p0": 1805.8, + "StackExchange|p1": 1994.3, + "StackExchange|p2": 2225.5, + "Github|p0": 2402.7, + "Github|p1": 2631.6, + "Github|p2": 2817.0, + "OOD_News_2024|p0": 3049.1, + "OOD_News_2024|p1": 3241.5, + "OOD_News_2024|p2": 3466.2, + "OOD_arXiv_2023|p0": 3641.8, + "OOD_arXiv_2023|p1": 3861.2, + "OOD_arXiv_2023|p2": 4052.7, + "OOD_CommonPile|p0": 4275.3, + "OOD_CommonPile|p1": 4449.4, + "OOD_CommonPile|p2": 4704.4, + "SHUFFLED_Pile-CC|p0": 4897.5, + "SHUFFLED_Pile-CC|p1": 5111.4, + "SHUFFLED_Pile-CC|p2": 5245.7 + }, + "status": "PRE-REGISTERED", + "question": "can a J-lens trained on an out-of-distribution sample read latent structure on an out-of-distribution corpus?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_operators": 24, + "n_items": 541, + "n_pairs": 893, + "prefix_tokens": 128, + "smoke": false, + "all_operators_share_one_fitter": "trainval.py N=200 --band 9,21 dim_batch=128 max_seq_len=128 skip_first=16 target_layer=-2 fp32", + "adjudication": { + "aggregation": "persist", + "min_does_not_vote": true, + "D": 0.000845372218579525, + "D_without_degenerate": 0.001007020774004221, + "D_hierarchical_ci": { + "point": 0.000845370814204216, + "ci_lo": -0.0008957392419688404, + "ci_hi": 0.0027882857248187065, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "degenerate_excluded_from_C2": [ + "Github" + ] + }, + "by_aggregation": { + "persist": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.1057175321996005, + "Wikipedia_en|USPTO_Backgrounds": 0.10812408523634076, + "Wikipedia_en|Pile-CC": 0.10244530709201677, + "Wikipedia_en|StackExchange": 0.08896334908850904, + "Wikipedia_en|Github": 0.09078200098075387, + "Wikipedia_en|OOD_News_2024": 0.10550682633183897, + "Wikipedia_en|OOD_arXiv_2023": 0.09642561516569306, + "Wikipedia_en|OOD_CommonPile": 0.09897492753031353, + "USPTO_Backgrounds|Wikipedia_en": 0.1134465583651844, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.11788387473206967, + "USPTO_Backgrounds|Pile-CC": 0.11393354688253668, + "USPTO_Backgrounds|StackExchange": 0.10075250616969747, + "USPTO_Backgrounds|Github": 0.0997736249739925, + "USPTO_Backgrounds|OOD_News_2024": 0.11525068599213328, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.10726413512602448, + "USPTO_Backgrounds|OOD_CommonPile": 0.10878850278548069, + "Pile-CC|Wikipedia_en": 0.11220522388401959, + "Pile-CC|USPTO_Backgrounds": 0.11493656668915517, + "Pile-CC|Pile-CC": 0.11208727546036243, + "Pile-CC|StackExchange": 0.0982377641963669, + "Pile-CC|Github": 0.09963205248220927, + "Pile-CC|OOD_News_2024": 0.11498919603311353, + "Pile-CC|OOD_arXiv_2023": 0.10682292872936361, + "Pile-CC|OOD_CommonPile": 0.107277470589098, + "StackExchange|Wikipedia_en": 0.1126410995092657, + "StackExchange|USPTO_Backgrounds": 0.11810599462025695, + "StackExchange|Pile-CC": 0.11142965133508874, + "StackExchange|StackExchange": 0.10109850274812844, + "StackExchange|Github": 0.10198042160417471, + "StackExchange|OOD_News_2024": 0.11420692674194773, + "StackExchange|OOD_arXiv_2023": 0.10534035389622053, + "StackExchange|OOD_CommonPile": 0.10708697298541665, + "Github|Wikipedia_en": 0.09720011671694616, + "Github|USPTO_Backgrounds": 0.09792679025170704, + "Github|Pile-CC": 0.09611689302449425, + "Github|StackExchange": 0.08193752315774974, + "Github|Github": 0.0831806702642805, + "Github|OOD_News_2024": 0.09784760768007901, + "Github|OOD_arXiv_2023": 0.09211890187321438, + "Github|OOD_CommonPile": 0.09339318144031697, + "OOD_News_2024|Wikipedia_en": 0.09874760847952631, + "OOD_News_2024|USPTO_Backgrounds": 0.10250586529728026, + "OOD_News_2024|Pile-CC": 0.09953238663470579, + "OOD_News_2024|StackExchange": 0.08627744833680077, + "OOD_News_2024|Github": 0.08632135616822376, + "OOD_News_2024|OOD_News_2024": 0.10254994873992271, + "OOD_News_2024|OOD_arXiv_2023": 0.09456200205927921, + "OOD_News_2024|OOD_CommonPile": 0.09503316304956873, + "OOD_arXiv_2023|Wikipedia_en": 0.10122454878356722, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.10528044994506572, + "OOD_arXiv_2023|Pile-CC": 0.10172094841642926, + "OOD_arXiv_2023|StackExchange": 0.08987056811909296, + "OOD_arXiv_2023|Github": 0.08733742833137512, + "OOD_arXiv_2023|OOD_News_2024": 0.10298278572348257, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.09594758972784298, + "OOD_arXiv_2023|OOD_CommonPile": 0.09871495900054773, + "OOD_CommonPile|Wikipedia_en": 0.1013734284038138, + "OOD_CommonPile|USPTO_Backgrounds": 0.10645713993969062, + "OOD_CommonPile|Pile-CC": 0.10359419542106076, + "OOD_CommonPile|StackExchange": 0.09089663302422397, + "OOD_CommonPile|Github": 0.08911381827977796, + "OOD_CommonPile|OOD_News_2024": 0.10473065648693591, + "OOD_CommonPile|OOD_arXiv_2023": 0.09662212647963316, + "OOD_CommonPile|OOD_CommonPile": 0.09897827323940064 + }, + "all8": { + "D": 0.000845372218579525, + "diag_mean": 0.0007397006912570839, + "offdiag_mean": -0.00010567152732244106, + "a_fit": { + "Wikipedia_en": -0.001823302244560579, + "USPTO_Backgrounds": 0.008195921680695994, + "Pile-CC": 0.006832802060267165, + "StackExchange": 0.007545482732368533, + "Github": -0.008975547146595389, + "OOD_News_2024": -0.00574953535203046, + "OOD_arXiv_2023": -0.0035558479417684558, + "OOD_CommonPile": -0.0024699737883767947 + }, + "b_read": { + "Wikipedia_en": 0.0038787568450465715, + "USPTO_Backgrounds": 0.007461838141251884, + "Pile-CC": 0.003666767835642931, + "StackExchange": -0.009186470842622738, + "Github": -0.00917558606209544, + "OOD_News_2024": 0.005817321518487822, + "OOD_arXiv_2023": -0.002052801065534973, + "OOD_CommonPile": -0.00040982637017603074 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002221319901420618, + "Wikipedia_en|USPTO_Backgrounds": 0.0010447916419555614, + "Wikipedia_en|Pile-CC": -0.0008389161967594788, + "Wikipedia_en|StackExchange": -0.0014676355220015341, + "Wikipedia_en|Github": 0.0003401315897159918, + "Wikipedia_en|OOD_News_2024": 7.204936021783237e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0011390392219052803, + "Wikipedia_en|OOD_CommonPile": -0.00023270155264375203, + "USPTO_Backgrounds|Wikipedia_en": -6.887785825206161e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0007853572124278951, + "USPTO_Backgrounds|Pile-CC": 0.0006300996685038585, + "USPTO_Backgrounds|StackExchange": 0.00030229763393031794, + "USPTO_Backgrounds|Github": -0.0006874683423019473, + "USPTO_Backgrounds|OOD_News_2024": -0.00020331490474442826, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00031974318683043734, + "USPTO_Backgrounds|OOD_CommonPile": -0.00043835022273316926, + "Pile-CC|Wikipedia_en": 5.290728101195463e-05, + "Pile-CC|USPTO_Backgrounds": -0.0007988312100577766, + "Pile-CC|Pile-CC": 0.0001469478667584373, + "Pile-CC|StackExchange": -0.0008493247189714226, + "Pile-CC|Github": 0.0005340787863436502, + "Pile-CC|OOD_News_2024": 0.0008983147566646449, + "Pile-CC|OOD_arXiv_2023": 0.0006021700369375216, + "Pile-CC|OOD_CommonPile": -0.0005862627986870234, + "StackExchange|Wikipedia_en": -0.00022389776584330612, + "StackExchange|USPTO_Backgrounds": 0.001657916048942637, + "StackExchange|Pile-CC": -0.0012233569306166203, + "StackExchange|StackExchange": 0.0012987331606887503, + "StackExchange|Github": 0.002169767236207723, + "StackExchange|OOD_News_2024": -0.0005966352066025249, + "StackExchange|OOD_arXiv_2023": -0.001593085468306929, + "StackExchange|OOD_CommonPile": -0.001489441074469744, + "Github|Wikipedia_en": 0.0008561493208010806, + "Github|USPTO_Backgrounds": -0.002000258440643346, + "Github|Pile-CC": -1.5085362247188594e-05, + "Github|StackExchange": -0.0013412165507260265, + "Github|Github": -0.00010895422472256833, + "Github|OOD_News_2024": -0.0004349243895073185, + "Github|OOD_arXiv_2023": 0.0017064923876508459, + "Github|OOD_CommonPile": 0.0013377972593944937, + "OOD_News_2024|Wikipedia_en": -0.0008223707111836981, + "OOD_News_2024|USPTO_Backgrounds": -0.0006471951896350608, + "OOD_News_2024|Pile-CC": 0.00017439645339942245, + "OOD_News_2024|StackExchange": -0.00022730316623992275, + "OOD_News_2024|Github": -0.00019428011534423917, + "OOD_News_2024|OOD_News_2024": 0.0010414048757714511, + "OOD_News_2024|OOD_arXiv_2023": 0.0009235807791507511, + "OOD_News_2024|OOD_CommonPile": -0.00024823292591867607, + "OOD_arXiv_2023|Wikipedia_en": -0.0005391178174047884, + "OOD_arXiv_2023|USPTO_Backgrounds": -6.629795211160472e-05, + "OOD_arXiv_2023|Pile-CC": 0.00016927082486088707, + "OOD_arXiv_2023|StackExchange": 0.0011721292057902538, + "OOD_arXiv_2023|Github": -0.0013718953624548835, + "OOD_arXiv_2023|OOD_News_2024": -0.0007194455509306891, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00011548103745251392, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012398756147983248, + "OOD_CommonPile|Wikipedia_en": -0.0014761123505498686, + "OOD_CommonPile|USPTO_Backgrounds": 2.4517889121639103e-05, + "OOD_CommonPile|Pile-CC": 0.0009566436761007241, + "OOD_CommonPile|StackExchange": 0.0011123199575296117, + "OOD_CommonPile|Github": -0.000681379567443699, + "OOD_CommonPile|OOD_News_2024": -5.744894086900931e-05, + "OOD_CommonPile|OOD_arXiv_2023": -0.000295856364148972, + "OOD_CommonPile|OOD_CommonPile": 0.000417315700259574 + } + }, + "without_degenerate": { + "D": 0.001007020774004221, + "diag_mean": 0.0008631606634321978, + "offdiag_mean": -0.00014386011057202305, + "a_fit": { + "Wikipedia_en": -0.0031518899369984976, + "USPTO_Backgrounds": 0.007014133978546375, + "Pile-CC": 0.005476507625453875, + "StackExchange": 0.005955518519003236, + "OOD_News_2024": -0.0070017785151740425, + "OOD_arXiv_2023": -0.004639860355324807, + "OOD_CommonPile": -0.00365263131550618 + }, + "b_read": { + "Wikipedia_en": 0.002447875345953629, + "USPTO_Backgrounds": 0.006439014893793865, + "Pile-CC": 0.0023603484341283287, + "StackExchange": -0.0103034429314975, + "OOD_News_2024": 0.00457087912086751, + "OOD_arXiv_2023": -0.0036051601456064397, + "OOD_CommonPile": -0.001909514717639449 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002389993618745076, + "Wikipedia_en|USPTO_Backgrounds": 0.0008054071076450953, + "Wikipedia_en|Pile-CC": -0.000794704577013361, + "Wikipedia_en|StackExchange": -0.0016128712148952562, + "Wikipedia_en|OOD_News_2024": 5.628397606965985e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0008488879236022984, + "Wikipedia_en|OOD_CommonPile": 4.779013051181558e-06, + "USPTO_Backgrounds|Wikipedia_en": -4.700413121590252e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0003991726878291302, + "USPTO_Backgrounds|Pile-CC": 0.0005275112979616775, + "USPTO_Backgrounds|StackExchange": 1.0261950748297055e-05, + "USPTO_Backgrounds|OOD_News_2024": -0.0003658802791808996, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00017639187881575424, + "USPTO_Backgrounds|OOD_CommonPile": -0.0003476696473265345, + "Pile-CC|Wikipedia_en": 0.0002492877407117844, + "Pile-CC|USPTO_Backgrounds": -0.0010105090019928709, + "Pile-CC|Pile-CC": 0.00021886622887992702, + "Pile-CC|StackExchange": -0.0009668536694897728, + "Pile-CC|OOD_News_2024": 0.0009102561148918442, + "Pile-CC|OOD_arXiv_2023": 0.0009200280776158754, + "Pile-CC|OOD_CommonPile": -0.0003210754906167179, + "StackExchange|Wikipedia_en": 0.00020615247240853007, + "StackExchange|USPTO_Backgrounds": 0.0016799080355595492, + "StackExchange|Pile-CC": -0.0009177687899431242, + "StackExchange|StackExchange": 0.0014148739887224066, + "StackExchange|OOD_News_2024": -0.00035102406982331913, + "StackExchange|OOD_arXiv_2023": -0.0010415576490765688, + "StackExchange|OOD_CommonPile": -0.000990583987847432, + "OOD_News_2024|Wikipedia_en": -0.0007300415231535767, + "OOD_News_2024|USPTO_Backgrounds": -0.0009629242532398635, + "OOD_News_2024|Pile-CC": 0.00014226354385120377, + "OOD_News_2024|StackExchange": -0.00044888338842798137, + "OOD_News_2024|OOD_News_2024": 0.0009492949623289421, + "OOD_News_2024|OOD_arXiv_2023": 0.0011373875481593965, + "OOD_News_2024|OOD_CommonPile": -8.709688951807903e-05, + "OOD_arXiv_2023|Wikipedia_en": -0.000615019378961898, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0005502577653036383, + "OOD_arXiv_2023|Pile-CC": -3.109283427456255e-05, + "OOD_arXiv_2023|StackExchange": 0.0007823182340149643, + "OOD_arXiv_2023|OOD_News_2024": -0.000979786213960429, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00016105705687392835, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012327809016116908, + "OOD_CommonPile|Wikipedia_en": -0.0014533687985339439, + "OOD_CommonPile|USPTO_Backgrounds": -0.00036079681049736023, + "OOD_CommonPile|Pile-CC": 0.0008549251305383088, + "OOD_CommonPile|StackExchange": 0.0008211540993273564, + "OOD_CommonPile|OOD_News_2024": -0.00021914449032571504, + "OOD_CommonPile|OOD_arXiv_2023": -0.00015163523115452326, + "OOD_CommonPile|OOD_CommonPile": 0.0005088661006459744 + } + }, + "C4_permutation": { + "null_mean": -9.644068875497913e-06, + "null_sd": 0.00040128992078327074, + "n_abs_ge_observed": 68, + "n_perm": 2000, + "p_two_sided": 0.034482758620689655 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.244750499708084e-06, + "null_sd": 0.00039985230956766146, + "n_abs_ge_observed": 24, + "n_perm": 2000, + "p_two_sided": 0.01249375312343828 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 9.441563525961398e-05, + "surrogate_D_without_degenerate": -3.09029476336121e-05, + "D_minus_surrogate_all8": 0.000750956583319911, + "D_minus_surrogate_without_degenerate": 0.001037923721637833, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + }, + "min": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.23895396097666688, + "Wikipedia_en|USPTO_Backgrounds": 0.2436402971131934, + "Wikipedia_en|Pile-CC": 0.24109223555359574, + "Wikipedia_en|StackExchange": 0.22112794599185387, + "Wikipedia_en|Github": 0.22357925524314246, + "Wikipedia_en|OOD_News_2024": 0.2387056555185053, + "Wikipedia_en|OOD_arXiv_2023": 0.2240878723975685, + "Wikipedia_en|OOD_CommonPile": 0.2209852614750465, + "USPTO_Backgrounds|Wikipedia_en": 0.262192660694321, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.2683731574151251, + "USPTO_Backgrounds|Pile-CC": 0.26694677480392986, + "USPTO_Backgrounds|StackExchange": 0.2489887709625893, + "USPTO_Backgrounds|Github": 0.24927170336660412, + "USPTO_Backgrounds|OOD_News_2024": 0.26407354954216217, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.2434683405690723, + "USPTO_Backgrounds|OOD_CommonPile": 0.24144293343027431, + "Pile-CC|Wikipedia_en": 0.26439758812387787, + "Pile-CC|USPTO_Backgrounds": 0.2711278670363956, + "Pile-CC|Pile-CC": 0.2672508102738195, + "Pile-CC|StackExchange": 0.23920876369294192, + "Pile-CC|Github": 0.24696667301985953, + "Pile-CC|OOD_News_2024": 0.2684913569026523, + "Pile-CC|OOD_arXiv_2023": 0.2459330331120226, + "Pile-CC|OOD_CommonPile": 0.2441944960090849, + "StackExchange|Wikipedia_en": 0.26712176754242845, + "StackExchange|USPTO_Backgrounds": 0.27237275782972575, + "StackExchange|Pile-CC": 0.2704890080831117, + "StackExchange|StackExchange": 0.24824307101468246, + "StackExchange|Github": 0.25470268204808233, + "StackExchange|OOD_News_2024": 0.27020158254437976, + "StackExchange|OOD_arXiv_2023": 0.24713207156293923, + "StackExchange|OOD_CommonPile": 0.24989844709634781, + "Github|Wikipedia_en": 0.2348206989467144, + "Github|USPTO_Backgrounds": 0.23774738440083132, + "Github|Pile-CC": 0.2397848286562496, + "Github|StackExchange": 0.22261133020122845, + "Github|Github": 0.22343710913426346, + "Github|OOD_News_2024": 0.2349901809874508, + "Github|OOD_arXiv_2023": 0.21322937744359174, + "Github|OOD_CommonPile": 0.2154477684862084, + "OOD_News_2024|Wikipedia_en": 0.251803253011571, + "OOD_News_2024|USPTO_Backgrounds": 0.2560335245397356, + "OOD_News_2024|Pile-CC": 0.25415840529733236, + "OOD_News_2024|StackExchange": 0.22414421873788037, + "OOD_News_2024|Github": 0.23448309062255754, + "OOD_News_2024|OOD_News_2024": 0.25150823344786966, + "OOD_News_2024|OOD_arXiv_2023": 0.23495631789167723, + "OOD_News_2024|OOD_CommonPile": 0.23010762880245844, + "OOD_arXiv_2023|Wikipedia_en": 0.25748952556815413, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.2689400637108419, + "OOD_arXiv_2023|Pile-CC": 0.2626505770410101, + "OOD_arXiv_2023|StackExchange": 0.2423444380155868, + "OOD_arXiv_2023|Github": 0.24505835103078022, + "OOD_arXiv_2023|OOD_News_2024": 0.2630884069121546, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.2443132063994805, + "OOD_arXiv_2023|OOD_CommonPile": 0.2444441412265102, + "OOD_CommonPile|Wikipedia_en": 0.2519655141565535, + "OOD_CommonPile|USPTO_Backgrounds": 0.26096289563510155, + "OOD_CommonPile|Pile-CC": 0.2566619581439429, + "OOD_CommonPile|StackExchange": 0.23913137020750178, + "OOD_CommonPile|Github": 0.2396458975970745, + "OOD_CommonPile|OOD_News_2024": 0.25631876453343366, + "OOD_CommonPile|OOD_arXiv_2023": 0.2370515153846807, + "OOD_CommonPile|OOD_CommonPile": 0.23404423404071067 + }, + "all8": { + "D": 0.0003741732225679442, + "diag_mean": 0.0003274015697469525, + "offdiag_mean": -4.6771652820991726e-05, + "a_fit": { + "Wikipedia_en": -0.015166510734383742, + "USPTO_Backgrounds": 0.00890666507992946, + "Pile-CC": 0.00925825225325147, + "StackExchange": 0.013332102197131851, + "Github": -0.01892948648601306, + "OOD_News_2024": -0.0045387372241950386, + "OOD_arXiv_2023": 0.006853017469984485, + "OOD_CommonPile": 0.0002846974442945738 + }, + "b_read": { + "Wikipedia_en": 0.0069050498594555865, + "USPTO_Backgrounds": 0.013211672192038476, + "Pile-CC": 0.010691253463543632, + "StackExchange": -0.010963082665047197, + "Github": -0.007044976010284809, + "OOD_News_2024": 0.009234145030495716, + "OOD_arXiv_2023": -0.010416604422951226, + "OOD_CommonPile": -0.011617457447250151 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0005273505835147185, + "Wikipedia_en|USPTO_Backgrounds": -0.0010929356125416456, + "Wikipedia_en|Pile-CC": -0.0011205784436444655, + "Wikipedia_en|StackExchange": 0.0005694681232044918, + "Wikipedia_en|Github": -0.00089732928026931, + "Wikipedia_en|OOD_News_2024": -0.0020500500456869797, + "Wikipedia_en|OOD_arXiv_2023": 0.0029829162868231507, + "Wikipedia_en|OOD_CommonPile": 0.0010811583886000675, + "USPTO_Backgrounds|Wikipedia_en": -0.00030712551314435177, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0004332511249231785, + "USPTO_Backgrounds|Pile-CC": 0.0006607849923764564, + "USPTO_Backgrounds|StackExchange": 0.004357117279626732, + "USPTO_Backgrounds|Github": 0.0007219430288791517, + "USPTO_Backgrounds|OOD_News_2024": -0.0007553318363433259, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0017097913559862565, + "USPTO_Backgrounds|OOD_CommonPile": -0.0025343454704853108, + "Pile-CC|Wikipedia_en": 0.001546214743090496, + "Pile-CC|USPTO_Backgrounds": 0.001969871323025313, + "Pile-CC|Pile-CC": 0.0006132332889440539, + "Pile-CC|StackExchange": -0.005774477163342673, + "Pile-CC|Github": -0.0019346744911874492, + "Pile-CC|OOD_News_2024": 0.0033108883508247977, + "Pile-CC|OOD_arXiv_2023": 0.00040331401364204766, + "Pile-CC|OOD_CommonPile": -0.000134370064996725, + "StackExchange|Wikipedia_en": 0.0001965442177606913, + "StackExchange|USPTO_Backgrounds": -0.0008590878275248903, + "StackExchange|Pile-CC": -0.00022241884564408454, + "StackExchange|StackExchange": -0.0008140197854825115, + "StackExchange|Github": 0.001727484593154971, + "StackExchange|OOD_News_2024": 0.0009472640486718753, + "StackExchange|OOD_arXiv_2023": -0.0024714974793217104, + "StackExchange|OOD_CommonPile": 0.001495731078385798, + "Github|Wikipedia_en": 0.000157064305191551, + "Github|USPTO_Backgrounds": -0.0032228725732744157, + "Github|Pile-CC": 0.0013349904106386978, + "Github|StackExchange": 0.005815828084208385, + "Github|Github": 0.0027235003624810072, + "Github|OOD_News_2024": -0.002002548825112166, + "Github|OOD_arXiv_2023": -0.004112602915524288, + "Github|OOD_CommonPile": -0.0006933588486087161, + "OOD_News_2024|Wikipedia_en": 0.002748869108230162, + "OOD_News_2024|USPTO_Backgrounds": 0.0006725183038118476, + "OOD_News_2024|Pile-CC": 0.0013178177899034471, + "OOD_News_2024|StackExchange": -0.007042032640957707, + "OOD_News_2024|Github": -0.0006212674110429306, + "OOD_News_2024|OOD_News_2024": 0.00012475437348866136, + "OOD_News_2024|OOD_arXiv_2023": 0.0032235882707431773, + "OOD_News_2024|OOD_CommonPile": -0.00042424779417668534, + "OOD_arXiv_2023|Wikipedia_en": -0.0029566130293662563, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0021873027807386147, + "OOD_arXiv_2023|Pile-CC": -0.001581765160598353, + "OOD_arXiv_2023|StackExchange": -0.00023356805743079967, + "OOD_arXiv_2023|Github": -0.001437761696999773, + "OOD_arXiv_2023|OOD_News_2024": 0.00031317314359410875, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0011887220843669333, + "OOD_arXiv_2023|OOD_CommonPile": 0.002520509935695553, + "OOD_CommonPile|Wikipedia_en": -0.001912304415276983, + "OOD_CommonPile|USPTO_Backgrounds": 0.0007784547306881884, + "OOD_CommonPile|Pile-CC": -0.0010020640319756413, + "OOD_CommonPile|StackExchange": 0.003121684160174082, + "OOD_CommonPile|Github": -0.0002818951050155838, + "OOD_CommonPile|OOD_News_2024": 0.00011185079056305613, + "OOD_CommonPile|OOD_arXiv_2023": 0.0004953510952570295, + "OOD_CommonPile|OOD_CommonPile": -0.0013110772244140645 + } + }, + "without_degenerate": { + "D": -8.222654747663485e-05, + "diag_mean": -7.047989783710653e-05, + "offdiag_mean": 1.1746649639528328e-05, + "a_fit": { + "Wikipedia_en": -0.017798114832397766, + "USPTO_Backgrounds": 0.006043736366322772, + "Pile-CC": 0.006774840328225729, + "StackExchange": 0.010325524688628629, + "OOD_News_2024": -0.0072097787320985485, + "OOD_arXiv_2023": 0.004298618002931964, + "OOD_CommonPile": -0.002434825821612835 + }, + "b_read": { + "Wikipedia_en": 0.0058206053171940275, + "USPTO_Backgrounds": 0.012610075775272012, + "Pile-CC": 0.009438533763361068, + "StackExchange": -0.012855922032882577, + "OOD_News_2024": 0.00845821664970614, + "OOD_arXiv_2023": -0.010891096505110498, + "OOD_CommonPile": -0.012580412967540255 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0004771800842685303, + "Wikipedia_en|USPTO_Backgrounds": -0.0016259542372829283, + "Wikipedia_en|Pile-CC": -0.0010024737849696486, + "Wikipedia_en|StackExchange": 0.0013276924495321252, + "Wikipedia_en|OOD_News_2024": -0.0024087367064051513, + "Wikipedia_en|OOD_arXiv_2023": 0.002322793327474676, + "Wikipedia_en|OOD_CommonPile": 0.0009094988673824245, + "USPTO_Backgrounds|Wikipedia_en": -0.00012597139679787484, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0007349451340717961, + "USPTO_Backgrounds|Pile-CC": 0.0010102142666439384, + "USPTO_Backgrounds|StackExchange": 0.005346666221547031, + "USPTO_Backgrounds|OOD_News_2024": -0.0008826938814688323, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.002138589699742066, + "USPTO_Backgrounds|OOD_CommonPile": -0.0024746803761102887, + "Pile-CC|Wikipedia_en": 0.001347852070856026, + "Pile-CC|USPTO_Backgrounds": 0.0012886605252957484, + "Pile-CC|Pile-CC": 0.000583145774630589, + "Pile-CC|StackExchange": -0.005164445010003321, + "Pile-CC|OOD_News_2024": 0.0028040095171183443, + "Pile-CC|OOD_arXiv_2023": -0.0004050011186947089, + "Pile-CC|OOD_CommonPile": -0.00045422175920264984, + "StackExchange|Wikipedia_en": 0.000521347129003702, + "StackExchange|USPTO_Backgrounds": -0.0010171330417769742, + "StackExchange|Pile-CC": 0.0002706592235199312, + "StackExchange|StackExchange": 0.0003191779513343207, + "StackExchange|OOD_News_2024": 0.0009635507984429026, + "StackExchange|OOD_arXiv_2023": -0.0027566470281809863, + "StackExchange|OOD_CommonPile": 0.0016990449676573538, + "OOD_News_2024|Wikipedia_en": 0.0027381360188734605, + "OOD_News_2024|USPTO_Backgrounds": 0.00017893708896005167, + "OOD_News_2024|Pile-CC": 0.0014753598584677508, + "OOD_News_2024|StackExchange": -0.006244370904740587, + "OOD_News_2024|OOD_News_2024": -0.00019449487734002346, + "OOD_News_2024|OOD_arXiv_2023": 0.0026029027212841893, + "OOD_News_2024|OOD_CommonPile": -0.0005564699055048417, + "OOD_arXiv_2023|Wikipedia_en": -0.003083988159573947, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0015770795250358294, + "OOD_arXiv_2023|Pile-CC": -0.0015408651328850387, + "OOD_arXiv_2023|StackExchange": 0.00044745163793533105, + "OOD_arXiv_2023|OOD_News_2024": -0.00012271814808556547, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00045139449405695586, + "OOD_arXiv_2023|OOD_CommonPile": 0.0022716457835164072, + "OOD_CommonPile|Wikipedia_en": -0.0018745557466297857, + "OOD_CommonPile|USPTO_Backgrounds": 0.0003333552738402912, + "OOD_CommonPile|Pile-CC": -0.0007960402054074389, + "OOD_CommonPile|StackExchange": 0.003967827654395101, + "OOD_CommonPile|OOD_News_2024": -0.00015891670226172994, + "OOD_CommonPile|OOD_arXiv_2023": -7.685269619805979e-05, + "OOD_CommonPile|OOD_CommonPile": -0.001394817577738322 + } + }, + "C4_permutation": { + "null_mean": -2.3054927495855173e-05, + "null_sd": 0.0009489086857509815, + "n_abs_ge_observed": 1426, + "n_perm": 2000, + "p_two_sided": 0.7131434282858571 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.452356661881763e-06, + "null_sd": 0.001010189712926274, + "n_abs_ge_observed": 1887, + "n_perm": 2000, + "p_two_sided": 0.9435282358820589 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": -1.0121748673588243e-05, + "surrogate_D_without_degenerate": -0.00011535759602394957, + "D_minus_surrogate_all8": 0.00038429497124153243, + "D_minus_surrogate_without_degenerate": 3.313104854731472e-05, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + } + }, + "per_set_D": { + "multihop": { + "per_prefix_seed": [ + 0.0011145231962603117, + 0.000726503469715161, + 0.0008255728420668415 + ], + "mean": 0.0008888665026807713, + "sd": 0.00016460942186982471 + }, + "multilingual": { + "per_prefix_seed": [ + -0.00024172088264354637, + 0.000781276895265494, + 0.0012064470377351557 + ], + "mean": 0.0005820010167857012, + "sd": 0.0006077722919051255 + }, + "order-ops": { + "per_prefix_seed": [ + -0.0012291302638394491, + -0.004769640043377876, + 0.0012368599751165935 + ], + "mean": -0.0015873034440335772, + "sd": 0.0024651878411595463 + }, + "poetry": { + "per_prefix_seed": [ + 5.206154725913491e-05, + -0.0002689851472886013, + 0.00020824664118533422 + ], + "mean": -2.892319614710725e-06, + "sd": 0.0001986663712455282 + }, + "typo": { + "per_prefix_seed": [ + 0.005978955621165889, + 0.0031976310004081044, + 0.0038619598905955043 + ], + "mean": 0.004346182170723166, + "sd": 0.0011859721958341132 + } + }, + "per_set_sign_split": { + "n_positive": 3, + "n_negative": 2, + "note": "if the sets split in sign, a null on the set-averaged D is cancellation, not absence of matching" + }, + "PARTIAL_UNBLINDING_DISCLOSED": "E36's stored ladder already contains a 5x5 in-stream fit x read matrix (E28 N=400 operators, same aggregation, same admitted sets). Applying this script's diag_excess to it gives D = +0.001214 with all five diagonal residuals positive and a permutation p of 0.134 \u2014 i.e. positive but NOT significant. That estimate was known before E52 ran and is recorded here so E52's primary is not presented as blind for the in-stream half. It also carries ~17% fit/prefix leakage (E28 fitted N=400 from the same 800-doc block), which is the confound Amendment 1 removes, so its sign is exactly what leakage predicts.", + "secondary_ood_rungs": { + "OOD_News_2024": { + "matched_J^q": 0.10254994873992271, + "logit": 0.07871548670809715, + "shuf_floor": 0.08219137775401274, + "best_instream_J^P": 0.11525068599213328, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.023834462031825557, + "delta_vs_best_instream": -0.012700737252210575 + }, + "OOD_arXiv_2023": { + "matched_J^q": 0.09594758972784298, + "logit": 0.06943058591180791, + "shuf_floor": 0.07357985358685255, + "best_instream_J^P": 0.10726413512602448, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.02651700381603507, + "delta_vs_best_instream": -0.011316545398181499 + }, + "OOD_CommonPile": { + "matched_J^q": 0.09897827323940064, + "logit": 0.07235888629996529, + "shuf_floor": 0.07529745303715268, + "best_instream_J^P": 0.10878850278548069, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.026619386939435358, + "delta_vs_best_instream": -0.009810229546080046 + } + }, + "controls": { + "C1_Q0_vs_e48": { + "applicable": false, + "why": "results/e48_crossover_410m.json was scored at the unstripped readout. Comparing a stripped Q0 rung against it would compare two conventions, not two runs. The stripped counterpart is R2 (results/e48_crossover_410m_rstrip.json).", + "fires": null + }, + "C1_fires": null, + "C2_shuf_floor_excl_degenerate": { + "below_floor": { + "Q0": [], + "Wikipedia_en": [], + "USPTO_Backgrounds": [], + "Pile-CC": [], + "StackExchange": [], + "Github": [], + "OOD_News_2024": [], + "OOD_arXiv_2023": [], + "OOD_CommonPile": [], + "SHUFFLED_Pile-CC": [] + }, + "fires": true + }, + "C3_capability": { + "q0_mean_rank": 3124.912653975364, + "per_rung": { + "Q0": 3124.912653975364, + "Wikipedia_en": 2912.431131019037, + "USPTO_Backgrounds": 2895.9477416946625, + "Pile-CC": 3041.0272489734975, + "StackExchange": 3170.777902202314, + "Github": 3232.4229189996267, + "OOD_News_2024": 3204.6147816349385, + "OOD_arXiv_2023": 3026.1086226203806, + "OOD_CommonPile": 3008.9858156028367, + "SHUFFLED_Pile-CC": 2797.349757372154 + }, + "collapsed": [], + "fires": true + }, + "C5_self_distance": { + "value": 0.0, + "fires": true + }, + "R1_C2_readout_divergence": { + "required_total": 157, + "required_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "observed_total": 157, + "observed_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "n_released_items": 551, + "examples": { + "multihop": { + "unstripped_token_id": 209, + "stripped_token_id": 253, + "unstripped": "' '", + "stripped": "' the'" + }, + "multilingual": { + "unstripped_token_id": 209, + "stripped_token_id": 10863, + "unstripped": "' '", + "stripped": "' ist'" + }, + "order-ops": { + "unstripped_token_id": 209, + "stripped_token_id": 426, + "unstripped": "' '", + "stripped": "' ='" + } + }, + "note": "computed over ALL released items in all six sets, independently of --rstrip: it describes the difference between the two conventions, not this run's arm.", + "fires": true + }, + "R1_C3_logit_constant_moves": { + "arm": "STRIPPED", + "observed_logit_Q0_persist": 0.08317913140635938, + "unstripped_reference": 0.0284395, + "reference_source": "E33/E36 agree to 2.6e-10 on the unstripped Q0 logit constant", + "abs_diff_vs_unstripped_reference": 0.05473963140635938, + "fires": true + }, + "R1_C4_derangement_floor": { + "alias_of": "C2_shuf_floor_excl_degenerate", + "required": "no non-degenerate operator at or below its rung's shuf floor under persist", + "fires": true + }, + "P0_C3_worker_threads": { + "workers": 2, + "observed_threads_in_cell": 1, + "required": "1 when --workers > 1", + "fires": true + } + }, + "VERDICT": "NO MATCHING (the publishable null) \u2014 D = +0.00085, CI [-0.00090,+0.00279] includes zero. The corpus effect established in E28/E33/E48 is a property of the FITTING CORPUS PER SE; the fit-read relationship is not the operative variable, and EQ2's ||J^P - J^Q|| gap framing is not what the data measures.", + "provenance": { + "schema_version": 1, + "experiment": "E52", + "utc": "2026-08-20T16:23:55+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "6d74ea7f4ef2a968b73923e3858cbafe48c5a8d735f3118ad31422b7d43a272d", + "n_lines": 851 + }, + "git": { + "commit": "32f753bad963a8493d1bbced178aafcc39d97dbb", + "commit_short": "32f753bad963", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/audit/", + "results/d2_device_crossvalidation.json", + "results/e52_factorial_410m_rstrip_w8.json", + "results/ladder410_cpu_rstrip/", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/r8_ladder_flatness.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "aee92baa8be53162e66b7c00eec04f9286b40c2afaf4dbb2dbe8c1782597cc28", + "describe": "32f753b-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--workers", + "2", + "--rstrip", + "--out", + "results/e52_factorial_410m_rstrip_w2.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "cb00c6c92899012c350c180d05e5928ea183811de386004899e6812fbacf37aa", + "bytes": 115278, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "5e5d58ac57efc4d429458c6300def2446e13ffcd219c6ce2b9e6b86acc7faaef" + } +} \ No newline at end of file diff --git a/results/e52_factorial_410m_rstrip_w8.json b/results/e52_factorial_410m_rstrip_w8.json new file mode 100644 index 0000000000000000000000000000000000000000..329c740dbb76273770b80c202915bd83f157a217 --- /dev/null +++ b/results/e52_factorial_410m_rstrip_w8.json @@ -0,0 +1,1093 @@ +{ + "experiment": "E52 \u2014 the fit x read factorial: J^Q read on Q, the missing cell", + "prereg": "specs/archive/prereg/PREREG_E52_FACTORIAL.md, written before this ran", + "prereg_files": [ + { + "path": "specs/archive/prereg/PREREG_E52_FACTORIAL.md", + "role": "E52, the original design", + "sha256": "740ddb1d6aeb35d8c0d93b626043e62c53d35d149ec9d6a2e7bb8817875902ce" + }, + { + "path": "specs/experiments/P0_t52_parallel_port.md", + "role": "P0, the pool port", + "sha256": "d2cd8cb4b3296249ace1ae995b238b2035b21d7884858596777202971869565e" + }, + { + "path": "specs/experiments/R1_grid_rstrip.md", + "role": "R1, the readout correction", + "sha256": "3b13a90cd3730455c6d1328a84d50653f3f146e39ed115f08dc2f77e872fcd07" + } + ], + "decision_rule_verbatim": { + "R1": "ACCEPT (headline robust): stripped fit_pct exceeds stripped read_pct under BOTH aggregations, and stripped fit_pct lies inside the published bootstrap interval [87.609, 92.171] under persist. | QUALIFIED (ordering survives, magnitude does not): stripped fit_pct still exceeds read_pct under both aggregations but falls outside the published interval. | REJECT (ordering does not survive): stripped fit_pct does not exceed read_pct under both aggregations. STOP. Alert the operator.", + "P0": "ACCEPT: the pooled run reproduces the serial run's stored matrix at max_abs_diff = 0.0 across all 128 cell values, and wallclock falls by at least 3x. | REJECT: any non-zero difference. STOP. Do not \"investigate the tolerance\" -- there is no tolerance." + }, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`, i.e. the final prompt token after .rstrip(). jacobian-lens/data/evaluations/README.md; src/anchor_evals.py:228", + "rstrip": true, + "workers": 8, + "wallclock_seconds": 3060.8, + "cell_completion_seconds": { + "Q0|p1": 407.9, + "Q0|p0": 411.1, + "Q0|p2": 411.8, + "USPTO_Backgrounds|p1": 790.2, + "Wikipedia_en|p0": 800.7, + "Wikipedia_en|p1": 813.4, + "Wikipedia_en|p2": 818.0, + "USPTO_Backgrounds|p0": 818.3, + "USPTO_Backgrounds|p2": 1214.0, + "Pile-CC|p1": 1224.3, + "Pile-CC|p0": 1228.5, + "Pile-CC|p2": 1686.5, + "StackExchange|p0": 1693.6, + "StackExchange|p1": 1719.0, + "StackExchange|p2": 1719.5, + "Github|p0": 1749.2, + "Github|p1": 2007.2, + "Github|p2": 2016.8, + "OOD_News_2024|p0": 2025.3, + "OOD_News_2024|p1": 2398.5, + "OOD_News_2024|p2": 2403.9, + "OOD_arXiv_2023|p0": 2667.4, + "OOD_arXiv_2023|p1": 2709.6, + "OOD_arXiv_2023|p2": 2735.1, + "OOD_CommonPile|p0": 2851.0, + "OOD_CommonPile|p2": 2854.2, + "OOD_CommonPile|p1": 2855.2, + "SHUFFLED_Pile-CC|p1": 3007.4, + "SHUFFLED_Pile-CC|p0": 3008.6, + "SHUFFLED_Pile-CC|p2": 3060.1 + }, + "status": "PRE-REGISTERED", + "question": "can a J-lens trained on an out-of-distribution sample read latent structure on an out-of-distribution corpus?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_operators": 24, + "n_items": 541, + "n_pairs": 893, + "prefix_tokens": 128, + "smoke": false, + "all_operators_share_one_fitter": "trainval.py N=200 --band 9,21 dim_batch=128 max_seq_len=128 skip_first=16 target_layer=-2 fp32", + "adjudication": { + "aggregation": "persist", + "min_does_not_vote": true, + "D": 0.000845372218579525, + "D_without_degenerate": 0.001007020774004221, + "D_hierarchical_ci": { + "point": 0.000845370814204216, + "ci_lo": -0.0008957392419688404, + "ci_hi": 0.0027882857248187065, + "n_sets": 5, + "n_seed_blocks": 3, + "n_boot": 4000, + "excludes_zero": false, + "unit": "eval set > seed block > item pair (three-level)", + "note": "seed variance is estimated from 3 clusters; honest about direction, not calibrated at the seed level" + }, + "degenerate_excluded_from_C2": [ + "Github" + ] + }, + "by_aggregation": { + "persist": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.1057175321996005, + "Wikipedia_en|USPTO_Backgrounds": 0.10812408523634076, + "Wikipedia_en|Pile-CC": 0.10244530709201677, + "Wikipedia_en|StackExchange": 0.08896334908850904, + "Wikipedia_en|Github": 0.09078200098075387, + "Wikipedia_en|OOD_News_2024": 0.10550682633183897, + "Wikipedia_en|OOD_arXiv_2023": 0.09642561516569306, + "Wikipedia_en|OOD_CommonPile": 0.09897492753031353, + "USPTO_Backgrounds|Wikipedia_en": 0.1134465583651844, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.11788387473206967, + "USPTO_Backgrounds|Pile-CC": 0.11393354688253668, + "USPTO_Backgrounds|StackExchange": 0.10075250616969747, + "USPTO_Backgrounds|Github": 0.0997736249739925, + "USPTO_Backgrounds|OOD_News_2024": 0.11525068599213328, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.10726413512602448, + "USPTO_Backgrounds|OOD_CommonPile": 0.10878850278548069, + "Pile-CC|Wikipedia_en": 0.11220522388401959, + "Pile-CC|USPTO_Backgrounds": 0.11493656668915517, + "Pile-CC|Pile-CC": 0.11208727546036243, + "Pile-CC|StackExchange": 0.0982377641963669, + "Pile-CC|Github": 0.09963205248220927, + "Pile-CC|OOD_News_2024": 0.11498919603311353, + "Pile-CC|OOD_arXiv_2023": 0.10682292872936361, + "Pile-CC|OOD_CommonPile": 0.107277470589098, + "StackExchange|Wikipedia_en": 0.1126410995092657, + "StackExchange|USPTO_Backgrounds": 0.11810599462025695, + "StackExchange|Pile-CC": 0.11142965133508874, + "StackExchange|StackExchange": 0.10109850274812844, + "StackExchange|Github": 0.10198042160417471, + "StackExchange|OOD_News_2024": 0.11420692674194773, + "StackExchange|OOD_arXiv_2023": 0.10534035389622053, + "StackExchange|OOD_CommonPile": 0.10708697298541665, + "Github|Wikipedia_en": 0.09720011671694616, + "Github|USPTO_Backgrounds": 0.09792679025170704, + "Github|Pile-CC": 0.09611689302449425, + "Github|StackExchange": 0.08193752315774974, + "Github|Github": 0.0831806702642805, + "Github|OOD_News_2024": 0.09784760768007901, + "Github|OOD_arXiv_2023": 0.09211890187321438, + "Github|OOD_CommonPile": 0.09339318144031697, + "OOD_News_2024|Wikipedia_en": 0.09874760847952631, + "OOD_News_2024|USPTO_Backgrounds": 0.10250586529728026, + "OOD_News_2024|Pile-CC": 0.09953238663470579, + "OOD_News_2024|StackExchange": 0.08627744833680077, + "OOD_News_2024|Github": 0.08632135616822376, + "OOD_News_2024|OOD_News_2024": 0.10254994873992271, + "OOD_News_2024|OOD_arXiv_2023": 0.09456200205927921, + "OOD_News_2024|OOD_CommonPile": 0.09503316304956873, + "OOD_arXiv_2023|Wikipedia_en": 0.10122454878356722, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.10528044994506572, + "OOD_arXiv_2023|Pile-CC": 0.10172094841642926, + "OOD_arXiv_2023|StackExchange": 0.08987056811909296, + "OOD_arXiv_2023|Github": 0.08733742833137512, + "OOD_arXiv_2023|OOD_News_2024": 0.10298278572348257, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.09594758972784298, + "OOD_arXiv_2023|OOD_CommonPile": 0.09871495900054773, + "OOD_CommonPile|Wikipedia_en": 0.1013734284038138, + "OOD_CommonPile|USPTO_Backgrounds": 0.10645713993969062, + "OOD_CommonPile|Pile-CC": 0.10359419542106076, + "OOD_CommonPile|StackExchange": 0.09089663302422397, + "OOD_CommonPile|Github": 0.08911381827977796, + "OOD_CommonPile|OOD_News_2024": 0.10473065648693591, + "OOD_CommonPile|OOD_arXiv_2023": 0.09662212647963316, + "OOD_CommonPile|OOD_CommonPile": 0.09897827323940064 + }, + "all8": { + "D": 0.000845372218579525, + "diag_mean": 0.0007397006912570839, + "offdiag_mean": -0.00010567152732244106, + "a_fit": { + "Wikipedia_en": -0.001823302244560579, + "USPTO_Backgrounds": 0.008195921680695994, + "Pile-CC": 0.006832802060267165, + "StackExchange": 0.007545482732368533, + "Github": -0.008975547146595389, + "OOD_News_2024": -0.00574953535203046, + "OOD_arXiv_2023": -0.0035558479417684558, + "OOD_CommonPile": -0.0024699737883767947 + }, + "b_read": { + "Wikipedia_en": 0.0038787568450465715, + "USPTO_Backgrounds": 0.007461838141251884, + "Pile-CC": 0.003666767835642931, + "StackExchange": -0.009186470842622738, + "Github": -0.00917558606209544, + "OOD_News_2024": 0.005817321518487822, + "OOD_arXiv_2023": -0.002052801065534973, + "OOD_CommonPile": -0.00040982637017603074 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002221319901420618, + "Wikipedia_en|USPTO_Backgrounds": 0.0010447916419555614, + "Wikipedia_en|Pile-CC": -0.0008389161967594788, + "Wikipedia_en|StackExchange": -0.0014676355220015341, + "Wikipedia_en|Github": 0.0003401315897159918, + "Wikipedia_en|OOD_News_2024": 7.204936021783237e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0011390392219052803, + "Wikipedia_en|OOD_CommonPile": -0.00023270155264375203, + "USPTO_Backgrounds|Wikipedia_en": -6.887785825206161e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0007853572124278951, + "USPTO_Backgrounds|Pile-CC": 0.0006300996685038585, + "USPTO_Backgrounds|StackExchange": 0.00030229763393031794, + "USPTO_Backgrounds|Github": -0.0006874683423019473, + "USPTO_Backgrounds|OOD_News_2024": -0.00020331490474442826, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00031974318683043734, + "USPTO_Backgrounds|OOD_CommonPile": -0.00043835022273316926, + "Pile-CC|Wikipedia_en": 5.290728101195463e-05, + "Pile-CC|USPTO_Backgrounds": -0.0007988312100577766, + "Pile-CC|Pile-CC": 0.0001469478667584373, + "Pile-CC|StackExchange": -0.0008493247189714226, + "Pile-CC|Github": 0.0005340787863436502, + "Pile-CC|OOD_News_2024": 0.0008983147566646449, + "Pile-CC|OOD_arXiv_2023": 0.0006021700369375216, + "Pile-CC|OOD_CommonPile": -0.0005862627986870234, + "StackExchange|Wikipedia_en": -0.00022389776584330612, + "StackExchange|USPTO_Backgrounds": 0.001657916048942637, + "StackExchange|Pile-CC": -0.0012233569306166203, + "StackExchange|StackExchange": 0.0012987331606887503, + "StackExchange|Github": 0.002169767236207723, + "StackExchange|OOD_News_2024": -0.0005966352066025249, + "StackExchange|OOD_arXiv_2023": -0.001593085468306929, + "StackExchange|OOD_CommonPile": -0.001489441074469744, + "Github|Wikipedia_en": 0.0008561493208010806, + "Github|USPTO_Backgrounds": -0.002000258440643346, + "Github|Pile-CC": -1.5085362247188594e-05, + "Github|StackExchange": -0.0013412165507260265, + "Github|Github": -0.00010895422472256833, + "Github|OOD_News_2024": -0.0004349243895073185, + "Github|OOD_arXiv_2023": 0.0017064923876508459, + "Github|OOD_CommonPile": 0.0013377972593944937, + "OOD_News_2024|Wikipedia_en": -0.0008223707111836981, + "OOD_News_2024|USPTO_Backgrounds": -0.0006471951896350608, + "OOD_News_2024|Pile-CC": 0.00017439645339942245, + "OOD_News_2024|StackExchange": -0.00022730316623992275, + "OOD_News_2024|Github": -0.00019428011534423917, + "OOD_News_2024|OOD_News_2024": 0.0010414048757714511, + "OOD_News_2024|OOD_arXiv_2023": 0.0009235807791507511, + "OOD_News_2024|OOD_CommonPile": -0.00024823292591867607, + "OOD_arXiv_2023|Wikipedia_en": -0.0005391178174047884, + "OOD_arXiv_2023|USPTO_Backgrounds": -6.629795211160472e-05, + "OOD_arXiv_2023|Pile-CC": 0.00016927082486088707, + "OOD_arXiv_2023|StackExchange": 0.0011721292057902538, + "OOD_arXiv_2023|Github": -0.0013718953624548835, + "OOD_arXiv_2023|OOD_News_2024": -0.0007194455509306891, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00011548103745251392, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012398756147983248, + "OOD_CommonPile|Wikipedia_en": -0.0014761123505498686, + "OOD_CommonPile|USPTO_Backgrounds": 2.4517889121639103e-05, + "OOD_CommonPile|Pile-CC": 0.0009566436761007241, + "OOD_CommonPile|StackExchange": 0.0011123199575296117, + "OOD_CommonPile|Github": -0.000681379567443699, + "OOD_CommonPile|OOD_News_2024": -5.744894086900931e-05, + "OOD_CommonPile|OOD_arXiv_2023": -0.000295856364148972, + "OOD_CommonPile|OOD_CommonPile": 0.000417315700259574 + } + }, + "without_degenerate": { + "D": 0.001007020774004221, + "diag_mean": 0.0008631606634321978, + "offdiag_mean": -0.00014386011057202305, + "a_fit": { + "Wikipedia_en": -0.0031518899369984976, + "USPTO_Backgrounds": 0.007014133978546375, + "Pile-CC": 0.005476507625453875, + "StackExchange": 0.005955518519003236, + "OOD_News_2024": -0.0070017785151740425, + "OOD_arXiv_2023": -0.004639860355324807, + "OOD_CommonPile": -0.00365263131550618 + }, + "b_read": { + "Wikipedia_en": 0.002447875345953629, + "USPTO_Backgrounds": 0.006439014893793865, + "Pile-CC": 0.0023603484341283287, + "StackExchange": -0.0103034429314975, + "OOD_News_2024": 0.00457087912086751, + "OOD_arXiv_2023": -0.0036051601456064397, + "OOD_CommonPile": -0.001909514717639449 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.002389993618745076, + "Wikipedia_en|USPTO_Backgrounds": 0.0008054071076450953, + "Wikipedia_en|Pile-CC": -0.000794704577013361, + "Wikipedia_en|StackExchange": -0.0016128712148952562, + "Wikipedia_en|OOD_News_2024": 5.628397606965985e-05, + "Wikipedia_en|OOD_arXiv_2023": -0.0008488879236022984, + "Wikipedia_en|OOD_CommonPile": 4.779013051181558e-06, + "USPTO_Backgrounds|Wikipedia_en": -4.700413121590252e-05, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.0003991726878291302, + "USPTO_Backgrounds|Pile-CC": 0.0005275112979616775, + "USPTO_Backgrounds|StackExchange": 1.0261950748297055e-05, + "USPTO_Backgrounds|OOD_News_2024": -0.0003658802791808996, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.00017639187881575424, + "USPTO_Backgrounds|OOD_CommonPile": -0.0003476696473265345, + "Pile-CC|Wikipedia_en": 0.0002492877407117844, + "Pile-CC|USPTO_Backgrounds": -0.0010105090019928709, + "Pile-CC|Pile-CC": 0.00021886622887992702, + "Pile-CC|StackExchange": -0.0009668536694897728, + "Pile-CC|OOD_News_2024": 0.0009102561148918442, + "Pile-CC|OOD_arXiv_2023": 0.0009200280776158754, + "Pile-CC|OOD_CommonPile": -0.0003210754906167179, + "StackExchange|Wikipedia_en": 0.00020615247240853007, + "StackExchange|USPTO_Backgrounds": 0.0016799080355595492, + "StackExchange|Pile-CC": -0.0009177687899431242, + "StackExchange|StackExchange": 0.0014148739887224066, + "StackExchange|OOD_News_2024": -0.00035102406982331913, + "StackExchange|OOD_arXiv_2023": -0.0010415576490765688, + "StackExchange|OOD_CommonPile": -0.000990583987847432, + "OOD_News_2024|Wikipedia_en": -0.0007300415231535767, + "OOD_News_2024|USPTO_Backgrounds": -0.0009629242532398635, + "OOD_News_2024|Pile-CC": 0.00014226354385120377, + "OOD_News_2024|StackExchange": -0.00044888338842798137, + "OOD_News_2024|OOD_News_2024": 0.0009492949623289421, + "OOD_News_2024|OOD_arXiv_2023": 0.0011373875481593965, + "OOD_News_2024|OOD_CommonPile": -8.709688951807903e-05, + "OOD_arXiv_2023|Wikipedia_en": -0.000615019378961898, + "OOD_arXiv_2023|USPTO_Backgrounds": -0.0005502577653036383, + "OOD_arXiv_2023|Pile-CC": -3.109283427456255e-05, + "OOD_arXiv_2023|StackExchange": 0.0007823182340149643, + "OOD_arXiv_2023|OOD_News_2024": -0.000979786213960429, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00016105705687392835, + "OOD_arXiv_2023|OOD_CommonPile": 0.0012327809016116908, + "OOD_CommonPile|Wikipedia_en": -0.0014533687985339439, + "OOD_CommonPile|USPTO_Backgrounds": -0.00036079681049736023, + "OOD_CommonPile|Pile-CC": 0.0008549251305383088, + "OOD_CommonPile|StackExchange": 0.0008211540993273564, + "OOD_CommonPile|OOD_News_2024": -0.00021914449032571504, + "OOD_CommonPile|OOD_arXiv_2023": -0.00015163523115452326, + "OOD_CommonPile|OOD_CommonPile": 0.0005088661006459744 + } + }, + "C4_permutation": { + "null_mean": -9.644068875497913e-06, + "null_sd": 0.00040128992078327074, + "n_abs_ge_observed": 68, + "n_perm": 2000, + "p_two_sided": 0.034482758620689655 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.244750499708084e-06, + "null_sd": 0.00039985230956766146, + "n_abs_ge_observed": 24, + "n_perm": 2000, + "p_two_sided": 0.01249375312343828 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": 9.441563525961398e-05, + "surrogate_D_without_degenerate": -3.09029476336121e-05, + "D_minus_surrogate_all8": 0.000750956583319911, + "D_minus_surrogate_without_degenerate": 0.001037923721637833, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + }, + "min": { + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.23895396097666688, + "Wikipedia_en|USPTO_Backgrounds": 0.2436402971131934, + "Wikipedia_en|Pile-CC": 0.24109223555359574, + "Wikipedia_en|StackExchange": 0.22112794599185387, + "Wikipedia_en|Github": 0.22357925524314246, + "Wikipedia_en|OOD_News_2024": 0.2387056555185053, + "Wikipedia_en|OOD_arXiv_2023": 0.2240878723975685, + "Wikipedia_en|OOD_CommonPile": 0.2209852614750465, + "USPTO_Backgrounds|Wikipedia_en": 0.262192660694321, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.2683731574151251, + "USPTO_Backgrounds|Pile-CC": 0.26694677480392986, + "USPTO_Backgrounds|StackExchange": 0.2489887709625893, + "USPTO_Backgrounds|Github": 0.24927170336660412, + "USPTO_Backgrounds|OOD_News_2024": 0.26407354954216217, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.2434683405690723, + "USPTO_Backgrounds|OOD_CommonPile": 0.24144293343027431, + "Pile-CC|Wikipedia_en": 0.26439758812387787, + "Pile-CC|USPTO_Backgrounds": 0.2711278670363956, + "Pile-CC|Pile-CC": 0.2672508102738195, + "Pile-CC|StackExchange": 0.23920876369294192, + "Pile-CC|Github": 0.24696667301985953, + "Pile-CC|OOD_News_2024": 0.2684913569026523, + "Pile-CC|OOD_arXiv_2023": 0.2459330331120226, + "Pile-CC|OOD_CommonPile": 0.2441944960090849, + "StackExchange|Wikipedia_en": 0.26712176754242845, + "StackExchange|USPTO_Backgrounds": 0.27237275782972575, + "StackExchange|Pile-CC": 0.2704890080831117, + "StackExchange|StackExchange": 0.24824307101468246, + "StackExchange|Github": 0.25470268204808233, + "StackExchange|OOD_News_2024": 0.27020158254437976, + "StackExchange|OOD_arXiv_2023": 0.24713207156293923, + "StackExchange|OOD_CommonPile": 0.24989844709634781, + "Github|Wikipedia_en": 0.2348206989467144, + "Github|USPTO_Backgrounds": 0.23774738440083132, + "Github|Pile-CC": 0.2397848286562496, + "Github|StackExchange": 0.22261133020122845, + "Github|Github": 0.22343710913426346, + "Github|OOD_News_2024": 0.2349901809874508, + "Github|OOD_arXiv_2023": 0.21322937744359174, + "Github|OOD_CommonPile": 0.2154477684862084, + "OOD_News_2024|Wikipedia_en": 0.251803253011571, + "OOD_News_2024|USPTO_Backgrounds": 0.2560335245397356, + "OOD_News_2024|Pile-CC": 0.25415840529733236, + "OOD_News_2024|StackExchange": 0.22414421873788037, + "OOD_News_2024|Github": 0.23448309062255754, + "OOD_News_2024|OOD_News_2024": 0.25150823344786966, + "OOD_News_2024|OOD_arXiv_2023": 0.23495631789167723, + "OOD_News_2024|OOD_CommonPile": 0.23010762880245844, + "OOD_arXiv_2023|Wikipedia_en": 0.25748952556815413, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.2689400637108419, + "OOD_arXiv_2023|Pile-CC": 0.2626505770410101, + "OOD_arXiv_2023|StackExchange": 0.2423444380155868, + "OOD_arXiv_2023|Github": 0.24505835103078022, + "OOD_arXiv_2023|OOD_News_2024": 0.2630884069121546, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.2443132063994805, + "OOD_arXiv_2023|OOD_CommonPile": 0.2444441412265102, + "OOD_CommonPile|Wikipedia_en": 0.2519655141565535, + "OOD_CommonPile|USPTO_Backgrounds": 0.26096289563510155, + "OOD_CommonPile|Pile-CC": 0.2566619581439429, + "OOD_CommonPile|StackExchange": 0.23913137020750178, + "OOD_CommonPile|Github": 0.2396458975970745, + "OOD_CommonPile|OOD_News_2024": 0.25631876453343366, + "OOD_CommonPile|OOD_arXiv_2023": 0.2370515153846807, + "OOD_CommonPile|OOD_CommonPile": 0.23404423404071067 + }, + "all8": { + "D": 0.0003741732225679442, + "diag_mean": 0.0003274015697469525, + "offdiag_mean": -4.6771652820991726e-05, + "a_fit": { + "Wikipedia_en": -0.015166510734383742, + "USPTO_Backgrounds": 0.00890666507992946, + "Pile-CC": 0.00925825225325147, + "StackExchange": 0.013332102197131851, + "Github": -0.01892948648601306, + "OOD_News_2024": -0.0045387372241950386, + "OOD_arXiv_2023": 0.006853017469984485, + "OOD_CommonPile": 0.0002846974442945738 + }, + "b_read": { + "Wikipedia_en": 0.0069050498594555865, + "USPTO_Backgrounds": 0.013211672192038476, + "Pile-CC": 0.010691253463543632, + "StackExchange": -0.010963082665047197, + "Github": -0.007044976010284809, + "OOD_News_2024": 0.009234145030495716, + "OOD_arXiv_2023": -0.010416604422951226, + "OOD_CommonPile": -0.011617457447250151 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0005273505835147185, + "Wikipedia_en|USPTO_Backgrounds": -0.0010929356125416456, + "Wikipedia_en|Pile-CC": -0.0011205784436444655, + "Wikipedia_en|StackExchange": 0.0005694681232044918, + "Wikipedia_en|Github": -0.00089732928026931, + "Wikipedia_en|OOD_News_2024": -0.0020500500456869797, + "Wikipedia_en|OOD_arXiv_2023": 0.0029829162868231507, + "Wikipedia_en|OOD_CommonPile": 0.0010811583886000675, + "USPTO_Backgrounds|Wikipedia_en": -0.00030712551314435177, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0004332511249231785, + "USPTO_Backgrounds|Pile-CC": 0.0006607849923764564, + "USPTO_Backgrounds|StackExchange": 0.004357117279626732, + "USPTO_Backgrounds|Github": 0.0007219430288791517, + "USPTO_Backgrounds|OOD_News_2024": -0.0007553318363433259, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.0017097913559862565, + "USPTO_Backgrounds|OOD_CommonPile": -0.0025343454704853108, + "Pile-CC|Wikipedia_en": 0.001546214743090496, + "Pile-CC|USPTO_Backgrounds": 0.001969871323025313, + "Pile-CC|Pile-CC": 0.0006132332889440539, + "Pile-CC|StackExchange": -0.005774477163342673, + "Pile-CC|Github": -0.0019346744911874492, + "Pile-CC|OOD_News_2024": 0.0033108883508247977, + "Pile-CC|OOD_arXiv_2023": 0.00040331401364204766, + "Pile-CC|OOD_CommonPile": -0.000134370064996725, + "StackExchange|Wikipedia_en": 0.0001965442177606913, + "StackExchange|USPTO_Backgrounds": -0.0008590878275248903, + "StackExchange|Pile-CC": -0.00022241884564408454, + "StackExchange|StackExchange": -0.0008140197854825115, + "StackExchange|Github": 0.001727484593154971, + "StackExchange|OOD_News_2024": 0.0009472640486718753, + "StackExchange|OOD_arXiv_2023": -0.0024714974793217104, + "StackExchange|OOD_CommonPile": 0.001495731078385798, + "Github|Wikipedia_en": 0.000157064305191551, + "Github|USPTO_Backgrounds": -0.0032228725732744157, + "Github|Pile-CC": 0.0013349904106386978, + "Github|StackExchange": 0.005815828084208385, + "Github|Github": 0.0027235003624810072, + "Github|OOD_News_2024": -0.002002548825112166, + "Github|OOD_arXiv_2023": -0.004112602915524288, + "Github|OOD_CommonPile": -0.0006933588486087161, + "OOD_News_2024|Wikipedia_en": 0.002748869108230162, + "OOD_News_2024|USPTO_Backgrounds": 0.0006725183038118476, + "OOD_News_2024|Pile-CC": 0.0013178177899034471, + "OOD_News_2024|StackExchange": -0.007042032640957707, + "OOD_News_2024|Github": -0.0006212674110429306, + "OOD_News_2024|OOD_News_2024": 0.00012475437348866136, + "OOD_News_2024|OOD_arXiv_2023": 0.0032235882707431773, + "OOD_News_2024|OOD_CommonPile": -0.00042424779417668534, + "OOD_arXiv_2023|Wikipedia_en": -0.0029566130293662563, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0021873027807386147, + "OOD_arXiv_2023|Pile-CC": -0.001581765160598353, + "OOD_arXiv_2023|StackExchange": -0.00023356805743079967, + "OOD_arXiv_2023|Github": -0.001437761696999773, + "OOD_arXiv_2023|OOD_News_2024": 0.00031317314359410875, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.0011887220843669333, + "OOD_arXiv_2023|OOD_CommonPile": 0.002520509935695553, + "OOD_CommonPile|Wikipedia_en": -0.001912304415276983, + "OOD_CommonPile|USPTO_Backgrounds": 0.0007784547306881884, + "OOD_CommonPile|Pile-CC": -0.0010020640319756413, + "OOD_CommonPile|StackExchange": 0.003121684160174082, + "OOD_CommonPile|Github": -0.0002818951050155838, + "OOD_CommonPile|OOD_News_2024": 0.00011185079056305613, + "OOD_CommonPile|OOD_arXiv_2023": 0.0004953510952570295, + "OOD_CommonPile|OOD_CommonPile": -0.0013110772244140645 + } + }, + "without_degenerate": { + "D": -8.222654747663485e-05, + "diag_mean": -7.047989783710653e-05, + "offdiag_mean": 1.1746649639528328e-05, + "a_fit": { + "Wikipedia_en": -0.017798114832397766, + "USPTO_Backgrounds": 0.006043736366322772, + "Pile-CC": 0.006774840328225729, + "StackExchange": 0.010325524688628629, + "OOD_News_2024": -0.0072097787320985485, + "OOD_arXiv_2023": 0.004298618002931964, + "OOD_CommonPile": -0.002434825821612835 + }, + "b_read": { + "Wikipedia_en": 0.0058206053171940275, + "USPTO_Backgrounds": 0.012610075775272012, + "Pile-CC": 0.009438533763361068, + "StackExchange": -0.012855922032882577, + "OOD_News_2024": 0.00845821664970614, + "OOD_arXiv_2023": -0.010891096505110498, + "OOD_CommonPile": -0.012580412967540255 + }, + "g": { + "Wikipedia_en|Wikipedia_en": 0.0004771800842685303, + "Wikipedia_en|USPTO_Backgrounds": -0.0016259542372829283, + "Wikipedia_en|Pile-CC": -0.0010024737849696486, + "Wikipedia_en|StackExchange": 0.0013276924495321252, + "Wikipedia_en|OOD_News_2024": -0.0024087367064051513, + "Wikipedia_en|OOD_arXiv_2023": 0.002322793327474676, + "Wikipedia_en|OOD_CommonPile": 0.0009094988673824245, + "USPTO_Backgrounds|Wikipedia_en": -0.00012597139679787484, + "USPTO_Backgrounds|USPTO_Backgrounds": -0.0007349451340717961, + "USPTO_Backgrounds|Pile-CC": 0.0010102142666439384, + "USPTO_Backgrounds|StackExchange": 0.005346666221547031, + "USPTO_Backgrounds|OOD_News_2024": -0.0008826938814688323, + "USPTO_Backgrounds|OOD_arXiv_2023": -0.002138589699742066, + "USPTO_Backgrounds|OOD_CommonPile": -0.0024746803761102887, + "Pile-CC|Wikipedia_en": 0.001347852070856026, + "Pile-CC|USPTO_Backgrounds": 0.0012886605252957484, + "Pile-CC|Pile-CC": 0.000583145774630589, + "Pile-CC|StackExchange": -0.005164445010003321, + "Pile-CC|OOD_News_2024": 0.0028040095171183443, + "Pile-CC|OOD_arXiv_2023": -0.0004050011186947089, + "Pile-CC|OOD_CommonPile": -0.00045422175920264984, + "StackExchange|Wikipedia_en": 0.000521347129003702, + "StackExchange|USPTO_Backgrounds": -0.0010171330417769742, + "StackExchange|Pile-CC": 0.0002706592235199312, + "StackExchange|StackExchange": 0.0003191779513343207, + "StackExchange|OOD_News_2024": 0.0009635507984429026, + "StackExchange|OOD_arXiv_2023": -0.0027566470281809863, + "StackExchange|OOD_CommonPile": 0.0016990449676573538, + "OOD_News_2024|Wikipedia_en": 0.0027381360188734605, + "OOD_News_2024|USPTO_Backgrounds": 0.00017893708896005167, + "OOD_News_2024|Pile-CC": 0.0014753598584677508, + "OOD_News_2024|StackExchange": -0.006244370904740587, + "OOD_News_2024|OOD_News_2024": -0.00019449487734002346, + "OOD_News_2024|OOD_arXiv_2023": 0.0026029027212841893, + "OOD_News_2024|OOD_CommonPile": -0.0005564699055048417, + "OOD_arXiv_2023|Wikipedia_en": -0.003083988159573947, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.0015770795250358294, + "OOD_arXiv_2023|Pile-CC": -0.0015408651328850387, + "OOD_arXiv_2023|StackExchange": 0.00044745163793533105, + "OOD_arXiv_2023|OOD_News_2024": -0.00012271814808556547, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.00045139449405695586, + "OOD_arXiv_2023|OOD_CommonPile": 0.0022716457835164072, + "OOD_CommonPile|Wikipedia_en": -0.0018745557466297857, + "OOD_CommonPile|USPTO_Backgrounds": 0.0003333552738402912, + "OOD_CommonPile|Pile-CC": -0.0007960402054074389, + "OOD_CommonPile|StackExchange": 0.003967827654395101, + "OOD_CommonPile|OOD_News_2024": -0.00015891670226172994, + "OOD_CommonPile|OOD_arXiv_2023": -7.685269619805979e-05, + "OOD_CommonPile|OOD_CommonPile": -0.001394817577738322 + } + }, + "C4_permutation": { + "null_mean": -2.3054927495855173e-05, + "null_sd": 0.0009489086857509815, + "n_abs_ge_observed": 1426, + "n_perm": 2000, + "p_two_sided": 0.7131434282858571 + }, + "C4_permutation_without_degenerate": { + "null_mean": -4.452356661881763e-06, + "null_sd": 0.001010189712926274, + "n_abs_ge_observed": 1887, + "n_perm": 2000, + "p_two_sided": 0.9435282358820589 + }, + "C6_link_artifact_null": { + "surrogate_D_all8": -1.0121748673588243e-05, + "surrogate_D_without_degenerate": -0.00011535759602394957, + "D_minus_surrogate_all8": 0.00038429497124153243, + "D_minus_surrogate_without_degenerate": 3.313104854731472e-05, + "note": "a multiplicative surrogate with the SAME margins has zero matching by construction; its D is the link artifact. D must exceed it, not 0." + } + } + }, + "per_set_D": { + "multihop": { + "per_prefix_seed": [ + 0.0011145231962603117, + 0.000726503469715161, + 0.0008255728420668415 + ], + "mean": 0.0008888665026807713, + "sd": 0.00016460942186982471 + }, + "multilingual": { + "per_prefix_seed": [ + -0.00024172088264354637, + 0.000781276895265494, + 0.0012064470377351557 + ], + "mean": 0.0005820010167857012, + "sd": 0.0006077722919051255 + }, + "order-ops": { + "per_prefix_seed": [ + -0.0012291302638394491, + -0.004769640043377876, + 0.0012368599751165935 + ], + "mean": -0.0015873034440335772, + "sd": 0.0024651878411595463 + }, + "poetry": { + "per_prefix_seed": [ + 5.206154725913491e-05, + -0.0002689851472886013, + 0.00020824664118533422 + ], + "mean": -2.892319614710725e-06, + "sd": 0.0001986663712455282 + }, + "typo": { + "per_prefix_seed": [ + 0.005978955621165889, + 0.0031976310004081044, + 0.0038619598905955043 + ], + "mean": 0.004346182170723166, + "sd": 0.0011859721958341132 + } + }, + "per_set_sign_split": { + "n_positive": 3, + "n_negative": 2, + "note": "if the sets split in sign, a null on the set-averaged D is cancellation, not absence of matching" + }, + "PARTIAL_UNBLINDING_DISCLOSED": "E36's stored ladder already contains a 5x5 in-stream fit x read matrix (E28 N=400 operators, same aggregation, same admitted sets). Applying this script's diag_excess to it gives D = +0.001214 with all five diagonal residuals positive and a permutation p of 0.134 \u2014 i.e. positive but NOT significant. That estimate was known before E52 ran and is recorded here so E52's primary is not presented as blind for the in-stream half. It also carries ~17% fit/prefix leakage (E28 fitted N=400 from the same 800-doc block), which is the confound Amendment 1 removes, so its sign is exactly what leakage predicts.", + "secondary_ood_rungs": { + "OOD_News_2024": { + "matched_J^q": 0.10254994873992271, + "logit": 0.07871548670809715, + "shuf_floor": 0.08219137775401274, + "best_instream_J^P": 0.11525068599213328, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.023834462031825557, + "delta_vs_best_instream": -0.012700737252210575 + }, + "OOD_arXiv_2023": { + "matched_J^q": 0.09594758972784298, + "logit": 0.06943058591180791, + "shuf_floor": 0.07357985358685255, + "best_instream_J^P": 0.10726413512602448, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.02651700381603507, + "delta_vs_best_instream": -0.011316545398181499 + }, + "OOD_CommonPile": { + "matched_J^q": 0.09897827323940064, + "logit": 0.07235888629996529, + "shuf_floor": 0.07529745303715268, + "best_instream_J^P": 0.10878850278548069, + "best_instream_corpus": "USPTO_Backgrounds", + "beats_logit": true, + "beats_shuf_floor": true, + "beats_best_instream": false, + "delta_vs_logit": 0.026619386939435358, + "delta_vs_best_instream": -0.009810229546080046 + } + }, + "controls": { + "C1_Q0_vs_e48": { + "applicable": false, + "why": "results/e48_crossover_410m.json was scored at the unstripped readout. Comparing a stripped Q0 rung against it would compare two conventions, not two runs. The stripped counterpart is R2 (results/e48_crossover_410m_rstrip.json).", + "fires": null + }, + "C1_fires": null, + "C2_shuf_floor_excl_degenerate": { + "below_floor": { + "Q0": [], + "Wikipedia_en": [], + "USPTO_Backgrounds": [], + "Pile-CC": [], + "StackExchange": [], + "Github": [], + "OOD_News_2024": [], + "OOD_arXiv_2023": [], + "OOD_CommonPile": [], + "SHUFFLED_Pile-CC": [] + }, + "fires": true + }, + "C3_capability": { + "q0_mean_rank": 3124.912653975364, + "per_rung": { + "Q0": 3124.912653975364, + "Wikipedia_en": 2912.431131019037, + "USPTO_Backgrounds": 2895.9477416946625, + "Pile-CC": 3041.0272489734975, + "StackExchange": 3170.777902202314, + "Github": 3232.4229189996267, + "OOD_News_2024": 3204.6147816349385, + "OOD_arXiv_2023": 3026.1086226203806, + "OOD_CommonPile": 3008.9858156028367, + "SHUFFLED_Pile-CC": 2797.349757372154 + }, + "collapsed": [], + "fires": true + }, + "C5_self_distance": { + "value": 0.0, + "fires": true + }, + "R1_C2_readout_divergence": { + "required_total": 157, + "required_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "observed_total": 157, + "observed_per_set": { + "multihop": 83, + "multilingual": 19, + "order-ops": 55, + "poetry": 0, + "typo": 0, + "association": 0 + }, + "n_released_items": 551, + "examples": { + "multihop": { + "unstripped_token_id": 209, + "stripped_token_id": 253, + "unstripped": "' '", + "stripped": "' the'" + }, + "multilingual": { + "unstripped_token_id": 209, + "stripped_token_id": 10863, + "unstripped": "' '", + "stripped": "' ist'" + }, + "order-ops": { + "unstripped_token_id": 209, + "stripped_token_id": 426, + "unstripped": "' '", + "stripped": "' ='" + } + }, + "note": "computed over ALL released items in all six sets, independently of --rstrip: it describes the difference between the two conventions, not this run's arm.", + "fires": true + }, + "R1_C3_logit_constant_moves": { + "arm": "STRIPPED", + "observed_logit_Q0_persist": 0.08317913140635938, + "unstripped_reference": 0.0284395, + "reference_source": "E33/E36 agree to 2.6e-10 on the unstripped Q0 logit constant", + "abs_diff_vs_unstripped_reference": 0.05473963140635938, + "fires": true + }, + "R1_C4_derangement_floor": { + "alias_of": "C2_shuf_floor_excl_degenerate", + "required": "no non-degenerate operator at or below its rung's shuf floor under persist", + "fires": true + }, + "P0_C3_worker_threads": { + "workers": 8, + "observed_threads_in_cell": 1, + "required": "1 when --workers > 1", + "fires": true + } + }, + "VERDICT": "NO MATCHING (the publishable null) \u2014 D = +0.00085, CI [-0.00090,+0.00279] includes zero. The corpus effect established in E28/E33/E48 is a property of the FITTING CORPUS PER SE; the fit-read relationship is not the operative variable, and EQ2's ||J^P - J^Q|| gap framing is not what the data measures.", + "provenance": { + "schema_version": 1, + "experiment": "E52", + "utc": "2026-08-20T14:56:11+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "6d74ea7f4ef2a968b73923e3858cbafe48c5a8d735f3118ad31422b7d43a272d", + "n_lines": 851 + }, + "git": { + "commit": "13c0a89e3f96566367a026a224e415910871fd2e", + "commit_short": "13c0a89e3f96", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/e28_ladder_410m.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "2d46c81e42b07a89c101e398f615af898c3ec2845b95740f4ba7ce92ca722e32", + "describe": "13c0a89-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--workers", + "8", + "--rstrip", + "--out", + "results/e52_factorial_410m_rstrip_w8.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m.json", + "sha256": "cb00c6c92899012c350c180d05e5928ea183811de386004899e6812fbacf37aa", + "bytes": 115278, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "a1f66ab7ea1752c990f94e907ff618842407c53bddfd0e92b90fdcb215640f9c" + } +} \ No newline at end of file diff --git a/results/e53_ladder_summary.json b/results/e53_ladder_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..04a89416609dd40b349210e8dbf8e1f22931f997 --- /dev/null +++ b/results/e53_ladder_summary.json @@ -0,0 +1,1371 @@ +{ + "experiment": "E53 \u2014 the ladder numbers the paper states and no file contains", + "why": "EXPERIMENT_VALIDITY_AUDIT v1 section 5 specified this script and it was never written (lab health H10 flagged it); v2 section 3.1 then found the paper's N-ladder sentence is contradicted by the file it cites.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json", + "results/e28_*_410m_n*_s*.pt (filenames only)", + "results/e36_qladder_410m.json", + "results/e54_aggregation_audit.json" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "asymptote_n_min": 75, + "grid_shape": { + "per_cell": { + "Github|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Github|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Github|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Wikipedia_en|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Wikipedia_en|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Wikipedia_en|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "StackExchange|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "StackExchange|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "StackExchange|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Pile-CC|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Pile-CC|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Pile-CC|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "USPTO_Backgrounds|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "USPTO_Backgrounds|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "USPTO_Backgrounds|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + } + }, + "n_cells": 15, + "n_cells_reaching_N800": 8, + "n_cells_stopping_at_N400": 7, + "total_fitted_lenses_410m": 83, + "C2_every_reconstruction_matches_its_fitted_blocks": true, + "PAPER_SAYS": "5 corpora x 3 seed blocks x 16 values of N in [25,800]", + "WHAT_IS_ON_DISK": "83 fitted lenses across 15 corpus-seed cells; 8 cells reach N=800 with 16 reconstructed values, 7 stop at N=400 with 11. The N values are consecutive-run sums of disjoint fitted blocks (e28_ladder_410m.py reconstruct()), not independent fits." + }, + "by_scale": { + "410m": { + "persist": { + "asymptote": { + "Github": 0.028084053793529264, + "Wikipedia_en": 0.03860496947484109, + "StackExchange": 0.0456756031007639, + "Pile-CC": 0.04571574828415005, + "USPTO_Backgrounds": 0.049735264846729854 + }, + "seed_sd": { + "Github": 0.0003725015802729375, + "Wikipedia_en": 0.0004975124529386809, + "StackExchange": 0.0002979844946588909, + "Pile-CC": 0.00018354240934833726, + "USPTO_Backgrounds": 0.0005027384624140496 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "StackExchange": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Pile-CC": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ] + }, + "asymptote_grids_identical_across_corpora": false, + "spread": 0.02165121105320059, + "pooled_seed_sd": 0.00037085587992657926, + "pooled_seed_sd_rms": 0.00039025750596544557, + "spread_over_seed_sd": 58.381738635199795 + }, + "min": { + "asymptote": { + "Github": 0.11288307417108089, + "Wikipedia_en": 0.13042243031696196, + "StackExchange": 0.14538949898881268, + "Pile-CC": 0.14493902100437375, + "USPTO_Backgrounds": 0.15533995895730282 + }, + "seed_sd": { + "Github": 0.000353933281277843, + "Wikipedia_en": 0.00010955838937670346, + "StackExchange": 0.0012802615956001497, + "Pile-CC": 0.00042533231079348594, + "USPTO_Backgrounds": 0.0016643039976484083 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "StackExchange": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Pile-CC": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ] + }, + "asymptote_grids_identical_across_corpora": false, + "spread": 0.042456884786221935, + "pooled_seed_sd": 0.0007666779149393181, + "pooled_seed_sd_rms": 0.0009723329607332006, + "spread_over_seed_sd": 55.37773289006552 + } + }, + "1b": { + "persist": { + "asymptote": { + "Github": 0.027256395461569938, + "Wikipedia_en": 0.02801095717586577, + "StackExchange": 0.05875215052316586, + "Pile-CC": 0.046013945933165296, + "USPTO_Backgrounds": 0.05470160290465824 + }, + "seed_sd": { + "Github": 0.0013762403288060878, + "Wikipedia_en": 0.00020285804501715254, + "StackExchange": 0.0007357119491078886, + "Pile-CC": 0.00039438237251602294, + "USPTO_Backgrounds": 0.0017248115034397009 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 200 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 200 + ], + "StackExchange": [ + 75, + 100, + 150, + 200 + ], + "Pile-CC": [ + 75, + 100, + 150, + 200 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 200 + ] + }, + "asymptote_grids_identical_across_corpora": true, + "spread": 0.03149575506159592, + "pooled_seed_sd": 0.0008868008397773706, + "pooled_seed_sd_rms": 0.0010589592134987906, + "spread_over_seed_sd": 35.51615385197748 + }, + "min": { + "asymptote": { + "Github": 0.13184866054604452, + "Wikipedia_en": 0.12168779909788144, + "StackExchange": 0.1760720702420388, + "Pile-CC": 0.14951401859566216, + "USPTO_Backgrounds": 0.16125627207968915 + }, + "seed_sd": { + "Github": 0.0011745417158351128, + "Wikipedia_en": 0.0006643603940748279, + "StackExchange": 0.000781853055555403, + "Pile-CC": 0.0016589081835269132, + "USPTO_Backgrounds": 0.0026333882425492983 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 200 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 200 + ], + "StackExchange": [ + 75, + 100, + 150, + 200 + ], + "Pile-CC": [ + 75, + 100, + 150, + 200 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 200 + ] + }, + "asymptote_grids_identical_across_corpora": true, + "spread": 0.054384271144157365, + "pooled_seed_sd": 0.001382610318308311, + "pooled_seed_sd_rms": 0.001556851127969718, + "spread_over_seed_sd": 39.33448956948266 + } + } + }, + "cross_scale": { + "persist": { + "order_410m": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "order_1b": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "spearman_of_asymptotes": 0.7, + "n": 5, + "worst_corpus_410m": "Github", + "worst_corpus_1b": "Github", + "same_worst_corpus": true, + "note": "n=5, so rho=+0.7 is p~0.19 and rho=+0.8 is p~0.10. 'The corpus effect replicates at 1B' is a claim that corpora DIFFER at both scales, which holds; it is not a claim that the SAME corpora are good, which this does not establish." + }, + "min": { + "order_410m": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "order_1b": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "spearman_of_asymptotes": 0.8, + "n": 5, + "worst_corpus_410m": "Github", + "worst_corpus_1b": "Wikipedia_en", + "same_worst_corpus": false, + "note": "n=5, so rho=+0.7 is p~0.19 and rho=+0.8 is p~0.10. 'The corpus effect replicates at 1B' is a claim that corpora DIFFER at both scales, which holds; it is not a claim that the SAME corpora are good, which this does not establish." + } + }, + "bands_declared_vs_used": { + "410m": { + "n_layers": 24, + "L38_L92": [ + 9, + 22 + ], + "target_layer_index": 22, + "declared_by_the_papers_own_rule": [ + 9, + 21 + ], + "used_in_the_ladder": [ + 9, + 21 + ], + "matches": true + }, + "1b": { + "n_layers": 16, + "L38_L92": [ + 6, + 15 + ], + "target_layer_index": 14, + "declared_by_the_papers_own_rule": [ + 6, + 13 + ], + "used_in_the_ladder": [ + 5, + 13 + ], + "matches": false + } + }, + "e36_containment_slopes": { + "persist": { + "logit": { + "slope_linear_containment": -0.003720899629554985, + "slope_log10_containment": -0.0005296062512720639, + "spearman": -0.2727272727272727, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.005342012891958096, + "StackExchange": -0.0033105996835403307, + "Wikipedia_en": -0.004159231116162598, + "USPTO_Backgrounds": -0.0034842593960081234, + "Github": -0.002234113859375035, + "CONTROL_PubMed_2023": -0.0037184028182850903, + "OOD_Wikipedia_2023": -0.0036969276410433907, + "TRAP_FineWeb": -0.002840794828560368, + "OOD_News_2024": -0.002472647958510993, + "OOD_arXiv_2023": -0.00535538400064562, + "OOD_CommonPile": -0.004356052515853489 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0008318596283442356, + "StackExchange": -0.000389681627275286, + "Wikipedia_en": -0.0005771961173907728, + "USPTO_Backgrounds": -0.00042768514003331125, + "Github": -0.00015929190525716992, + "CONTROL_PubMed_2023": -0.0005051741014559277, + "OOD_Wikipedia_2023": -0.0005653796365463086, + "TRAP_FineWeb": -0.0005079235074284086, + "OOD_News_2024": 9.839815919873245e-06, + "OOD_arXiv_2023": -0.0015902638125509932, + "OOD_CommonPile": -0.000617386511644501 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 1 + }, + "J|Github": { + "slope_linear_containment": -0.0014550596366756292, + "slope_log10_containment": -1.0939804428585397e-06, + "spearman": -0.19090909090909092, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.001561401546835692, + "StackExchange": -0.0005760183119989624, + "Wikipedia_en": -0.001992270380004623, + "USPTO_Backgrounds": -0.0017519561244663739, + "Github": -0.0008909328785794681, + "CONTROL_PubMed_2023": -0.0016397555863833476, + "OOD_Wikipedia_2023": -0.0009673012084557685, + "TRAP_FineWeb": -0.0011688572755800576, + "OOD_News_2024": -0.0014455593119102068, + "OOD_arXiv_2023": -0.0019401073419784616, + "OOD_CommonPile": -0.0021622055526610198 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 7.386689134070697e-06, + "StackExchange": 0.00022555885965485492, + "Wikipedia_en": -8.869649789138863e-05, + "USPTO_Backgrounds": -3.429243288848448e-05, + "Github": 0.0001519590333369196, + "CONTROL_PubMed_2023": -4.069669530389137e-05, + "OOD_Wikipedia_2023": -9.637614420813418e-05, + "TRAP_FineWeb": 6.554606456226265e-06, + "OOD_News_2024": 9.277410192215612e-05, + "OOD_arXiv_2023": -0.00015958374768649015, + "OOD_CommonPile": -0.00013358642358098752 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 5 + }, + "J|Wikipedia_en": { + "slope_linear_containment": -0.0009354827291769033, + "slope_log10_containment": 0.0002869798762885821, + "spearman": 0.06363636363636363, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0016945220408105522, + "StackExchange": 0.0007790027011743094, + "Wikipedia_en": -0.001943307383106595, + "USPTO_Backgrounds": -0.0014773995230767255, + "Github": 0.0006114770190120221, + "CONTROL_PubMed_2023": -0.001345574896548144, + "OOD_Wikipedia_2023": -0.00013865249525168055, + "TRAP_FineWeb": -0.0008008900076050542, + "OOD_News_2024": -0.00043233948548224637, + "OOD_arXiv_2023": -0.0017250513630531405, + "OOD_CommonPile": -0.00225182718285068 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0001587231312785397, + "StackExchange": 0.0007053078107235931, + "Wikipedia_en": 0.00010082406161996977, + "USPTO_Backgrounds": 0.00020565756093412292, + "Github": 0.0006557206071300077, + "CONTROL_PubMed_2023": 0.00018259944505365897, + "OOD_Wikipedia_2023": 0.00014469565594808258, + "TRAP_FineWeb": 0.00029123958467279065, + "OOD_News_2024": 0.000590124287887275, + "OOD_arXiv_2023": 5.458070632956026e-05, + "OOD_CommonPile": -6.767144880380829e-07 + }, + "n_loo_sign_flips_linear": 2, + "n_loo_sign_flips_log10": 1 + }, + "J|StackExchange": { + "slope_linear_containment": 0.0002240976423130857, + "slope_log10_containment": 0.00043591080099479097, + "spearman": 0.2818181818181818, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": 9.734162192914718e-05, + "StackExchange": 0.0013572724997980054, + "Wikipedia_en": -0.00024598000465200266, + "USPTO_Backgrounds": -0.00032751857982107153, + "Github": 0.000645747151939162, + "CONTROL_PubMed_2023": 0.00014738936959018587, + "OOD_Wikipedia_2023": 0.0008552523402857944, + "TRAP_FineWeb": 0.00026812738983809003, + "OOD_News_2024": 0.00010939484488637846, + "OOD_arXiv_2023": -0.0003280293027039663, + "OOD_CommonPile": -0.00022378417326704018 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0004285719021829549, + "StackExchange": 0.0007086432089750901, + "Wikipedia_en": 0.0003522684629005442, + "USPTO_Backgrounds": 0.00033586356876891434, + "Github": 0.0005465238526106352, + "CONTROL_PubMed_2023": 0.00043017554862703327, + "OOD_Wikipedia_2023": 0.0003323733409809342, + "TRAP_FineWeb": 0.00043633093680939597, + "OOD_News_2024": 0.0004872682870979624, + "OOD_arXiv_2023": 0.00033084731233546635, + "OOD_CommonPile": 0.0003710199870615957 + }, + "n_loo_sign_flips_linear": 4, + "n_loo_sign_flips_log10": 0 + }, + "J|Pile-CC": { + "slope_linear_containment": -0.0011535435993366304, + "slope_log10_containment": 0.00013903741903277346, + "spearman": -0.00909090909090909, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0018561479020940102, + "StackExchange": 0.0003276404302305153, + "Wikipedia_en": -0.0022811577240861203, + "USPTO_Backgrounds": -0.0015056028936971267, + "Github": 0.00014389080343519365, + "CONTROL_PubMed_2023": -0.0013506750771647544, + "OOD_Wikipedia_2023": -0.0005025777305976444, + "TRAP_FineWeb": -0.0010021430954018857, + "OOD_News_2024": -0.0007940013046808708, + "OOD_arXiv_2023": -0.0018366231525598165, + "OOD_CommonPile": -0.0021429228225167247 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 1.9282762995631993e-05, + "StackExchange": 0.000501527682090862, + "Wikipedia_en": -7.739278687463401e-05, + "USPTO_Backgrounds": 9.533123564495353e-05, + "Github": 0.00045059330345505465, + "CONTROL_PubMed_2023": 9.841175561313877e-05, + "OOD_Wikipedia_2023": 1.894809683383725e-05, + "TRAP_FineWeb": 0.0001439618487261618, + "OOD_News_2024": 0.00036998076199487877, + "OOD_arXiv_2023": -9.674744982390889e-05, + "OOD_CommonPile": -6.733770272352924e-05 + }, + "n_loo_sign_flips_linear": 2, + "n_loo_sign_flips_log10": 3 + }, + "J|USPTO_Backgrounds": { + "slope_linear_containment": -0.000885040945743085, + "slope_log10_containment": 0.0002350879878021014, + "spearman": 0.20909090909090908, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0014123147101777196, + "StackExchange": 0.00024297344343503642, + "Wikipedia_en": -0.001286481013089218, + "USPTO_Backgrounds": -0.0014672707299009076, + "Github": 7.746754680233447e-05, + "CONTROL_PubMed_2023": -0.0010450754171928793, + "OOD_Wikipedia_2023": -0.00014922575853216864, + "TRAP_FineWeb": -0.0007833716787412385, + "OOD_News_2024": -0.0010267371441971883, + "OOD_arXiv_2023": -0.0015151819788109704, + "OOD_CommonPile": -0.0014965239401716014 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.00015317413921890838, + "StackExchange": 0.0005186653686300515, + "Wikipedia_en": 0.0001788193096829291, + "USPTO_Backgrounds": 0.00014089125945786622, + "Github": 0.00047451609661586626, + "CONTROL_PubMed_2023": 0.00020754413654669994, + "OOD_Wikipedia_2023": 0.00010314967326905822, + "TRAP_FineWeb": 0.0002386930570768142, + "OOD_News_2024": 0.0002977371626300231, + "OOD_arXiv_2023": 7.147629354977188e-05, + "OOD_CommonPile": 0.00014381364955460372 + }, + "n_loo_sign_flips_linear": 2, + "n_loo_sign_flips_log10": 0 + }, + "_summary": { + "n_rungs": 11, + "max_abs_spearman": 0.2818181818181818, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_linear": 3, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_log10": 1 + } + }, + "min": { + "logit": { + "slope_linear_containment": -0.0010699323813758122, + "slope_log10_containment": 0.0009282505416221876, + "spearman": 0.00909090909090909, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0031277114593503412, + "StackExchange": -0.0005606867995279083, + "Wikipedia_en": -0.0018411056179029237, + "USPTO_Backgrounds": -0.00022972811415334008, + "Github": 0.00048355835055198017, + "CONTROL_PubMed_2023": -0.0005893318825439976, + "OOD_Wikipedia_2023": 0.00013717442831089735, + "TRAP_FineWeb": 0.0009480040767784513, + "OOD_News_2024": -0.0011114299140829592, + "OOD_arXiv_2023": -0.002789304049721013, + "OOD_CommonPile": -0.0033985900411396585 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0005572752682302916, + "StackExchange": 0.001117413951950436, + "Wikipedia_en": 0.0008339949157585983, + "USPTO_Backgrounds": 0.0011881567350198682, + "Github": 0.0013385498123661536, + "CONTROL_PubMed_2023": 0.0011432343029709941, + "OOD_Wikipedia_2023": 0.0007217489988221277, + "TRAP_FineWeb": 0.0009630678891223896, + "OOD_News_2024": 0.0012048674710092267, + "OOD_arXiv_2023": 0.0005171717376496735, + "OOD_CommonPile": 0.00046510911732908856 + }, + "n_loo_sign_flips_linear": 3, + "n_loo_sign_flips_log10": 0 + }, + "J|Github": { + "slope_linear_containment": -0.0035714446311215964, + "slope_log10_containment": -0.00029611400443362804, + "spearman": -0.09090909090909091, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0052687368331600405, + "StackExchange": -0.001960639289563647, + "Wikipedia_en": -0.0034644178667903418, + "USPTO_Backgrounds": -0.0048044144955547655, + "Github": -0.002052368335791041, + "CONTROL_PubMed_2023": -0.0035556823516738945, + "OOD_Wikipedia_2023": -0.002989236397777608, + "TRAP_FineWeb": -0.0026493610346831294, + "OOD_News_2024": -0.004848794207343968, + "OOD_arXiv_2023": -0.004352390461459243, + "OOD_CommonPile": -0.00348374819833562 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0006047413787830261, + "StackExchange": 0.00012156774546108781, + "Wikipedia_en": -0.0002130582540511553, + "USPTO_Backgrounds": -0.0005044491607809716, + "Github": 9.133692692012993e-05, + "CONTROL_PubMed_2023": -0.0002577355308012017, + "OOD_Wikipedia_2023": -0.00042586913211386726, + "TRAP_FineWeb": -0.00027397588588984654, + "OOD_News_2024": -0.0005900652750617479, + "OOD_arXiv_2023": -0.0006067527223211215, + "OOD_CommonPile": -0.00013541611648333051 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 2 + }, + "J|Wikipedia_en": { + "slope_linear_containment": -0.004603344091202535, + "slope_log10_containment": -0.000310141050106287, + "spearman": 0.05454545454545454, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.007037656959177343, + "StackExchange": -0.00033718111815656036, + "Wikipedia_en": -0.005967997333887068, + "USPTO_Backgrounds": -0.005772717284704184, + "Github": -0.0016207231641492746, + "CONTROL_PubMed_2023": -0.005800630286563434, + "OOD_Wikipedia_2023": -0.003278371513152543, + "TRAP_FineWeb": -0.003957832832162028, + "OOD_News_2024": -0.004794875269979331, + "OOD_arXiv_2023": -0.00468132498800471, + "OOD_CommonPile": -0.007606565142089651 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0007574823472158516, + "StackExchange": 0.0007204277251684816, + "Wikipedia_en": -0.0005308878981940796, + "USPTO_Backgrounds": -0.00048300317363772916, + "Github": 0.0004128726742000434, + "CONTROL_PubMed_2023": -0.0006472133229897882, + "OOD_Wikipedia_2023": -0.0005774356935193184, + "TRAP_FineWeb": -0.00029021934148038225, + "OOD_News_2024": -0.00016452947655907251, + "OOD_arXiv_2023": -4.779378759709274e-06, + "OOD_CommonPile": -0.0010117930336119925 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 2 + }, + "J|StackExchange": { + "slope_linear_containment": -0.0035589979944138225, + "slope_log10_containment": -0.00028825210444100556, + "spearman": 0.00909090909090909, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.005354689643722768, + "StackExchange": -7.861127536555136e-05, + "Wikipedia_en": -0.00400213687233838, + "USPTO_Backgrounds": -0.004980725535569202, + "Github": -0.0018976731424252432, + "CONTROL_PubMed_2023": -0.00422844192966735, + "OOD_Wikipedia_2023": -0.0023323168584560294, + "TRAP_FineWeb": -0.0033621771436820732, + "OOD_News_2024": -0.0057165041295091585, + "OOD_arXiv_2023": -0.00295331038636373, + "OOD_CommonPile": -0.004445761072997944 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0006181090101206631, + "StackExchange": 0.0005463290236336597, + "Wikipedia_en": -0.00032653417926189267, + "USPTO_Backgrounds": -0.0005375644780560232, + "Github": 0.00012984701853845188, + "CONTROL_PubMed_2023": -0.0004691915081592516, + "OOD_Wikipedia_2023": -0.0005311457658765471, + "TRAP_FineWeb": -0.0002776836972149278, + "OOD_News_2024": -0.0008879258086593867, + "OOD_arXiv_2023": 0.00038247464336330864, + "OOD_CommonPile": -0.00041503279685644066 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 3 + }, + "J|Pile-CC": { + "slope_linear_containment": -0.004014659692830054, + "slope_log10_containment": -0.00033377197965353794, + "spearman": -0.01818181818181818, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.006127892387626296, + "StackExchange": 2.76804226033701e-05, + "Wikipedia_en": -0.005243503813823139, + "USPTO_Backgrounds": -0.0047696458834969166, + "Github": -0.002156511734341, + "CONTROL_PubMed_2023": -0.005012628579589455, + "OOD_Wikipedia_2023": -0.003101921604069063, + "TRAP_FineWeb": -0.003264071432890173, + "OOD_News_2024": -0.004697706531445889, + "OOD_arXiv_2023": -0.004054662748765874, + "OOD_CommonPile": -0.005922931885080463 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0007254269721705388, + "StackExchange": 0.0006331040930860547, + "Wikipedia_en": -0.0005384660407427376, + "USPTO_Backgrounds": -0.00043042208791283605, + "Github": 0.0001339564204317549, + "CONTROL_PubMed_2023": -0.0006161089168561895, + "OOD_Wikipedia_2023": -0.000525059227267405, + "TRAP_FineWeb": -0.0003134391082028514, + "OOD_News_2024": -0.0004019695039494006, + "OOD_arXiv_2023": -9.308719321018015e-05, + "OOD_CommonPile": -0.0007469416250329605 + }, + "n_loo_sign_flips_linear": 1, + "n_loo_sign_flips_log10": 2 + }, + "J|USPTO_Backgrounds": { + "slope_linear_containment": -4.247891056195874e-05, + "slope_log10_containment": 0.0010588073618437773, + "spearman": 0.15454545454545454, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0026059268745792716, + "StackExchange": 0.002348159889371986, + "Wikipedia_en": -0.0007002231730581452, + "USPTO_Backgrounds": -0.0003369993638281012, + "Github": 0.0026729119488506235, + "CONTROL_PubMed_2023": -0.0006377075981444336, + "OOD_Wikipedia_2023": 0.0013686035810469748, + "TRAP_FineWeb": 0.0016655550887958454, + "OOD_News_2024": -0.0005666647124356315, + "OOD_arXiv_2023": 4.532180181529704e-05, + "OOD_CommonPile": -0.004017114631374387 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0005629508738604782, + "StackExchange": 0.0016530366826223108, + "Wikipedia_en": 0.0009751134880302337, + "USPTO_Backgrounds": 0.0010564825770613415, + "Github": 0.0017027487188217658, + "CONTROL_PubMed_2023": 0.0009217765107029212, + "OOD_Wikipedia_2023": 0.0008249363625129528, + "TRAP_FineWeb": 0.0010865656480485888, + "OOD_News_2024": 0.001134563899364181, + "OOD_arXiv_2023": 0.0018849705502802568, + "OOD_CommonPile": 7.827160798897051e-05 + }, + "n_loo_sign_flips_linear": 5, + "n_loo_sign_flips_log10": 0 + }, + "_summary": { + "n_rungs": 11, + "max_abs_spearman": 0.15454545454545454, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_linear": 1, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_log10": 4 + } + } + }, + "control_C1_agrees_with_e54": { + "n_quantities_checked": 24, + "max_abs_diff": 7.105427357601002e-15, + "tolerance": 1e-12, + "fires": true, + "reference": "results/e54_aggregation_audit.json" + }, + "provenance": { + "schema_version": 1, + "experiment": "E53", + "utc": "2026-08-17T00:42:42+00:00", + "script": { + "path": "experiments/t53_ladder_summary.py", + "sha256": "b4b320a9af601f6fca3b6603649b4b4502778c724cff528d6ca7b82d45d8c894", + "n_lines": 298 + }, + "git": { + "commit": "f8ef435b29365012c0492e023fb6b4dfece5f0f3", + "commit_short": "f8ef435b2936", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t53_ladder_summary.py", + "results/e35_containment.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/experiments/E57_grid_variance_ci.md", + "xperiments/t52_factorial.py" + ], + "dirty_diff_sha256": "afc7a07f9036dd20688df568954065b1dc6c531b68598e663d0da07e7055fb6a", + "describe": "f8ef435-dirty" + }, + "argv": [ + "experiments/t53_ladder_summary.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e54_aggregation_audit.json", + "sha256": "886885b41660dd01808aa626d5f87dddf6d5402f2c9b2f8241ac4470df82276b", + "bytes": 69079, + "exists": true + }, + { + "path": "results/e36_qladder_410m.json", + "sha256": "d0cc7b6381ac515f6d53401705e246d8c20c67884cda25e4b1d35d5312f45153", + "bytes": 80656, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "46e5f8fbda0c07fd68c01d04bab37b61a26bdac23fe0814604a1ad21c86f882c" + } +} \ No newline at end of file diff --git a/results/e54_aggregation_audit.json b/results/e54_aggregation_audit.json new file mode 100644 index 0000000000000000000000000000000000000000..05c63c740c6f290a69f8ddc74e7d2c17f0ddb024 --- /dev/null +++ b/results/e54_aggregation_audit.json @@ -0,0 +1,2433 @@ +{ + "experiment": "E54 \u2014 the aggregation audit: every headline under `min` and `persist`", + "why": "`persist` is ours; `min` is the anchor's. A claim that exists only under `persist` is a claim about our metric. Nothing had ever tabulated which claims those are.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json", + "results/e33_logit_baseline_410m_v2.json", + "results/e36_qladder_410m.json", + "results/e48_crossover_410m.json", + "results/e48c_exposure_vs_read.json", + "results/e52_factorial_410m.json" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "asymptote_n_min": 75, + "classification_thresholds": { + "REVERSES": "sign of the decisive quantity flips, or a p-value crosses 0.05", + "WEAKENS": "decisive ratio falls by more than 2x, sign kept", + "HOLDS-STRONGER": "decisive ratio rises by more than 1.2x", + "HOLDS": "otherwise" + }, + "ladder": { + "410m": { + "persist": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.028084053793529264, + "Wikipedia_en": 0.03860496947484109, + "StackExchange": 0.0456756031007639, + "Pile-CC": 0.04571574828415005, + "USPTO_Backgrounds": 0.049735264846729854 + }, + "per_corpus_seed_sd": { + "Github": 0.0003725015802729375, + "Wikipedia_en": 0.000497512452938681, + "StackExchange": 0.0002979844946588909, + "Pile-CC": 0.0001835424093483373, + "USPTO_Backgrounds": 0.0005027384624140496 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.02165121105320059, + "pooled_seed_sd": 0.00037085587992657926, + "spread_over_seed_sd": 58.381738635199795, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 12.031761970089196, + 73.54092188255042 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -29.32008778867865, + "df": 3.706181480116789, + "p": 1.5934823642012846e-05, + "separates": true + }, + "Github|StackExchange": { + "t": -63.87401426888638, + "df": 3.816032427092647, + "p": 6.351744966106404e-07, + "separates": true + }, + "Github|Pile-CC": { + "t": -73.54092188255042, + "df": 2.9170731313531917, + "p": 7.344546283218717e-06, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -59.934247643859756, + "df": 3.687411568613282, + "p": 1.198506487554617e-06, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -21.11771264108646, + "df": 3.2713441661282854, + "p": 0.00013067655363204733, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -23.225504298699857, + "df": 2.534507444454648, + "p": 0.0005080331049453861, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -27.256333209790494, + "df": 3.9995633119295007, + "p": 1.0784912038682538e-05, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": -0.1986812662848506, + "df": 3.326610968599927, + "p": 0.8540456327496845, + "separates": false + }, + "StackExchange|USPTO_Backgrounds": { + "t": -12.031761970089196, + "df": 3.250888721535667, + "p": 0.0008327630169994736, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -13.008353627023636, + "df": 2.5238427055378474, + "p": 0.002227348310144479, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0028340971186047517, + "range_over_seed_sd": 2.5341568525777767, + "argmax_N": 75 + }, + "Wikipedia_en": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0020914917545659176, + "range_over_seed_sd": 3.6204764630199766, + "argmax_N": 25 + }, + "StackExchange": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0017882611513847374, + "range_over_seed_sd": 3.6981860639992252, + "argmax_N": 300 + }, + "Pile-CC": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.000687981006645022, + "range_over_seed_sd": 1.2278526816817164, + "argmax_N": 75 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0025357189366505273, + "range_over_seed_sd": 3.083030892326044, + "argmax_N": 400 + } + }, + "flat_range_over_seed_sd": [ + 1.2278526816817164, + 3.6981860639992252 + ], + "max_movement_in_N": 0.0028340971186047517 + }, + "min": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.11288307417108089, + "Wikipedia_en": 0.13042243031696196, + "StackExchange": 0.14538949898881268, + "Pile-CC": 0.14493902100437375, + "USPTO_Backgrounds": 0.15533995895730282 + }, + "per_corpus_seed_sd": { + "Github": 0.00035393328127784305, + "Wikipedia_en": 0.00010955838937670346, + "StackExchange": 0.00128026159560015, + "Pile-CC": 0.00042533231079348594, + "USPTO_Backgrounds": 0.0016643039976484083 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.042456884786221935, + "pooled_seed_sd": 0.0007666779149393181, + "spread_over_seed_sd": 55.37773289006552, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 8.207952613477497, + 100.34215439417814 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -81.99428757919529, + "df": 2.3797863362681175, + "p": 3.6533723385372034e-05, + "separates": true + }, + "Github|StackExchange": { + "t": -42.38760474793699, + "df": 2.303931704222899, + "p": 0.00022021273158969959, + "separates": true + }, + "Github|Pile-CC": { + "t": -100.34215439417814, + "df": 3.8721318177057205, + "p": 9.295015311877957e-08, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -43.21865282672605, + "df": 2.1805303117974812, + "p": 0.00030647101881163866, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -20.175033663366225, + "df": 2.0292907534920257, + "p": 0.002284948539916303, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -57.24627016230037, + "df": 2.264232884694126, + "p": 0.00012573058813856007, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -25.87581218897987, + "df": 2.017333162219172, + "p": 0.0014245310770301593, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 0.5783639426857416, + "df": 2.4361749261412093, + "p": 0.6120606018893571, + "separates": false + }, + "StackExchange|USPTO_Backgrounds": { + "t": -8.207952613477497, + "df": 3.7531031511904787, + "p": 0.0015712415434009322, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -10.487261913538559, + "df": 2.2601374216530616, + "p": 0.005805838608934484, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0034548367684086173, + "range_over_seed_sd": 2.2622282558307703, + "argmax_N": 375 + }, + "Wikipedia_en": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0014686014769332845, + "range_over_seed_sd": 1.1840692303542948, + "argmax_N": 375 + }, + "StackExchange": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0019650841930083096, + "range_over_seed_sd": 1.302208655813546, + "argmax_N": 25 + }, + "Pile-CC": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.002348728105425829, + "range_over_seed_sd": 2.106425607322571, + "argmax_N": 100 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0026533089932941067, + "range_over_seed_sd": 1.2335055892097606, + "argmax_N": 400 + } + }, + "flat_range_over_seed_sd": [ + 1.1840692303542948, + 2.2622282558307703 + ], + "max_movement_in_N": 0.0034548367684086173 + } + }, + "1b": { + "persist": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.027256395461569938, + "Wikipedia_en": 0.02801095717586577, + "StackExchange": 0.05875215052316586, + "Pile-CC": 0.046013945933165296, + "USPTO_Backgrounds": 0.05470160290465824 + }, + "per_corpus_seed_sd": { + "Github": 0.0013762403288060878, + "Wikipedia_en": 0.00020285804501715256, + "StackExchange": 0.0007357119491078887, + "Pile-CC": 0.00039438237251602294, + "USPTO_Backgrounds": 0.0017248115034397009 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.03149575506159592, + "pooled_seed_sd": 0.0008868008397773706, + "spread_over_seed_sd": 35.51615385197748, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 3.7414047636770564, + 70.3097211819473 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -0.9394933684708586, + "df": 2.0868662234127733, + "p": 0.4430895229085523, + "separates": false + }, + "Github|StackExchange": { + "t": -34.957113847459645, + "df": 3.056800239042669, + "p": 4.430261747534506e-05, + "separates": true + }, + "Github|Pile-CC": { + "t": -22.693676218822052, + "df": 2.3262778089342686, + "p": 0.0008783542616057277, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -21.543026480272037, + "df": 3.8121178219340974, + "p": 4.004856453522681e-05, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -69.76890715662432, + "df": 2.30236101278019, + "p": 7.029390923616283e-05, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -70.3097211819473, + "df": 2.9890668231741286, + "p": 6.576476832894556e-06, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -26.619197602231548, + "df": 2.0553194805721473, + "p": 0.0012181637277215592, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 26.43089120460415, + "df": 3.0617496325437474, + "p": 0.00010269188216586204, + "separates": true + }, + "StackExchange|USPTO_Backgrounds": { + "t": 3.7414047636770564, + "df": 2.7044477621250116, + "p": 0.03971364135834821, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -8.504633312249457, + "df": 2.2085576959071216, + "p": 0.009952014794920177, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.006032132942761694, + "range_over_seed_sd": 2.0832274009134886, + "argmax_N": 10 + }, + "Wikipedia_en": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0023527308677633585, + "range_over_seed_sd": 1.9704177680409294, + "argmax_N": 150 + }, + "StackExchange": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.006524874713449244, + "range_over_seed_sd": 3.7745797870225046, + "argmax_N": 150 + }, + "Pile-CC": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.003513823981795991, + "range_over_seed_sd": 1.6205093198035128, + "argmax_N": 50 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.004342931668673243, + "range_over_seed_sd": 2.298899257873663, + "argmax_N": 50 + } + }, + "flat_range_over_seed_sd": [ + 1.6205093198035128, + 3.7745797870225046 + ], + "max_movement_in_N": 0.006524874713449244 + }, + "min": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.13184866054604452, + "Wikipedia_en": 0.12168779909788144, + "StackExchange": 0.1760720702420388, + "Pile-CC": 0.14951401859566216, + "USPTO_Backgrounds": 0.16125627207968915 + }, + "per_corpus_seed_sd": { + "Github": 0.001174541715835113, + "Wikipedia_en": 0.000664360394074828, + "StackExchange": 0.000781853055555403, + "Pile-CC": 0.0016589081835269135, + "USPTO_Backgrounds": 0.0026333882425492983 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "between_corpus_spread": 0.054384271144157365, + "pooled_seed_sd": 0.0013826103183083112, + "spread_over_seed_sd": 39.334489569482656, + "n_pairs_separating_p05": 10, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 6.534674049686413, + 91.80958763345963 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": 13.042035228685885, + "df": 3.1609302063989753, + "p": 0.0007454506503368391, + "separates": true + }, + "Github|StackExchange": { + "t": -54.28682048115797, + "df": 3.481548250716117, + "p": 3.1857647786542707e-06, + "separates": true + }, + "Github|Pile-CC": { + "t": -15.05316770337511, + "df": 3.6024790741888855, + "p": 0.0002207550470131397, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -17.664770849542823, + "df": 2.7654406589883376, + "p": 0.0006315346383159637, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -91.80958763345963, + "df": 3.8984238567221983, + "p": 1.1973006104680652e-07, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -26.97065532927737, + "df": 2.6254499585287703, + "p": 0.00027785032815772143, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -25.234591190044668, + "df": 2.2535606927828096, + "p": 0.0008231019913061611, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 25.082795391343012, + "df": 2.846737462807291, + "p": 0.00019868054152209687, + "separates": true + }, + "StackExchange|USPTO_Backgrounds": { + "t": 9.341712101684202, + "df": 2.349879826655504, + "p": 0.00655414759865613, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -6.534674049686413, + "df": 3.3713894640736504, + "p": 0.005037196334992217, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0030341794803029065, + "range_over_seed_sd": 1.1210924731652319, + "argmax_N": 10 + }, + "Wikipedia_en": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.002315773735088955, + "range_over_seed_sd": 0.9972203747816083, + "argmax_N": 25 + }, + "StackExchange": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.008386632312266606, + "range_over_seed_sd": 5.729404685250978, + "argmax_N": 150 + }, + "Pile-CC": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.00464149415493012, + "range_over_seed_sd": 1.2370669965583059, + "argmax_N": 50 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0042105066279570225, + "range_over_seed_sd": 1.535072587315563, + "argmax_N": 50 + } + }, + "flat_range_over_seed_sd": [ + 0.9972203747816083, + 5.729404685250978 + ], + "max_movement_in_N": 0.008386632312266606 + } + } + }, + "variance_decomposition": { + "410m": { + "persist": { + "frac_set_main": 91.27829399393488, + "frac_corpus_main": 1.9064501148312785, + "frac_interaction": 6.780626428099203, + "frac_residual_seed": 0.034629463134648465, + "F": 611.8910219722385, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 3.55667655573526 + }, + "min": { + "frac_set_main": 96.06544098391524, + "frac_corpus_main": 1.911884671540871, + "frac_interaction": 1.9989092146925351, + "frac_residual_seed": 0.023765129851367858, + "F": 262.84692467416227, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 1.0455176739722105 + } + }, + "1b": { + "persist": { + "frac_set_main": 88.57950497345804, + "frac_corpus_main": 2.2117202251311614, + "frac_interaction": 9.158325214755475, + "frac_residual_seed": 0.05044958665530746, + "F": 567.2943663870428, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 4.1408154208167804 + }, + "min": { + "frac_set_main": 94.14618752120755, + "frac_corpus_main": 1.9252567551293123, + "frac_interaction": 3.896419437299789, + "frac_residual_seed": 0.03213628636333453, + "F": 378.8960119379021, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 2.0238440545235648 + } + } + }, + "e33": { + "persist": { + "logit_constant": 0.02843950316309929, + "by_corpus": { + "Github": { + "asymptote": 0.028084053793529264, + "seed_sd": 0.0003725015802729375, + "pct_vs_logit": -1.2498438089144464, + "t_vs_logit_df2": -1.6527617605340326, + "separates_from_logit": false, + "below_logit": true + }, + "Wikipedia_en": { + "asymptote": 0.03860496947484109, + "seed_sd": 0.000497512452938681, + "pct_vs_logit": 35.74417687061305, + "t_vs_logit_df2": 35.39027823437558, + "separates_from_logit": true, + "below_logit": false + }, + "StackExchange": { + "asymptote": 0.0456756031007639, + "seed_sd": 0.0002979844946588909, + "pct_vs_logit": 60.606192164526725, + "t_vs_logit_df2": 100.18575245179832, + "separates_from_logit": true, + "below_logit": false + }, + "Pile-CC": { + "asymptote": 0.04571574828415005, + "seed_sd": 0.0001835424093483373, + "pct_vs_logit": 60.747352096737636, + "t_vs_logit_df2": 163.03226278828905, + "separates_from_logit": true, + "below_logit": false + }, + "USPTO_Backgrounds": { + "asymptote": 0.049735264846729854, + "seed_sd": 0.0005027384624140496, + "pct_vs_logit": 74.88092025201817, + "t_vs_logit_df2": 73.368846785286, + "separates_from_logit": true, + "below_logit": false + } + }, + "n_corpora_indistinguishable_from_logit": 1, + "n_corpora_significantly_below_logit": 0 + }, + "min": { + "logit_constant": 0.12473962224487747, + "by_corpus": { + "Github": { + "asymptote": 0.11288307417108089, + "seed_sd": 0.00035393328127784305, + "pct_vs_logit": -9.505037661987538, + "t_vs_logit_df2": -58.02264085495087, + "separates_from_logit": true, + "below_logit": true + }, + "Wikipedia_en": { + "asymptote": 0.13042243031696196, + "seed_sd": 0.00010955838937670346, + "pct_vs_logit": 4.55573615649446, + "t_vs_logit_df2": 89.8417032827052, + "separates_from_logit": true, + "below_logit": false + }, + "StackExchange": { + "asymptote": 0.14538949898881268, + "seed_sd": 0.00128026159560015, + "pct_vs_logit": 16.554384543026146, + "t_vs_logit_df2": 27.93697461007129, + "separates_from_logit": true, + "below_logit": false + }, + "Pile-CC": { + "asymptote": 0.14493902100437375, + "seed_sd": 0.00042533231079348594, + "pct_vs_logit": 16.193249904062284, + "t_vs_logit_df2": 82.25658866245516, + "separates_from_logit": true, + "below_logit": false + }, + "USPTO_Backgrounds": { + "asymptote": 0.15533995895730282, + "seed_sd": 0.0016643039976484083, + "pct_vs_logit": 24.53136875174558, + "t_vs_logit_df2": 31.84594760904531, + "separates_from_logit": true, + "below_logit": false + } + }, + "n_corpora_indistinguishable_from_logit": 0, + "n_corpora_significantly_below_logit": 1 + } + }, + "e48c": { + "persist": { + "logit_constant": 0.028439503419212996, + "by_rung": { + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "read": 0.03820955517391364, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "read": 0.04961537096338967, + "tier": "in-stream" + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "read": 0.04546591031830758, + "tier": "in-stream" + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "read": 0.045729962767412265, + "tier": "in-stream" + }, + "Github": { + "containment_k32": 0.9340004374453194, + "read": 0.02789688172439734, + "tier": "github" + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "read": 0.0337636109519129, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "read": 0.04105739652489623, + "tier": "OOD" + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "read": 0.03823967572922508, + "tier": "OOD" + } + }, + "spearman_all8": 0.0, + "spearman_without_github": 0.5, + "containment_span": 10861.817987204726, + "tier_gap_instream_minus_ood": 0.007068305403744389, + "tier_gap_t": 2.2164522110159894, + "tier_gap_df": 4.961001162329945, + "tier_gap_p": 0.07789643681450693, + "n_ood_rungs_above_instream_wikipedia": 2 + }, + "min": { + "logit_constant": 0.1247396256774664, + "by_rung": { + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "read": 0.1304755893846353, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "read": 0.15502665042877198, + "tier": "in-stream" + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "read": 0.14452077796061835, + "tier": "in-stream" + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "read": 0.1455607984215021, + "tier": "in-stream" + }, + "Github": { + "containment_k32": 0.9340004374453194, + "read": 0.1126159243285656, + "tier": "github" + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "read": 0.13940734267234803, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "read": 0.1407180975501736, + "tier": "OOD" + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "read": 0.1375763162970543, + "tier": "OOD" + } + }, + "spearman_all8": -0.09523809523809523, + "spearman_without_github": 0.35714285714285715, + "containment_span": 10861.817987204726, + "tier_gap_instream_minus_ood": 0.0046620352090233075, + "tier_gap_t": 0.9068870057058248, + "tier_gap_df": 3.192694536183569, + "tier_gap_p": 0.42763534967408634, + "n_ood_rungs_above_instream_wikipedia": 3 + } + }, + "e36": { + "persist": { + "rungs": [ + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "containment": [ + 0.9340004374453194, + 0.9276060773863086, + 0.9267616923398619, + 0.9215951727072293, + 0.9179914195561899, + 0.7927273508312213, + 0.26957534299550123, + 0.06309446548070848, + 0.00125, + 0.0007487928942136383, + 8.598932872430531e-05 + ], + "logit": { + "slope": -0.003720899629554985, + "spearman": -0.2727272727272727, + "mean": 0.031253348919562995 + }, + "shuf_arm": { + "mean": 0.02982744561000304, + "above_logit": false, + "n_rungs_above_logit": 5 + }, + "by_fit_corpus": { + "Github": { + "slope": -0.0014550596366756292, + "spearman": -0.19090909090909092, + "worst_loo_slope": -0.0021622055526610198, + "loo_slope_by_dropped_rung": { + "Github": -0.0008909328785794681, + "Pile-CC": -0.001561401546835692, + "USPTO_Backgrounds": -0.0017519561244663739, + "Wikipedia_en": -0.001992270380004623, + "StackExchange": -0.0005760183119989624, + "CONTROL_PubMed_2023": -0.0016397555863833476, + "OOD_Wikipedia_2023": -0.0009673012084557685, + "TRAP_FineWeb": -0.0011688572755800576, + "OOD_CommonPile": -0.0021622055526610198, + "OOD_News_2024": -0.0014455593119102068, + "OOD_arXiv_2023": -0.0019401073419784616 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 10, + "rungs_below_logit": [ + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "steeper_than_logit": false + }, + "Wikipedia_en": { + "slope": -0.0009354827291769033, + "spearman": 0.06363636363636363, + "worst_loo_slope": -0.00225182718285068, + "loo_slope_by_dropped_rung": { + "Github": 0.0006114770190120221, + "Pile-CC": -0.0016945220408105522, + "USPTO_Backgrounds": -0.0014773995230767255, + "Wikipedia_en": -0.001943307383106595, + "StackExchange": 0.0007790027011743094, + "CONTROL_PubMed_2023": -0.001345574896548144, + "OOD_Wikipedia_2023": -0.00013865249525168055, + "TRAP_FineWeb": -0.0008008900076050542, + "OOD_CommonPile": -0.00225182718285068, + "OOD_News_2024": -0.00043233948548224637, + "OOD_arXiv_2023": -0.0017250513630531405 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + }, + "StackExchange": { + "slope": 0.0002240976423130857, + "spearman": 0.2818181818181818, + "worst_loo_slope": 0.0013572724997980054, + "loo_slope_by_dropped_rung": { + "Github": 0.000645747151939162, + "Pile-CC": 9.734162192914718e-05, + "USPTO_Backgrounds": -0.00032751857982107153, + "Wikipedia_en": -0.00024598000465200266, + "StackExchange": 0.0013572724997980054, + "CONTROL_PubMed_2023": 0.00014738936959018587, + "OOD_Wikipedia_2023": 0.0008552523402857944, + "TRAP_FineWeb": 0.00026812738983809003, + "OOD_CommonPile": -0.00022378417326704018, + "OOD_News_2024": 0.00010939484488637846, + "OOD_arXiv_2023": -0.0003280293027039663 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + }, + "Pile-CC": { + "slope": -0.0011535435993366304, + "spearman": -0.00909090909090909, + "worst_loo_slope": -0.0022811577240861203, + "loo_slope_by_dropped_rung": { + "Github": 0.00014389080343519365, + "Pile-CC": -0.0018561479020940102, + "USPTO_Backgrounds": -0.0015056028936971267, + "Wikipedia_en": -0.0022811577240861203, + "StackExchange": 0.0003276404302305153, + "CONTROL_PubMed_2023": -0.0013506750771647544, + "OOD_Wikipedia_2023": -0.0005025777305976444, + "TRAP_FineWeb": -0.0010021430954018857, + "OOD_CommonPile": -0.0021429228225167247, + "OOD_News_2024": -0.0007940013046808708, + "OOD_arXiv_2023": -0.0018366231525598165 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + }, + "USPTO_Backgrounds": { + "slope": -0.000885040945743085, + "spearman": 0.20909090909090908, + "worst_loo_slope": -0.0015151819788109704, + "loo_slope_by_dropped_rung": { + "Github": 7.746754680233447e-05, + "Pile-CC": -0.0014123147101777196, + "USPTO_Backgrounds": -0.0014672707299009076, + "Wikipedia_en": -0.001286481013089218, + "StackExchange": 0.00024297344343503642, + "CONTROL_PubMed_2023": -0.0010450754171928793, + "OOD_Wikipedia_2023": -0.00014922575853216864, + "TRAP_FineWeb": -0.0007833716787412385, + "OOD_CommonPile": -0.0014965239401716014, + "OOD_News_2024": -0.0010267371441971883, + "OOD_arXiv_2023": -0.0015151819788109704 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + } + }, + "n_fitted_steeper_than_logit": 0, + "n_fitted_crossing_below_logit_at_any_rung": 1 + }, + "min": { + "rungs": [ + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "containment": [ + 0.9340004374453194, + 0.9276060773863086, + 0.9267616923398619, + 0.9215951727072293, + 0.9179914195561899, + 0.7927273508312213, + 0.26957534299550123, + 0.06309446548070848, + 0.00125, + 0.0007487928942136383, + 8.598932872430531e-05 + ], + "logit": { + "slope": -0.0010699323813758122, + "spearman": 0.00909090909090909, + "mean": 0.12512224580753933 + }, + "shuf_arm": { + "mean": 0.15574212485190594, + "above_logit": true, + "n_rungs_above_logit": 11 + }, + "by_fit_corpus": { + "Github": { + "slope": -0.0035714446311215964, + "spearman": -0.09090909090909091, + "worst_loo_slope": -0.0052687368331600405, + "loo_slope_by_dropped_rung": { + "Github": -0.002052368335791041, + "Pile-CC": -0.0052687368331600405, + "USPTO_Backgrounds": -0.0048044144955547655, + "Wikipedia_en": -0.0034644178667903418, + "StackExchange": -0.001960639289563647, + "CONTROL_PubMed_2023": -0.0035556823516738945, + "OOD_Wikipedia_2023": -0.002989236397777608, + "TRAP_FineWeb": -0.0026493610346831294, + "OOD_CommonPile": -0.00348374819833562, + "OOD_News_2024": -0.004848794207343968, + "OOD_arXiv_2023": -0.004352390461459243 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 11, + "rungs_below_logit": [ + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "steeper_than_logit": true + }, + "Wikipedia_en": { + "slope": -0.004603344091202535, + "spearman": 0.05454545454545454, + "worst_loo_slope": -0.007606565142089651, + "loo_slope_by_dropped_rung": { + "Github": -0.0016207231641492746, + "Pile-CC": -0.007037656959177343, + "USPTO_Backgrounds": -0.005772717284704184, + "Wikipedia_en": -0.005967997333887068, + "StackExchange": -0.00033718111815656036, + "CONTROL_PubMed_2023": -0.005800630286563434, + "OOD_Wikipedia_2023": -0.003278371513152543, + "TRAP_FineWeb": -0.003957832832162028, + "OOD_CommonPile": -0.007606565142089651, + "OOD_News_2024": -0.004794875269979331, + "OOD_arXiv_2023": -0.00468132498800471 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 1, + "rungs_below_logit": [ + "StackExchange" + ], + "steeper_than_logit": true + }, + "StackExchange": { + "slope": -0.0035589979944138225, + "spearman": 0.00909090909090909, + "worst_loo_slope": -0.0057165041295091585, + "loo_slope_by_dropped_rung": { + "Github": -0.0018976731424252432, + "Pile-CC": -0.005354689643722768, + "USPTO_Backgrounds": -0.004980725535569202, + "Wikipedia_en": -0.00400213687233838, + "StackExchange": -7.861127536555136e-05, + "CONTROL_PubMed_2023": -0.00422844192966735, + "OOD_Wikipedia_2023": -0.0023323168584560294, + "TRAP_FineWeb": -0.0033621771436820732, + "OOD_CommonPile": -0.004445761072997944, + "OOD_News_2024": -0.0057165041295091585, + "OOD_arXiv_2023": -0.00295331038636373 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "Pile-CC": { + "slope": -0.004014659692830054, + "spearman": -0.01818181818181818, + "worst_loo_slope": -0.006127892387626296, + "loo_slope_by_dropped_rung": { + "Github": -0.002156511734341, + "Pile-CC": -0.006127892387626296, + "USPTO_Backgrounds": -0.0047696458834969166, + "Wikipedia_en": -0.005243503813823139, + "StackExchange": 2.76804226033701e-05, + "CONTROL_PubMed_2023": -0.005012628579589455, + "OOD_Wikipedia_2023": -0.003101921604069063, + "TRAP_FineWeb": -0.003264071432890173, + "OOD_CommonPile": -0.005922931885080463, + "OOD_News_2024": -0.004697706531445889, + "OOD_arXiv_2023": -0.004054662748765874 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "USPTO_Backgrounds": { + "slope": -4.247891056195874e-05, + "spearman": 0.15454545454545454, + "worst_loo_slope": -0.004017114631374387, + "loo_slope_by_dropped_rung": { + "Github": 0.0026729119488506235, + "Pile-CC": -0.0026059268745792716, + "USPTO_Backgrounds": -0.0003369993638281012, + "Wikipedia_en": -0.0007002231730581452, + "StackExchange": 0.002348159889371986, + "CONTROL_PubMed_2023": -0.0006377075981444336, + "OOD_Wikipedia_2023": 0.0013686035810469748, + "TRAP_FineWeb": 0.0016655550887958454, + "OOD_CommonPile": -0.004017114631374387, + "OOD_News_2024": -0.0005666647124356315, + "OOD_arXiv_2023": 4.532180181529704e-05 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + } + }, + "n_fitted_steeper_than_logit": 4, + "n_fitted_crossing_below_logit_at_any_rung": 2 + } + }, + "e52": { + "persist": { + "D_all8": 0.0010325341964960988, + "D_without_degenerate": 0.0009875729428136123, + "permutation_p": 0.005497251374312844, + "link_artifact_surrogate": 0.00013936889285791815, + "D_minus_surrogate": 0.0008931653036381806, + "sign_survives_dropping_github": true, + "significant_at_5pct": true + }, + "min": { + "D_all8": 0.0005479490122063032, + "D_without_degenerate": -0.00031888559115705905, + "permutation_p": 0.6636681659170415, + "link_artifact_surrogate": 0.0003985587486577635, + "D_minus_surrogate": 0.00014939026354853966, + "sign_survives_dropping_github": false, + "significant_at_5pct": false + } + }, + "quadrants": { + "persist": { + "P_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "Q_corpora": [ + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "quadrant_means": { + "JP_at_P": 0.04062002426613536, + "JP_at_Q": 0.040890102766419724, + "JQ_at_P": 0.03799831597306938, + "JQ_at_Q": 0.038442899579653675 + }, + "fit_row_means": { + "Wikipedia_en": 0.03836092785899786, + "USPTO_Backgrounds": 0.04748104946532597, + "Pile-CC": 0.04716437525704451, + "StackExchange": 0.04518624561864676, + "Github": 0.02541392031869489, + "OOD_News_2024": 0.036911535940857394, + "OOD_arXiv_2023": 0.03942143082547571, + "OOD_CommonPile": 0.03816213771028237 + }, + "read_col_means": { + "Wikipedia_en": 0.04101473994604829, + "USPTO_Backgrounds": 0.04202875193182586, + "Pile-CC": 0.041234459214805, + "StackExchange": 0.03650194659324674, + "Github": 0.037404520595252205, + "OOD_News_2024": 0.041391930579427735, + "OOD_arXiv_2023": 0.039012664472102186, + "OOD_CommonPile": 0.03951260966261745 + }, + "diagonal_mean": 0.04066617029634977, + "offdiagonal_mean": 0.039633636099853674, + "fit_axis_span": 0.02206712914663108, + "read_axis_span": 0.005526805338579126 + }, + "min": { + "P_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "Q_corpora": [ + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "quadrant_means": { + "JP_at_P": 0.13736161468591956, + "JP_at_Q": 0.1386651265759159, + "JQ_at_P": 0.1382778434096663, + "JQ_at_Q": 0.13890393505355825 + }, + "fit_row_means": { + "Wikipedia_en": 0.12989506709337648, + "USPTO_Backgrounds": 0.15617140837841564, + "Pile-CC": 0.14402559753248675, + "StackExchange": 0.14620193947727483, + "Github": 0.11295814574178722, + "OOD_News_2024": 0.13518428391156098, + "OOD_arXiv_2023": 0.14045638353305145, + "OOD_CommonPile": 0.13989721588376494 + }, + "read_col_means": { + "Wikipedia_en": 0.13891349073706402, + "USPTO_Backgrounds": 0.14661414006517992, + "Pile-CC": 0.14461656864473804, + "StackExchange": 0.13126294978428632, + "Github": 0.1271188530553546, + "OOD_News_2024": 0.14362420086593677, + "OOD_arXiv_2023": 0.14041273188518566, + "OOD_CommonPile": 0.1322271065139729 + }, + "diagonal_mean": 0.1385782105796453, + "offdiagonal_mean": 0.138030261567439, + "fit_axis_span": 0.04321326263662842, + "read_axis_span": 0.019495287009825307 + } + }, + "matrix_structure": { + "persist": { + "variance_fit_axis_pct": 91.21993064640641, + "variance_read_axis_pct": 7.133265187892407, + "variance_fit_x_read_interaction_pct": 1.6468041657011812, + "note_on_interaction": "this is the fit x READ interaction. It is NOT the corpus x CONCEPT-SET interaction reported in E51; the concept sets are averaged out of this matrix.", + "own_corpus": { + "Wikipedia_en": { + "rank_of_own_corpus": 3, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "USPTO_Backgrounds": { + "rank_of_own_corpus": 1, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "Pile-CC": { + "rank_of_own_corpus": 3, + "best_read_context": "OOD_News_2024", + "worst_read_context": "StackExchange" + }, + "StackExchange": { + "rank_of_own_corpus": 8, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "Github": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_News_2024": { + "rank_of_own_corpus": 1, + "best_read_context": "OOD_News_2024", + "worst_read_context": "StackExchange" + }, + "OOD_arXiv_2023": { + "rank_of_own_corpus": 6, + "best_read_context": "Pile-CC", + "worst_read_context": "Github" + }, + "OOD_CommonPile": { + "rank_of_own_corpus": 4, + "best_read_context": "Pile-CC", + "worst_read_context": "Github" + } + }, + "mean_rank_of_own_corpus": 4.125, + "chance_rank": 4.5, + "n_operators_reading_own_corpus_best": 2, + "fit_axis_order": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "OOD_arXiv_2023", + "Wikipedia_en", + "OOD_CommonPile", + "OOD_News_2024", + "Github" + ], + "read_axis_order": [ + "USPTO_Backgrounds", + "OOD_News_2024", + "Pile-CC", + "Wikipedia_en", + "OOD_CommonPile", + "OOD_arXiv_2023", + "Github", + "StackExchange" + ], + "n_Q_operators_above_worst_P_operator": 3, + "n_Q_operators_above_second_worst_P_operator": 1, + "worst_read_context": "StackExchange", + "worst_read_context_is_P": true, + "github_cost_as_fit_corpus_pct": -36.08603419400133, + "github_cost_as_read_context_pct": -5.930638786330567 + }, + "min": { + "variance_fit_axis_pct": 73.41700738536099, + "variance_read_axis_pct": 22.67267017800786, + "variance_fit_x_read_interaction_pct": 3.910322436631175, + "note_on_interaction": "this is the fit x READ interaction. It is NOT the corpus x CONCEPT-SET interaction reported in E51; the concept sets are averaged out of this matrix.", + "own_corpus": { + "Wikipedia_en": { + "rank_of_own_corpus": 4, + "best_read_context": "Pile-CC", + "worst_read_context": "Github" + }, + "USPTO_Backgrounds": { + "rank_of_own_corpus": 1, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_CommonPile" + }, + "Pile-CC": { + "rank_of_own_corpus": 3, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "StackExchange": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "Github": { + "rank_of_own_corpus": 8, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "OOD_News_2024": { + "rank_of_own_corpus": 3, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_arXiv_2023": { + "rank_of_own_corpus": 4, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "OOD_CommonPile": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + } + }, + "mean_rank_of_own_corpus": 4.625, + "chance_rank": 4.5, + "n_operators_reading_own_corpus_best": 1, + "fit_axis_order": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "OOD_arXiv_2023", + "OOD_CommonPile", + "OOD_News_2024", + "Wikipedia_en", + "Github" + ], + "read_axis_order": [ + "USPTO_Backgrounds", + "Pile-CC", + "OOD_News_2024", + "OOD_arXiv_2023", + "Wikipedia_en", + "OOD_CommonPile", + "StackExchange", + "Github" + ], + "n_Q_operators_above_worst_P_operator": 3, + "n_Q_operators_above_second_worst_P_operator": 3, + "worst_read_context": "Github", + "worst_read_context_is_P": true, + "github_cost_as_fit_corpus_pct": -18.204805261905975, + "github_cost_as_read_context_pct": -7.9507611224941765 + } + }, + "baseline_movement": { + "persist": { + "n_read_contexts": 12, + "logit_min": 0.023829053342342376, + "logit_max": 0.036840112763457, + "logit_range": 0.013011059421114624, + "largest_fitted_advantage": 0.018084099576420463, + "logit_range_as_pct_of_largest_advantage": 71.94751038685672, + "per_operator": { + "Github": { + "range_across_read_contexts": 0.00793257928453386, + "mean_gap_over_logit": -0.004125156249049015, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "Wikipedia_en": { + "range_across_read_contexts": 0.013182362999456625, + "mean_gap_over_logit": 0.008602426227621942, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "StackExchange": { + "range_across_read_contexts": 0.008880645781755445, + "mean_gap_over_logit": 0.015815443232552046, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "Pile-CC": { + "range_across_read_contexts": 0.011356671992689374, + "mean_gap_over_logit": 0.016980879426571643, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "USPTO_Backgrounds": { + "range_across_read_contexts": 0.010293584906806548, + "mean_gap_over_logit": 0.018084099576420463, + "n_contexts_scrambled_beats_this_operator": 0 + } + }, + "n_contexts_scrambled_beats_logit": 6 + }, + "min": { + "n_read_contexts": 12, + "logit_min": 0.11852239407598972, + "logit_max": 0.13364263189335665, + "logit_range": 0.015120237817366933, + "largest_fitted_advantage": 0.03470899688286914, + "logit_range_as_pct_of_largest_advantage": 43.56287756858116, + "per_operator": { + "Github": { + "range_across_read_contexts": 0.01352383928994337, + "mean_gap_over_logit": -0.01108258850872517, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "Wikipedia_en": { + "range_across_read_contexts": 0.029782165338595706, + "mean_gap_over_logit": 0.007636598011271817, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "StackExchange": { + "range_across_read_contexts": 0.025165284735461063, + "mean_gap_over_logit": 0.02323210664714376, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "Pile-CC": { + "range_across_read_contexts": 0.025652496019999166, + "mean_gap_over_logit": 0.02104439205076131, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "USPTO_Backgrounds": { + "range_across_read_contexts": 0.02193380519747734, + "mean_gap_over_logit": 0.03470899688286914, + "n_contexts_scrambled_beats_this_operator": 2 + } + }, + "n_contexts_scrambled_beats_logit": 12 + } + }, + "matrix_loo": { + "persist": { + "full": { + "fit_pct": 91.21993064640641, + "read_pct": 7.133265187892407, + "interaction_pct": 1.6468041657011812, + "fit_span": 0.02206712914663108, + "read_span": 0.005526805338579126, + "span_ratio": 3.992745862170038, + "fit_dominates_read": true + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 92.62271892860798, + "read_pct": 6.044474474909023, + "interaction_pct": 1.3328065964830005, + "fit_span": 0.021921292635878283, + "read_span": 0.0052411437245263245, + "span_ratio": 4.182539878327693, + "fit_dominates_read": true + }, + "USPTO_Backgrounds": { + "fit_pct": 90.93746718003844, + "read_pct": 6.993844859630081, + "interaction_pct": 2.0686879603315065, + "fit_span": 0.021692538208195142, + "read_span": 0.004853508410797944, + "span_ratio": 4.469455159474787, + "fit_dominates_read": true + }, + "Pile-CC": { + "fit_pct": 90.84291026481041, + "read_pct": 7.2900814225666135, + "interaction_pct": 1.8670083126229458, + "fit_span": 0.021950742357321792, + "read_span": 0.005468765466237473, + "span_ratio": 4.0138386794678125, + "fit_dominates_read": true + }, + "StackExchange": { + "fit_pct": 93.33692623435269, + "read_pct": 5.156511202775666, + "interaction_pct": 1.5065625628716701, + "fit_span": 0.022061346833121092, + "read_span": 0.004746862740758508, + "span_ratio": 4.647563672674445, + "fit_dominates_read": true + }, + "Github": { + "fit_pct": 81.04881620398683, + "read_pct": 15.567785798307998, + "interaction_pct": 3.3833979977051234, + "fit_span": 0.010373391022550915, + "read_span": 0.005564901774882204, + "span_ratio": 1.8640744153602775, + "fit_dominates_read": true + }, + "OOD_News_2024": { + "fit_pct": 92.43636904450099, + "read_pct": 6.27734357951022, + "interaction_pct": 1.2862873759887745, + "fit_span": 0.02194255113727339, + "read_span": 0.005361218094116167, + "span_ratio": 4.092829418999932, + "fit_dominates_read": true + }, + "OOD_arXiv_2023": { + "fit_pct": 91.1983253417951, + "read_pct": 7.343503025106673, + "interaction_pct": 1.4581716330982468, + "fit_span": 0.022309378342747335, + "read_span": 0.005842901524969392, + "span_ratio": 3.818202009294381, + "fit_dominates_read": true + }, + "OOD_CommonPile": { + "fit_pct": 91.10865204439243, + "read_pct": 7.5957558760329, + "interaction_pct": 1.2955920795746776, + "fit_span": 0.022281274635950844, + "read_span": 0.005781825323633492, + "span_ratio": 3.853674815265527, + "fit_dominates_read": true + } + }, + "drop_both_extremes_github_and_uspto": { + "fit_pct": 78.67789669144574, + "read_pct": 16.686938451159083, + "interaction_pct": 4.63516485739523, + "fit_span": 0.010451911164758111, + "read_span": 0.0049434692887761825, + "span_ratio": 2.114286658660645, + "fit_dominates_read": true + }, + "worst_loo_span_ratio": 1.8640744153602775, + "worst_loo_corpus": "Github", + "fit_dominates_in_all_loo": true, + "fit_dominates_with_both_extremes_dropped": true + }, + "min": { + "full": { + "fit_pct": 73.41700738536099, + "read_pct": 22.67267017800786, + "interaction_pct": 3.910322436631175, + "fit_span": 0.04321326263662842, + "read_span": 0.019495287009825307, + "span_ratio": 2.2166004847658636, + "fit_dominates_read": true + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 72.21409286150445, + "read_pct": 23.71515132910453, + "interaction_pct": 4.070755809391008, + "fit_span": 0.042136523452779615, + "read_span": 0.01932690815911406, + "span_ratio": 2.180199911226315, + "fit_dominates_read": true + }, + "USPTO_Backgrounds": { + "fit_pct": 69.97297180223329, + "read_pct": 24.704987971694027, + "interaction_pct": 5.3220402260727235, + "fit_span": 0.032951310528294434, + "read_span": 0.017268920689821263, + "span_ratio": 1.9081279670081968, + "fit_dominates_read": true + }, + "Pile-CC": { + "fit_pct": 75.22745446862925, + "read_pct": 20.50190993884228, + "interaction_pct": 4.270635592528498, + "fit_span": 0.042820826634055106, + "read_span": 0.01925233122375275, + "span_ratio": 2.2241891715027475, + "fit_dominates_read": true + }, + "StackExchange": { + "fit_pct": 74.77416486260653, + "read_pct": 22.702208386947053, + "interaction_pct": 2.5236267504463816, + "fit_span": 0.043067101315255193, + "read_span": 0.019554403409480114, + "span_ratio": 2.202424712910237, + "fit_dominates_read": true + }, + "Github": { + "fit_pct": 58.59657103734628, + "read_pct": 34.84817420406899, + "interaction_pct": 6.5552547585846535, + "fit_span": 0.026079021388339618, + "read_span": 0.016088034097282655, + "span_ratio": 1.6210197734939216, + "fit_dominates_read": true + }, + "OOD_News_2024": { + "fit_pct": 75.8043220820044, + "read_pct": 20.54338347833448, + "interaction_pct": 3.652294439661085, + "fit_span": 0.04277797218944346, + "read_span": 0.019508810660668785, + "span_ratio": 2.192751415425187, + "fit_dominates_read": true + }, + "OOD_arXiv_2023": { + "fit_pct": 74.1434371562919, + "read_pct": 22.09754567085689, + "interaction_pct": 3.759017172851185, + "fit_span": 0.04287713592842457, + "read_span": 0.019068447991259507, + "span_ratio": 2.248590758308089, + "fit_dominates_read": true + }, + "OOD_CommonPile": { + "fit_pct": 77.20909139763098, + "read_pct": 20.313039456100718, + "interaction_pct": 2.477869146268303, + "fit_span": 0.04500458402529596, + "read_span": 0.01934977030706786, + "span_ratio": 2.3258459046853495, + "fit_dominates_read": true + } + }, + "drop_both_extremes_github_and_uspto": { + "fit_pct": 42.0114078749595, + "read_pct": 47.61343085472334, + "interaction_pct": 10.375161270317184, + "fit_span": 0.01565969237298878, + "read_span": 0.014799208839044536, + "span_ratio": 1.0581438875079623, + "fit_dominates_read": false + }, + "worst_loo_span_ratio": 1.6210197734939216, + "worst_loo_corpus": "Github", + "fit_dominates_in_all_loo": true, + "fit_dominates_with_both_extremes_dropped": false + } + }, + "C2_derangement": { + "persist": { + "n_draws": 120, + "shuf_beats_jp_paired_by_seed": 0, + "ties": 0, + "shuf_beats_jp_vs_corpus_mean": 0, + "n_corpora_where_shuf_beats_jp_on_the_mean": 0, + "shuf_sd_range": [ + 0.0015449648806984003, + 0.002914492891994465 + ], + "gap_range": [ + 0.010767463998248177, + 0.021970295277424154 + ], + "per_corpus": { + "Github": { + "jp_mean": 0.02789688172439734, + "shuf_mean": 0.017129417726149163, + "gap": 0.010767463998248177, + "shuf_sd_over_15_draws": 0.0018638386960376496, + "shuf_beats_jp_paired": 0 + }, + "OOD_CommonPile": { + "jp_mean": 0.03823967572922508, + "shuf_mean": 0.02355857611944278, + "gap": 0.014681099609782297, + "shuf_sd_over_15_draws": 0.002912470775340541, + "shuf_beats_jp_paired": 0 + }, + "OOD_News_2024": { + "jp_mean": 0.0337636109519129, + "shuf_mean": 0.018717034761793913, + "gap": 0.015046576190118989, + "shuf_sd_over_15_draws": 0.0016517842631268904, + "shuf_beats_jp_paired": 0 + }, + "OOD_arXiv_2023": { + "jp_mean": 0.04105739652489623, + "shuf_mean": 0.024700007438659667, + "gap": 0.01635738908623656, + "shuf_sd_over_15_draws": 0.0028135854091460816, + "shuf_beats_jp_paired": 0 + }, + "Pile-CC": { + "jp_mean": 0.04546591031830758, + "shuf_mean": 0.02692041976687809, + "gap": 0.01854549055142949, + "shuf_sd_over_15_draws": 0.0020402960046508106, + "shuf_beats_jp_paired": 0 + }, + "StackExchange": { + "jp_mean": 0.045729962767412265, + "shuf_mean": 0.026390449275883535, + "gap": 0.01933951349152873, + "shuf_sd_over_15_draws": 0.002914492891994465, + "shuf_beats_jp_paired": 0 + }, + "USPTO_Backgrounds": { + "jp_mean": 0.04961537096338967, + "shuf_mean": 0.02764507568596552, + "gap": 0.021970295277424154, + "shuf_sd_over_15_draws": 0.002039146369823083, + "shuf_beats_jp_paired": 0 + }, + "Wikipedia_en": { + "jp_mean": 0.03820955517391364, + "shuf_mean": 0.019931373049815498, + "gap": 0.018278182124098142, + "shuf_sd_over_15_draws": 0.0015449648806984003, + "shuf_beats_jp_paired": 0 + } + }, + "admissible_as_an_operator_comparator": true + }, + "min": { + "n_draws": 120, + "shuf_beats_jp_paired_by_seed": 104, + "ties": 0, + "shuf_beats_jp_vs_corpus_mean": 103, + "n_corpora_where_shuf_beats_jp_on_the_mean": 7, + "shuf_sd_range": [ + 0.006642094422897706, + 0.012379500382090904 + ], + "gap_range": [ + -0.03211234564582505, + 0.0011352714647849538 + ], + "per_corpus": { + "Github": { + "jp_mean": 0.1126159243285656, + "shuf_mean": 0.11933546006679535, + "gap": -0.006719535738229754, + "shuf_sd_over_15_draws": 0.012379500382090904, + "shuf_beats_jp_paired": 12 + }, + "OOD_CommonPile": { + "jp_mean": 0.1375763162970543, + "shuf_mean": 0.14552284215887387, + "gap": -0.00794652586181957, + "shuf_sd_over_15_draws": 0.007826596012088445, + "shuf_beats_jp_paired": 14 + }, + "OOD_News_2024": { + "jp_mean": 0.13940734267234803, + "shuf_mean": 0.17151968831817307, + "gap": -0.03211234564582505, + "shuf_sd_over_15_draws": 0.007718225133197829, + "shuf_beats_jp_paired": 15 + }, + "OOD_arXiv_2023": { + "jp_mean": 0.1407180975501736, + "shuf_mean": 0.14946635278562706, + "gap": -0.008748255235453473, + "shuf_sd_over_15_draws": 0.011114347019169522, + "shuf_beats_jp_paired": 12 + }, + "Pile-CC": { + "jp_mean": 0.14452077796061835, + "shuf_mean": 0.16368520364165307, + "gap": -0.019164425681034714, + "shuf_sd_over_15_draws": 0.007386723679365784, + "shuf_beats_jp_paired": 15 + }, + "StackExchange": { + "jp_mean": 0.1455607984215021, + "shuf_mean": 0.15189976240197817, + "gap": -0.0063389639804760645, + "shuf_sd_over_15_draws": 0.006642094422897706, + "shuf_beats_jp_paired": 12 + }, + "USPTO_Backgrounds": { + "jp_mean": 0.15502665042877198, + "shuf_mean": 0.15389137896398702, + "gap": 0.0011352714647849538, + "shuf_sd_over_15_draws": 0.008418568188510181, + "shuf_beats_jp_paired": 9 + }, + "Wikipedia_en": { + "jp_mean": 0.1304755893846353, + "shuf_mean": 0.15765395919481914, + "gap": -0.027178369810183844, + "shuf_sd_over_15_draws": 0.010262702456552721, + "shuf_beats_jp_paired": 15 + } + }, + "admissible_as_an_operator_comparator": false + } + }, + "C1_persist_column_reproduces_stored_headlines": { + "e51_410m_interaction_pct": { + "recomputed": 6.780626428099203, + "stored": 6.780626428099203, + "abs_diff": 0.0, + "ok": true + }, + "e51_410m_corpus_pct": { + "recomputed": 1.9064501148312785, + "stored": 1.9064501148312791, + "abs_diff": 6.661338147750939e-16, + "ok": true + }, + "e51_1b_interaction_pct": { + "recomputed": 9.158325214755475, + "stored": 9.158325214755473, + "abs_diff": 1.7763568394002505e-15, + "ok": true + }, + "e51_410m_F": { + "recomputed": 611.8910219722385, + "stored": 611.8910219722385, + "abs_diff": 0.0, + "ok": true + }, + "e33_logit_constant": { + "recomputed": 0.02843950316309929, + "stored": 0.02843950316309929, + "abs_diff": 0.0, + "ok": true + }, + "e33_uspto_asymptote": { + "recomputed": 0.049735264846729854, + "stored": 0.049735264846729854, + "abs_diff": 0.0, + "ok": true + }, + "e48c_spearman_all8": { + "recomputed": 0.0, + "stored": 0.0, + "abs_diff": 0.0, + "ok": true + }, + "e52_D": { + "recomputed": 0.0010325341964960988, + "stored": 0.0010325341964960988, + "abs_diff": 0.0, + "ok": true + } + }, + "C1_fires": true, + "VERDICT_BY_CLAIM": { + "corpus_identity_decides_the_read_410m": { + "persist": 58.381738635199795, + "min": 55.37773289006552, + "verdict": "HOLDS" + }, + "reads_are_flat_in_N_410m": { + "persist": [ + 1.2278526816817164, + 3.6981860639992252 + ], + "min": [ + 1.1840692303542948, + 2.2622282558307703 + ], + "verdict": "HOLDS" + }, + "github_indistinguishable_from_not_fitting": { + "persist": -1.6527617605340326, + "min": -58.02264085495087, + "verdict": "REVERSES", + "note": "under min Github is not merely indistinguishable from the logit lens, it is significantly BELOW it \u2014 a stronger form of the same claim, but a different sentence" + }, + "effect_is_an_interaction_not_a_level_410m": { + "persist": 3.55667655573526, + "min": 1.0455176739722105, + "verdict": "WEAKENS" + }, + "effect_is_an_interaction_not_a_level_1b": { + "persist": 4.1408154208167804, + "min": 2.0238440545235648, + "verdict": "WEAKENS" + }, + "exposure_does_not_order_the_read": { + "persist": 0.0, + "min": -0.09523809523809523, + "verdict": "HOLDS", + "note": "both are indistinguishable from zero at n=8; all three OOD rungs beat in-stream Wikipedia under both" + }, + "e36_no_fitted_operator_crosses_below_logit": { + "persist": 1, + "min": 2, + "verdict": "HOLDS", + "note": "the count includes Github, which sits below the logit lens at every rung under both aggregations and is degenerate by E33. Excluding it: 0 of 4 under persist, 1 of 4 (Wikipedia, at 1 rung of 11) under min. The pre-registration states that a crossing appearing only under min does not count." + }, + "e36_fitted_operators_are_flatter_than_logit": { + "persist": 0, + "min": 4, + "verdict": "REVERSES", + "note": "0 of 5 fitted operators are steeper than the logit lens under persist; 4 of 5 are under min. This secondary is aggregation-dependent AND single-rung-fragile under LOO." + }, + "e52_matched_fit_read_helps": { + "persist": 0.005497251374312844, + "min": 0.6636681659170415, + "verdict": "REVERSES", + "note": "p = 0.0055 under persist, 0.6637 under min; and D changes sign without Github under min (-3.19e-04)" + } + }, + "VERDICT": "THE SPINE IS AGGREGATION-ROBUST; THREE SECONDARIES ARE NOT. Corpus identity deciding the read, reads being flat in N, and exposure not ordering the read all hold under the anchor's own `min` \u2014 the first with a LARGER absolute spread. What does not survive: the interaction's dominance over the corpus main effect at 410M (3.56x -> 1.05x), E36's slope secondary (0 of 5 fitted operators steeper -> 4 of 5), and E52's matched-diagonal effect (p=0.0055 -> p=0.66, and D changes sign without Github). Separately, C2 shows why `min` cannot be the adjudicating metric: under it the layer-deranged operator beats the real one on 104 of 120 paired draws and outscores the logit lens, while under `persist` it loses 120 of 120. Both facts must be reported together \u2014 `min` is inadmissible as an operator comparator AND three of our claims depend on not using it.", + "provenance": { + "schema_version": 1, + "experiment": "E54", + "utc": "2026-08-16T22:56:32+00:00", + "script": { + "path": "experiments/t54_aggregation_audit.py", + "sha256": "e407bc2a61da00b83e0e01301649ba3a7952b77944b9a9d9f9711b468956b9a7", + "n_lines": 673 + }, + "git": { + "commit": "1340e766a6152bc2f7f069222613f4733570ee5b", + "commit_short": "1340e766a615", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t54_aggregation_audit.py", + "results/e54_aggregation_audit.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md" + ], + "dirty_diff_sha256": null, + "describe": "1340e76" + }, + "argv": [ + "experiments/t54_aggregation_audit.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + }, + { + "path": "results/e36_qladder_410m.json", + "sha256": "d0cc7b6381ac515f6d53401705e246d8c20c67884cda25e4b1d35d5312f45153", + "bytes": 80656, + "exists": true + }, + { + "path": "results/e48_crossover_410m.json", + "sha256": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", + "bytes": 116036, + "exists": true + }, + { + "path": "results/e48c_exposure_vs_read.json", + "sha256": "ff344efc91604138b698251c630e0d4f320d5b5b27ac9f87e2914d848ab1c374", + "bytes": 9875, + "exists": true + }, + { + "path": "results/e51_interaction_variance.json", + "sha256": "2559e4017660a6a53949cb2a739fb3e7884e712851f2a04c8e04273dd217f62b", + "bytes": 29478, + "exists": true + }, + { + "path": "results/e52_factorial_410m.json", + "sha256": "6e81f3cae37d0c1906f8912af4bf8e75db0a0f259b2869f209701b74b6682590", + "bytes": 41190, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "3d8af6bd411d5767e86f3de415d112a72fa19373e8ab7d82d027305bc8da2d8d" + } +} \ No newline at end of file diff --git a/results/e54_aggregation_audit_rstrip.json b/results/e54_aggregation_audit_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..dcb924ca8b950cd388cda15c56a12bcc3c45f2be --- /dev/null +++ b/results/e54_aggregation_audit_rstrip.json @@ -0,0 +1,2450 @@ +{ + "experiment": "E54 \u2014 the aggregation audit: every headline under `min` and `persist`", + "why": "`persist` is ours; `min` is the anchor's. A claim that exists only under `persist` is a claim about our metric. Nothing had ever tabulated which claims those are.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json", + "results/e33_logit_baseline_410m_v2.json", + "results/e36_qladder_410m.json", + "results/e48_crossover_410m.json", + "results/e48c_exposure_vs_read.json", + "results/e52_factorial_410m.json" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "asymptote_n_min": 75, + "classification_thresholds": { + "REVERSES": "sign of the decisive quantity flips, or a p-value crosses 0.05", + "WEAKENS": "decisive ratio falls by more than 2x, sign kept", + "HOLDS-STRONGER": "decisive ratio rises by more than 1.2x", + "HOLDS": "otherwise" + }, + "ladder": { + "410m": { + "persist": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.028084053793529264, + "Wikipedia_en": 0.03860496947484109, + "StackExchange": 0.0456756031007639, + "Pile-CC": 0.04571574828415005, + "USPTO_Backgrounds": 0.049735264846729854 + }, + "per_corpus_seed_sd": { + "Github": 0.0003725015802729375, + "Wikipedia_en": 0.000497512452938681, + "StackExchange": 0.0002979844946588909, + "Pile-CC": 0.0001835424093483373, + "USPTO_Backgrounds": 0.0005027384624140496 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.02165121105320059, + "pooled_seed_sd": 0.00037085587992657926, + "spread_over_seed_sd": 58.381738635199795, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 12.031761970089196, + 73.54092188255042 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -29.32008778867865, + "df": 3.706181480116789, + "p": 1.5934823642012846e-05, + "separates": true + }, + "Github|StackExchange": { + "t": -63.87401426888638, + "df": 3.816032427092647, + "p": 6.351744966106404e-07, + "separates": true + }, + "Github|Pile-CC": { + "t": -73.54092188255042, + "df": 2.9170731313531917, + "p": 7.344546283218717e-06, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -59.934247643859756, + "df": 3.687411568613282, + "p": 1.198506487554617e-06, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -21.11771264108646, + "df": 3.2713441661282854, + "p": 0.00013067655363204733, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -23.225504298699857, + "df": 2.534507444454648, + "p": 0.0005080331049453861, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -27.256333209790494, + "df": 3.9995633119295007, + "p": 1.0784912038682538e-05, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": -0.1986812662848506, + "df": 3.326610968599927, + "p": 0.8540456327496845, + "separates": false + }, + "StackExchange|USPTO_Backgrounds": { + "t": -12.031761970089196, + "df": 3.250888721535667, + "p": 0.0008327630169994736, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -13.008353627023636, + "df": 2.5238427055378474, + "p": 0.002227348310144479, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0028340971186047517, + "range_over_seed_sd": 2.5341568525777767, + "argmax_N": 75 + }, + "Wikipedia_en": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0020914917545659176, + "range_over_seed_sd": 3.6204764630199766, + "argmax_N": 25 + }, + "StackExchange": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0017882611513847374, + "range_over_seed_sd": 3.6981860639992252, + "argmax_N": 300 + }, + "Pile-CC": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.000687981006645022, + "range_over_seed_sd": 1.2278526816817164, + "argmax_N": 75 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0025357189366505273, + "range_over_seed_sd": 3.083030892326044, + "argmax_N": 400 + } + }, + "flat_range_over_seed_sd": [ + 1.2278526816817164, + 3.6981860639992252 + ], + "max_movement_in_N": 0.0028340971186047517 + }, + "min": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.11288307417108089, + "Wikipedia_en": 0.13042243031696196, + "StackExchange": 0.14538949898881268, + "Pile-CC": 0.14493902100437375, + "USPTO_Backgrounds": 0.15533995895730282 + }, + "per_corpus_seed_sd": { + "Github": 0.00035393328127784305, + "Wikipedia_en": 0.00010955838937670346, + "StackExchange": 0.00128026159560015, + "Pile-CC": 0.00042533231079348594, + "USPTO_Backgrounds": 0.0016643039976484083 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.042456884786221935, + "pooled_seed_sd": 0.0007666779149393181, + "spread_over_seed_sd": 55.37773289006552, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 8.207952613477497, + 100.34215439417814 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -81.99428757919529, + "df": 2.3797863362681175, + "p": 3.6533723385372034e-05, + "separates": true + }, + "Github|StackExchange": { + "t": -42.38760474793699, + "df": 2.303931704222899, + "p": 0.00022021273158969959, + "separates": true + }, + "Github|Pile-CC": { + "t": -100.34215439417814, + "df": 3.8721318177057205, + "p": 9.295015311877957e-08, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -43.21865282672605, + "df": 2.1805303117974812, + "p": 0.00030647101881163866, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -20.175033663366225, + "df": 2.0292907534920257, + "p": 0.002284948539916303, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -57.24627016230037, + "df": 2.264232884694126, + "p": 0.00012573058813856007, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -25.87581218897987, + "df": 2.017333162219172, + "p": 0.0014245310770301593, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 0.5783639426857416, + "df": 2.4361749261412093, + "p": 0.6120606018893571, + "separates": false + }, + "StackExchange|USPTO_Backgrounds": { + "t": -8.207952613477497, + "df": 3.7531031511904787, + "p": 0.0015712415434009322, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -10.487261913538559, + "df": 2.2601374216530616, + "p": 0.005805838608934484, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0034548367684086173, + "range_over_seed_sd": 2.2622282558307703, + "argmax_N": 375 + }, + "Wikipedia_en": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0014686014769332845, + "range_over_seed_sd": 1.1840692303542948, + "argmax_N": 375 + }, + "StackExchange": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0019650841930083096, + "range_over_seed_sd": 1.302208655813546, + "argmax_N": 25 + }, + "Pile-CC": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.002348728105425829, + "range_over_seed_sd": 2.106425607322571, + "argmax_N": 100 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0026533089932941067, + "range_over_seed_sd": 1.2335055892097606, + "argmax_N": 400 + } + }, + "flat_range_over_seed_sd": [ + 1.1840692303542948, + 2.2622282558307703 + ], + "max_movement_in_N": 0.0034548367684086173 + } + }, + "1b": { + "persist": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.027256395461569938, + "Wikipedia_en": 0.02801095717586577, + "StackExchange": 0.05875215052316586, + "Pile-CC": 0.046013945933165296, + "USPTO_Backgrounds": 0.05470160290465824 + }, + "per_corpus_seed_sd": { + "Github": 0.0013762403288060878, + "Wikipedia_en": 0.00020285804501715256, + "StackExchange": 0.0007357119491078887, + "Pile-CC": 0.00039438237251602294, + "USPTO_Backgrounds": 0.0017248115034397009 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.03149575506159592, + "pooled_seed_sd": 0.0008868008397773706, + "spread_over_seed_sd": 35.51615385197748, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 3.7414047636770564, + 70.3097211819473 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -0.9394933684708586, + "df": 2.0868662234127733, + "p": 0.4430895229085523, + "separates": false + }, + "Github|StackExchange": { + "t": -34.957113847459645, + "df": 3.056800239042669, + "p": 4.430261747534506e-05, + "separates": true + }, + "Github|Pile-CC": { + "t": -22.693676218822052, + "df": 2.3262778089342686, + "p": 0.0008783542616057277, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -21.543026480272037, + "df": 3.8121178219340974, + "p": 4.004856453522681e-05, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -69.76890715662432, + "df": 2.30236101278019, + "p": 7.029390923616283e-05, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -70.3097211819473, + "df": 2.9890668231741286, + "p": 6.576476832894556e-06, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -26.619197602231548, + "df": 2.0553194805721473, + "p": 0.0012181637277215592, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 26.43089120460415, + "df": 3.0617496325437474, + "p": 0.00010269188216586204, + "separates": true + }, + "StackExchange|USPTO_Backgrounds": { + "t": 3.7414047636770564, + "df": 2.7044477621250116, + "p": 0.03971364135834821, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -8.504633312249457, + "df": 2.2085576959071216, + "p": 0.009952014794920177, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.006032132942761694, + "range_over_seed_sd": 2.0832274009134886, + "argmax_N": 10 + }, + "Wikipedia_en": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0023527308677633585, + "range_over_seed_sd": 1.9704177680409294, + "argmax_N": 150 + }, + "StackExchange": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.006524874713449244, + "range_over_seed_sd": 3.7745797870225046, + "argmax_N": 150 + }, + "Pile-CC": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.003513823981795991, + "range_over_seed_sd": 1.6205093198035128, + "argmax_N": 50 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.004342931668673243, + "range_over_seed_sd": 2.298899257873663, + "argmax_N": 50 + } + }, + "flat_range_over_seed_sd": [ + 1.6205093198035128, + 3.7745797870225046 + ], + "max_movement_in_N": 0.006524874713449244 + }, + "min": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.13184866054604452, + "Wikipedia_en": 0.12168779909788144, + "StackExchange": 0.1760720702420388, + "Pile-CC": 0.14951401859566216, + "USPTO_Backgrounds": 0.16125627207968915 + }, + "per_corpus_seed_sd": { + "Github": 0.001174541715835113, + "Wikipedia_en": 0.000664360394074828, + "StackExchange": 0.000781853055555403, + "Pile-CC": 0.0016589081835269135, + "USPTO_Backgrounds": 0.0026333882425492983 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "between_corpus_spread": 0.054384271144157365, + "pooled_seed_sd": 0.0013826103183083112, + "spread_over_seed_sd": 39.334489569482656, + "n_pairs_separating_p05": 10, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 6.534674049686413, + 91.80958763345963 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": 13.042035228685885, + "df": 3.1609302063989753, + "p": 0.0007454506503368391, + "separates": true + }, + "Github|StackExchange": { + "t": -54.28682048115797, + "df": 3.481548250716117, + "p": 3.1857647786542707e-06, + "separates": true + }, + "Github|Pile-CC": { + "t": -15.05316770337511, + "df": 3.6024790741888855, + "p": 0.0002207550470131397, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -17.664770849542823, + "df": 2.7654406589883376, + "p": 0.0006315346383159637, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -91.80958763345963, + "df": 3.8984238567221983, + "p": 1.1973006104680652e-07, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -26.97065532927737, + "df": 2.6254499585287703, + "p": 0.00027785032815772143, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -25.234591190044668, + "df": 2.2535606927828096, + "p": 0.0008231019913061611, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 25.082795391343012, + "df": 2.846737462807291, + "p": 0.00019868054152209687, + "separates": true + }, + "StackExchange|USPTO_Backgrounds": { + "t": 9.341712101684202, + "df": 2.349879826655504, + "p": 0.00655414759865613, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -6.534674049686413, + "df": 3.3713894640736504, + "p": 0.005037196334992217, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0030341794803029065, + "range_over_seed_sd": 1.1210924731652319, + "argmax_N": 10 + }, + "Wikipedia_en": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.002315773735088955, + "range_over_seed_sd": 0.9972203747816083, + "argmax_N": 25 + }, + "StackExchange": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.008386632312266606, + "range_over_seed_sd": 5.729404685250978, + "argmax_N": 150 + }, + "Pile-CC": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.00464149415493012, + "range_over_seed_sd": 1.2370669965583059, + "argmax_N": 50 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0042105066279570225, + "range_over_seed_sd": 1.535072587315563, + "argmax_N": 50 + } + }, + "flat_range_over_seed_sd": [ + 0.9972203747816083, + 5.729404685250978 + ], + "max_movement_in_N": 0.008386632312266606 + } + } + }, + "variance_decomposition": { + "410m": { + "persist": { + "frac_set_main": 91.27829399393488, + "frac_corpus_main": 1.9064501148312785, + "frac_interaction": 6.780626428099203, + "frac_residual_seed": 0.034629463134648465, + "F": 611.8910219722385, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 3.55667655573526 + }, + "min": { + "frac_set_main": 96.06544098391524, + "frac_corpus_main": 1.911884671540871, + "frac_interaction": 1.9989092146925351, + "frac_residual_seed": 0.023765129851367858, + "F": 262.84692467416227, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 1.0455176739722105 + } + }, + "1b": { + "persist": { + "frac_set_main": 88.57950497345804, + "frac_corpus_main": 2.2117202251311614, + "frac_interaction": 9.158325214755475, + "frac_residual_seed": 0.05044958665530746, + "F": 567.2943663870428, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 4.1408154208167804 + }, + "min": { + "frac_set_main": 94.14618752120755, + "frac_corpus_main": 1.9252567551293123, + "frac_interaction": 3.896419437299789, + "frac_residual_seed": 0.03213628636333453, + "F": 378.8960119379021, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 2.0238440545235648 + } + } + }, + "e33": { + "persist": { + "logit_constant": 0.02843950316309929, + "by_corpus": { + "Github": { + "asymptote": 0.028084053793529264, + "seed_sd": 0.0003725015802729375, + "pct_vs_logit": -1.2498438089144464, + "t_vs_logit_df2": -1.6527617605340326, + "separates_from_logit": false, + "below_logit": true + }, + "Wikipedia_en": { + "asymptote": 0.03860496947484109, + "seed_sd": 0.000497512452938681, + "pct_vs_logit": 35.74417687061305, + "t_vs_logit_df2": 35.39027823437558, + "separates_from_logit": true, + "below_logit": false + }, + "StackExchange": { + "asymptote": 0.0456756031007639, + "seed_sd": 0.0002979844946588909, + "pct_vs_logit": 60.606192164526725, + "t_vs_logit_df2": 100.18575245179832, + "separates_from_logit": true, + "below_logit": false + }, + "Pile-CC": { + "asymptote": 0.04571574828415005, + "seed_sd": 0.0001835424093483373, + "pct_vs_logit": 60.747352096737636, + "t_vs_logit_df2": 163.03226278828905, + "separates_from_logit": true, + "below_logit": false + }, + "USPTO_Backgrounds": { + "asymptote": 0.049735264846729854, + "seed_sd": 0.0005027384624140496, + "pct_vs_logit": 74.88092025201817, + "t_vs_logit_df2": 73.368846785286, + "separates_from_logit": true, + "below_logit": false + } + }, + "n_corpora_indistinguishable_from_logit": 1, + "n_corpora_significantly_below_logit": 0 + }, + "min": { + "logit_constant": 0.12473962224487747, + "by_corpus": { + "Github": { + "asymptote": 0.11288307417108089, + "seed_sd": 0.00035393328127784305, + "pct_vs_logit": -9.505037661987538, + "t_vs_logit_df2": -58.02264085495087, + "separates_from_logit": true, + "below_logit": true + }, + "Wikipedia_en": { + "asymptote": 0.13042243031696196, + "seed_sd": 0.00010955838937670346, + "pct_vs_logit": 4.55573615649446, + "t_vs_logit_df2": 89.8417032827052, + "separates_from_logit": true, + "below_logit": false + }, + "StackExchange": { + "asymptote": 0.14538949898881268, + "seed_sd": 0.00128026159560015, + "pct_vs_logit": 16.554384543026146, + "t_vs_logit_df2": 27.93697461007129, + "separates_from_logit": true, + "below_logit": false + }, + "Pile-CC": { + "asymptote": 0.14493902100437375, + "seed_sd": 0.00042533231079348594, + "pct_vs_logit": 16.193249904062284, + "t_vs_logit_df2": 82.25658866245516, + "separates_from_logit": true, + "below_logit": false + }, + "USPTO_Backgrounds": { + "asymptote": 0.15533995895730282, + "seed_sd": 0.0016643039976484083, + "pct_vs_logit": 24.53136875174558, + "t_vs_logit_df2": 31.84594760904531, + "separates_from_logit": true, + "below_logit": false + } + }, + "n_corpora_indistinguishable_from_logit": 0, + "n_corpora_significantly_below_logit": 1 + } + }, + "e48c": { + "persist": { + "logit_constant": 0.028439503419212996, + "by_rung": { + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "read": 0.03820955517391364, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "read": 0.04961537096338967, + "tier": "in-stream" + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "read": 0.04546591031830758, + "tier": "in-stream" + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "read": 0.045729962767412265, + "tier": "in-stream" + }, + "Github": { + "containment_k32": 0.9340004374453194, + "read": 0.02789688172439734, + "tier": "github" + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "read": 0.0337636109519129, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "read": 0.04105739652489623, + "tier": "OOD" + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "read": 0.03823967572922508, + "tier": "OOD" + } + }, + "spearman_all8": 0.0, + "spearman_without_github": 0.5, + "containment_span": 10861.817987204726, + "tier_gap_instream_minus_ood": 0.007068305403744389, + "tier_gap_t": 2.2164522110159894, + "tier_gap_df": 4.961001162329945, + "tier_gap_p": 0.07789643681450693, + "n_ood_rungs_above_instream_wikipedia": 2 + }, + "min": { + "logit_constant": 0.1247396256774664, + "by_rung": { + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "read": 0.1304755893846353, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "read": 0.15502665042877198, + "tier": "in-stream" + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "read": 0.14452077796061835, + "tier": "in-stream" + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "read": 0.1455607984215021, + "tier": "in-stream" + }, + "Github": { + "containment_k32": 0.9340004374453194, + "read": 0.1126159243285656, + "tier": "github" + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "read": 0.13940734267234803, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "read": 0.1407180975501736, + "tier": "OOD" + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "read": 0.1375763162970543, + "tier": "OOD" + } + }, + "spearman_all8": -0.09523809523809523, + "spearman_without_github": 0.35714285714285715, + "containment_span": 10861.817987204726, + "tier_gap_instream_minus_ood": 0.0046620352090233075, + "tier_gap_t": 0.9068870057058248, + "tier_gap_df": 3.192694536183569, + "tier_gap_p": 0.42763534967408634, + "n_ood_rungs_above_instream_wikipedia": 3 + } + }, + "e36": { + "persist": { + "rungs": [ + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "containment": [ + 0.9340004374453194, + 0.9276060773863086, + 0.9267616923398619, + 0.9215951727072293, + 0.9179914195561899, + 0.7927273508312213, + 0.26957534299550123, + 0.06309446548070848, + 0.00125, + 0.0007487928942136383, + 8.598932872430531e-05 + ], + "logit": { + "slope": -0.0037284686515085515, + "spearman": 0.03636363636363636, + "mean": 0.07387016320417664 + }, + "shuf_arm": { + "mean": 0.08303366410263786, + "above_logit": true, + "n_rungs_above_logit": 11 + }, + "by_fit_corpus": { + "Github": { + "slope": -0.006302421723152167, + "spearman": -0.1, + "worst_loo_slope": -0.008441645016381174, + "loo_slope_by_dropped_rung": { + "Github": -0.0024168897694502177, + "Pile-CC": -0.008441645016381174, + "USPTO_Backgrounds": -0.007384536525218063, + "Wikipedia_en": -0.007728880892965402, + "StackExchange": -0.0043434026002888825, + "CONTROL_PubMed_2023": -0.006713846696496293, + "OOD_Wikipedia_2023": -0.005112283764729826, + "TRAP_FineWeb": -0.00582211622752024, + "OOD_CommonPile": -0.007829920487629368, + "OOD_News_2024": -0.00555404368229907, + "OOD_arXiv_2023": -0.008185001025962615 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "Wikipedia_en": { + "slope": -0.003751919371114022, + "spearman": 0.05454545454545454, + "worst_loo_slope": -0.005678867843274564, + "loo_slope_by_dropped_rung": { + "Github": 0.0003016115498988717, + "Pile-CC": -0.005491148554964415, + "USPTO_Backgrounds": -0.005534449398737498, + "Wikipedia_en": -0.005665184321947394, + "StackExchange": -0.001235347641740309, + "CONTROL_PubMed_2023": -0.004183852962555575, + "OOD_Wikipedia_2023": -0.0027372203566205155, + "TRAP_FineWeb": -0.0032927560949970527, + "OOD_CommonPile": -0.005678867843274564, + "OOD_News_2024": -0.0031015921835953306, + "OOD_arXiv_2023": -0.004826130927711405 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "StackExchange": { + "slope": -0.0036664399547795337, + "spearman": -0.00909090909090909, + "worst_loo_slope": -0.0056006463421927315, + "loo_slope_by_dropped_rung": { + "Github": -0.0006790836836824708, + "Pile-CC": -0.004990725423007533, + "USPTO_Backgrounds": -0.0056006463421927315, + "Wikipedia_en": -0.0050372393557747835, + "StackExchange": -0.00151881318439263, + "CONTROL_PubMed_2023": -0.003640588143018499, + "OOD_Wikipedia_2023": -0.0027296844073786066, + "TRAP_FineWeb": -0.0025106958885208, + "OOD_CommonPile": -0.005407453174742336, + "OOD_News_2024": -0.003153774919972062, + "OOD_arXiv_2023": -0.00527199455255791 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + }, + "Pile-CC": { + "slope": -0.0048877667147762035, + "spearman": 0.0, + "worst_loo_slope": -0.006724107905736161, + "loo_slope_by_dropped_rung": { + "Github": -0.0009967740515735804, + "Pile-CC": -0.006724107905736161, + "USPTO_Backgrounds": -0.006463482017118128, + "Wikipedia_en": -0.006690320408810834, + "StackExchange": -0.0025393937647051157, + "CONTROL_PubMed_2023": -0.005261524362022698, + "OOD_Wikipedia_2023": -0.003884067551259087, + "TRAP_FineWeb": -0.004879048784359732, + "OOD_CommonPile": -0.006598440375986708, + "OOD_News_2024": -0.0039224128489219676, + "OOD_arXiv_2023": -0.005960032823002878 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "USPTO_Backgrounds": { + "slope": -0.004609631566017049, + "spearman": -0.00909090909090909, + "worst_loo_slope": -0.006625617543198389, + "loo_slope_by_dropped_rung": { + "Github": -0.000513467266396934, + "Pile-CC": -0.006556779660893942, + "USPTO_Backgrounds": -0.006625617543198389, + "Wikipedia_en": -0.005387748642851779, + "StackExchange": -0.0027711663748551606, + "CONTROL_PubMed_2023": -0.00507721409229556, + "OOD_Wikipedia_2023": -0.0036012922990186122, + "TRAP_FineWeb": -0.00393341135225128, + "OOD_CommonPile": -0.0060053386906768515, + "OOD_News_2024": -0.004495785776616605, + "OOD_arXiv_2023": -0.0059195173698972075 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + } + }, + "n_fitted_steeper_than_logit": 4, + "n_fitted_crossing_below_logit_at_any_rung": 0 + }, + "min": { + "rungs": [ + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "containment": [ + 0.9340004374453194, + 0.9276060773863086, + 0.9267616923398619, + 0.9215951727072293, + 0.9179914195561899, + 0.7927273508312213, + 0.26957534299550123, + 0.06309446548070848, + 0.00125, + 0.0007487928942136383, + 8.598932872430531e-05 + ], + "logit": { + "slope": 0.0021193657987416048, + "spearman": 0.19090909090909092, + "mean": 0.1990344447394212 + }, + "shuf_arm": { + "mean": 0.26733599201296315, + "above_logit": true, + "n_rungs_above_logit": 11 + }, + "by_fit_corpus": { + "Github": { + "slope": 0.00374649675334448, + "spearman": 0.3181818181818182, + "worst_loo_slope": 0.00692427999778951, + "loo_slope_by_dropped_rung": { + "Github": 0.00692427999778951, + "Pile-CC": 0.001850419039028697, + "USPTO_Backgrounds": 0.0027376935014466397, + "Wikipedia_en": 0.0015034475594117858, + "StackExchange": 0.005411913387345525, + "CONTROL_PubMed_2023": 0.004867735619613101, + "OOD_Wikipedia_2023": 0.00583858625340628, + "TRAP_FineWeb": 0.006388740901425092, + "OOD_CommonPile": 0.0006282266015373978, + "OOD_News_2024": 0.004283113705859323, + "OOD_arXiv_2023": 0.0002689178441613544 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + }, + "Wikipedia_en": { + "slope": -0.0017812876146258553, + "spearman": 0.05454545454545454, + "worst_loo_slope": -0.005772345907007004, + "loo_slope_by_dropped_rung": { + "Github": 0.0012169867001940442, + "Pile-CC": -0.004183024011643131, + "USPTO_Backgrounds": -0.0032090508105418825, + "Wikipedia_en": -0.0037107514032504317, + "StackExchange": 0.0016006596714169714, + "CONTROL_PubMed_2023": -0.001612386654700412, + "OOD_Wikipedia_2023": -0.00031422614152677345, + "TRAP_FineWeb": 0.000538800428554092, + "OOD_CommonPile": -0.005772345907007004, + "OOD_News_2024": -0.0006546885625461061, + "OOD_arXiv_2023": -0.003854520399759611 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "StackExchange": { + "slope": 0.00014062084151355905, + "spearman": 0.16363636363636364, + "worst_loo_slope": -0.0039263860180292175, + "loo_slope_by_dropped_rung": { + "Github": 0.002285386508006283, + "Pile-CC": -0.002374064436702188, + "USPTO_Backgrounds": -0.0009179314206659777, + "Wikipedia_en": -0.0015158815390440086, + "StackExchange": 0.0038188850917921765, + "CONTROL_PubMed_2023": 0.00041641391023375135, + "OOD_Wikipedia_2023": 0.0017804207252592253, + "TRAP_FineWeb": 0.0033469708661429077, + "OOD_CommonPile": -0.0039263860180292175, + "OOD_News_2024": 0.0012011239916363539, + "OOD_arXiv_2023": -0.003005678217388545 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "Pile-CC": { + "slope": -0.001888259914374931, + "spearman": 0.1, + "worst_loo_slope": -0.0066902792082571445, + "loo_slope_by_dropped_rung": { + "Github": 0.00047872924608353826, + "Pile-CC": -0.005057749174065081, + "USPTO_Backgrounds": -0.0035113200534188493, + "Wikipedia_en": -0.0036818181668935262, + "StackExchange": 0.0028447736299541636, + "CONTROL_PubMed_2023": -0.0016658009635072274, + "OOD_Wikipedia_2023": -0.0004948629479700113, + "TRAP_FineWeb": 0.001043891426156754, + "OOD_CommonPile": -0.0066902792082571445, + "OOD_News_2024": 2.336202287363774e-05, + "OOD_arXiv_2023": -0.0044432618897068624 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "USPTO_Backgrounds": { + "slope": 0.0028406448262535064, + "spearman": 0.24545454545454545, + "worst_loo_slope": 0.005819661371718266, + "loo_slope_by_dropped_rung": { + "Github": 0.005710533487887158, + "Pile-CC": 9.863395130239031e-05, + "USPTO_Backgrounds": 0.001356311640778398, + "Wikipedia_en": 0.0016527905617019472, + "StackExchange": 0.004714116898481311, + "CONTROL_PubMed_2023": 0.004101508487886535, + "OOD_Wikipedia_2023": 0.004572727249792843, + "TRAP_FineWeb": 0.005819661371718266, + "OOD_CommonPile": -0.0018155905788073956, + "OOD_News_2024": 0.004428249630030372, + "OOD_arXiv_2023": 0.00013908058997158366 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + } + }, + "n_fitted_steeper_than_logit": 3, + "n_fitted_crossing_below_logit_at_any_rung": 0 + } + }, + "e52": { + "persist": { + "D_all8": 0.000845372218579525, + "D_without_degenerate": 0.001007020774004221, + "permutation_p": 0.034482758620689655, + "link_artifact_surrogate": 9.441563525961398e-05, + "D_minus_surrogate": 0.000750956583319911, + "sign_survives_dropping_github": true, + "significant_at_5pct": true + }, + "min": { + "D_all8": 0.0003741732225679442, + "D_without_degenerate": -8.222654747663485e-05, + "permutation_p": 0.7131434282858571, + "link_artifact_surrogate": -1.0121748673588243e-05, + "D_minus_surrogate": 0.00038429497124153243, + "sign_survives_dropping_github": false, + "significant_at_5pct": false + } + }, + "quadrants": { + "persist": { + "P_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "Q_corpora": [ + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "quadrant_means": { + "JP_at_P": 0.10322155726659629, + "JP_at_Q": 0.10475294886001696, + "JQ_at_P": 0.09668358823870896, + "JQ_at_Q": 0.09890238938962374 + }, + "fit_row_means": { + "Wikipedia_en": 0.09961745545313332, + "USPTO_Backgrounds": 0.10963667937838989, + "Pile-CC": 0.10827355975796106, + "StackExchange": 0.10898624043006243, + "Github": 0.0924652105510985, + "OOD_News_2024": 0.09569122234566343, + "OOD_arXiv_2023": 0.09788490975592544, + "OOD_CommonPile": 0.0989707839093171 + }, + "read_col_means": { + "Wikipedia_en": 0.10531951454274047, + "USPTO_Backgrounds": 0.10890259583894578, + "Pile-CC": 0.10510752553333683, + "StackExchange": 0.09225428685507116, + "Github": 0.09226517163559846, + "OOD_News_2024": 0.10725807921618172, + "OOD_arXiv_2023": 0.09938795663215892, + "OOD_CommonPile": 0.10103093132751786 + }, + "diagonal_mean": 0.10218045838895098, + "offdiagonal_mean": 0.10133508617037146, + "fit_axis_span": 0.017171468827291383, + "read_axis_span": 0.01664830898387462 + }, + "min": { + "P_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "Q_corpora": [ + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "quadrant_means": { + "JP_at_P": 0.24897796408500936, + "JP_at_Q": 0.2414854618051538, + "JQ_at_P": 0.24969820555437494, + "JQ_at_Q": 0.2439813831821084 + }, + "fit_row_means": { + "Wikipedia_en": 0.23152156053369657, + "USPTO_Backgrounds": 0.2555947363480098, + "Pile-CC": 0.2559463235213318, + "StackExchange": 0.26002017346521217, + "Github": 0.22775858478206726, + "OOD_News_2024": 0.24214933404388528, + "OOD_arXiv_2023": 0.2535410887380648, + "OOD_CommonPile": 0.2469727687123749 + }, + "read_col_means": { + "Wikipedia_en": 0.2535931211275359, + "USPTO_Backgrounds": 0.2598997434601188, + "Pile-CC": 0.25737932473162395, + "StackExchange": 0.23572498860303312, + "Github": 0.2396430952577955, + "OOD_News_2024": 0.25592221629857603, + "OOD_arXiv_2023": 0.2362714668451291, + "OOD_CommonPile": 0.23507061382083017 + }, + "diagonal_mean": 0.24701547283782727, + "offdiagonal_mean": 0.24664129961525932, + "fit_axis_span": 0.03226158868314491, + "read_axis_span": 0.024829129639288627 + } + }, + "matrix_structure": { + "persist": { + "variance_fit_axis_pct": 50.697184677169346, + "variance_read_axis_pct": 48.13211438274321, + "variance_fit_x_read_interaction_pct": 1.1707009400874848, + "note_on_interaction": "this is the fit x READ interaction. It is NOT the corpus x CONCEPT-SET interaction reported in E51; the concept sets are averaged out of this matrix.", + "own_corpus": { + "Wikipedia_en": { + "rank_of_own_corpus": 2, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "USPTO_Backgrounds": { + "rank_of_own_corpus": 1, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "Pile-CC": { + "rank_of_own_corpus": 4, + "best_read_context": "OOD_News_2024", + "worst_read_context": "StackExchange" + }, + "StackExchange": { + "rank_of_own_corpus": 8, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "Github": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_News_2024": { + "rank_of_own_corpus": 1, + "best_read_context": "OOD_News_2024", + "worst_read_context": "StackExchange" + }, + "OOD_arXiv_2023": { + "rank_of_own_corpus": 6, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "OOD_CommonPile": { + "rank_of_own_corpus": 5, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + } + }, + "mean_rank_of_own_corpus": 4.25, + "chance_rank": 4.5, + "n_operators_reading_own_corpus_best": 2, + "fit_axis_order": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "OOD_CommonPile", + "OOD_arXiv_2023", + "OOD_News_2024", + "Github" + ], + "read_axis_order": [ + "USPTO_Backgrounds", + "OOD_News_2024", + "Wikipedia_en", + "Pile-CC", + "OOD_CommonPile", + "OOD_arXiv_2023", + "Github", + "StackExchange" + ], + "n_Q_operators_above_worst_P_operator": 3, + "n_Q_operators_above_second_worst_P_operator": 0, + "worst_read_context": "StackExchange", + "worst_read_context_is_P": true, + "github_cost_as_fit_corpus_pct": -8.84806792684223, + "github_cost_as_read_context_pct": -9.045265700242332 + }, + "min": { + "variance_fit_axis_pct": 53.35344735234028, + "variance_read_axis_pct": 44.60936175509925, + "variance_fit_x_read_interaction_pct": 2.037190892560503, + "note_on_interaction": "this is the fit x READ interaction. It is NOT the corpus x CONCEPT-SET interaction reported in E51; the concept sets are averaged out of this matrix.", + "own_corpus": { + "Wikipedia_en": { + "rank_of_own_corpus": 3, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_CommonPile" + }, + "USPTO_Backgrounds": { + "rank_of_own_corpus": 1, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_CommonPile" + }, + "Pile-CC": { + "rank_of_own_corpus": 3, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "StackExchange": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_arXiv_2023" + }, + "Github": { + "rank_of_own_corpus": 5, + "best_read_context": "Pile-CC", + "worst_read_context": "OOD_arXiv_2023" + }, + "OOD_News_2024": { + "rank_of_own_corpus": 4, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_arXiv_2023": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_CommonPile": { + "rank_of_own_corpus": 8, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_CommonPile" + } + }, + "mean_rank_of_own_corpus": 4.75, + "chance_rank": 4.5, + "n_operators_reading_own_corpus_best": 1, + "fit_axis_order": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "OOD_arXiv_2023", + "OOD_CommonPile", + "OOD_News_2024", + "Wikipedia_en", + "Github" + ], + "read_axis_order": [ + "USPTO_Backgrounds", + "Pile-CC", + "OOD_News_2024", + "Wikipedia_en", + "Github", + "OOD_arXiv_2023", + "StackExchange", + "OOD_CommonPile" + ], + "n_Q_operators_above_worst_P_operator": 3, + "n_Q_operators_above_second_worst_P_operator": 3, + "worst_read_context": "OOD_CommonPile", + "worst_read_context_is_P": false, + "github_cost_as_fit_corpus_pct": -7.6734502761757195, + "github_cost_as_read_context_pct": -2.855823540259028 + } + }, + "baseline_movement": { + "persist": { + "n_read_contexts": 12, + "logit_min": 0.05914952543874582, + "logit_max": 0.08317913140635938, + "logit_range": 0.02402960596761356, + "largest_fitted_advantage": 0.0374568174547878, + "logit_range_as_pct_of_largest_advantage": 64.15282343893328, + "per_operator": { + "Github": { + "range_across_read_contexts": 0.027728552355741454, + "mean_gap_over_logit": 0.019731618306832385, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "Wikipedia_en": { + "range_across_read_contexts": 0.025501235644333056, + "mean_gap_over_logit": 0.026948396621608278, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "StackExchange": { + "range_across_read_contexts": 0.020683898093799738, + "mean_gap_over_logit": 0.037169504749019526, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "Pile-CC": { + "range_across_read_contexts": 0.025536208258320897, + "mean_gap_over_logit": 0.035217998431633334, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "USPTO_Backgrounds": { + "range_across_read_contexts": 0.027602271906410653, + "mean_gap_over_logit": 0.0374568174547878, + "n_contexts_scrambled_beats_this_operator": 0 + } + }, + "n_contexts_scrambled_beats_logit": 12 + }, + "min": { + "n_read_contexts": 12, + "logit_min": 0.1873030010610819, + "logit_max": 0.2132010769098997, + "logit_range": 0.025898075848817814, + "largest_fitted_advantage": 0.06672917040478853, + "logit_range_as_pct_of_largest_advantage": 38.810726540906245, + "per_operator": { + "Github": { + "range_across_read_contexts": 0.02543728364010653, + "mean_gap_over_logit": 0.03134944018804365, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "Wikipedia_en": { + "range_across_read_contexts": 0.02524586891134578, + "mean_gap_over_logit": 0.037084430507901646, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "StackExchange": { + "range_across_read_contexts": 0.026412486347059394, + "mean_gap_over_logit": 0.06672917040478853, + "n_contexts_scrambled_beats_this_operator": 9 + }, + "Pile-CC": { + "range_across_read_contexts": 0.0324745271354914, + "mean_gap_over_logit": 0.06034543172249364, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "USPTO_Backgrounds": { + "range_across_read_contexts": 0.02731785501043002, + "mean_gap_over_logit": 0.060505543856157196, + "n_contexts_scrambled_beats_this_operator": 12 + } + }, + "n_contexts_scrambled_beats_logit": 12 + } + }, + "matrix_loo": { + "persist": { + "full": { + "fit_pct": 50.697184677169346, + "read_pct": 48.13211438274321, + "interaction_pct": 1.1707009400874848, + "fit_span": 0.017171468827291383, + "read_span": 0.01664830898387462, + "span_ratio": 1.031424203138198, + "fit_dominates_read": true + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 52.373250427828836, + "read_pct": 46.61005338540753, + "interaction_pct": 1.0166961867635491, + "fit_span": 0.017303615567156122, + "read_span": 0.0165367584815988, + "span_ratio": 1.0463728781194113, + "fit_dominates_read": true + }, + "USPTO_Backgrounds": { + "fit_pct": 48.900137854184116, + "read_pct": 49.70701457618453, + "interaction_pct": 1.3928475696313072, + "fit_span": 0.015998433523308778, + "read_span": 0.015076022723778365, + "span_ratio": 1.061183961873151, + "fit_dominates_read": false + }, + "Pile-CC": { + "fit_pct": 47.46625999643371, + "read_pct": 51.23363164594214, + "interaction_pct": 1.3001083576241463, + "fit_span": 0.017079299537184095, + "read_span": 0.016827839917118947, + "span_ratio": 1.0149430717967158, + "fit_dominates_read": false + }, + "StackExchange": { + "fit_pct": 54.302935606304814, + "read_pct": 44.71750309530783, + "interaction_pct": 0.9795612983873323, + "fit_span": 0.0169366810866262, + "read_span": 0.01671054580152803, + "span_ratio": 1.0135324894700621, + "fit_dominates_read": true + }, + "Github": { + "fit_pct": 50.83451434240633, + "read_pct": 47.965622476031534, + "interaction_pct": 1.199863181562132, + "fit_span": 0.014015912493720417, + "read_span": 0.016742457825291365, + "span_ratio": 0.8371478453150293, + "fit_dominates_read": true + }, + "OOD_News_2024": { + "fit_pct": 50.49955454200501, + "read_pct": 48.12259484369173, + "interaction_pct": 1.3778506143031934, + "fit_span": 0.017138381758039542, + "read_span": 0.016708293558645337, + "span_ratio": 1.0257410008918395, + "fit_dominates_read": true + }, + "OOD_arXiv_2023": { + "fit_pct": 50.57242596342314, + "read_pct": 48.44478039653153, + "interaction_pct": 0.9827936400453459, + "fit_span": 0.0174609310522173, + "read_span": 0.016825227149289163, + "span_ratio": 1.0377827827991608, + "fit_dominates_read": true + }, + "OOD_CommonPile": { + "fit_pct": 50.852977854777855, + "read_pct": 48.11818900866957, + "interaction_pct": 1.028833136552585, + "fit_span": 0.01742520418188105, + "read_span": 0.016803709279361465, + "span_ratio": 1.0369855781355914, + "fit_dominates_read": true + } + }, + "drop_both_extremes_github_and_uspto": { + "fit_pct": 49.077827715914886, + "read_pct": 49.432990888192485, + "interaction_pct": 1.4891813958926687, + "fit_span": 0.012516824986044048, + "read_span": 0.014937012424019888, + "span_ratio": 0.8379737949414845, + "fit_dominates_read": false + }, + "worst_loo_span_ratio": 0.8371478453150293, + "worst_loo_corpus": "Github", + "fit_dominates_in_all_loo": false, + "fit_dominates_with_both_extremes_dropped": false + }, + "min": { + "full": { + "fit_pct": 53.35344735234028, + "read_pct": 44.60936175509925, + "interaction_pct": 2.037190892560503, + "fit_span": 0.03226158868314491, + "read_span": 0.024829129639288627, + "span_ratio": 1.2993443246635377, + "fit_dominates_read": true + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 46.81110557693417, + "read_pct": 50.85837600168466, + "interaction_pct": 2.330518421381303, + "fit_span": 0.032255948695635056, + "read_span": 0.025139714496594606, + "span_ratio": 1.283067423061006, + "fit_dominates_read": false + }, + "USPTO_Backgrounds": { + "fit_pct": 56.67920183763611, + "read_pct": 41.06419873316167, + "interaction_pct": 2.2565994292022826, + "fit_span": 0.03192390514803786, + "read_span": 0.02218238359819802, + "span_ratio": 1.4391557609990657, + "fit_dominates_read": true + }, + "Pile-CC": { + "fit_pct": 56.50763481529161, + "read_pct": 41.4252808563404, + "interaction_pct": 2.0670843283679354, + "fit_span": 0.03248407571975673, + "read_span": 0.024528523726714047, + "span_ratio": 1.3243388017020479, + "fit_dominates_read": true + }, + "StackExchange": { + "fit_pct": 53.53726007459881, + "read_pct": 45.32738406925986, + "interaction_pct": 1.1353558561412547, + "fit_span": 0.0298434966317718, + "read_span": 0.025165532340132996, + "span_ratio": 1.1858877542669173, + "fit_dominates_read": true + }, + "Github": { + "fit_pct": 41.358656075065184, + "read_pct": 56.523659412915094, + "interaction_pct": 2.1176845120195984, + "fit_span": 0.028123639521026395, + "read_span": 0.02546599780815459, + "span_ratio": 1.1043603998120501, + "fit_dominates_read": false + }, + "OOD_News_2024": { + "fit_pct": 56.05510540136367, + "read_pct": 42.24118054522517, + "interaction_pct": 1.7037140534111448, + "fit_span": 0.03184018684403292, + "read_span": 0.024672448768147365, + "span_ratio": 1.2905158763624327, + "fit_dominates_read": true + }, + "OOD_arXiv_2023": { + "fit_pct": 56.005471966512765, + "read_pct": 42.12050880191629, + "interaction_pct": 1.8740192315709754, + "fit_span": 0.032027145049401684, + "read_span": 0.024876730661425345, + "span_ratio": 1.2874338467258482, + "fit_dominates_read": true + }, + "OOD_CommonPile": { + "fit_pct": 57.55685418677556, + "read_pct": 40.24133072996441, + "interaction_pct": 2.2018150832599948, + "fit_span": 0.031948861550717134, + "read_span": 0.024509501918440785, + "span_ratio": 1.3035296130060898, + "fit_dominates_read": true + } + }, + "drop_both_extremes_github_and_uspto": { + "fit_pct": 45.23962835179833, + "read_pct": 52.609332663665406, + "interaction_pct": 2.151038984536295, + "fit_span": 0.028022169321775425, + "read_span": 0.023017197788727517, + "span_ratio": 1.2174448679195455, + "fit_dominates_read": false + }, + "worst_loo_span_ratio": 1.1043603998120501, + "worst_loo_corpus": "Github", + "fit_dominates_in_all_loo": false, + "fit_dominates_with_both_extremes_dropped": false + } + }, + "C2_derangement": { + "persist": { + "n_draws": 120, + "shuf_beats_jp_paired_by_seed": 0, + "ties": 0, + "shuf_beats_jp_vs_corpus_mean": 0, + "n_corpora_where_shuf_beats_jp_on_the_mean": 0, + "shuf_sd_range": [ + 0.0015449648806984003, + 0.002914492891994465 + ], + "gap_range": [ + 0.010767463998248177, + 0.021970295277424154 + ], + "per_corpus": { + "Github": { + "jp_mean": 0.02789688172439734, + "shuf_mean": 0.017129417726149163, + "gap": 0.010767463998248177, + "shuf_sd_over_15_draws": 0.0018638386960376496, + "shuf_beats_jp_paired": 0 + }, + "OOD_CommonPile": { + "jp_mean": 0.03823967572922508, + "shuf_mean": 0.02355857611944278, + "gap": 0.014681099609782297, + "shuf_sd_over_15_draws": 0.002912470775340541, + "shuf_beats_jp_paired": 0 + }, + "OOD_News_2024": { + "jp_mean": 0.0337636109519129, + "shuf_mean": 0.018717034761793913, + "gap": 0.015046576190118989, + "shuf_sd_over_15_draws": 0.0016517842631268904, + "shuf_beats_jp_paired": 0 + }, + "OOD_arXiv_2023": { + "jp_mean": 0.04105739652489623, + "shuf_mean": 0.024700007438659667, + "gap": 0.01635738908623656, + "shuf_sd_over_15_draws": 0.0028135854091460816, + "shuf_beats_jp_paired": 0 + }, + "Pile-CC": { + "jp_mean": 0.04546591031830758, + "shuf_mean": 0.02692041976687809, + "gap": 0.01854549055142949, + "shuf_sd_over_15_draws": 0.0020402960046508106, + "shuf_beats_jp_paired": 0 + }, + "StackExchange": { + "jp_mean": 0.045729962767412265, + "shuf_mean": 0.026390449275883535, + "gap": 0.01933951349152873, + "shuf_sd_over_15_draws": 0.002914492891994465, + "shuf_beats_jp_paired": 0 + }, + "USPTO_Backgrounds": { + "jp_mean": 0.04961537096338967, + "shuf_mean": 0.02764507568596552, + "gap": 0.021970295277424154, + "shuf_sd_over_15_draws": 0.002039146369823083, + "shuf_beats_jp_paired": 0 + }, + "Wikipedia_en": { + "jp_mean": 0.03820955517391364, + "shuf_mean": 0.019931373049815498, + "gap": 0.018278182124098142, + "shuf_sd_over_15_draws": 0.0015449648806984003, + "shuf_beats_jp_paired": 0 + } + }, + "admissible_as_an_operator_comparator": true + }, + "min": { + "n_draws": 120, + "shuf_beats_jp_paired_by_seed": 104, + "ties": 0, + "shuf_beats_jp_vs_corpus_mean": 103, + "n_corpora_where_shuf_beats_jp_on_the_mean": 7, + "shuf_sd_range": [ + 0.006642094422897706, + 0.012379500382090904 + ], + "gap_range": [ + -0.03211234564582505, + 0.0011352714647849538 + ], + "per_corpus": { + "Github": { + "jp_mean": 0.1126159243285656, + "shuf_mean": 0.11933546006679535, + "gap": -0.006719535738229754, + "shuf_sd_over_15_draws": 0.012379500382090904, + "shuf_beats_jp_paired": 12 + }, + "OOD_CommonPile": { + "jp_mean": 0.1375763162970543, + "shuf_mean": 0.14552284215887387, + "gap": -0.00794652586181957, + "shuf_sd_over_15_draws": 0.007826596012088445, + "shuf_beats_jp_paired": 14 + }, + "OOD_News_2024": { + "jp_mean": 0.13940734267234803, + "shuf_mean": 0.17151968831817307, + "gap": -0.03211234564582505, + "shuf_sd_over_15_draws": 0.007718225133197829, + "shuf_beats_jp_paired": 15 + }, + "OOD_arXiv_2023": { + "jp_mean": 0.1407180975501736, + "shuf_mean": 0.14946635278562706, + "gap": -0.008748255235453473, + "shuf_sd_over_15_draws": 0.011114347019169522, + "shuf_beats_jp_paired": 12 + }, + "Pile-CC": { + "jp_mean": 0.14452077796061835, + "shuf_mean": 0.16368520364165307, + "gap": -0.019164425681034714, + "shuf_sd_over_15_draws": 0.007386723679365784, + "shuf_beats_jp_paired": 15 + }, + "StackExchange": { + "jp_mean": 0.1455607984215021, + "shuf_mean": 0.15189976240197817, + "gap": -0.0063389639804760645, + "shuf_sd_over_15_draws": 0.006642094422897706, + "shuf_beats_jp_paired": 12 + }, + "USPTO_Backgrounds": { + "jp_mean": 0.15502665042877198, + "shuf_mean": 0.15389137896398702, + "gap": 0.0011352714647849538, + "shuf_sd_over_15_draws": 0.008418568188510181, + "shuf_beats_jp_paired": 9 + }, + "Wikipedia_en": { + "jp_mean": 0.1304755893846353, + "shuf_mean": 0.15765395919481914, + "gap": -0.027178369810183844, + "shuf_sd_over_15_draws": 0.010262702456552721, + "shuf_beats_jp_paired": 15 + } + }, + "admissible_as_an_operator_comparator": false + } + }, + "C1_persist_column_reproduces_stored_headlines": { + "e51_410m_interaction_pct": { + "recomputed": 6.780626428099203, + "stored": 6.780626428099203, + "abs_diff": 0.0, + "ok": true + }, + "e51_410m_corpus_pct": { + "recomputed": 1.9064501148312785, + "stored": 1.9064501148312791, + "abs_diff": 6.661338147750939e-16, + "ok": true + }, + "e51_1b_interaction_pct": { + "recomputed": 9.158325214755475, + "stored": 9.158325214755473, + "abs_diff": 1.7763568394002505e-15, + "ok": true + }, + "e51_410m_F": { + "recomputed": 611.8910219722385, + "stored": 611.8910219722385, + "abs_diff": 0.0, + "ok": true + }, + "e33_logit_constant": { + "recomputed": 0.02843950316309929, + "stored": 0.02843950316309929, + "abs_diff": 0.0, + "ok": true + }, + "e33_uspto_asymptote": { + "recomputed": 0.049735264846729854, + "stored": 0.049735264846729854, + "abs_diff": 0.0, + "ok": true + }, + "e48c_spearman_all8": { + "recomputed": 0.0, + "stored": 0.0, + "abs_diff": 0.0, + "ok": true + }, + "e52_D": { + "recomputed": 0.000845372218579525, + "stored": 0.000845372218579525, + "abs_diff": 0.0, + "ok": true + } + }, + "C1_fires": true, + "VERDICT_BY_CLAIM": { + "corpus_identity_decides_the_read_410m": { + "persist": 58.381738635199795, + "min": 55.37773289006552, + "verdict": "HOLDS" + }, + "reads_are_flat_in_N_410m": { + "persist": [ + 1.2278526816817164, + 3.6981860639992252 + ], + "min": [ + 1.1840692303542948, + 2.2622282558307703 + ], + "verdict": "HOLDS" + }, + "github_indistinguishable_from_not_fitting": { + "persist": -1.6527617605340326, + "min": -58.02264085495087, + "verdict": "REVERSES", + "note": "under min Github is not merely indistinguishable from the logit lens, it is significantly BELOW it \u2014 a stronger form of the same claim, but a different sentence" + }, + "effect_is_an_interaction_not_a_level_410m": { + "persist": 3.55667655573526, + "min": 1.0455176739722105, + "verdict": "WEAKENS" + }, + "effect_is_an_interaction_not_a_level_1b": { + "persist": 4.1408154208167804, + "min": 2.0238440545235648, + "verdict": "WEAKENS" + }, + "exposure_does_not_order_the_read": { + "persist": 0.0, + "min": -0.09523809523809523, + "verdict": "HOLDS", + "note": "both are indistinguishable from zero at n=8; all three OOD rungs beat in-stream Wikipedia under both" + }, + "e36_no_fitted_operator_crosses_below_logit": { + "persist": 0, + "min": 0, + "verdict": "HOLDS", + "note": "the count includes Github, which sits below the logit lens at every rung under both aggregations and is degenerate by E33. Excluding it: 0 of 4 under persist, 1 of 4 (Wikipedia, at 1 rung of 11) under min. The pre-registration states that a crossing appearing only under min does not count." + }, + "e36_fitted_operators_are_flatter_than_logit": { + "persist": 4, + "min": 3, + "verdict": "REVERSES", + "note": "0 of 5 fitted operators are steeper than the logit lens under persist; 3 of 5 are under min. This secondary is aggregation-dependent AND single-rung-fragile under LOO." + }, + "e52_matched_fit_read_helps": { + "persist": 0.034482758620689655, + "min": 0.7131434282858571, + "verdict": "REVERSES", + "note": "p = 0.0345 under persist, 0.7131 under min; and D changes sign without Github under min (-8.22e-05)" + } + }, + "VERDICT": "THE SPINE IS AGGREGATION-ROBUST; THREE SECONDARIES ARE NOT. Corpus identity deciding the read, reads being flat in N, and exposure not ordering the read all hold under the anchor's own `min` \u2014 the first with a LARGER absolute spread. What does not survive: the interaction's dominance over the corpus main effect at 410M (3.56x -> 1.05x), E36's slope secondary (0 of 5 fitted operators steeper -> 4 of 5), and E52's matched-diagonal effect (p=0.0055 -> p=0.66, and D changes sign without Github). Separately, C2 shows why `min` cannot be the adjudicating metric: under it the layer-deranged operator beats the real one on 104 of 120 paired draws and outscores the logit lens, while under `persist` it loses 120 of 120. Both facts must be reported together \u2014 `min` is inadmissible as an operator comparator AND three of our claims depend on not using it.", + "provenance": { + "schema_version": 1, + "experiment": "E54", + "utc": "2026-08-20T14:04:09+00:00", + "script": { + "path": "experiments/t54_aggregation_audit.py", + "sha256": "f97a3a04761016555e7c159c98c6c836bae774f1e325ba9d65c09558a13e184a", + "n_lines": 683 + }, + "git": { + "commit": "0085188b142f807a95aed2394063dae7009cf1b7", + "commit_short": "0085188b142f", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/r6_within_source.py", + "experiments/r7_matched_pools.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/e36_qladder_410m_c1arm.json", + "results/e36_qladder_410m_rstrip.json", + "results/e52_factorial_410m_rstrip_w1.json", + "results/r4b_e36_flatness.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "aee92baa8be53162e66b7c00eec04f9286b40c2afaf4dbb2dbe8c1782597cc28", + "describe": "0085188-dirty" + }, + "argv": [ + "experiments/t54_aggregation_audit.py", + "--e52", + "results/e52_factorial_410m_rstrip.json", + "--e36", + "results/e36_qladder_410m_rstrip.json", + "--out", + "results/e54_aggregation_audit_rstrip.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + }, + { + "path": "results/e36_qladder_410m_rstrip.json", + "sha256": "6a40a858516cc56473bc4b9145a7d232a5a64bee8cfb9e2a405e78b4bd8362d2", + "bytes": 111957, + "exists": true + }, + { + "path": "results/e48_crossover_410m.json", + "sha256": "cb00c6c92899012c350c180d05e5928ea183811de386004899e6812fbacf37aa", + "bytes": 115278, + "exists": true + }, + { + "path": "results/e48c_exposure_vs_read.json", + "sha256": "ff344efc91604138b698251c630e0d4f320d5b5b27ac9f87e2914d848ab1c374", + "bytes": 9875, + "exists": true + }, + { + "path": "results/e51_interaction_variance.json", + "sha256": "2559e4017660a6a53949cb2a739fb3e7884e712851f2a04c8e04273dd217f62b", + "bytes": 29478, + "exists": true + }, + { + "path": "results/e52_factorial_410m_rstrip.json", + "sha256": "d95f684866c8d7830d1a1a3ca7efd8a43a923d52b99aa65c3d4ad4818b0d446e", + "bytes": 43359, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "3cf2de44dae86fa5cd02084697e76a2b476ec1f7f95aecb0fac7c83acf6872c1" + } +} \ No newline at end of file diff --git a/results/e54_aggregation_audit_rstrip_v2.json b/results/e54_aggregation_audit_rstrip_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..d7440a140fe567e3839edd603a9084483454fc45 --- /dev/null +++ b/results/e54_aggregation_audit_rstrip_v2.json @@ -0,0 +1,2484 @@ +{ + "experiment": "E54 \u2014 the aggregation audit: every headline under `min` and `persist`", + "why": "`persist` is ours; `min` is the anchor's. A claim that exists only under `persist` is a claim about our metric. Nothing had ever tabulated which claims those are.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json", + "results/e33_logit_baseline_410m_v2.json", + "results/e36_qladder_410m_rstrip.json", + "results/e48_crossover_410m_rstrip.json", + "results/e48c_exposure_vs_read.json", + "results/e52_factorial_410m_rstrip.json" + ], + "readout_convention_note": "e48c_exposure_vs_read.json has no corrected arm and is read unstripped in every run. Only its containment_k32 column is used here, which is an n-gram overlap between eval items and the training stream and carries no readout position; the READ values it is paired against come from --e48. Declared, not assumed away.", + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "asymptote_n_min": 75, + "classification_thresholds": { + "REVERSES": "sign of the decisive quantity flips, or a p-value crosses 0.05", + "WEAKENS": "decisive ratio falls by more than 2x, sign kept", + "HOLDS-STRONGER": "decisive ratio rises by more than 1.2x", + "HOLDS": "otherwise" + }, + "ladder": { + "410m": { + "persist": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.028084053793529264, + "Wikipedia_en": 0.03860496947484109, + "StackExchange": 0.0456756031007639, + "Pile-CC": 0.04571574828415005, + "USPTO_Backgrounds": 0.049735264846729854 + }, + "per_corpus_seed_sd": { + "Github": 0.0003725015802729375, + "Wikipedia_en": 0.000497512452938681, + "StackExchange": 0.0002979844946588909, + "Pile-CC": 0.0001835424093483373, + "USPTO_Backgrounds": 0.0005027384624140496 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.02165121105320059, + "pooled_seed_sd": 0.00037085587992657926, + "spread_over_seed_sd": 58.381738635199795, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 12.031761970089196, + 73.54092188255042 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -29.32008778867865, + "df": 3.706181480116789, + "p": 1.5934823642012846e-05, + "separates": true + }, + "Github|StackExchange": { + "t": -63.87401426888638, + "df": 3.816032427092647, + "p": 6.351744966106404e-07, + "separates": true + }, + "Github|Pile-CC": { + "t": -73.54092188255042, + "df": 2.9170731313531917, + "p": 7.344546283218717e-06, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -59.934247643859756, + "df": 3.687411568613282, + "p": 1.198506487554617e-06, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -21.11771264108646, + "df": 3.2713441661282854, + "p": 0.00013067655363204733, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -23.225504298699857, + "df": 2.534507444454648, + "p": 0.0005080331049453861, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -27.256333209790494, + "df": 3.9995633119295007, + "p": 1.0784912038682538e-05, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": -0.1986812662848506, + "df": 3.326610968599927, + "p": 0.8540456327496845, + "separates": false + }, + "StackExchange|USPTO_Backgrounds": { + "t": -12.031761970089196, + "df": 3.250888721535667, + "p": 0.0008327630169994736, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -13.008353627023636, + "df": 2.5238427055378474, + "p": 0.002227348310144479, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0028340971186047517, + "range_over_seed_sd": 2.5341568525777767, + "argmax_N": 75 + }, + "Wikipedia_en": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0020914917545659176, + "range_over_seed_sd": 3.6204764630199766, + "argmax_N": 25 + }, + "StackExchange": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0017882611513847374, + "range_over_seed_sd": 3.6981860639992252, + "argmax_N": 300 + }, + "Pile-CC": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.000687981006645022, + "range_over_seed_sd": 1.2278526816817164, + "argmax_N": 75 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0025357189366505273, + "range_over_seed_sd": 3.083030892326044, + "argmax_N": 400 + } + }, + "flat_range_over_seed_sd": [ + 1.2278526816817164, + 3.6981860639992252 + ], + "max_movement_in_N": 0.0028340971186047517 + }, + "min": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.11288307417108089, + "Wikipedia_en": 0.13042243031696196, + "StackExchange": 0.14538949898881268, + "Pile-CC": 0.14493902100437375, + "USPTO_Backgrounds": 0.15533995895730282 + }, + "per_corpus_seed_sd": { + "Github": 0.00035393328127784305, + "Wikipedia_en": 0.00010955838937670346, + "StackExchange": 0.00128026159560015, + "Pile-CC": 0.00042533231079348594, + "USPTO_Backgrounds": 0.0016643039976484083 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.042456884786221935, + "pooled_seed_sd": 0.0007666779149393181, + "spread_over_seed_sd": 55.37773289006552, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 8.207952613477497, + 100.34215439417814 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -81.99428757919529, + "df": 2.3797863362681175, + "p": 3.6533723385372034e-05, + "separates": true + }, + "Github|StackExchange": { + "t": -42.38760474793699, + "df": 2.303931704222899, + "p": 0.00022021273158969959, + "separates": true + }, + "Github|Pile-CC": { + "t": -100.34215439417814, + "df": 3.8721318177057205, + "p": 9.295015311877957e-08, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -43.21865282672605, + "df": 2.1805303117974812, + "p": 0.00030647101881163866, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -20.175033663366225, + "df": 2.0292907534920257, + "p": 0.002284948539916303, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -57.24627016230037, + "df": 2.264232884694126, + "p": 0.00012573058813856007, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -25.87581218897987, + "df": 2.017333162219172, + "p": 0.0014245310770301593, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 0.5783639426857416, + "df": 2.4361749261412093, + "p": 0.6120606018893571, + "separates": false + }, + "StackExchange|USPTO_Backgrounds": { + "t": -8.207952613477497, + "df": 3.7531031511904787, + "p": 0.0015712415434009322, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -10.487261913538559, + "df": 2.2601374216530616, + "p": 0.005805838608934484, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0034548367684086173, + "range_over_seed_sd": 2.2622282558307703, + "argmax_N": 375 + }, + "Wikipedia_en": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0014686014769332845, + "range_over_seed_sd": 1.1840692303542948, + "argmax_N": 375 + }, + "StackExchange": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "range": 0.0019650841930083096, + "range_over_seed_sd": 1.302208655813546, + "argmax_N": 25 + }, + "Pile-CC": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.002348728105425829, + "range_over_seed_sd": 2.106425607322571, + "argmax_N": 100 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "range": 0.0026533089932941067, + "range_over_seed_sd": 1.2335055892097606, + "argmax_N": 400 + } + }, + "flat_range_over_seed_sd": [ + 1.1840692303542948, + 2.2622282558307703 + ], + "max_movement_in_N": 0.0034548367684086173 + } + }, + "1b": { + "persist": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.027256395461569938, + "Wikipedia_en": 0.02801095717586577, + "StackExchange": 0.05875215052316586, + "Pile-CC": 0.046013945933165296, + "USPTO_Backgrounds": 0.05470160290465824 + }, + "per_corpus_seed_sd": { + "Github": 0.0013762403288060878, + "Wikipedia_en": 0.00020285804501715256, + "StackExchange": 0.0007357119491078887, + "Pile-CC": 0.00039438237251602294, + "USPTO_Backgrounds": 0.0017248115034397009 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "between_corpus_spread": 0.03149575506159592, + "pooled_seed_sd": 0.0008868008397773706, + "spread_over_seed_sd": 35.51615385197748, + "n_pairs_separating_p05": 9, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 3.7414047636770564, + 70.3097211819473 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": -0.9394933684708586, + "df": 2.0868662234127733, + "p": 0.4430895229085523, + "separates": false + }, + "Github|StackExchange": { + "t": -34.957113847459645, + "df": 3.056800239042669, + "p": 4.430261747534506e-05, + "separates": true + }, + "Github|Pile-CC": { + "t": -22.693676218822052, + "df": 2.3262778089342686, + "p": 0.0008783542616057277, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -21.543026480272037, + "df": 3.8121178219340974, + "p": 4.004856453522681e-05, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -69.76890715662432, + "df": 2.30236101278019, + "p": 7.029390923616283e-05, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -70.3097211819473, + "df": 2.9890668231741286, + "p": 6.576476832894556e-06, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -26.619197602231548, + "df": 2.0553194805721473, + "p": 0.0012181637277215592, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 26.43089120460415, + "df": 3.0617496325437474, + "p": 0.00010269188216586204, + "separates": true + }, + "StackExchange|USPTO_Backgrounds": { + "t": 3.7414047636770564, + "df": 2.7044477621250116, + "p": 0.03971364135834821, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -8.504633312249457, + "df": 2.2085576959071216, + "p": 0.009952014794920177, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.006032132942761694, + "range_over_seed_sd": 2.0832274009134886, + "argmax_N": 10 + }, + "Wikipedia_en": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0023527308677633585, + "range_over_seed_sd": 1.9704177680409294, + "argmax_N": 150 + }, + "StackExchange": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.006524874713449244, + "range_over_seed_sd": 3.7745797870225046, + "argmax_N": 150 + }, + "Pile-CC": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.003513823981795991, + "range_over_seed_sd": 1.6205093198035128, + "argmax_N": 50 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.004342931668673243, + "range_over_seed_sd": 2.298899257873663, + "argmax_N": 50 + } + }, + "flat_range_over_seed_sd": [ + 1.6205093198035128, + 3.7745797870225046 + ], + "max_movement_in_N": 0.006524874713449244 + }, + "min": { + "asymptote_definition": "mean over N >= 75, 5 admitted sets, seeds separate", + "per_corpus_mean": { + "Github": 0.13184866054604452, + "Wikipedia_en": 0.12168779909788144, + "StackExchange": 0.1760720702420388, + "Pile-CC": 0.14951401859566216, + "USPTO_Backgrounds": 0.16125627207968915 + }, + "per_corpus_seed_sd": { + "Github": 0.001174541715835113, + "Wikipedia_en": 0.000664360394074828, + "StackExchange": 0.000781853055555403, + "Pile-CC": 0.0016589081835269135, + "USPTO_Backgrounds": 0.0026333882425492983 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "between_corpus_spread": 0.054384271144157365, + "pooled_seed_sd": 0.0013826103183083112, + "spread_over_seed_sd": 39.334489569482656, + "n_pairs_separating_p05": 10, + "n_pairs": 10, + "abs_t_range_over_separating_pairs": [ + 6.534674049686413, + 91.80958763345963 + ], + "pairs": { + "Github|Wikipedia_en": { + "t": 13.042035228685885, + "df": 3.1609302063989753, + "p": 0.0007454506503368391, + "separates": true + }, + "Github|StackExchange": { + "t": -54.28682048115797, + "df": 3.481548250716117, + "p": 3.1857647786542707e-06, + "separates": true + }, + "Github|Pile-CC": { + "t": -15.05316770337511, + "df": 3.6024790741888855, + "p": 0.0002207550470131397, + "separates": true + }, + "Github|USPTO_Backgrounds": { + "t": -17.664770849542823, + "df": 2.7654406589883376, + "p": 0.0006315346383159637, + "separates": true + }, + "Wikipedia_en|StackExchange": { + "t": -91.80958763345963, + "df": 3.8984238567221983, + "p": 1.1973006104680652e-07, + "separates": true + }, + "Wikipedia_en|Pile-CC": { + "t": -26.97065532927737, + "df": 2.6254499585287703, + "p": 0.00027785032815772143, + "separates": true + }, + "Wikipedia_en|USPTO_Backgrounds": { + "t": -25.234591190044668, + "df": 2.2535606927828096, + "p": 0.0008231019913061611, + "separates": true + }, + "StackExchange|Pile-CC": { + "t": 25.082795391343012, + "df": 2.846737462807291, + "p": 0.00019868054152209687, + "separates": true + }, + "StackExchange|USPTO_Backgrounds": { + "t": 9.341712101684202, + "df": 2.349879826655504, + "p": 0.00655414759865613, + "separates": true + }, + "Pile-CC|USPTO_Backgrounds": { + "t": -6.534674049686413, + "df": 3.3713894640736504, + "p": 0.005037196334992217, + "separates": true + } + }, + "flatness_in_N": { + "Github": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0030341794803029065, + "range_over_seed_sd": 1.1210924731652319, + "argmax_N": 10 + }, + "Wikipedia_en": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.002315773735088955, + "range_over_seed_sd": 0.9972203747816083, + "argmax_N": 25 + }, + "StackExchange": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.008386632312266606, + "range_over_seed_sd": 5.729404685250978, + "argmax_N": 150 + }, + "Pile-CC": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.00464149415493012, + "range_over_seed_sd": 1.2370669965583059, + "argmax_N": 50 + }, + "USPTO_Backgrounds": { + "n_grid": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "range": 0.0042105066279570225, + "range_over_seed_sd": 1.535072587315563, + "argmax_N": 50 + } + }, + "flat_range_over_seed_sd": [ + 0.9972203747816083, + 5.729404685250978 + ], + "max_movement_in_N": 0.008386632312266606 + } + } + }, + "variance_decomposition": { + "410m": { + "persist": { + "frac_set_main": 91.27829399393488, + "frac_corpus_main": 1.9064501148312785, + "frac_interaction": 6.780626428099203, + "frac_residual_seed": 0.034629463134648465, + "F": 611.8910219722385, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 3.55667655573526 + }, + "min": { + "frac_set_main": 96.06544098391524, + "frac_corpus_main": 1.911884671540871, + "frac_interaction": 1.9989092146925351, + "frac_residual_seed": 0.023765129851367858, + "F": 262.84692467416227, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 1.0455176739722105 + } + }, + "1b": { + "persist": { + "frac_set_main": 88.57950497345804, + "frac_corpus_main": 2.2117202251311614, + "frac_interaction": 9.158325214755475, + "frac_residual_seed": 0.05044958665530746, + "F": 567.2943663870428, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 4.1408154208167804 + }, + "min": { + "frac_set_main": 94.14618752120755, + "frac_corpus_main": 1.9252567551293123, + "frac_interaction": 3.896419437299789, + "frac_residual_seed": 0.03213628636333453, + "F": 378.8960119379021, + "F_df": [ + 16, + 50 + ], + "F_crit_0p001": 3.3, + "interaction_over_corpus_main": 2.0238440545235648 + } + } + }, + "e33": { + "persist": { + "logit_constant": 0.02843950316309929, + "by_corpus": { + "Github": { + "asymptote": 0.028084053793529264, + "seed_sd": 0.0003725015802729375, + "pct_vs_logit": -1.2498438089144464, + "t_vs_logit_df2": -1.6527617605340326, + "separates_from_logit": false, + "below_logit": true + }, + "Wikipedia_en": { + "asymptote": 0.03860496947484109, + "seed_sd": 0.000497512452938681, + "pct_vs_logit": 35.74417687061305, + "t_vs_logit_df2": 35.39027823437558, + "separates_from_logit": true, + "below_logit": false + }, + "StackExchange": { + "asymptote": 0.0456756031007639, + "seed_sd": 0.0002979844946588909, + "pct_vs_logit": 60.606192164526725, + "t_vs_logit_df2": 100.18575245179832, + "separates_from_logit": true, + "below_logit": false + }, + "Pile-CC": { + "asymptote": 0.04571574828415005, + "seed_sd": 0.0001835424093483373, + "pct_vs_logit": 60.747352096737636, + "t_vs_logit_df2": 163.03226278828905, + "separates_from_logit": true, + "below_logit": false + }, + "USPTO_Backgrounds": { + "asymptote": 0.049735264846729854, + "seed_sd": 0.0005027384624140496, + "pct_vs_logit": 74.88092025201817, + "t_vs_logit_df2": 73.368846785286, + "separates_from_logit": true, + "below_logit": false + } + }, + "n_corpora_indistinguishable_from_logit": 1, + "n_corpora_significantly_below_logit": 0 + }, + "min": { + "logit_constant": 0.12473962224487747, + "by_corpus": { + "Github": { + "asymptote": 0.11288307417108089, + "seed_sd": 0.00035393328127784305, + "pct_vs_logit": -9.505037661987538, + "t_vs_logit_df2": -58.02264085495087, + "separates_from_logit": true, + "below_logit": true + }, + "Wikipedia_en": { + "asymptote": 0.13042243031696196, + "seed_sd": 0.00010955838937670346, + "pct_vs_logit": 4.55573615649446, + "t_vs_logit_df2": 89.8417032827052, + "separates_from_logit": true, + "below_logit": false + }, + "StackExchange": { + "asymptote": 0.14538949898881268, + "seed_sd": 0.00128026159560015, + "pct_vs_logit": 16.554384543026146, + "t_vs_logit_df2": 27.93697461007129, + "separates_from_logit": true, + "below_logit": false + }, + "Pile-CC": { + "asymptote": 0.14493902100437375, + "seed_sd": 0.00042533231079348594, + "pct_vs_logit": 16.193249904062284, + "t_vs_logit_df2": 82.25658866245516, + "separates_from_logit": true, + "below_logit": false + }, + "USPTO_Backgrounds": { + "asymptote": 0.15533995895730282, + "seed_sd": 0.0016643039976484083, + "pct_vs_logit": 24.53136875174558, + "t_vs_logit_df2": 31.84594760904531, + "separates_from_logit": true, + "below_logit": false + } + }, + "n_corpora_indistinguishable_from_logit": 0, + "n_corpora_significantly_below_logit": 1 + } + }, + "e48c": { + "persist": { + "logit_constant": 0.08317913140635938, + "by_rung": { + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "read": 0.10422163270413876, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "read": 0.1200522772812595, + "tier": "in-stream" + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "read": 0.11109575630010417, + "tier": "in-stream" + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "read": 0.11614819277698794, + "tier": "in-stream" + }, + "Github": { + "containment_k32": 0.9340004374453194, + "read": 0.10075108803187807, + "tier": "github" + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "read": 0.09791991202315936, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "read": 0.10589559425910314, + "tier": "OOD" + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "read": 0.10663878470659256, + "tier": "OOD" + } + }, + "spearman_all8": 0.19047619047619047, + "spearman_without_github": 0.5714285714285714, + "containment_span": 10861.817987204726, + "tier_gap_instream_minus_ood": 0.009394701102670905, + "tier_gap_t": 2.128715930849548, + "tier_gap_df": 4.9999994956672085, + "tier_gap_p": 0.08654474865638569, + "n_ood_rungs_above_instream_wikipedia": 2 + }, + "min": { + "logit_constant": 0.19810852520167826, + "by_rung": { + "Wikipedia_en": { + "containment_k32": 0.9215951727072293, + "read": 0.24160845403869946, + "tier": "in-stream" + }, + "USPTO_Backgrounds": { + "containment_k32": 0.9267616923398619, + "read": 0.26112590978542966, + "tier": "in-stream" + }, + "Pile-CC": { + "containment_k32": 0.9276060773863086, + "read": 0.2623990322152774, + "tier": "in-stream" + }, + "StackExchange": { + "containment_k32": 0.9179914195561899, + "read": 0.26430599875748156, + "tier": "in-stream" + }, + "Github": { + "containment_k32": 0.9340004374453194, + "read": 0.2404973489542802, + "tier": "github" + }, + "OOD_News_2024": { + "containment_k32": 0.0007487928942136383, + "read": 0.25006329963604607, + "tier": "OOD" + }, + "OOD_arXiv_2023": { + "containment_k32": 8.598932872430531e-05, + "read": 0.26927303777386746, + "tier": "OOD" + }, + "OOD_CommonPile": { + "containment_k32": 0.00125, + "read": 0.2528811464707057, + "tier": "OOD" + } + }, + "spearman_all8": -0.42857142857142855, + "spearman_without_github": -0.14285714285714285, + "containment_span": 10861.817987204726, + "tier_gap_instream_minus_ood": -4.597926098437455e-05, + "tier_gap_t": -0.005753537927105847, + "tier_gap_df": 4.509035362342915, + "tier_gap_p": 0.6701544274309155, + "n_ood_rungs_above_instream_wikipedia": 3 + } + }, + "e36": { + "persist": { + "rungs": [ + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "containment": [ + 0.9340004374453194, + 0.9276060773863086, + 0.9267616923398619, + 0.9215951727072293, + 0.9179914195561899, + 0.7927273508312213, + 0.26957534299550123, + 0.06309446548070848, + 0.00125, + 0.0007487928942136383, + 8.598932872430531e-05 + ], + "logit": { + "slope": -0.0037284686515085515, + "spearman": 0.03636363636363636, + "mean": 0.07387016320417664 + }, + "shuf_arm": { + "mean": 0.08303366410263786, + "above_logit": true, + "n_rungs_above_logit": 11 + }, + "by_fit_corpus": { + "Github": { + "slope": -0.006302421723152167, + "spearman": -0.1, + "worst_loo_slope": -0.008441645016381174, + "loo_slope_by_dropped_rung": { + "Github": -0.0024168897694502177, + "Pile-CC": -0.008441645016381174, + "USPTO_Backgrounds": -0.007384536525218063, + "Wikipedia_en": -0.007728880892965402, + "StackExchange": -0.0043434026002888825, + "CONTROL_PubMed_2023": -0.006713846696496293, + "OOD_Wikipedia_2023": -0.005112283764729826, + "TRAP_FineWeb": -0.00582211622752024, + "OOD_CommonPile": -0.007829920487629368, + "OOD_News_2024": -0.00555404368229907, + "OOD_arXiv_2023": -0.008185001025962615 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "Wikipedia_en": { + "slope": -0.003751919371114022, + "spearman": 0.05454545454545454, + "worst_loo_slope": -0.005678867843274564, + "loo_slope_by_dropped_rung": { + "Github": 0.0003016115498988717, + "Pile-CC": -0.005491148554964415, + "USPTO_Backgrounds": -0.005534449398737498, + "Wikipedia_en": -0.005665184321947394, + "StackExchange": -0.001235347641740309, + "CONTROL_PubMed_2023": -0.004183852962555575, + "OOD_Wikipedia_2023": -0.0027372203566205155, + "TRAP_FineWeb": -0.0032927560949970527, + "OOD_CommonPile": -0.005678867843274564, + "OOD_News_2024": -0.0031015921835953306, + "OOD_arXiv_2023": -0.004826130927711405 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "StackExchange": { + "slope": -0.0036664399547795337, + "spearman": -0.00909090909090909, + "worst_loo_slope": -0.0056006463421927315, + "loo_slope_by_dropped_rung": { + "Github": -0.0006790836836824708, + "Pile-CC": -0.004990725423007533, + "USPTO_Backgrounds": -0.0056006463421927315, + "Wikipedia_en": -0.0050372393557747835, + "StackExchange": -0.00151881318439263, + "CONTROL_PubMed_2023": -0.003640588143018499, + "OOD_Wikipedia_2023": -0.0027296844073786066, + "TRAP_FineWeb": -0.0025106958885208, + "OOD_CommonPile": -0.005407453174742336, + "OOD_News_2024": -0.003153774919972062, + "OOD_arXiv_2023": -0.00527199455255791 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + }, + "Pile-CC": { + "slope": -0.0048877667147762035, + "spearman": 0.0, + "worst_loo_slope": -0.006724107905736161, + "loo_slope_by_dropped_rung": { + "Github": -0.0009967740515735804, + "Pile-CC": -0.006724107905736161, + "USPTO_Backgrounds": -0.006463482017118128, + "Wikipedia_en": -0.006690320408810834, + "StackExchange": -0.0025393937647051157, + "CONTROL_PubMed_2023": -0.005261524362022698, + "OOD_Wikipedia_2023": -0.003884067551259087, + "TRAP_FineWeb": -0.004879048784359732, + "OOD_CommonPile": -0.006598440375986708, + "OOD_News_2024": -0.0039224128489219676, + "OOD_arXiv_2023": -0.005960032823002878 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "USPTO_Backgrounds": { + "slope": -0.004609631566017049, + "spearman": -0.00909090909090909, + "worst_loo_slope": -0.006625617543198389, + "loo_slope_by_dropped_rung": { + "Github": -0.000513467266396934, + "Pile-CC": -0.006556779660893942, + "USPTO_Backgrounds": -0.006625617543198389, + "Wikipedia_en": -0.005387748642851779, + "StackExchange": -0.0027711663748551606, + "CONTROL_PubMed_2023": -0.00507721409229556, + "OOD_Wikipedia_2023": -0.0036012922990186122, + "TRAP_FineWeb": -0.00393341135225128, + "OOD_CommonPile": -0.0060053386906768515, + "OOD_News_2024": -0.004495785776616605, + "OOD_arXiv_2023": -0.0059195173698972075 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + } + }, + "n_fitted_steeper_than_logit": 4, + "n_fitted_crossing_below_logit_at_any_rung": 0 + }, + "min": { + "rungs": [ + "Github", + "Pile-CC", + "USPTO_Backgrounds", + "Wikipedia_en", + "StackExchange", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "containment": [ + 0.9340004374453194, + 0.9276060773863086, + 0.9267616923398619, + 0.9215951727072293, + 0.9179914195561899, + 0.7927273508312213, + 0.26957534299550123, + 0.06309446548070848, + 0.00125, + 0.0007487928942136383, + 8.598932872430531e-05 + ], + "logit": { + "slope": 0.0021193657987416048, + "spearman": 0.19090909090909092, + "mean": 0.1990344447394212 + }, + "shuf_arm": { + "mean": 0.26733599201296315, + "above_logit": true, + "n_rungs_above_logit": 11 + }, + "by_fit_corpus": { + "Github": { + "slope": 0.00374649675334448, + "spearman": 0.3181818181818182, + "worst_loo_slope": 0.00692427999778951, + "loo_slope_by_dropped_rung": { + "Github": 0.00692427999778951, + "Pile-CC": 0.001850419039028697, + "USPTO_Backgrounds": 0.0027376935014466397, + "Wikipedia_en": 0.0015034475594117858, + "StackExchange": 0.005411913387345525, + "CONTROL_PubMed_2023": 0.004867735619613101, + "OOD_Wikipedia_2023": 0.00583858625340628, + "TRAP_FineWeb": 0.006388740901425092, + "OOD_CommonPile": 0.0006282266015373978, + "OOD_News_2024": 0.004283113705859323, + "OOD_arXiv_2023": 0.0002689178441613544 + }, + "slope_sign_flips_under_loo": false, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + }, + "Wikipedia_en": { + "slope": -0.0017812876146258553, + "spearman": 0.05454545454545454, + "worst_loo_slope": -0.005772345907007004, + "loo_slope_by_dropped_rung": { + "Github": 0.0012169867001940442, + "Pile-CC": -0.004183024011643131, + "USPTO_Backgrounds": -0.0032090508105418825, + "Wikipedia_en": -0.0037107514032504317, + "StackExchange": 0.0016006596714169714, + "CONTROL_PubMed_2023": -0.001612386654700412, + "OOD_Wikipedia_2023": -0.00031422614152677345, + "TRAP_FineWeb": 0.000538800428554092, + "OOD_CommonPile": -0.005772345907007004, + "OOD_News_2024": -0.0006546885625461061, + "OOD_arXiv_2023": -0.003854520399759611 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "StackExchange": { + "slope": 0.00014062084151355905, + "spearman": 0.16363636363636364, + "worst_loo_slope": -0.0039263860180292175, + "loo_slope_by_dropped_rung": { + "Github": 0.002285386508006283, + "Pile-CC": -0.002374064436702188, + "USPTO_Backgrounds": -0.0009179314206659777, + "Wikipedia_en": -0.0015158815390440086, + "StackExchange": 0.0038188850917921765, + "CONTROL_PubMed_2023": 0.00041641391023375135, + "OOD_Wikipedia_2023": 0.0017804207252592253, + "TRAP_FineWeb": 0.0033469708661429077, + "OOD_CommonPile": -0.0039263860180292175, + "OOD_News_2024": 0.0012011239916363539, + "OOD_arXiv_2023": -0.003005678217388545 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "Pile-CC": { + "slope": -0.001888259914374931, + "spearman": 0.1, + "worst_loo_slope": -0.0066902792082571445, + "loo_slope_by_dropped_rung": { + "Github": 0.00047872924608353826, + "Pile-CC": -0.005057749174065081, + "USPTO_Backgrounds": -0.0035113200534188493, + "Wikipedia_en": -0.0036818181668935262, + "StackExchange": 0.0028447736299541636, + "CONTROL_PubMed_2023": -0.0016658009635072274, + "OOD_Wikipedia_2023": -0.0004948629479700113, + "TRAP_FineWeb": 0.001043891426156754, + "OOD_CommonPile": -0.0066902792082571445, + "OOD_News_2024": 2.336202287363774e-05, + "OOD_arXiv_2023": -0.0044432618897068624 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": true + }, + "USPTO_Backgrounds": { + "slope": 0.0028406448262535064, + "spearman": 0.24545454545454545, + "worst_loo_slope": 0.005819661371718266, + "loo_slope_by_dropped_rung": { + "Github": 0.005710533487887158, + "Pile-CC": 9.863395130239031e-05, + "USPTO_Backgrounds": 0.001356311640778398, + "Wikipedia_en": 0.0016527905617019472, + "StackExchange": 0.004714116898481311, + "CONTROL_PubMed_2023": 0.004101508487886535, + "OOD_Wikipedia_2023": 0.004572727249792843, + "TRAP_FineWeb": 0.005819661371718266, + "OOD_CommonPile": -0.0018155905788073956, + "OOD_News_2024": 0.004428249630030372, + "OOD_arXiv_2023": 0.00013908058997158366 + }, + "slope_sign_flips_under_loo": true, + "n_rungs_below_logit": 0, + "rungs_below_logit": [], + "steeper_than_logit": false + } + }, + "n_fitted_steeper_than_logit": 3, + "n_fitted_crossing_below_logit_at_any_rung": 0 + } + }, + "e52": { + "persist": { + "D_all8": 0.000845372218579525, + "D_without_degenerate": 0.001007020774004221, + "permutation_p": 0.034482758620689655, + "link_artifact_surrogate": 9.441563525961398e-05, + "D_minus_surrogate": 0.000750956583319911, + "sign_survives_dropping_github": true, + "significant_at_5pct": true + }, + "min": { + "D_all8": 0.0003741732225679442, + "D_without_degenerate": -8.222654747663485e-05, + "permutation_p": 0.7131434282858571, + "link_artifact_surrogate": -1.0121748673588243e-05, + "D_minus_surrogate": 0.00038429497124153243, + "sign_survives_dropping_github": false, + "significant_at_5pct": false + } + }, + "quadrants": { + "persist": { + "P_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "Q_corpora": [ + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "quadrant_means": { + "JP_at_P": 0.10322155726659629, + "JP_at_Q": 0.10475294886001696, + "JQ_at_P": 0.09668358823870896, + "JQ_at_Q": 0.09890238938962374 + }, + "fit_row_means": { + "Wikipedia_en": 0.09961745545313332, + "USPTO_Backgrounds": 0.10963667937838989, + "Pile-CC": 0.10827355975796106, + "StackExchange": 0.10898624043006243, + "Github": 0.0924652105510985, + "OOD_News_2024": 0.09569122234566343, + "OOD_arXiv_2023": 0.09788490975592544, + "OOD_CommonPile": 0.0989707839093171 + }, + "read_col_means": { + "Wikipedia_en": 0.10531951454274047, + "USPTO_Backgrounds": 0.10890259583894578, + "Pile-CC": 0.10510752553333683, + "StackExchange": 0.09225428685507116, + "Github": 0.09226517163559846, + "OOD_News_2024": 0.10725807921618172, + "OOD_arXiv_2023": 0.09938795663215892, + "OOD_CommonPile": 0.10103093132751786 + }, + "diagonal_mean": 0.10218045838895098, + "offdiagonal_mean": 0.10133508617037146, + "fit_axis_span": 0.017171468827291383, + "read_axis_span": 0.01664830898387462 + }, + "min": { + "P_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github" + ], + "Q_corpora": [ + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "quadrant_means": { + "JP_at_P": 0.24897796408500936, + "JP_at_Q": 0.2414854618051538, + "JQ_at_P": 0.24969820555437494, + "JQ_at_Q": 0.2439813831821084 + }, + "fit_row_means": { + "Wikipedia_en": 0.23152156053369657, + "USPTO_Backgrounds": 0.2555947363480098, + "Pile-CC": 0.2559463235213318, + "StackExchange": 0.26002017346521217, + "Github": 0.22775858478206726, + "OOD_News_2024": 0.24214933404388528, + "OOD_arXiv_2023": 0.2535410887380648, + "OOD_CommonPile": 0.2469727687123749 + }, + "read_col_means": { + "Wikipedia_en": 0.2535931211275359, + "USPTO_Backgrounds": 0.2598997434601188, + "Pile-CC": 0.25737932473162395, + "StackExchange": 0.23572498860303312, + "Github": 0.2396430952577955, + "OOD_News_2024": 0.25592221629857603, + "OOD_arXiv_2023": 0.2362714668451291, + "OOD_CommonPile": 0.23507061382083017 + }, + "diagonal_mean": 0.24701547283782727, + "offdiagonal_mean": 0.24664129961525932, + "fit_axis_span": 0.03226158868314491, + "read_axis_span": 0.024829129639288627 + } + }, + "matrix_structure": { + "persist": { + "variance_fit_axis_pct": 50.697184677169346, + "variance_read_axis_pct": 48.13211438274321, + "variance_fit_x_read_interaction_pct": 1.1707009400874848, + "note_on_interaction": "this is the fit x READ interaction. It is NOT the corpus x CONCEPT-SET interaction reported in E51; the concept sets are averaged out of this matrix.", + "own_corpus": { + "Wikipedia_en": { + "rank_of_own_corpus": 2, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "USPTO_Backgrounds": { + "rank_of_own_corpus": 1, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "Pile-CC": { + "rank_of_own_corpus": 4, + "best_read_context": "OOD_News_2024", + "worst_read_context": "StackExchange" + }, + "StackExchange": { + "rank_of_own_corpus": 8, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "Github": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_News_2024": { + "rank_of_own_corpus": 1, + "best_read_context": "OOD_News_2024", + "worst_read_context": "StackExchange" + }, + "OOD_arXiv_2023": { + "rank_of_own_corpus": 6, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + }, + "OOD_CommonPile": { + "rank_of_own_corpus": 5, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "Github" + } + }, + "mean_rank_of_own_corpus": 4.25, + "chance_rank": 4.5, + "n_operators_reading_own_corpus_best": 2, + "fit_axis_order": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "OOD_CommonPile", + "OOD_arXiv_2023", + "OOD_News_2024", + "Github" + ], + "read_axis_order": [ + "USPTO_Backgrounds", + "OOD_News_2024", + "Wikipedia_en", + "Pile-CC", + "OOD_CommonPile", + "OOD_arXiv_2023", + "Github", + "StackExchange" + ], + "n_Q_operators_above_worst_P_operator": 3, + "n_Q_operators_above_second_worst_P_operator": 0, + "worst_read_context": "StackExchange", + "worst_read_context_is_P": true, + "github_cost_as_fit_corpus_pct": -8.84806792684223, + "github_cost_as_read_context_pct": -9.045265700242332 + }, + "min": { + "variance_fit_axis_pct": 53.35344735234028, + "variance_read_axis_pct": 44.60936175509925, + "variance_fit_x_read_interaction_pct": 2.037190892560503, + "note_on_interaction": "this is the fit x READ interaction. It is NOT the corpus x CONCEPT-SET interaction reported in E51; the concept sets are averaged out of this matrix.", + "own_corpus": { + "Wikipedia_en": { + "rank_of_own_corpus": 3, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_CommonPile" + }, + "USPTO_Backgrounds": { + "rank_of_own_corpus": 1, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_CommonPile" + }, + "Pile-CC": { + "rank_of_own_corpus": 3, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "StackExchange": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_arXiv_2023" + }, + "Github": { + "rank_of_own_corpus": 5, + "best_read_context": "Pile-CC", + "worst_read_context": "OOD_arXiv_2023" + }, + "OOD_News_2024": { + "rank_of_own_corpus": 4, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_arXiv_2023": { + "rank_of_own_corpus": 7, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "StackExchange" + }, + "OOD_CommonPile": { + "rank_of_own_corpus": 8, + "best_read_context": "USPTO_Backgrounds", + "worst_read_context": "OOD_CommonPile" + } + }, + "mean_rank_of_own_corpus": 4.75, + "chance_rank": 4.5, + "n_operators_reading_own_corpus_best": 1, + "fit_axis_order": [ + "StackExchange", + "Pile-CC", + "USPTO_Backgrounds", + "OOD_arXiv_2023", + "OOD_CommonPile", + "OOD_News_2024", + "Wikipedia_en", + "Github" + ], + "read_axis_order": [ + "USPTO_Backgrounds", + "Pile-CC", + "OOD_News_2024", + "Wikipedia_en", + "Github", + "OOD_arXiv_2023", + "StackExchange", + "OOD_CommonPile" + ], + "n_Q_operators_above_worst_P_operator": 3, + "n_Q_operators_above_second_worst_P_operator": 3, + "worst_read_context": "OOD_CommonPile", + "worst_read_context_is_P": false, + "github_cost_as_fit_corpus_pct": -7.6734502761757195, + "github_cost_as_read_context_pct": -2.855823540259028 + } + }, + "baseline_movement": { + "persist": { + "n_read_contexts": 12, + "logit_min": 0.05914952543874582, + "logit_max": 0.08317913140635938, + "logit_range": 0.02402960596761356, + "largest_fitted_advantage": 0.0374568174547878, + "logit_range_as_pct_of_largest_advantage": 64.15282343893328, + "per_operator": { + "Github": { + "range_across_read_contexts": 0.027728552355741454, + "mean_gap_over_logit": 0.019731618306832385, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "Wikipedia_en": { + "range_across_read_contexts": 0.025501235644333056, + "mean_gap_over_logit": 0.026948396621608278, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "StackExchange": { + "range_across_read_contexts": 0.020683898093799738, + "mean_gap_over_logit": 0.037169504749019526, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "Pile-CC": { + "range_across_read_contexts": 0.025536208258320897, + "mean_gap_over_logit": 0.035217998431633334, + "n_contexts_scrambled_beats_this_operator": 0 + }, + "USPTO_Backgrounds": { + "range_across_read_contexts": 0.027602271906410653, + "mean_gap_over_logit": 0.0374568174547878, + "n_contexts_scrambled_beats_this_operator": 0 + } + }, + "n_contexts_scrambled_beats_logit": 12 + }, + "min": { + "n_read_contexts": 12, + "logit_min": 0.1873030010610819, + "logit_max": 0.2132010769098997, + "logit_range": 0.025898075848817814, + "largest_fitted_advantage": 0.06672917040478853, + "logit_range_as_pct_of_largest_advantage": 38.810726540906245, + "per_operator": { + "Github": { + "range_across_read_contexts": 0.02543728364010653, + "mean_gap_over_logit": 0.03134944018804365, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "Wikipedia_en": { + "range_across_read_contexts": 0.02524586891134578, + "mean_gap_over_logit": 0.037084430507901646, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "StackExchange": { + "range_across_read_contexts": 0.026412486347059394, + "mean_gap_over_logit": 0.06672917040478853, + "n_contexts_scrambled_beats_this_operator": 9 + }, + "Pile-CC": { + "range_across_read_contexts": 0.0324745271354914, + "mean_gap_over_logit": 0.06034543172249364, + "n_contexts_scrambled_beats_this_operator": 12 + }, + "USPTO_Backgrounds": { + "range_across_read_contexts": 0.02731785501043002, + "mean_gap_over_logit": 0.060505543856157196, + "n_contexts_scrambled_beats_this_operator": 12 + } + }, + "n_contexts_scrambled_beats_logit": 12 + } + }, + "matrix_loo": { + "persist": { + "full": { + "fit_pct": 50.697184677169346, + "read_pct": 48.13211438274321, + "interaction_pct": 1.1707009400874848, + "fit_span": 0.017171468827291383, + "read_span": 0.01664830898387462, + "span_ratio": 1.031424203138198, + "fit_dominates_read": true + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 52.373250427828836, + "read_pct": 46.61005338540753, + "interaction_pct": 1.0166961867635491, + "fit_span": 0.017303615567156122, + "read_span": 0.0165367584815988, + "span_ratio": 1.0463728781194113, + "fit_dominates_read": true + }, + "USPTO_Backgrounds": { + "fit_pct": 48.900137854184116, + "read_pct": 49.70701457618453, + "interaction_pct": 1.3928475696313072, + "fit_span": 0.015998433523308778, + "read_span": 0.015076022723778365, + "span_ratio": 1.061183961873151, + "fit_dominates_read": false + }, + "Pile-CC": { + "fit_pct": 47.46625999643371, + "read_pct": 51.23363164594214, + "interaction_pct": 1.3001083576241463, + "fit_span": 0.017079299537184095, + "read_span": 0.016827839917118947, + "span_ratio": 1.0149430717967158, + "fit_dominates_read": false + }, + "StackExchange": { + "fit_pct": 54.302935606304814, + "read_pct": 44.71750309530783, + "interaction_pct": 0.9795612983873323, + "fit_span": 0.0169366810866262, + "read_span": 0.01671054580152803, + "span_ratio": 1.0135324894700621, + "fit_dominates_read": true + }, + "Github": { + "fit_pct": 50.83451434240633, + "read_pct": 47.965622476031534, + "interaction_pct": 1.199863181562132, + "fit_span": 0.014015912493720417, + "read_span": 0.016742457825291365, + "span_ratio": 0.8371478453150293, + "fit_dominates_read": true + }, + "OOD_News_2024": { + "fit_pct": 50.49955454200501, + "read_pct": 48.12259484369173, + "interaction_pct": 1.3778506143031934, + "fit_span": 0.017138381758039542, + "read_span": 0.016708293558645337, + "span_ratio": 1.0257410008918395, + "fit_dominates_read": true + }, + "OOD_arXiv_2023": { + "fit_pct": 50.57242596342314, + "read_pct": 48.44478039653153, + "interaction_pct": 0.9827936400453459, + "fit_span": 0.0174609310522173, + "read_span": 0.016825227149289163, + "span_ratio": 1.0377827827991608, + "fit_dominates_read": true + }, + "OOD_CommonPile": { + "fit_pct": 50.852977854777855, + "read_pct": 48.11818900866957, + "interaction_pct": 1.028833136552585, + "fit_span": 0.01742520418188105, + "read_span": 0.016803709279361465, + "span_ratio": 1.0369855781355914, + "fit_dominates_read": true + } + }, + "drop_both_extremes_github_and_uspto": { + "fit_pct": 49.077827715914886, + "read_pct": 49.432990888192485, + "interaction_pct": 1.4891813958926687, + "fit_span": 0.012516824986044048, + "read_span": 0.014937012424019888, + "span_ratio": 0.8379737949414845, + "fit_dominates_read": false + }, + "worst_loo_span_ratio": 0.8371478453150293, + "worst_loo_corpus": "Github", + "fit_dominates_in_all_loo": false, + "fit_dominates_with_both_extremes_dropped": false + }, + "min": { + "full": { + "fit_pct": 53.35344735234028, + "read_pct": 44.60936175509925, + "interaction_pct": 2.037190892560503, + "fit_span": 0.03226158868314491, + "read_span": 0.024829129639288627, + "span_ratio": 1.2993443246635377, + "fit_dominates_read": true + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 46.81110557693417, + "read_pct": 50.85837600168466, + "interaction_pct": 2.330518421381303, + "fit_span": 0.032255948695635056, + "read_span": 0.025139714496594606, + "span_ratio": 1.283067423061006, + "fit_dominates_read": false + }, + "USPTO_Backgrounds": { + "fit_pct": 56.67920183763611, + "read_pct": 41.06419873316167, + "interaction_pct": 2.2565994292022826, + "fit_span": 0.03192390514803786, + "read_span": 0.02218238359819802, + "span_ratio": 1.4391557609990657, + "fit_dominates_read": true + }, + "Pile-CC": { + "fit_pct": 56.50763481529161, + "read_pct": 41.4252808563404, + "interaction_pct": 2.0670843283679354, + "fit_span": 0.03248407571975673, + "read_span": 0.024528523726714047, + "span_ratio": 1.3243388017020479, + "fit_dominates_read": true + }, + "StackExchange": { + "fit_pct": 53.53726007459881, + "read_pct": 45.32738406925986, + "interaction_pct": 1.1353558561412547, + "fit_span": 0.0298434966317718, + "read_span": 0.025165532340132996, + "span_ratio": 1.1858877542669173, + "fit_dominates_read": true + }, + "Github": { + "fit_pct": 41.358656075065184, + "read_pct": 56.523659412915094, + "interaction_pct": 2.1176845120195984, + "fit_span": 0.028123639521026395, + "read_span": 0.02546599780815459, + "span_ratio": 1.1043603998120501, + "fit_dominates_read": false + }, + "OOD_News_2024": { + "fit_pct": 56.05510540136367, + "read_pct": 42.24118054522517, + "interaction_pct": 1.7037140534111448, + "fit_span": 0.03184018684403292, + "read_span": 0.024672448768147365, + "span_ratio": 1.2905158763624327, + "fit_dominates_read": true + }, + "OOD_arXiv_2023": { + "fit_pct": 56.005471966512765, + "read_pct": 42.12050880191629, + "interaction_pct": 1.8740192315709754, + "fit_span": 0.032027145049401684, + "read_span": 0.024876730661425345, + "span_ratio": 1.2874338467258482, + "fit_dominates_read": true + }, + "OOD_CommonPile": { + "fit_pct": 57.55685418677556, + "read_pct": 40.24133072996441, + "interaction_pct": 2.2018150832599948, + "fit_span": 0.031948861550717134, + "read_span": 0.024509501918440785, + "span_ratio": 1.3035296130060898, + "fit_dominates_read": true + } + }, + "drop_both_extremes_github_and_uspto": { + "fit_pct": 45.23962835179833, + "read_pct": 52.609332663665406, + "interaction_pct": 2.151038984536295, + "fit_span": 0.028022169321775425, + "read_span": 0.023017197788727517, + "span_ratio": 1.2174448679195455, + "fit_dominates_read": false + }, + "worst_loo_span_ratio": 1.1043603998120501, + "worst_loo_corpus": "Github", + "fit_dominates_in_all_loo": false, + "fit_dominates_with_both_extremes_dropped": false + } + }, + "C2_derangement": { + "persist": { + "n_draws": 120, + "shuf_beats_jp_paired_by_seed": 0, + "ties": 0, + "shuf_beats_jp_vs_corpus_mean": 0, + "n_corpora_where_shuf_beats_jp_on_the_mean": 0, + "shuf_sd_range": [ + 0.0016611376499874374, + 0.003904935305054106 + ], + "gap_range": [ + 0.0190823487068216, + 0.033592421001133826 + ], + "per_corpus": { + "Github": { + "jp_mean": 0.10075108803187807, + "shuf_mean": 0.08166873932505647, + "gap": 0.0190823487068216, + "shuf_sd_over_15_draws": 0.0027560662324541676, + "shuf_beats_jp_paired": 0 + }, + "OOD_CommonPile": { + "jp_mean": 0.10663878470659256, + "shuf_mean": 0.07937348173310359, + "gap": 0.027265302973488975, + "shuf_sd_over_15_draws": 0.0016835173291184415, + "shuf_beats_jp_paired": 0 + }, + "OOD_News_2024": { + "jp_mean": 0.09791991202315936, + "shuf_mean": 0.07102687652378033, + "gap": 0.026893035499379034, + "shuf_sd_over_15_draws": 0.0016611376499874374, + "shuf_beats_jp_paired": 0 + }, + "OOD_arXiv_2023": { + "jp_mean": 0.10589559425910314, + "shuf_mean": 0.07916459808126092, + "gap": 0.02673099617784222, + "shuf_sd_over_15_draws": 0.0023569520613490307, + "shuf_beats_jp_paired": 0 + }, + "Pile-CC": { + "jp_mean": 0.11109575630010417, + "shuf_mean": 0.08255375813382368, + "gap": 0.02854199816628049, + "shuf_sd_over_15_draws": 0.0018510910218030826, + "shuf_beats_jp_paired": 0 + }, + "StackExchange": { + "jp_mean": 0.11614819277698794, + "shuf_mean": 0.08374283629314352, + "gap": 0.03240535648384442, + "shuf_sd_over_15_draws": 0.003904935305054106, + "shuf_beats_jp_paired": 0 + }, + "USPTO_Backgrounds": { + "jp_mean": 0.1200522772812595, + "shuf_mean": 0.08645985628012567, + "gap": 0.033592421001133826, + "shuf_sd_over_15_draws": 0.0027942992021020227, + "shuf_beats_jp_paired": 0 + }, + "Wikipedia_en": { + "jp_mean": 0.10422163270413876, + "shuf_mean": 0.07876515122130513, + "gap": 0.025456481482833623, + "shuf_sd_over_15_draws": 0.0022273356138562688, + "shuf_beats_jp_paired": 0 + } + }, + "admissible_as_an_operator_comparator": true + }, + "min": { + "n_draws": 120, + "shuf_beats_jp_paired_by_seed": 84, + "ties": 0, + "shuf_beats_jp_vs_corpus_mean": 84, + "n_corpora_where_shuf_beats_jp_on_the_mean": 7, + "shuf_sd_range": [ + 0.0058957991568539675, + 0.014118417628430236 + ], + "gap_range": [ + -0.01677957579493522, + 0.008090302099784202 + ], + "per_corpus": { + "Github": { + "jp_mean": 0.2404973489542802, + "shuf_mean": 0.232407046854496, + "gap": 0.008090302099784202, + "shuf_sd_over_15_draws": 0.014118417628430236, + "shuf_beats_jp_paired": 5 + }, + "OOD_CommonPile": { + "jp_mean": 0.2528811464707057, + "shuf_mean": 0.2607669298350811, + "gap": -0.007885783364375398, + "shuf_sd_over_15_draws": 0.011553615813550225, + "shuf_beats_jp_paired": 11 + }, + "OOD_News_2024": { + "jp_mean": 0.25006329963604607, + "shuf_mean": 0.2571750570833683, + "gap": -0.007111757447322231, + "shuf_sd_over_15_draws": 0.009827221125735572, + "shuf_beats_jp_paired": 10 + }, + "OOD_arXiv_2023": { + "jp_mean": 0.26927303777386746, + "shuf_mean": 0.2703329712400834, + "gap": -0.0010599334662159121, + "shuf_sd_over_15_draws": 0.01244684477098846, + "shuf_beats_jp_paired": 8 + }, + "Pile-CC": { + "jp_mean": 0.2623990322152774, + "shuf_mean": 0.2748033052186171, + "gap": -0.012404273003339739, + "shuf_sd_over_15_draws": 0.006460454237689801, + "shuf_beats_jp_paired": 15 + }, + "StackExchange": { + "jp_mean": 0.26430599875748156, + "shuf_mean": 0.26492709482709564, + "gap": -0.0006210960696140799, + "shuf_sd_over_15_draws": 0.0070726797713334365, + "shuf_beats_jp_paired": 8 + }, + "USPTO_Backgrounds": { + "jp_mean": 0.26112590978542966, + "shuf_mean": 0.2651418822258711, + "gap": -0.00401597244044144, + "shuf_sd_over_15_draws": 0.0058957991568539675, + "shuf_beats_jp_paired": 12 + }, + "Wikipedia_en": { + "jp_mean": 0.24160845403869946, + "shuf_mean": 0.2583880298336347, + "gap": -0.01677957579493522, + "shuf_sd_over_15_draws": 0.010394572119940946, + "shuf_beats_jp_paired": 15 + } + }, + "admissible_as_an_operator_comparator": false + } + }, + "C1_persist_column_reproduces_stored_headlines": { + "e51_410m_interaction_pct": { + "recomputed": 6.780626428099203, + "stored": 6.780626428099203, + "abs_diff": 0.0, + "ok": true + }, + "e51_410m_corpus_pct": { + "recomputed": 1.9064501148312785, + "stored": 1.9064501148312791, + "abs_diff": 6.661338147750939e-16, + "ok": true + }, + "e51_1b_interaction_pct": { + "recomputed": 9.158325214755475, + "stored": 9.158325214755473, + "abs_diff": 1.7763568394002505e-15, + "ok": true + }, + "e51_410m_F": { + "recomputed": 611.8910219722385, + "stored": 611.8910219722385, + "abs_diff": 0.0, + "ok": true + }, + "e33_logit_constant": { + "recomputed": 0.02843950316309929, + "stored": 0.02843950316309929, + "abs_diff": 0.0, + "ok": true + }, + "e33_uspto_asymptote": { + "recomputed": 0.049735264846729854, + "stored": 0.049735264846729854, + "abs_diff": 0.0, + "ok": true + }, + "e48c_spearman_all8": { + "recomputed": 0.19047619047619047, + "stored": 0.0, + "abs_diff": 0.19047619047619047, + "ok": false + }, + "e52_D": { + "recomputed": 0.000845372218579525, + "stored": 0.000845372218579525, + "abs_diff": 0.0, + "ok": true + } + }, + "C1_fires": false, + "VERDICT_BY_CLAIM": { + "corpus_identity_decides_the_read_410m": { + "persist": 58.381738635199795, + "min": 55.37773289006552, + "verdict": "HOLDS" + }, + "reads_are_flat_in_N_410m": { + "persist": [ + 1.2278526816817164, + 3.6981860639992252 + ], + "min": [ + 1.1840692303542948, + 2.2622282558307703 + ], + "verdict": "HOLDS" + }, + "github_indistinguishable_from_not_fitting": { + "persist": -1.6527617605340326, + "min": -58.02264085495087, + "verdict": "REVERSES", + "note": "under min Github is not merely indistinguishable from the logit lens, it is significantly BELOW it \u2014 a stronger form of the same claim, but a different sentence" + }, + "effect_is_an_interaction_not_a_level_410m": { + "persist": 3.55667655573526, + "min": 1.0455176739722105, + "verdict": "WEAKENS" + }, + "effect_is_an_interaction_not_a_level_1b": { + "persist": 4.1408154208167804, + "min": 2.0238440545235648, + "verdict": "WEAKENS" + }, + "exposure_does_not_order_the_read": { + "persist": 0.19047619047619047, + "min": -0.42857142857142855, + "verdict": "HOLDS", + "note": "both are indistinguishable from zero at n=8; all three OOD rungs beat in-stream Wikipedia under both" + }, + "e36_no_fitted_operator_crosses_below_logit": { + "persist": 0, + "min": 0, + "verdict": "HOLDS", + "note": "the count includes Github, which sits below the logit lens at every rung under both aggregations and is degenerate by E33. Excluding it: 0 of 4 under persist, 1 of 4 (Wikipedia, at 1 rung of 11) under min. The pre-registration states that a crossing appearing only under min does not count." + }, + "e36_fitted_operators_are_flatter_than_logit": { + "persist": 4, + "min": 3, + "verdict": "REVERSES", + "note": "0 of 5 fitted operators are steeper than the logit lens under persist; 3 of 5 are under min. This secondary is aggregation-dependent AND single-rung-fragile under LOO." + }, + "e52_matched_fit_read_helps": { + "persist": 0.034482758620689655, + "min": 0.7131434282858571, + "verdict": "REVERSES", + "note": "p = 0.0345 under persist, 0.7131 under min; and D changes sign without Github under min (-8.22e-05)" + } + }, + "VERDICT": "THE SPINE IS AGGREGATION-ROBUST; THREE SECONDARIES ARE NOT. Corpus identity deciding the read, reads being flat in N, and exposure not ordering the read all hold under the anchor's own `min` \u2014 the first with a LARGER absolute spread. What does not survive: the interaction's dominance over the corpus main effect at 410M (3.56x -> 1.05x), E36's slope secondary (0 of 5 fitted operators steeper -> 4 of 5), and E52's matched-diagonal effect (p=0.0055 -> p=0.66, and D changes sign without Github). Separately, C2 shows why `min` cannot be the adjudicating metric: under it the layer-deranged operator beats the real one on 104 of 120 paired draws and outscores the logit lens, while under `persist` it loses 120 of 120. Both facts must be reported together \u2014 `min` is inadmissible as an operator comparator AND three of our claims depend on not using it.", + "provenance": { + "schema_version": 1, + "experiment": "E54", + "utc": "2026-08-31T23:20:08+00:00", + "script": { + "path": "experiments/t54_aggregation_audit.py", + "sha256": "529bf053fb5af4e3cf6cdb4022053b061f7110730418d1e10ef98fee03afcae6", + "n_lines": 700 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/t54_aggregation_audit.py", + "--e52", + "results/e52_factorial_410m_rstrip.json", + "--e36", + "results/e36_qladder_410m_rstrip.json", + "--e48", + "results/e48_crossover_410m_rstrip.json", + "--out", + "results/e54_aggregation_audit_rstrip_v2.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e33_logit_baseline_410m_v2.json", + "sha256": "48ee37f6526b78919176fcba03c1a49dd05f6337237f16b6b5a321990ed7a418", + "bytes": 5912, + "exists": true + }, + { + "path": "results/e36_qladder_410m_rstrip.json", + "sha256": "540bf21811bb5f522c0e9b22965d0d289e5ade98cfcc06fa2bdf966dbff1c185", + "bytes": 111158, + "exists": true + }, + { + "path": "results/e48_crossover_410m_rstrip.json", + "sha256": "eef75b627898125b090cb0b8b351a398885b1162f27c334bcef895d980064cc6", + "bytes": 111498, + "exists": true + }, + { + "path": "results/e48c_exposure_vs_read.json", + "sha256": "d784d85d3194a8b2298b13ef1b0fb0c74fceadef8217e935c98126507b902fc6", + "bytes": 11217, + "exists": true + }, + { + "path": "results/e51_interaction_variance.json", + "sha256": "f03956141844c2784ee4da109a51f528e3c8965ab9474a3b619062cf21a709de", + "bytes": 33212, + "exists": true + }, + { + "path": "results/e52_factorial_410m_rstrip.json", + "sha256": "e3e42850ce7f436eafb50fa2f6ba7f94559b321540075023badd6e1cd6ef7e19", + "bytes": 45765, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "5d0a4c2a4e3e95c9468504e97a1031b381e608aadb312af64f9594b530fd0796" + } +} \ No newline at end of file diff --git a/results/e55_matrix_robustness.json b/results/e55_matrix_robustness.json new file mode 100644 index 0000000000000000000000000000000000000000..83b47dcefb3676346d3fc1b58ee6dae362fff940 --- /dev/null +++ b/results/e55_matrix_robustness.json @@ -0,0 +1,2689 @@ +{ + "experiment": "E55 \u2014 leave-k-out robustness of the fit>read ordering, and its replication on an independent operator population", + "why": "the paper defends its headline with 8 leave-one-out splits; leave-two-out is stronger and free, the read axis has an unreported leverage point of its own, and an independent 5x5 on N=400 fastfit operators has been on disk unanalysed.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/e52_factorial_410m.json", + "results/e36_qladder_410m.json", + "results/e54_aggregation_audit.json" + ], + "note_on_the_decomposition": "one observation per cell, so fit+read+residual = 100 exactly and the residual is NOT an interaction estimate. See E57.", + "e52_8x8": { + "persist": { + "full": { + "fit_pct": 91.21993064640641, + "read_pct": 7.133265187892407, + "residual_pct": 1.6468041657011878, + "fit_span": 0.02206712914663108, + "read_span": 0.005526805338579126, + "row_means": { + "Wikipedia_en": 0.03836092785899786, + "USPTO_Backgrounds": 0.04748104946532597, + "Pile-CC": 0.04716437525704451, + "StackExchange": 0.04518624561864676, + "Github": 0.02541392031869489, + "OOD_News_2024": 0.036911535940857394, + "OOD_arXiv_2023": 0.03942143082547571, + "OOD_CommonPile": 0.03816213771028237 + }, + "col_means": { + "Wikipedia_en": 0.04101473994604829, + "USPTO_Backgrounds": 0.04202875193182586, + "Pile-CC": 0.041234459214805, + "StackExchange": 0.03650194659324674, + "Github": 0.037404520595252205, + "OOD_News_2024": 0.041391930579427735, + "OOD_arXiv_2023": 0.039012664472102186, + "OOD_CommonPile": 0.03951260966261745 + }, + "n": 8 + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 92.62271892860798, + "read_pct": 6.044474474909023, + "residual_pct": 1.332806596483, + "fit_span": 0.021921292635878283, + "read_span": 0.0052411437245263245, + "row_means": { + "USPTO_Backgrounds": 0.04725084222320999, + "Pile-CC": 0.04680624548013189, + "StackExchange": 0.04515486964512439, + "Github": 0.02532954958733171, + "OOD_News_2024": 0.03664244358679132, + "OOD_arXiv_2023": 0.03926486926308523, + "OOD_CommonPile": 0.03819589886614787 + }, + "col_means": { + "USPTO_Backgrounds": 0.042100674369268946, + "Pile-CC": 0.04137512479820067, + "StackExchange": 0.03685953064474262, + "Github": 0.03777074500814908, + "OOD_News_2024": 0.041382435287555884, + "OOD_arXiv_2023": 0.03940636313091668, + "OOD_CommonPile": 0.03974984541298851 + }, + "n": 7 + }, + "USPTO_Backgrounds": { + "fit_pct": 90.93746718003844, + "read_pct": 6.993844859630081, + "residual_pct": 2.0686879603314856, + "fit_span": 0.021692538208195142, + "read_span": 0.004853508410797944, + "row_means": { + "Wikipedia_en": 0.03790887542889409, + "Pile-CC": 0.04689759177946678, + "StackExchange": 0.04474780149610033, + "Github": 0.025205053571271636, + "OOD_News_2024": 0.03646105016426494, + "OOD_arXiv_2023": 0.039222009030969014, + "OOD_CommonPile": 0.03793605904922717 + }, + "col_means": { + "Wikipedia_en": 0.03986077420117836, + "Pile-CC": 0.040136986062122834, + "StackExchange": 0.03547030144836753, + "Github": 0.0362140714704606, + "OOD_News_2024": 0.04032380985916548, + "OOD_arXiv_2023": 0.03792402107556838, + "OOD_CommonPile": 0.03844847640333077 + }, + "n": 7 + }, + "Pile-CC": { + "fit_pct": 90.84291026481041, + "read_pct": 7.2900814225666135, + "residual_pct": 1.867008312622964, + "fit_span": 0.021950742357321792, + "read_span": 0.005468765466237473, + "row_means": { + "Wikipedia_en": 0.03809108896299251, + "USPTO_Backgrounds": 0.04727594634843251, + "StackExchange": 0.04512235012186307, + "Github": 0.02532520399111072, + "OOD_News_2024": 0.03667749583144628, + "OOD_arXiv_2023": 0.03921038585286292, + "OOD_CommonPile": 0.03790781696157027 + }, + "col_means": { + "Wikipedia_en": 0.03977809083899335, + "USPTO_Backgrounds": 0.04102830797921689, + "StackExchange": 0.035559542512979415, + "Github": 0.03644236385903602, + "OOD_News_2024": 0.04018177904728209, + "OOD_arXiv_2023": 0.038016044529199246, + "OOD_CommonPile": 0.03860415930357126 + }, + "n": 7 + }, + "StackExchange": { + "fit_pct": 93.33692623435269, + "read_pct": 5.156511202775666, + "residual_pct": 1.5065625628716504, + "fit_span": 0.022061346833121092, + "read_span": 0.004746862740758508, + "row_means": { + "Wikipedia_en": 0.03898408066274391, + "USPTO_Backgrounds": 0.048017847587886665, + "Pile-CC": 0.047745175271605454, + "Github": 0.025956500754765573, + "OOD_News_2024": 0.03741618102852492, + "OOD_arXiv_2023": 0.03969545646437577, + "OOD_CommonPile": 0.03847058268396982 + }, + "col_means": { + "Wikipedia_en": 0.04038743459072614, + "USPTO_Backgrounds": 0.04113923728259073, + "Pile-CC": 0.04060602280318678, + "Github": 0.03639237454183222, + "OOD_News_2024": 0.04079898492366608, + "OOD_arXiv_2023": 0.03825799364278773, + "OOD_CommonPile": 0.038703776669082424 + }, + "n": 7 + }, + "Github": { + "fit_pct": 81.04881620398683, + "read_pct": 15.567785798307998, + "residual_pct": 3.383397997705165, + "fit_span": 0.010373391022550915, + "read_span": 0.005564901774882204, + "row_means": { + "Wikipedia_en": 0.03886378188100126, + "USPTO_Backgrounds": 0.04773010446483062, + "Pile-CC": 0.04759648347251295, + "StackExchange": 0.04528577456856885, + "OOD_News_2024": 0.037356713442279706, + "OOD_arXiv_2023": 0.039837152181592374, + "OOD_CommonPile": 0.038538662659832176 + }, + "col_means": { + "Wikipedia_en": 0.04315905773287846, + "USPTO_Backgrounds": 0.04419343255770703, + "Pile-CC": 0.043405819872379424, + "StackExchange": 0.03862853078282483, + "OOD_News_2024": 0.04360083488947047, + "OOD_arXiv_2023": 0.04083421890039943, + "OOD_CommonPile": 0.0413867779349583 + }, + "n": 7 + }, + "OOD_News_2024": { + "fit_pct": 92.43636904450099, + "read_pct": 6.2773435795102195, + "residual_pct": 1.2862873759887863, + "fit_span": 0.02194255113727339, + "read_span": 0.005361218094116167, + "row_means": { + "Wikipedia_en": 0.037918432178493174, + "USPTO_Backgrounds": 0.04728280287162061, + "Pile-CC": 0.04677885867884412, + "StackExchange": 0.04513534496848782, + "Github": 0.025340251734347216, + "OOD_arXiv_2023": 0.039317994554054055, + "OOD_CommonPile": 0.03802593106751345 + }, + "col_means": { + "Wikipedia_en": 0.04133181959272377, + "USPTO_Backgrounds": 0.04230929701108604, + "Pile-CC": 0.041617979573100684, + "StackExchange": 0.03694807891696987, + "Github": 0.03792012447587377, + "OOD_arXiv_2023": 0.03945838889912776, + "OOD_CommonPile": 0.040213927584478545 + }, + "n": 7 + }, + "OOD_arXiv_2023": { + "fit_pct": 91.1983253417951, + "read_pct": 7.343503025106673, + "residual_pct": 1.4581716330982242, + "fit_span": 0.022309378342747335, + "read_span": 0.005842901524969392, + "row_means": { + "Wikipedia_en": 0.038661521287368876, + "USPTO_Backgrounds": 0.047602175353538424, + "Pile-CC": 0.04733228542627619, + "StackExchange": 0.045313514953124384, + "Github": 0.02529279701079109, + "OOD_News_2024": 0.037057099149133715, + "OOD_CommonPile": 0.038195824074292825 + }, + "col_means": { + "Wikipedia_en": 0.04108579397231104, + "USPTO_Backgrounds": 0.042201804581083475, + "Pile-CC": 0.04128241829781069, + "StackExchange": 0.036358903056114084, + "Github": 0.03753211191847979, + "OOD_News_2024": 0.04156999427285637, + "OOD_CommonPile": 0.03942419115587004 + }, + "n": 7 + }, + "OOD_CommonPile": { + "fit_pct": 91.10865204439243, + "read_pct": 7.5957558760329, + "residual_pct": 1.2955920795746683, + "fit_span": 0.022281274635950844, + "read_span": 0.005781825323633492, + "row_means": { + "Wikipedia_en": 0.03843363763742326, + "USPTO_Backgrounds": 0.04755526474928336, + "Pile-CC": 0.047349034268630755, + "StackExchange": 0.04518793204740163, + "Github": 0.025273990113332513, + "OOD_News_2024": 0.03724127190246705, + "OOD_arXiv_2023": 0.039319986770565196 + }, + "col_means": { + "Wikipedia_en": 0.0414560157070232, + "USPTO_Backgrounds": 0.04235504673099116, + "Pile-CC": 0.04141904153816757, + "StackExchange": 0.036573221407357664, + "Github": 0.037672814528369125, + "OOD_News_2024": 0.04171712291796529, + "OOD_arXiv_2023": 0.039167854659229756 + }, + "n": 7 + } + }, + "n_loo_fit_gt_read": "8/8", + "leave_two_out": { + "Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 92.90422389516985, + "read_pct": 5.441867087941937, + "residual_pct": 1.6539090168882127, + "fit_span": 0.02136349986669504, + "read_span": 0.004419169914305077, + "row_means": { + "Pile-CC": 0.04643530979347243, + "StackExchange": 0.044638122173233166, + "Github": 0.025071809926777388, + "OOD_News_2024": 0.036072028121755764, + "OOD_arXiv_2023": 0.039006116909095674, + "OOD_CommonPile": 0.03793776728756105 + }, + "col_means": { + "Pile-CC": 0.04011818371730408, + "StackExchange": 0.03571554198429954, + "Github": 0.036442925098041694, + "OOD_News_2024": 0.04013471189860462, + "OOD_arXiv_2023": 0.038201895611429657, + "OOD_CommonPile": 0.038547895902215884 + }, + "n": 6 + }, + "Wikipedia_en+Pile-CC": { + "fit_pct": 92.78940527673387, + "read_pct": 5.805123560121626, + "residual_pct": 1.4054711631444974, + "fit_span": 0.021761201963225306, + "read_span": 0.005125820271118922, + "row_means": { + "USPTO_Backgrounds": 0.04697318737981496, + "StackExchange": 0.045075095569956355, + "Github": 0.02521198541658965, + "OOD_News_2024": 0.03632454806680067, + "OOD_arXiv_2023": 0.03899255653463856, + "OOD_CommonPile": 0.03790481818529467 + }, + "col_means": { + "USPTO_Backgrounds": 0.040945476830798994, + "StackExchange": 0.03581965655968007, + "Github": 0.03670926621804635, + "OOD_News_2024": 0.03996900928474066, + "OOD_arXiv_2023": 0.038309256307332326, + "OOD_CommonPile": 0.038729525952496464 + }, + "n": 6 + }, + "Wikipedia_en+StackExchange": { + "fit_pct": 94.49208659572241, + "read_pct": 4.277500631331475, + "residual_pct": 1.230412772946112, + "fit_span": 0.021890240518324493, + "read_span": 0.004423949003426561, + "row_means": { + "USPTO_Backgrounds": 0.04783873882584481, + "Pile-CC": 0.04742415720096755, + "Github": 0.025948498307520315, + "OOD_News_2024": 0.03718634746339241, + "OOD_arXiv_2023": 0.03955847224807022, + "OOD_CommonPile": 0.038561378194760805 + }, + "col_means": { + "USPTO_Backgrounds": 0.04107489435140181, + "Pile-CC": 0.04066539324854535, + "Github": 0.03665094534797525, + "OOD_News_2024": 0.040689082807188644, + "OOD_arXiv_2023": 0.038591530273185556, + "OOD_CommonPile": 0.03884574621225949 + }, + "n": 6 + }, + "Wikipedia_en+Github": { + "fit_pct": 82.09091532665984, + "read_pct": 14.917080642721988, + "residual_pct": 2.992004030618184, + "fit_span": 0.010386069902839759, + "read_span": 0.0052379792978711134, + "row_means": { + "USPTO_Backgrounds": 0.04750303851561276, + "Pile-CC": 0.047250683435359625, + "StackExchange": 0.04526575742444644, + "OOD_News_2024": 0.037116968612773, + "OOD_arXiv_2023": 0.03972378391815626, + "OOD_CommonPile": 0.03864080483326689 + }, + "col_means": { + "USPTO_Backgrounds": 0.04463812217237083, + "Pile-CC": 0.04393182316260343, + "StackExchange": 0.039400142874499715, + "OOD_News_2024": 0.04395790776729377, + "OOD_arXiv_2023": 0.04159712640706588, + "OOD_CommonPile": 0.041975914355781345 + }, + "n": 6 + }, + "Wikipedia_en+OOD_News_2024": { + "fit_pct": 93.9502366304117, + "read_pct": 5.198166807781096, + "residual_pct": 0.8515965618072017, + "fit_span": 0.02175164553983551, + "read_span": 0.005000348336977406, + "row_means": { + "USPTO_Backgrounds": 0.04698118665686774, + "Pile-CC": 0.046296787842746, + "StackExchange": 0.0450902562243519, + "Github": 0.025229541117032232, + "OOD_arXiv_2023": 0.03911810001936155, + "OOD_CommonPile": 0.03804261797556171 + }, + "col_means": { + "USPTO_Backgrounds": 0.04243996403464634, + "Pile-CC": 0.04184600948011158, + "StackExchange": 0.03743961569766893, + "Github": 0.038433320271023724, + "OOD_arXiv_2023": 0.03999199140558226, + "OOD_CommonPile": 0.04060758894688829 + }, + "n": 6 + }, + "Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 92.50118296926041, + "read_pct": 6.285402740424156, + "residual_pct": 1.2134142903154328, + "fit_span": 0.022179610612888437, + "read_span": 0.005562312339639497, + "row_means": { + "USPTO_Backgrounds": 0.04735378788577185, + "Pile-CC": 0.04694245238141674, + "StackExchange": 0.04529812120642789, + "Github": 0.025174177272883416, + "OOD_News_2024": 0.036767418604102674, + "OOD_CommonPile": 0.03824082648347098 + }, + "col_means": { + "USPTO_Backgrounds": 0.04231455619964335, + "Pile-CC": 0.04145452132560658, + "StackExchange": 0.03675224386000385, + "Github": 0.03798063895406408, + "OOD_News_2024": 0.04158859371457732, + "OOD_CommonPile": 0.03968622978017838 + }, + "n": 6 + }, + "Wikipedia_en+OOD_CommonPile": { + "fit_pct": 92.64058010271503, + "read_pct": 6.302906156312658, + "residual_pct": 1.0565137409723027, + "fit_span": 0.0221468229549592, + "read_span": 0.005491056771414288, + "row_means": { + "USPTO_Backgrounds": 0.047299058847474276, + "Pile-CC": 0.04696199269749707, + "StackExchange": 0.04515160781641801, + "Github": 0.025152235892515078, + "OOD_News_2024": 0.0369822868163249, + "OOD_arXiv_2023": 0.03912042427195788 + }, + "col_means": { + "USPTO_Backgrounds": 0.04249333870786898, + "Pile-CC": 0.041613915106022935, + "StackExchange": 0.03700228193645469, + "Github": 0.03814479199893497, + "OOD_News_2024": 0.04176024380053773, + "OOD_arXiv_2023": 0.03965303479236792 + }, + "n": 6 + }, + "USPTO_Backgrounds+Pile-CC": { + "fit_pct": 90.14600839627873, + "read_pct": 7.155713139154008, + "residual_pct": 2.6982784645672604, + "fit_span": 0.01953344266490873, + "read_span": 0.00453505712104272, + "row_means": { + "Wikipedia_en": 0.03751872131187055, + "StackExchange": 0.04460018272942829, + "Github": 0.02506674006451956, + "OOD_News_2024": 0.036112922407186554, + "OOD_arXiv_2023": 0.03894255293050298, + "OOD_CommonPile": 0.037601671732220524 + }, + "col_means": { + "Wikipedia_en": 0.03822568928546927, + "StackExchange": 0.03419888916390913, + "Github": 0.03489314709074312, + "OOD_News_2024": 0.03873394628495185, + "OOD_arXiv_2023": 0.03657985724275931, + "OOD_CommonPile": 0.03721126210789576 + }, + "n": 6 + }, + "USPTO_Backgrounds+StackExchange": { + "fit_pct": 91.91641433529908, + "read_pct": 6.028536093227016, + "residual_pct": 2.0550495714739028, + "fit_span": 0.021727474928074687, + "read_span": 0.004619193586727806, + "row_means": { + "Wikipedia_en": 0.038560544961580526, + "Pile-CC": 0.047530727883524915, + "Github": 0.025803252955450228, + "OOD_News_2024": 0.036974721803778296, + "OOD_arXiv_2023": 0.03950846864393464, + "OOD_CommonPile": 0.03825823174168666 + }, + "col_means": { + "Wikipedia_en": 0.038936590329157535, + "Pile-CC": 0.03922089805645454, + "Github": 0.03483482622067203, + "OOD_News_2024": 0.039454019807399834, + "OOD_arXiv_2023": 0.03686213120861255, + "OOD_CommonPile": 0.03732748236765878 + }, + "n": 6 + }, + "USPTO_Backgrounds+Github": { + "fit_pct": 78.67789669144574, + "read_pct": 16.686938451159083, + "residual_pct": 4.63516485739518, + "fit_span": 0.010451911164758111, + "read_span": 0.0049434692887761825, + "row_means": { + "Wikipedia_en": 0.03842019638288076, + "Pile-CC": 0.047357254117917, + "StackExchange": 0.044790844583918375, + "OOD_News_2024": 0.036905342953158886, + "OOD_arXiv_2023": 0.039673780314020676, + "OOD_CommonPile": 0.03833765838019274 + }, + "col_means": { + "Wikipedia_en": 0.042170150661668564, + "Pile-CC": 0.04248732797051263, + "StackExchange": 0.037779375478728776, + "OOD_News_2024": 0.04272284476750496, + "OOD_arXiv_2023": 0.039867727342492866, + "OOD_CommonPile": 0.04045765051118064 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 92.8584738297847, + "read_pct": 5.603648447513692, + "residual_pct": 1.537877722701608, + "fit_span": 0.02131906276034122, + "read_span": 0.004582665986122766, + "row_means": { + "Wikipedia_en": 0.03731728839662133, + "Pile-CC": 0.04640335852530336, + "StackExchange": 0.04461534338382383, + "Github": 0.025084295764962142, + "OOD_arXiv_2023": 0.03906809641522597, + "OOD_CommonPile": 0.037739471522487564 + }, + "col_means": { + "Wikipedia_en": 0.040038372831488096, + "Pile-CC": 0.040401514288020766, + "StackExchange": 0.035818848301898, + "Github": 0.036617201143720496, + "OOD_arXiv_2023": 0.038262592341009254, + "OOD_CommonPile": 0.03908932510228759 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 90.83454709308151, + "read_pct": 7.262154654564543, + "residual_pct": 1.9032982523539506, + "fit_span": 0.022020091143160787, + "read_span": 0.005222054250555601, + "row_means": { + "Wikipedia_en": 0.038184225690309645, + "Pile-CC": 0.04704902306397411, + "StackExchange": 0.04482320836589982, + "Github": 0.025028931920813326, + "OOD_News_2024": 0.03655579294448856, + "OOD_CommonPile": 0.03793768003039683 + }, + "col_means": { + "Wikipedia_en": 0.03975134294100657, + "Pile-CC": 0.040010026133515765, + "StackExchange": 0.03513147646423291, + "Github": 0.03616451982676086, + "OOD_News_2024": 0.04035353071478851, + "OOD_CommonPile": 0.038167965935577676 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 90.88325953331656, + "read_pct": 7.4767248885812, + "residual_pct": 1.6400155781022396, + "fit_span": 0.022061572839609453, + "read_span": 0.005143666260065169, + "row_means": { + "Wikipedia_en": 0.03791836143203976, + "Pile-CC": 0.04706856338005444, + "StackExchange": 0.04467669497588994, + "Github": 0.025006990540444987, + "OOD_News_2024": 0.036770661156710786, + "OOD_arXiv_2023": 0.0390704206678223 + }, + "col_means": { + "Wikipedia_en": 0.04018326829817077, + "Pile-CC": 0.04016941991393213, + "StackExchange": 0.03538151454068375, + "Github": 0.036328672871631745, + "OOD_News_2024": 0.04052518080074892, + "OOD_arXiv_2023": 0.03792363572779491 + }, + "n": 6 + }, + "Pile-CC+StackExchange": { + "fit_pct": 92.30547577664414, + "read_pct": 5.759735551025383, + "residual_pct": 1.934788672330485, + "fit_span": 0.02192459852667525, + "read_span": 0.004722632888997734, + "row_means": { + "Wikipedia_en": 0.03877312741802867, + "USPTO_Backgrounds": 0.04786802697193774, + "Github": 0.02594342844526249, + "OOD_News_2024": 0.0372272417488232, + "OOD_arXiv_2023": 0.039494908269477526, + "OOD_CommonPile": 0.03822528263942028 + }, + "col_means": { + "Wikipedia_en": 0.038840126406608354, + "USPTO_Backgrounds": 0.03982380022967441, + "Github": 0.03510116734067668, + "OOD_News_2024": 0.03928831719353588, + "OOD_arXiv_2023": 0.03696949190451522, + "OOD_CommonPile": 0.03750911241793936 + }, + "n": 6 + }, + "Pile-CC+Github": { + "fit_pct": 78.22601673224129, + "read_pct": 17.505815035390924, + "residual_pct": 4.268168232367786, + "fit_span": 0.010374463763501908, + "read_span": 0.005503537996534122, + "row_means": { + "Wikipedia_en": 0.03863277883932891, + "USPTO_Backgrounds": 0.04753232666170569, + "StackExchange": 0.04522781798064157, + "OOD_News_2024": 0.03715786289820378, + "OOD_arXiv_2023": 0.03966021993956356, + "OOD_CommonPile": 0.038304709277926355 + }, + "col_means": { + "Wikipedia_en": 0.04207368673911939, + "USPTO_Backgrounds": 0.04338702805064343, + "StackExchange": 0.037883490054109306, + "OOD_News_2024": 0.042557142153641, + "OOD_arXiv_2023": 0.039975088038395536, + "OOD_CommonPile": 0.04063928056146122 + }, + "n": 6 + }, + "Pile-CC+OOD_News_2024": { + "fit_pct": 92.69565552535016, + "read_pct": 6.039823374018541, + "residual_pct": 1.2645211006312875, + "fit_span": 0.02178600354818627, + "read_span": 0.005265907035640401, + "row_means": { + "Wikipedia_en": 0.03752987085306948, + "USPTO_Backgrounds": 0.047010474802960674, + "StackExchange": 0.045052316780547025, + "Github": 0.025224471254774405, + "OOD_arXiv_2023": 0.039054536040768854, + "OOD_CommonPile": 0.037706522420221184 + }, + "col_means": { + "Wikipedia_en": 0.039941908908938915, + "USPTO_Backgrounds": 0.04118886991291893, + "StackExchange": 0.03592296287727853, + "Github": 0.03688354226372515, + "OOD_arXiv_2023": 0.03836995303691192, + "OOD_CommonPile": 0.03927095515256816 + }, + "n": 6 + }, + "Pile-CC+OOD_arXiv_2023": { + "fit_pct": 90.71979950869466, + "read_pct": 7.634466926636535, + "residual_pct": 1.6457335646687892, + "fit_span": 0.022213968621239203, + "read_span": 0.005827871038302505, + "row_means": { + "Wikipedia_en": 0.0383968081467578, + "USPTO_Backgrounds": 0.04738307603186479, + "StackExchange": 0.04526018176262302, + "Github": 0.02516910741062559, + "OOD_News_2024": 0.036808312889533464, + "OOD_CommonPile": 0.037904730928130445 + }, + "col_means": { + "Wikipedia_en": 0.0396548790184574, + "USPTO_Backgrounds": 0.041063462077915946, + "StackExchange": 0.03523559103961344, + "Github": 0.036430860946765516, + "OOD_News_2024": 0.040187828100924555, + "OOD_CommonPile": 0.03834959598585825 + }, + "n": 6 + }, + "Pile-CC+OOD_CommonPile": { + "fit_pct": 90.62675022807012, + "read_pct": 7.8865495143733275, + "residual_pct": 1.4867002575565618, + "fit_span": 0.022181180963309965, + "read_span": 0.005756615470077296, + "row_means": { + "Wikipedia_en": 0.038130943888487916, + "USPTO_Backgrounds": 0.047328346993567215, + "StackExchange": 0.04511366837261314, + "Github": 0.02514716603025725, + "OOD_News_2024": 0.03702318110175569, + "OOD_arXiv_2023": 0.039056860293365185 + }, + "col_means": { + "Wikipedia_en": 0.04008680437562159, + "USPTO_Backgrounds": 0.04124224458614158, + "StackExchange": 0.03548562911606428, + "Github": 0.036595013991636394, + "OOD_News_2024": 0.04035947818688496, + "OOD_arXiv_2023": 0.03803099642369758 + }, + "n": 6 + }, + "StackExchange+Github": { + "fit_pct": 89.32817149258182, + "read_pct": 7.622681009904747, + "residual_pct": 3.049147497513449, + "fit_span": 0.01037821581294001, + "read_span": 0.0032590835669974727, + "row_means": { + "Wikipedia_en": 0.03967460248903889, + "USPTO_Backgrounds": 0.04839787810773554, + "Pile-CC": 0.04834610152541211, + "OOD_News_2024": 0.03801966229479553, + "OOD_arXiv_2023": 0.040226135652995224, + "OOD_CommonPile": 0.0389612692873925 + }, + "col_means": { + "Wikipedia_en": 0.042784587782807644, + "USPTO_Backgrounds": 0.043516445571246246, + "Pile-CC": 0.04303453750175389, + "OOD_News_2024": 0.04327721567608899, + "OOD_arXiv_2023": 0.04025736200424877, + "OOD_CommonPile": 0.04075550082122424 + }, + "n": 6 + }, + "StackExchange+OOD_News_2024": { + "fit_pct": 94.57279616379387, + "read_pct": 4.319917957104046, + "residual_pct": 1.1072858791020976, + "fit_span": 0.021915042103285452, + "read_span": 0.004493066039867703, + "row_means": { + "Wikipedia_en": 0.038571694502779454, + "USPTO_Backgrounds": 0.047876026248990525, + "Pile-CC": 0.047392205932798484, + "Github": 0.025960984145705072, + "OOD_arXiv_2023": 0.039620451754200516, + "OOD_CommonPile": 0.03836308242968732 + }, + "col_means": { + "Wikipedia_en": 0.04065280995262718, + "USPTO_Backgrounds": 0.041318287433521755, + "Pile-CC": 0.04094872381926204, + "Github": 0.03682522139365405, + "OOD_arXiv_2023": 0.038652227002765153, + "OOD_CommonPile": 0.03938717541233119 + }, + "n": 6 + }, + "StackExchange+OOD_arXiv_2023": { + "fit_pct": 93.87631217339211, + "read_pct": 4.839335951027906, + "residual_pct": 1.2843518755799832, + "fit_span": 0.022343007176338386, + "read_span": 0.0048203395218243475, + "row_means": { + "Wikipedia_en": 0.03943863179646777, + "USPTO_Backgrounds": 0.04824862747789464, + "Pile-CC": 0.04803787047146923, + "Github": 0.025905620301556256, + "OOD_News_2024": 0.037670112286125206, + "OOD_CommonPile": 0.03856129093759659 + }, + "col_means": { + "Wikipedia_en": 0.040365780062145654, + "USPTO_Backgrounds": 0.041192879598518764, + "Pile-CC": 0.04055723566475704, + "Github": 0.036372540076694417, + "OOD_News_2024": 0.04090790162337254, + "OOD_CommonPile": 0.03846581624562128 + }, + "n": 6 + }, + "StackExchange+OOD_CommonPile": { + "fit_pct": 93.76170317509077, + "read_pct": 5.2406725818332065, + "residual_pct": 0.9976242430760142, + "fit_span": 0.02231021951840914, + "read_span": 0.004834968985179093, + "row_means": { + "Wikipedia_en": 0.039172767538197886, + "USPTO_Backgrounds": 0.04819389843959706, + "Pile-CC": 0.04805741078754956, + "Github": 0.025883678921187918, + "OOD_News_2024": 0.03788498049834743, + "OOD_arXiv_2023": 0.039622776006796846 + }, + "col_means": { + "Wikipedia_en": 0.04079770541930985, + "USPTO_Backgrounds": 0.041371662106744395, + "Pile-CC": 0.040716629445173395, + "Github": 0.0365366931215653, + "OOD_News_2024": 0.04107955170933295, + "OOD_arXiv_2023": 0.03831327038955081 + }, + "n": 6 + }, + "Github+OOD_News_2024": { + "fit_pct": 80.24195935005882, + "read_pct": 16.67283677372139, + "residual_pct": 3.085203876219787, + "fit_span": 0.009108980014678782, + "read_span": 0.005378066062392599, + "row_means": { + "Wikipedia_en": 0.03843134592407969, + "USPTO_Backgrounds": 0.047540325938758475, + "Pile-CC": 0.04721873216719056, + "StackExchange": 0.04524297863503711, + "OOD_arXiv_2023": 0.03978576342428655, + "OOD_CommonPile": 0.038442509068193395 + }, + "col_means": { + "Wikipedia_en": 0.04388637028513821, + "USPTO_Backgrounds": 0.044881515254490766, + "Pile-CC": 0.04421515373332012, + "StackExchange": 0.03950344919209817, + "OOD_arXiv_2023": 0.04165782313664547, + "OOD_CommonPile": 0.04251734355585304 + }, + "n": 6 + }, + "Github+OOD_arXiv_2023": { + "fit_pct": 79.78959397518167, + "read_pct": 17.096919581278616, + "residual_pct": 3.1134864435397085, + "fit_span": 0.010312193732156798, + "read_span": 0.005940030065054697, + "row_means": { + "Wikipedia_en": 0.039298283217768015, + "USPTO_Backgrounds": 0.04791292716766259, + "Pile-CC": 0.0478643967058613, + "StackExchange": 0.0454508436171131, + "OOD_News_2024": 0.037600733435505795, + "OOD_CommonPile": 0.038640717576102664 + }, + "col_means": { + "Wikipedia_en": 0.04359934039465669, + "USPTO_Backgrounds": 0.04475610741948778, + "Pile-CC": 0.04382366557881512, + "StackExchange": 0.038816077354433084, + "OOD_News_2024": 0.04417672658347766, + "OOD_CommonPile": 0.04159598438914314 + }, + "n": 6 + }, + "Github+OOD_CommonPile": { + "fit_pct": 78.33153134124592, + "read_pct": 19.179427920928127, + "residual_pct": 2.4890407378259503, + "fit_span": 0.010068335374213623, + "read_span": 0.0058687744968294875, + "row_means": { + "Wikipedia_en": 0.03903241895949813, + "USPTO_Backgrounds": 0.04785819812936501, + "Pile-CC": 0.04788393702194164, + "StackExchange": 0.04530433022710322, + "OOD_News_2024": 0.037815601647728014, + "OOD_arXiv_2023": 0.03978808767688288 + }, + "col_means": { + "Wikipedia_en": 0.04403126575182088, + "USPTO_Backgrounds": 0.04493488992771341, + "Pile-CC": 0.04398305935923148, + "StackExchange": 0.039066115430883924, + "OOD_News_2024": 0.04434837666943807, + "OOD_arXiv_2023": 0.04131886652343113 + }, + "n": 6 + }, + "OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 92.42390828541038, + "read_pct": 6.452502495109725, + "residual_pct": 1.1235892194799069, + "fit_span": 0.0222044121978494, + "read_span": 0.005702399104160975, + "row_means": { + "Wikipedia_en": 0.03819537523150858, + "USPTO_Backgrounds": 0.04739107530891757, + "Pile-CC": 0.046910501113247675, + "StackExchange": 0.045275342417018555, + "Github": 0.02518666311106817, + "OOD_CommonPile": 0.038042530718397485 + }, + "col_means": { + "Wikipedia_en": 0.04146756256447622, + "USPTO_Backgrounds": 0.042557949281763284, + "Pile-CC": 0.041737851896323265, + "StackExchange": 0.03685555017760231, + "Github": 0.03815491499974289, + "OOD_CommonPile": 0.04022765898025008 + }, + "n": 6 + }, + "OOD_News_2024+OOD_CommonPile": { + "fit_pct": 92.21951857698689, + "read_pct": 6.671225766742414, + "residual_pct": 1.1092556562707094, + "fit_span": 0.02217162453992016, + "read_span": 0.005631143535935766, + "row_means": { + "Wikipedia_en": 0.0379295109732387, + "USPTO_Backgrounds": 0.04733634627061999, + "Pile-CC": 0.04693004142932801, + "StackExchange": 0.04512882902700868, + "Github": 0.02516472173069983, + "OOD_arXiv_2023": 0.039182403778088175 + }, + "col_means": { + "Wikipedia_en": 0.041899487921640416, + "USPTO_Backgrounds": 0.042736731789988915, + "Pile-CC": 0.04189724567673962, + "StackExchange": 0.03710558825405315, + "Github": 0.03831906804461377, + "OOD_arXiv_2023": 0.039713731521947515 + }, + "n": 6 + }, + "OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 91.13337272113088, + "read_pct": 7.871049745782995, + "residual_pct": 0.9955775330861203, + "fit_span": 0.022599589612973096, + "read_span": 0.006193107538597864, + "row_means": { + "Wikipedia_en": 0.03879644826692701, + "USPTO_Backgrounds": 0.04770894749952411, + "Pile-CC": 0.04757570596799875, + "StackExchange": 0.04533669400908467, + "Github": 0.025109357886551015, + "OOD_News_2024": 0.037466051639057696 + }, + "col_means": { + "Wikipedia_en": 0.04161245803115889, + "USPTO_Backgrounds": 0.04261132395498592, + "Pile-CC": 0.04150575752223462, + "StackExchange": 0.03641821641638806, + "Github": 0.03786638672765413, + "OOD_News_2024": 0.04197906261672162 + }, + "n": 6 + } + }, + "n_l2o_fit_gt_read": "28/28", + "l2o_failures": [], + "l2o_worst_split": { + "dropped": "Github+OOD_CommonPile", + "fit_pct": 78.33153134124592, + "read_pct": 19.179427920928127, + "residual_pct": 2.4890407378259503, + "fit_span": 0.010068335374213623, + "read_span": 0.0058687744968294875, + "row_means": { + "Wikipedia_en": 0.03903241895949813, + "USPTO_Backgrounds": 0.04785819812936501, + "Pile-CC": 0.04788393702194164, + "StackExchange": 0.04530433022710322, + "OOD_News_2024": 0.037815601647728014, + "OOD_arXiv_2023": 0.03978808767688288 + }, + "col_means": { + "Wikipedia_en": 0.04403126575182088, + "USPTO_Backgrounds": 0.04493488992771341, + "Pile-CC": 0.04398305935923148, + "StackExchange": 0.039066115430883924, + "OOD_News_2024": 0.04434837666943807, + "OOD_arXiv_2023": 0.04131886652343113 + }, + "n": 6 + }, + "fit_axis_leverage_point": "Github", + "read_axis_leverage_point": "StackExchange", + "drop_fit_leverage_only": { + "fit_pct": 81.04881620398683, + "read_pct": 15.567785798307998, + "residual_pct": 3.383397997705165, + "fit_span": 0.010373391022550915, + "read_span": 0.005564901774882204, + "row_means": { + "Wikipedia_en": 0.03886378188100126, + "USPTO_Backgrounds": 0.04773010446483062, + "Pile-CC": 0.04759648347251295, + "StackExchange": 0.04528577456856885, + "OOD_News_2024": 0.037356713442279706, + "OOD_arXiv_2023": 0.039837152181592374, + "OOD_CommonPile": 0.038538662659832176 + }, + "col_means": { + "Wikipedia_en": 0.04315905773287846, + "USPTO_Backgrounds": 0.04419343255770703, + "Pile-CC": 0.043405819872379424, + "StackExchange": 0.03862853078282483, + "OOD_News_2024": 0.04360083488947047, + "OOD_arXiv_2023": 0.04083421890039943, + "OOD_CommonPile": 0.0413867779349583 + }, + "n": 7 + }, + "drop_read_leverage_only": { + "fit_pct": 93.33692623435269, + "read_pct": 5.156511202775666, + "residual_pct": 1.5065625628716504, + "fit_span": 0.022061346833121092, + "read_span": 0.004746862740758508, + "row_means": { + "Wikipedia_en": 0.03898408066274391, + "USPTO_Backgrounds": 0.048017847587886665, + "Pile-CC": 0.047745175271605454, + "Github": 0.025956500754765573, + "OOD_News_2024": 0.03741618102852492, + "OOD_arXiv_2023": 0.03969545646437577, + "OOD_CommonPile": 0.03847058268396982 + }, + "col_means": { + "Wikipedia_en": 0.04038743459072614, + "USPTO_Backgrounds": 0.04113923728259073, + "Pile-CC": 0.04060602280318678, + "Github": 0.03639237454183222, + "OOD_News_2024": 0.04079898492366608, + "OOD_arXiv_2023": 0.03825799364278773, + "OOD_CommonPile": 0.038703776669082424 + }, + "n": 7 + }, + "asymmetry_note": "dropping the fit axis's weakest member (Github) moves fit share to 81.0%; dropping the read axis's weakest member (StackExchange) moves it to 93.3%. The paper shows only the first." + }, + "min": { + "full": { + "fit_pct": 73.41700738536099, + "read_pct": 22.67267017800786, + "residual_pct": 3.910322436631158, + "fit_span": 0.04321326263662842, + "read_span": 0.019495287009825307, + "row_means": { + "Wikipedia_en": 0.12989506709337648, + "USPTO_Backgrounds": 0.15617140837841564, + "Pile-CC": 0.14402559753248675, + "StackExchange": 0.14620193947727483, + "Github": 0.11295814574178722, + "OOD_News_2024": 0.13518428391156098, + "OOD_arXiv_2023": 0.14045638353305145, + "OOD_CommonPile": 0.13989721588376494 + }, + "col_means": { + "Wikipedia_en": 0.13891349073706402, + "USPTO_Backgrounds": 0.14661414006517992, + "Pile-CC": 0.14461656864473804, + "StackExchange": 0.13126294978428632, + "Github": 0.1271188530553546, + "OOD_News_2024": 0.14362420086593677, + "OOD_arXiv_2023": 0.14041273188518566, + "OOD_CommonPile": 0.1322271065139729 + }, + "n": 8 + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 72.21409286150445, + "read_pct": 23.71515132910453, + "residual_pct": 4.070755809391016, + "fit_span": 0.042136523452779615, + "read_span": 0.01932690815911406, + "row_means": { + "USPTO_Backgrounds": 0.15578141137366258, + "Pile-CC": 0.14378966328111434, + "StackExchange": 0.14641918519304858, + "Github": 0.11364488792088297, + "OOD_News_2024": 0.13500802957467617, + "OOD_arXiv_2023": 0.1400439641394076, + "OOD_CommonPile": 0.13976089046114967 + }, + "col_means": { + "USPTO_Backgrounds": 0.1478670597372074, + "Pile-CC": 0.1455141710294854, + "StackExchange": 0.13293830791990907, + "Github": 0.12854015157809334, + "OOD_News_2024": 0.14454323585248655, + "OOD_arXiv_2023": 0.1415780169711936, + "OOD_CommonPile": 0.1334670888555665 + }, + "n": 7 + }, + "USPTO_Backgrounds": { + "fit_pct": 69.97297180223329, + "read_pct": 24.704987971694027, + "residual_pct": 5.322040226072688, + "fit_span": 0.032951310528294434, + "read_span": 0.017268920689821263, + "row_means": { + "Wikipedia_en": 0.12875954776943202, + "Pile-CC": 0.14263539841428163, + "StackExchange": 0.144902804916695, + "Github": 0.11195149438840056, + "OOD_News_2024": 0.13389897656937438, + "OOD_arXiv_2023": 0.1391904472062985, + "OOD_CommonPile": 0.13872995732559099 + }, + "col_means": { + "Wikipedia_en": 0.13605807692640356, + "Pile-CC": 0.14185798933049518, + "StackExchange": 0.12800796924838945, + "Github": 0.12458906864067391, + "OOD_News_2024": 0.1410433505587871, + "OOD_arXiv_2023": 0.1379470965011962, + "OOD_CommonPile": 0.1305650753841277 + }, + "n": 7 + }, + "Pile-CC": { + "fit_pct": 75.22745446862925, + "read_pct": 20.50190993884228, + "residual_pct": 4.270635592528474, + "fit_span": 0.042820826634055106, + "read_span": 0.01925233122375275, + "row_means": { + "Wikipedia_en": 0.12868959782792935, + "USPTO_Backgrounds": 0.15506352045469815, + "StackExchange": 0.14566133950378687, + "Github": 0.11224269382064304, + "OOD_News_2024": 0.1340909593221214, + "OOD_arXiv_2023": 0.13940309948982701, + "OOD_CommonPile": 0.1387944360367126 + }, + "col_means": { + "Wikipedia_en": 0.1379472555149169, + "USPTO_Backgrounds": 0.14559373273735954, + "StackExchange": 0.13081471575867562, + "Github": 0.1263414015136068, + "OOD_News_2024": 0.14262957148076524, + "OOD_arXiv_2023": 0.13970409101023087, + "OOD_CommonPile": 0.13091487844016345 + }, + "n": 7 + }, + "StackExchange": { + "fit_pct": 74.77416486260654, + "read_pct": 22.702208386947053, + "residual_pct": 2.5236267504464065, + "fit_span": 0.043067101315255193, + "read_span": 0.019554403409480114, + "row_means": { + "Wikipedia_en": 0.13137501341601213, + "USPTO_Backgrounds": 0.1564747790702515, + "Pile-CC": 0.14540059889947612, + "Github": 0.11340767775499631, + "OOD_News_2024": 0.13723920306989124, + "OOD_arXiv_2023": 0.14105494373019725, + "OOD_CommonPile": 0.1402442828973844 + }, + "col_means": { + "Wikipedia_en": 0.1380895294899505, + "USPTO_Backgrounds": 0.1453738913028723, + "Pile-CC": 0.14384948712374482, + "Github": 0.12581948789339217, + "OOD_News_2024": 0.14296236081669728, + "OOD_arXiv_2023": 0.13881822755589845, + "OOD_CommonPile": 0.13028351465565344 + }, + "n": 7 + }, + "Github": { + "fit_pct": 58.596571037346294, + "read_pct": 34.84817420406899, + "residual_pct": 6.555254758584715, + "fit_span": 0.026079021388339618, + "read_span": 0.016088034097282655, + "row_means": { + "Wikipedia_en": 0.13171296762154688, + "USPTO_Backgrounds": 0.1577919890098865, + "Pile-CC": 0.14566339520175778, + "StackExchange": 0.14762872951844383, + "OOD_News_2024": 0.13667049391993455, + "OOD_arXiv_2023": 0.14240232722626792, + "OOD_CommonPile": 0.14166051502975205 + }, + "col_means": { + "Wikipedia_en": 0.14330813934405645, + "USPTO_Backgrounds": 0.15041548790084938, + "Pile-CC": 0.14842374856687254, + "StackExchange": 0.13432745380356673, + "OOD_News_2024": 0.14765196423977614, + "OOD_arXiv_2023": 0.14445654644320408, + "OOD_CommonPile": 0.13494707722926425 + }, + "n": 7 + }, + "OOD_News_2024": { + "fit_pct": 75.8043220820044, + "read_pct": 20.54338347833448, + "residual_pct": 3.6522944396611114, + "fit_span": 0.04277797218944346, + "read_span": 0.019508810660668785, + "row_means": { + "Wikipedia_en": 0.1288527972552748, + "USPTO_Backgrounds": 0.1553830162873344, + "Pile-CC": 0.1430883105282509, + "StackExchange": 0.1459083478010836, + "Github": 0.11260504409789093, + "OOD_arXiv_2023": 0.13963580267237766, + "OOD_CommonPile": 0.13888340706212654 + }, + "col_means": { + "Wikipedia_en": 0.13926998023239393, + "USPTO_Backgrounds": 0.1469616693163675, + "Pile-CC": 0.1448707133028952, + "StackExchange": 0.13275767835300595, + "Github": 0.12745285865569872, + "OOD_arXiv_2023": 0.14054039034637666, + "OOD_CommonPile": 0.13250343549760088 + }, + "n": 7 + }, + "OOD_arXiv_2023": { + "fit_pct": 74.1434371562919, + "read_pct": 22.097545670856885, + "residual_pct": 3.759017172851229, + "fit_span": 0.04287713592842457, + "read_span": 0.019068447991259507, + "row_means": { + "Wikipedia_en": 0.12955782863769738, + "USPTO_Backgrounds": 0.155957012493459, + "Pile-CC": 0.1438330803214321, + "StackExchange": 0.14543446480400032, + "Github": 0.11307987656503443, + "OOD_News_2024": 0.13456502123366273, + "OOD_CommonPile": 0.13985437491524313 + }, + "col_means": { + "Wikipedia_en": 0.13828065808685053, + "USPTO_Backgrounds": 0.14622788324301678, + "Pile-CC": 0.14415759676032597, + "StackExchange": 0.13054816230303712, + "Github": 0.12715943525175727, + "OOD_News_2024": 0.14325616533853233, + "OOD_CommonPile": 0.13265175798700915 + }, + "n": 7 + }, + "OOD_CommonPile": { + "fit_pct": 77.20909139763098, + "read_pct": 20.313039456100714, + "residual_pct": 2.4778691462682962, + "fit_span": 0.04500458402529596, + "read_span": 0.01934977030706786, + "row_means": { + "Wikipedia_en": 0.13080190094631342, + "USPTO_Backgrounds": 0.1579299918006337, + "Pile-CC": 0.14439886817560782, + "StackExchange": 0.14625475232799848, + "Github": 0.11292540777533773, + "OOD_News_2024": 0.1358830668091301, + "OOD_arXiv_2023": 0.14205664600881318 + }, + "col_means": { + "Wikipedia_en": 0.1386366331506343, + "USPTO_Backgrounds": 0.14640644210435097, + "Pile-CC": 0.14418798204925326, + "StackExchange": 0.13037655021226596, + "Github": 0.1270566717972831, + "OOD_News_2024": 0.1431428184703229, + "OOD_arXiv_2023": 0.14044353605972396 + }, + "n": 7 + } + }, + "n_loo_fit_gt_read": "8/8", + "leave_two_out": { + "Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 69.70911059376193, + "read_pct": 24.806620056599378, + "residual_pct": 5.484269349638686, + "fit_span": 0.03235481745321993, + "read_span": 0.016619809378904338, + "row_means": { + "Pile-CC": 0.14212844193464627, + "StackExchange": 0.14493973582500108, + "Github": 0.11258491837178115, + "OOD_News_2024": 0.13347912861931102, + "OOD_arXiv_2023": 0.13849830185925519, + "OOD_CommonPile": 0.13837636790617747 + }, + "col_means": { + "Pile-CC": 0.14244542889365996, + "StackExchange": 0.12942005698396652, + "Github": 0.12582561951475563, + "OOD_News_2024": 0.1416854163252369, + "OOD_arXiv_2023": 0.13889565653754055, + "OOD_CommonPile": 0.13173471626101269 + }, + "n": 6 + }, + "Wikipedia_en+Pile-CC": { + "fit_pct": 74.27637442756014, + "read_pct": 21.35911579521616, + "residual_pct": 4.364509777223703, + "fit_span": 0.041499224919135935, + "read_span": 0.019015396600244205, + "row_means": { + "USPTO_Backgrounds": 0.15442387596186666, + "StackExchange": 0.14582469284327493, + "Github": 0.11292465104273072, + "OOD_News_2024": 0.13370310849751588, + "OOD_arXiv_2023": 0.13874639619003842, + "OOD_CommonPile": 0.1384515930691527 + }, + "col_means": { + "USPTO_Backgrounds": 0.14688540446675485, + "StackExchange": 0.13269459457930038, + "Github": 0.12787000786651065, + "OOD_News_2024": 0.14353600740087805, + "OOD_arXiv_2023": 0.14094548346474767, + "OOD_CommonPile": 0.1321428198263877 + }, + "n": 6 + }, + "Wikipedia_en+StackExchange": { + "fit_pct": 73.4016665053439, + "read_pct": 24.123595386290692, + "residual_pct": 2.4747381083654205, + "fit_span": 0.04178654538053607, + "read_span": 0.019367814150259455, + "row_means": { + "USPTO_Backgrounds": 0.15607034434667894, + "Pile-CC": 0.1453545091673732, + "Github": 0.11428379896614287, + "OOD_News_2024": 0.1373760595365807, + "OOD_arXiv_2023": 0.14067354780380373, + "OOD_CommonPile": 0.14014308107326978 + }, + "col_means": { + "USPTO_Backgrounds": 0.14662892279318637, + "Pile-CC": 0.14476884298578457, + "Github": 0.12726110864292692, + "OOD_News_2024": 0.14392426162613212, + "OOD_arXiv_2023": 0.13991197610135983, + "OOD_CommonPile": 0.13140622874445937 + }, + "n": 6 + }, + "Wikipedia_en+Github": { + "fit_pct": 47.1372449404656, + "read_pct": 43.845072157538624, + "residual_pct": 9.017682901995776, + "fit_span": 0.020894523747955224, + "read_span": 0.01571799652581965, + "row_means": { + "USPTO_Backgrounds": 0.15760708927625308, + "Pile-CC": 0.14566110485336847, + "StackExchange": 0.14811998119370806, + "OOD_News_2024": 0.13671256552829786, + "OOD_arXiv_2023": 0.14224549521588617, + "OOD_CommonPile": 0.1417953518943654 + }, + "col_means": { + "USPTO_Backgrounds": 0.15251078549082633, + "Pile-CC": 0.15010548133610024, + "StackExchange": 0.13679278896500668, + "OOD_News_2024": 0.1493954656197241, + "OOD_arXiv_2023": 0.1464900148032164, + "OOD_CommonPile": 0.1368470517470053 + }, + "n": 6 + }, + "Wikipedia_en+OOD_News_2024": { + "fit_pct": 74.58834495659279, + "read_pct": 21.867082182884847, + "residual_pct": 3.5445728605223614, + "fit_span": 0.04144922806708902, + "read_span": 0.01931462260997957, + "row_means": { + "USPTO_Backgrounds": 0.15479662109994227, + "Pile-CC": 0.14265683940094379, + "StackExchange": 0.14611286919012115, + "Github": 0.11334739303285325, + "OOD_arXiv_2023": 0.13901788323634753, + "OOD_CommonPile": 0.13855539259880228 + }, + "col_means": { + "USPTO_Backgrounds": 0.14848133047559747, + "Pile-CC": 0.14596027352812666, + "StackExchange": 0.13496138427268575, + "Github": 0.1291667078656179, + "OOD_arXiv_2023": 0.14192116602358443, + "OOD_CommonPile": 0.13399613639339805 + }, + "n": 6 + }, + "Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 73.1737398117515, + "read_pct": 22.973417055456252, + "residual_pct": 3.852843132792238, + "fit_span": 0.04156491909590035, + "read_span": 0.01880086616233545, + "row_means": { + "USPTO_Backgrounds": 0.155466283340421, + "Pile-CC": 0.1435257374929885, + "StackExchange": 0.14556000569352398, + "Github": 0.11390136424452066, + "OOD_News_2024": 0.13425618072764742, + "OOD_CommonPile": 0.13968818842743833 + }, + "col_means": { + "USPTO_Backgrounds": 0.14762524672335497, + "Pile-CC": 0.14512830422846257, + "StackExchange": 0.13238361554772213, + "Github": 0.12882438056101952, + "OOD_News_2024": 0.14426703356827297, + "OOD_CommonPile": 0.1341691792977077 + }, + "n": 6 + }, + "Wikipedia_en+OOD_CommonPile": { + "fit_pct": 76.65092905025313, + "read_pct": 20.941908114529095, + "residual_pct": 2.40716283521778, + "fit_span": 0.0440469418755836, + "read_span": 0.019129075530778472, + "row_means": { + "USPTO_Backgrounds": 0.1577680925321248, + "Pile-CC": 0.14418582332286017, + "StackExchange": 0.14651700780485516, + "Github": 0.11372115065654119, + "OOD_News_2024": 0.13579390056569268, + "OOD_arXiv_2023": 0.14184220046218898 + }, + "col_means": { + "USPTO_Backgrounds": 0.1478335653949115, + "Pile-CC": 0.14516375373221108, + "StackExchange": 0.13218340144182245, + "Github": 0.12870448986413302, + "OOD_News_2024": 0.144134795555362, + "OOD_arXiv_2023": 0.14180816935582294 + }, + "n": 6 + }, + "USPTO_Backgrounds+Pile-CC": { + "fit_pct": 72.42964843924064, + "read_pct": 21.054112387579057, + "residual_pct": 6.516239173180304, + "fit_span": 0.03310655726602785, + "read_span": 0.01619239678537404, + "row_means": { + "Wikipedia_en": 0.1271639137390863, + "StackExchange": 0.14405558252086242, + "Github": 0.11094902525483458, + "OOD_News_2024": 0.13240921332466382, + "OOD_arXiv_2023": 0.1377506264347445, + "OOD_CommonPile": 0.13724883774433422 + }, + "col_means": { + "Wikipedia_en": 0.13445490019878856, + "StackExchange": 0.12694253279586082, + "Github": 0.12326041110618799, + "OOD_News_2024": 0.13945280789156203, + "OOD_arXiv_2023": 0.136709409583084, + "OOD_CommonPile": 0.12875713744304246 + }, + "n": 6 + }, + "USPTO_Backgrounds+StackExchange": { + "fit_pct": 69.88418071540721, + "read_pct": 26.35581157270119, + "residual_pct": 3.7600077118916078, + "fit_span": 0.03169969364448831, + "read_span": 0.017851785787691685, + "row_means": { + "Wikipedia_en": 0.13029689859184954, + "Pile-CC": 0.14400786682273503, + "Github": 0.11230817317824672, + "OOD_News_2024": 0.13608216436372864, + "OOD_arXiv_2023": 0.1396777780485098, + "OOD_CommonPile": 0.13894032574845133 + }, + "col_means": { + "Wikipedia_en": 0.13462088650299442, + "Pile-CC": 0.14050329767029596, + "Github": 0.12265151188260427, + "OOD_News_2024": 0.13984106211681607, + "OOD_arXiv_2023": 0.13567590221969617, + "OOD_CommonPile": 0.12802054636111415 + }, + "n": 6 + }, + "USPTO_Backgrounds+Github": { + "fit_pct": 42.0114078749595, + "read_pct": 47.61343085472334, + "residual_pct": 10.375161270317161, + "fit_span": 0.01565969237298878, + "read_span": 0.014799208839044536, + "row_means": { + "Wikipedia_en": 0.13069117849830675, + "Pile-CC": 0.1443144625087303, + "StackExchange": 0.14635087087129553, + "OOD_News_2024": 0.1354186703554458, + "OOD_arXiv_2023": 0.14124972546059225, + "OOD_CommonPile": 0.14059259656954695 + }, + "col_means": { + "Wikipedia_en": 0.140709264666118, + "Pile-CC": 0.14583993602061163, + "StackExchange": 0.1310407271815671, + "OOD_News_2024": 0.1453122661104081, + "OOD_arXiv_2023": 0.14225394092155275, + "OOD_CommonPile": 0.13346136936366007 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 74.93567784036671, + "read_pct": 19.95982228631515, + "residual_pct": 5.104499873318141, + "fit_span": 0.03297199162275151, + "read_span": 0.017137617107342795, + "row_means": { + "Wikipedia_en": 0.12735431307098932, + "Pile-CC": 0.14131019705630563, + "StackExchange": 0.14434375886770862, + "Github": 0.1113717672449571, + "OOD_arXiv_2023": 0.1380221134810536, + "OOD_CommonPile": 0.13735263727398384 + }, + "col_means": { + "Wikipedia_en": 0.13599807903584507, + "Pile-CC": 0.14169472821263804, + "StackExchange": 0.1292093224892462, + "Github": 0.12455711110529524, + "OOD_arXiv_2023": 0.13768509214192076, + "OOD_CommonPile": 0.1306104540100528 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 70.82400576139757, + "read_pct": 24.290616152385706, + "residual_pct": 4.8853780862167255, + "fit_span": 0.03186515691448692, + "read_span": 0.016647975112277058, + "row_means": { + "Wikipedia_en": 0.12817684968381568, + "Pile-CC": 0.14217909514835037, + "StackExchange": 0.14379089537111145, + "Github": 0.11192573845662453, + "OOD_News_2024": 0.13296228555479536, + "OOD_CommonPile": 0.13848543310261988 + }, + "col_means": { + "Wikipedia_en": 0.13484386986604444, + "Pile-CC": 0.14086275891297395, + "StackExchange": 0.12663155376428256, + "Github": 0.12421478380069689, + "OOD_News_2024": 0.14018383405895696, + "OOD_CommonPile": 0.13078349691436247 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 72.08814955375458, + "read_pct": 24.241553046524665, + "residual_pct": 3.670297399720758, + "fit_span": 0.03300237261379761, + "read_span": 0.016803315312912057, + "row_means": { + "Wikipedia_en": 0.12962826737720107, + "Pile-CC": 0.142839180978222, + "StackExchange": 0.14474789748244266, + "Github": 0.11174552486864504, + "OOD_News_2024": 0.1345000053928406, + "OOD_arXiv_2023": 0.14084643070689506 + }, + "col_means": { + "Wikipedia_en": 0.1352591741071255, + "Pile-CC": 0.14089820841672243, + "StackExchange": 0.12643133965838288, + "Github": 0.12409489310381037, + "OOD_News_2024": 0.14005159604604597, + "OOD_arXiv_2023": 0.13757209547415927 + }, + "n": 6 + }, + "Pile-CC+StackExchange": { + "fit_pct": 75.2683107770375, + "read_pct": 21.959603367863302, + "residual_pct": 2.7720858550991916, + "fit_span": 0.04258489909202412, + "read_span": 0.019280807725671256, + "row_means": { + "Wikipedia_en": 0.13021529032676307, + "USPTO_Backgrounds": 0.1552328049412204, + "Github": 0.11264790584919629, + "OOD_News_2024": 0.13630614424193346, + "OOD_arXiv_2023": 0.13992587237929305, + "OOD_CommonPile": 0.13901555091142653 + }, + "col_means": { + "Wikipedia_en": 0.13682492818959333, + "USPTO_Backgrounds": 0.14397670796003054, + "Github": 0.12469590023435928, + "OOD_News_2024": 0.14169165319245722, + "OOD_arXiv_2023": 0.13772572914690331, + "OOD_CommonPile": 0.12842864992648914 + }, + "n": 6 + }, + "Pile-CC+Github": { + "fit_pct": 61.13057575432661, + "read_pct": 32.15027784254386, + "residual_pct": 6.71914640312953, + "fit_span": 0.02615997963757427, + "read_span": 0.015989097728635437, + "row_means": { + "Wikipedia_en": 0.1306095702332203, + "USPTO_Backgrounds": 0.15676954987079458, + "StackExchange": 0.14723582788956938, + "OOD_News_2024": 0.13564265023365063, + "OOD_arXiv_2023": 0.1414978197913755, + "OOD_CommonPile": 0.14066782173252215 + }, + "col_means": { + "Wikipedia_en": 0.1429133063527169, + "USPTO_Backgrounds": 0.1498585706576705, + "StackExchange": 0.13431526477690098, + "OOD_News_2024": 0.1471628571860492, + "OOD_arXiv_2023": 0.1443037678487599, + "OOD_CommonPile": 0.13386947292903506 + }, + "n": 6 + }, + "Pile-CC+OOD_News_2024": { + "fit_pct": 78.70428343626931, + "read_pct": 17.49805926335739, + "residual_pct": 3.7976573003732943, + "fit_span": 0.04224758177857707, + "read_span": 0.0192276161853914, + "row_means": { + "Wikipedia_en": 0.12727270480590286, + "USPTO_Backgrounds": 0.15395908169448375, + "StackExchange": 0.14522871588598246, + "Github": 0.11171149991590668, + "OOD_arXiv_2023": 0.13827020781183685, + "OOD_CommonPile": 0.13742786243695904 + }, + "col_means": { + "Wikipedia_en": 0.13820212072244398, + "USPTO_Backgrounds": 0.14582911564244164, + "StackExchange": 0.13248386008458005, + "Github": 0.12660149945705024, + "OOD_arXiv_2023": 0.1397349190691279, + "OOD_CommonPile": 0.13101855757542782 + }, + "n": 6 + }, + "Pile-CC+OOD_arXiv_2023": { + "fit_pct": 76.32740589735464, + "read_pct": 19.434271366595024, + "residual_pct": 4.238322736050335, + "fit_span": 0.042363272807388386, + "read_span": 0.01871385973774725, + "row_means": { + "Wikipedia_en": 0.12809524141872922, + "USPTO_Backgrounds": 0.15462874393496248, + "StackExchange": 0.1446758523893853, + "Github": 0.1122654711275741, + "OOD_News_2024": 0.1331862654330002, + "OOD_CommonPile": 0.13856065826559508 + }, + "col_means": { + "Wikipedia_en": 0.13704791155264334, + "USPTO_Backgrounds": 0.14497303189019914, + "StackExchange": 0.12990609135961642, + "Github": 0.1262591721524519, + "OOD_News_2024": 0.1420344251345981, + "OOD_CommonPile": 0.13119160047973746 + }, + "n": 6 + }, + "Pile-CC+OOD_CommonPile": { + "fit_pct": 79.0793245863189, + "read_pct": 18.16182035962983, + "residual_pct": 2.758855054051259, + "fit_span": 0.044845295587071676, + "read_span": 0.0190420691061903, + "row_means": { + "Wikipedia_en": 0.1295466591121146, + "USPTO_Backgrounds": 0.1569305531266663, + "StackExchange": 0.1456328545007165, + "Github": 0.11208525753959461, + "OOD_News_2024": 0.13472398527104545, + "OOD_arXiv_2023": 0.1410945250376783 + }, + "col_means": { + "Wikipedia_en": 0.13746321579372442, + "USPTO_Backgrounds": 0.14518135056175568, + "StackExchange": 0.12970587725371674, + "Github": 0.12613928145556538, + "OOD_News_2024": 0.14190218712168712, + "OOD_arXiv_2023": 0.1396219224013664 + }, + "n": 6 + }, + "StackExchange+Github": { + "fit_pct": 64.19576519470036, + "read_pct": 33.101664859298566, + "residual_pct": 2.7025699460010824, + "fit_span": 0.024673463169623322, + "read_span": 0.016469207136995262, + "row_means": { + "Wikipedia_en": 0.13374255508598354, + "USPTO_Backgrounds": 0.15841601825560686, + "Pile-CC": 0.1475405297414572, + "OOD_News_2024": 0.13931560127271547, + "OOD_arXiv_2023": 0.1434249714051408, + "OOD_CommonPile": 0.14235930973663927 + }, + "col_means": { + "Wikipedia_en": 0.14307929265692276, + "USPTO_Backgrounds": 0.14960208898410202, + "Pile-CC": 0.14816335011273624, + "OOD_News_2024": 0.14755111141130328, + "OOD_arXiv_2023": 0.14327026048537206, + "OOD_CommonPile": 0.13313288184710675 + }, + "n": 6 + }, + "StackExchange+OOD_News_2024": { + "fit_pct": 75.68119691496648, + "read_pct": 21.501457192553758, + "residual_pct": 2.8173458924797576, + "fit_span": 0.042534902239977204, + "read_span": 0.01958003373540662, + "row_means": { + "Wikipedia_en": 0.13040568965866609, + "USPTO_Backgrounds": 0.15560555007929602, + "Pile-CC": 0.14453626428903255, + "Github": 0.11307064783931882, + "OOD_arXiv_2023": 0.14019735942560213, + "OOD_CommonPile": 0.13911935044107615 + }, + "col_means": { + "Wikipedia_en": 0.13836810702664984, + "USPTO_Backgrounds": 0.14557263396887316, + "Pile-CC": 0.14401814230476265, + "Github": 0.12599260023346653, + "OOD_arXiv_2023": 0.13870141170574007, + "OOD_CommonPile": 0.1302819664934995 + }, + "n": 6 + }, + "StackExchange+OOD_arXiv_2023": { + "fit_pct": 75.66308318285165, + "read_pct": 21.88171045120262, + "residual_pct": 2.4552063659457226, + "fit_span": 0.04265059326878852, + "read_span": 0.019066277287762473, + "row_means": { + "Wikipedia_en": 0.13122822627149247, + "USPTO_Backgrounds": 0.15627521231977476, + "Pile-CC": 0.14540516238107726, + "Github": 0.11362461905098624, + "OOD_News_2024": 0.136859216472065, + "OOD_CommonPile": 0.1402521462697122 + }, + "col_means": { + "Wikipedia_en": 0.1372138978568492, + "USPTO_Backgrounds": 0.14471655021663066, + "Pile-CC": 0.14318617300509856, + "Github": 0.12565027292886818, + "OOD_News_2024": 0.14242267935985217, + "OOD_CommonPile": 0.13045500939780916 + }, + "n": 6 + }, + "StackExchange+OOD_CommonPile": { + "fit_pct": 81.17827115221291, + "read_pct": 17.532211187475802, + "residual_pct": 1.2895176603112795, + "fit_span": 0.04513261604847181, + "read_span": 0.019394486656205523, + "row_means": { + "Wikipedia_en": 0.13267964396487783, + "USPTO_Backgrounds": 0.15857702151147857, + "Pile-CC": 0.14606524821094893, + "Github": 0.11344440546300676, + "OOD_News_2024": 0.13839693631011027, + "OOD_arXiv_2023": 0.1430216766514436 + }, + "col_means": { + "Wikipedia_en": 0.13762920209793028, + "USPTO_Backgrounds": 0.1449248688881872, + "Pile-CC": 0.14322162250884704, + "Github": 0.12553038223198168, + "OOD_News_2024": 0.1422904413469412, + "OOD_arXiv_2023": 0.13858841503797859 + }, + "n": 6 + }, + "Github+OOD_News_2024": { + "fit_pct": 60.09234275665318, + "read_pct": 33.53263046705697, + "residual_pct": 6.37502677628985, + "fit_span": 0.026342325443746872, + "read_span": 0.015731707170467707, + "row_means": { + "Wikipedia_en": 0.13079996956512333, + "USPTO_Backgrounds": 0.1571422950088702, + "Pile-CC": 0.14484285997502783, + "StackExchange": 0.1475240042364156, + "OOD_arXiv_2023": 0.14176930683768457, + "OOD_CommonPile": 0.14077162126217174 + }, + "col_means": { + "Wikipedia_en": 0.14445648518977341, + "USPTO_Backgrounds": 0.15145449666651312, + "Pile-CC": 0.14935478065507832, + "StackExchange": 0.13658205447028632, + "OOD_arXiv_2023": 0.14527945040759666, + "OOD_CommonPile": 0.1357227894960454 + }, + "n": 6 + }, + "Github+OOD_arXiv_2023": { + "fit_pct": 59.36821556956008, + "read_pct": 34.438925881511416, + "residual_pct": 6.192858548928506, + "fit_span": 0.02618945107139925, + "read_span": 0.016594127168947925, + "row_means": { + "Wikipedia_en": 0.13162250617794968, + "USPTO_Backgrounds": 0.15781195724934893, + "Pile-CC": 0.14571175806707254, + "StackExchange": 0.14697114073981843, + "OOD_News_2024": 0.1361957224637822, + "OOD_CommonPile": 0.1419044170908078 + }, + "col_means": { + "Wikipedia_en": 0.14330227601997278, + "USPTO_Backgrounds": 0.15059841291427062, + "Pile-CC": 0.14852281135541423, + "StackExchange": 0.1340042857453227, + "OOD_News_2024": 0.14789388335344417, + "OOD_CommonPile": 0.13589583240035508 + }, + "n": 6 + }, + "Github+OOD_CommonPile": { + "fit_pct": 67.95813838086735, + "read_pct": 28.340415884299983, + "residual_pct": 3.7014457348326655, + "fit_span": 0.02703984256971767, + "read_span": 0.017002659946404125, + "row_means": { + "Wikipedia_en": 0.13307392387133504, + "USPTO_Backgrounds": 0.1601137664410527, + "Pile-CC": 0.14637184389694421, + "StackExchange": 0.1479281428511496, + "OOD_News_2024": 0.13773344230182744, + "OOD_arXiv_2023": 0.14459362406352605 + }, + "col_means": { + "Wikipedia_en": 0.14371758026105386, + "USPTO_Backgrounds": 0.15080673158582714, + "Pile-CC": 0.1485582608591627, + "StackExchange": 0.133804071639423, + "OOD_News_2024": 0.14776164534053318, + "OOD_arXiv_2023": 0.14516645373983517 + }, + "n": 6 + }, + "OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 76.94110075840524, + "read_pct": 19.531209121719506, + "residual_pct": 3.527690119875254, + "fit_span": 0.042313275955341484, + "read_span": 0.019013085747482616, + "row_means": { + "Wikipedia_en": 0.12828564075063223, + "USPTO_Backgrounds": 0.1550014890730381, + "Pile-CC": 0.14270749261464785, + "StackExchange": 0.14496402873623151, + "Github": 0.11268821311769661, + "OOD_CommonPile": 0.13866445779524467 + }, + "col_means": { + "Wikipedia_en": 0.13859109038969986, + "USPTO_Backgrounds": 0.14656895789904176, + "Pile-CC": 0.14437760354744064, + "StackExchange": 0.1321728810530018, + "Github": 0.12755587215155914, + "OOD_CommonPile": 0.1330449170467478 + }, + "n": 6 + }, + "OOD_News_2024+OOD_CommonPile": { + "fit_pct": 79.70567974665822, + "read_pct": 18.245451187320302, + "residual_pct": 2.048869066021478, + "fit_span": 0.04479529873502476, + "read_span": 0.019341295115925666, + "row_means": { + "Wikipedia_en": 0.12973705844401762, + "USPTO_Backgrounds": 0.1573032982647419, + "Pile-CC": 0.14336757844451953, + "StackExchange": 0.1459210308475627, + "Github": 0.11250799952971714, + "OOD_arXiv_2023": 0.14136601208398739 + }, + "col_means": { + "Wikipedia_en": 0.13900639463078093, + "USPTO_Backgrounds": 0.1467772765705983, + "Pile-CC": 0.14441305305118915, + "StackExchange": 0.1319726669471021, + "Github": 0.12743598145467264, + "OOD_arXiv_2023": 0.14059760496020318 + }, + "n": 6 + }, + "OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 77.40134945672342, + "read_pct": 20.26441408833755, + "residual_pct": 2.3342364549390195, + "fit_span": 0.044910989763836076, + "read_span": 0.01882753866828149, + "row_means": { + "Wikipedia_en": 0.13055959505684397, + "USPTO_Backgrounds": 0.15797296050522064, + "Pile-CC": 0.14423647653656424, + "StackExchange": 0.14536816735096553, + "Github": 0.11306197074138456, + "OOD_News_2024": 0.135277057501177 + }, + "col_means": { + "Wikipedia_en": 0.1378521854609803, + "USPTO_Backgrounds": 0.14592119281835578, + "Pile-CC": 0.14358108375152503, + "StackExchange": 0.12939489822213848, + "Github": 0.1270936541500743, + "OOD_News_2024": 0.14263321328908204 + }, + "n": 6 + } + }, + "n_l2o_fit_gt_read": "27/28", + "l2o_failures": [ + "USPTO_Backgrounds+Github" + ], + "l2o_worst_split": { + "dropped": "USPTO_Backgrounds+Github", + "fit_pct": 42.0114078749595, + "read_pct": 47.61343085472334, + "residual_pct": 10.375161270317161, + "fit_span": 0.01565969237298878, + "read_span": 0.014799208839044536, + "row_means": { + "Wikipedia_en": 0.13069117849830675, + "Pile-CC": 0.1443144625087303, + "StackExchange": 0.14635087087129553, + "OOD_News_2024": 0.1354186703554458, + "OOD_arXiv_2023": 0.14124972546059225, + "OOD_CommonPile": 0.14059259656954695 + }, + "col_means": { + "Wikipedia_en": 0.140709264666118, + "Pile-CC": 0.14583993602061163, + "StackExchange": 0.1310407271815671, + "OOD_News_2024": 0.1453122661104081, + "OOD_arXiv_2023": 0.14225394092155275, + "OOD_CommonPile": 0.13346136936366007 + }, + "n": 6 + }, + "fit_axis_leverage_point": "Github", + "read_axis_leverage_point": "Github", + "drop_fit_leverage_only": { + "fit_pct": 58.596571037346294, + "read_pct": 34.84817420406899, + "residual_pct": 6.555254758584715, + "fit_span": 0.026079021388339618, + "read_span": 0.016088034097282655, + "row_means": { + "Wikipedia_en": 0.13171296762154688, + "USPTO_Backgrounds": 0.1577919890098865, + "Pile-CC": 0.14566339520175778, + "StackExchange": 0.14762872951844383, + "OOD_News_2024": 0.13667049391993455, + "OOD_arXiv_2023": 0.14240232722626792, + "OOD_CommonPile": 0.14166051502975205 + }, + "col_means": { + "Wikipedia_en": 0.14330813934405645, + "USPTO_Backgrounds": 0.15041548790084938, + "Pile-CC": 0.14842374856687254, + "StackExchange": 0.13432745380356673, + "OOD_News_2024": 0.14765196423977614, + "OOD_arXiv_2023": 0.14445654644320408, + "OOD_CommonPile": 0.13494707722926425 + }, + "n": 7 + }, + "drop_read_leverage_only": { + "fit_pct": 58.596571037346294, + "read_pct": 34.84817420406899, + "residual_pct": 6.555254758584715, + "fit_span": 0.026079021388339618, + "read_span": 0.016088034097282655, + "row_means": { + "Wikipedia_en": 0.13171296762154688, + "USPTO_Backgrounds": 0.1577919890098865, + "Pile-CC": 0.14566339520175778, + "StackExchange": 0.14762872951844383, + "OOD_News_2024": 0.13667049391993455, + "OOD_arXiv_2023": 0.14240232722626792, + "OOD_CommonPile": 0.14166051502975205 + }, + "col_means": { + "Wikipedia_en": 0.14330813934405645, + "USPTO_Backgrounds": 0.15041548790084938, + "Pile-CC": 0.14842374856687254, + "StackExchange": 0.13432745380356673, + "OOD_News_2024": 0.14765196423977614, + "OOD_arXiv_2023": 0.14445654644320408, + "OOD_CommonPile": 0.13494707722926425 + }, + "n": 7 + }, + "asymmetry_note": "dropping the fit axis's weakest member (Github) moves fit share to 58.6%; dropping the read axis's weakest member (Github) moves it to 58.6%. The paper shows only the first." + } + }, + "e36_5x5_independent": { + "persist": { + "operators": "e28__410m_n400_s0.pt \u2014 N=400, seed block 0, fitted by fastfit", + "replication_unit": "prefix seed (3 draws), ONE operator per corpus", + "seed_averaged": { + "fit_pct": 84.96076305196873, + "read_pct": 13.323573572207149, + "residual_pct": 1.7156633758241149, + "fit_span": 0.022305954536423086, + "read_span": 0.00818338589587559, + "row_means": { + "Github": 0.025804355496851106, + "Wikipedia_en": 0.03849926081641267, + "StackExchange": 0.046645578785489, + "Pile-CC": 0.04729283005620043, + "USPTO_Backgrounds": 0.04811031003327419 + }, + "col_means": { + "Github": 0.037957433629781004, + "Wikipedia_en": 0.04468518639138589, + "StackExchange": 0.0365018004955103, + "Pile-CC": 0.0435597403269882, + "USPTO_Backgrounds": 0.043648174344561995 + }, + "n": 5 + }, + "per_prefix_seed_draw": [ + { + "fit_pct": 85.77569903846643, + "read_pct": 12.513361594819434, + "residual_pct": 1.7109393667141362, + "fit_span": 0.022982529508881272, + "read_span": 0.009132107505574819, + "row_means": { + "Github": 0.02567847983445972, + "Wikipedia_en": 0.03957081381231546, + "StackExchange": 0.04720505081117153, + "Pile-CC": 0.04818122926168144, + "USPTO_Backgrounds": 0.04866100934334099 + }, + "col_means": { + "Github": 0.037733071045950056, + "Wikipedia_en": 0.046865178551524875, + "StackExchange": 0.038667550338432194, + "Pile-CC": 0.043633068460039795, + "USPTO_Backgrounds": 0.04239771466702223 + }, + "n": 5 + }, + { + "fit_pct": 78.46345798160242, + "read_pct": 18.840010767864634, + "residual_pct": 2.696531250532943, + "fit_span": 0.021756873317062858, + "read_span": 0.011026522000320255, + "row_means": { + "Github": 0.025798564204014836, + "Wikipedia_en": 0.03782766117714345, + "StackExchange": 0.0465714720916003, + "Pile-CC": 0.04684677483979613, + "USPTO_Backgrounds": 0.047555437521077694 + }, + "col_means": { + "Github": 0.03815336768515408, + "Wikipedia_en": 0.04274981981609017, + "StackExchange": 0.03431733370758593, + "Pile-CC": 0.04534385570790619, + "USPTO_Backgrounds": 0.04403553291689605 + }, + "n": 5 + }, + { + "fit_pct": 84.05117974949819, + "read_pct": 13.040227802118029, + "residual_pct": 2.908592448383781, + "fit_span": 0.022178460783325138, + "read_span": 0.007990758009254938, + "row_means": { + "Github": 0.02593602245207876, + "Wikipedia_en": 0.03809930745977908, + "StackExchange": 0.04616021345369518, + "Pile-CC": 0.04685048606712371, + "USPTO_Backgrounds": 0.0481144832354039 + }, + "col_means": { + "Github": 0.03798586215823889, + "Wikipedia_en": 0.044440560806542635, + "StackExchange": 0.036520517440512774, + "Pile-CC": 0.041702296813018624, + "USPTO_Backgrounds": 0.04451127544976771 + }, + "n": 5 + } + ], + "fit_gt_read_in_every_draw": true, + "leave_one_out": { + "Github": { + "fit_pct": 50.06157285079295, + "read_pct": 46.53699457539348, + "residual_pct": 3.4014325738135747, + "fit_span": 0.00910597042335818, + "read_span": 0.008764024045861643, + "row_means": { + "Wikipedia_en": 0.039842049123641726, + "StackExchange": 0.046977921579188356, + "Pile-CC": 0.04846147454809398, + "USPTO_Backgrounds": 0.048948019546999906 + }, + "col_means": { + "Wikipedia_en": 0.04874888522123608, + "StackExchange": 0.039984861175374434, + "Pile-CC": 0.047789373867756994, + "USPTO_Backgrounds": 0.047706344533556454 + }, + "n": 4 + }, + "Wikipedia_en": { + "fit_pct": 90.37547394485536, + "read_pct": 8.700329663501021, + "residual_pct": 0.924196391643609, + "fit_span": 0.022430326708126815, + "read_span": 0.006603540251186736, + "row_means": { + "Github": 0.02514784660306759, + "StackExchange": 0.04607711071924617, + "Pile-CC": 0.04601185917854309, + "USPTO_Backgrounds": 0.047578173311194405 + }, + "col_means": { + "Github": 0.03916476514035215, + "StackExchange": 0.03759048900101334, + "Pile-CC": 0.04386570641848569, + "USPTO_Backgrounds": 0.044194029252200076 + }, + "n": 4 + }, + "StackExchange": { + "fit_pct": 89.40370004644036, + "read_pct": 9.295411121242426, + "residual_pct": 1.3008888323172074, + "fit_span": 0.022550062000906715, + "read_span": 0.0075088800920639145, + "row_means": { + "Github": 0.026613054927050447, + "Wikipedia_en": 0.0400873144021413, + "Pile-CC": 0.04871066633301477, + "USPTO_Backgrounds": 0.04916311692795716 + }, + "col_means": { + "Github": 0.036117740134553364, + "Wikipedia_en": 0.04362662022661728, + "Pile-CC": 0.042574912594864144, + "USPTO_Backgrounds": 0.042254879634128883 + }, + "n": 4 + }, + "Pile-CC": { + "fit_pct": 86.40740022165774, + "read_pct": 12.216495019148402, + "residual_pct": 1.376104759193859, + "fit_span": 0.021864569419994952, + "read_span": 0.00753042521537281, + "row_means": { + "Github": 0.02559514283008563, + "Wikipedia_en": 0.03754010703026627, + "StackExchange": 0.04643221066799015, + "USPTO_Backgrounds": 0.04745971225008058 + }, + "col_means": { + "Github": 0.036792229015069705, + "Wikipedia_en": 0.04275230459752492, + "StackExchange": 0.03522187938215211, + "USPTO_Backgrounds": 0.0422607597836759 + }, + "n": 4 + }, + "USPTO_Backgrounds": { + "fit_pct": 83.04335115144399, + "read_pct": 14.719316877423793, + "residual_pct": 2.237331971132233, + "fit_span": 0.02141500844930609, + "read_span": 0.008644288753081736, + "row_means": { + "Github": 0.02540157097391784, + "Wikipedia_en": 0.037757887342013415, + "StackExchange": 0.046001635185287645, + "Pile-CC": 0.04681657942322393 + }, + "col_means": { + "Github": 0.03625692404263342, + "Wikipedia_en": 0.04329676875883403, + "StackExchange": 0.034652480005752294, + "Pile-CC": 0.04177150011722309 + }, + "n": 4 + } + }, + "n_loo_fit_gt_read": "5/5", + "same_five_corpora_from_E52_operators": { + "fit_pct": 92.17543763701067, + "read_pct": 7.003845590259521, + "residual_pct": 0.8207167727297995, + "fit_span": 0.022531672493140732, + "read_span": 0.0060945458363534655, + "row_means": { + "Github": 0.0249453093820355, + "Wikipedia_en": 0.03826405839580629, + "StackExchange": 0.04529552277694974, + "Pile-CC": 0.04711824890070905, + "USPTO_Backgrounds": 0.04747698187517623 + }, + "col_means": { + "Github": 0.03868060538700471, + "Wikipedia_en": 0.04217591315352668, + "StackExchange": 0.03702605563422872, + "Pile-CC": 0.0420969456853345, + "USPTO_Backgrounds": 0.043120601470582186 + }, + "n": 5 + }, + "E52_same_five_minus_github": { + "fit_pct": 66.05459534803015, + "read_pct": 31.359355167255384, + "residual_pct": 2.5860494847144593, + "fit_span": 0.008791975658257595, + "read_span": 0.006303149724327446, + "row_means": { + "Wikipedia_en": 0.03911983556851434, + "StackExchange": 0.04549701772888916, + "Pile-CC": 0.047862906688694945, + "USPTO_Backgrounds": 0.04791181122677193 + }, + "col_means": { + "Wikipedia_en": 0.04621876258234907, + "StackExchange": 0.040878605226235876, + "Pile-CC": 0.04611244845372211, + "USPTO_Backgrounds": 0.04718175495056332 + }, + "n": 4 + }, + "E36_same_five_minus_github": { + "fit_pct": 50.06157285079295, + "read_pct": 46.53699457539348, + "residual_pct": 3.4014325738135747, + "fit_span": 0.00910597042335818, + "read_span": 0.008764024045861643, + "row_means": { + "Wikipedia_en": 0.039842049123641726, + "StackExchange": 0.046977921579188356, + "Pile-CC": 0.04846147454809398, + "USPTO_Backgrounds": 0.048948019546999906 + }, + "col_means": { + "Wikipedia_en": 0.04874888522123608, + "StackExchange": 0.039984861175374434, + "Pile-CC": 0.047789373867756994, + "USPTO_Backgrounds": 0.047706344533556454 + }, + "n": 4 + }, + "offdiagonal_only": { + "why": "E36's five diagonal rungs carry 17.0% prefix/fit overlap (E58). Dropping them removes the contamination entirely.", + "E36_5x5": { + "fit_pct": 85.51107109718984, + "read_pct": 13.005048217851625, + "n_cells": 20, + "fit_gt_read": true + }, + "E52_5x5": { + "fit_pct": 92.00507372166676, + "read_pct": 8.469880721749831, + "n_cells": 20, + "fit_gt_read": true + }, + "E36_4x4_minus_github": { + "fit_pct": 56.91343895602421, + "read_pct": 55.87288649698407, + "n_cells": 12, + "fit_gt_read": true + }, + "E52_4x4_minus_github": { + "fit_pct": 65.98432550467183, + "read_pct": 41.201011115479226, + "n_cells": 12, + "fit_gt_read": true + } + } + }, + "min": { + "operators": "e28__410m_n400_s0.pt \u2014 N=400, seed block 0, fitted by fastfit", + "replication_unit": "prefix seed (3 draws), ONE operator per corpus", + "seed_averaged": { + "fit_pct": 77.40097983200543, + "read_pct": 20.68267597391707, + "residual_pct": 1.9163441940774903, + "fit_span": 0.046371082440018654, + "read_span": 0.021674229626854252, + "row_means": { + "Github": 0.11237846309940021, + "Wikipedia_en": 0.1290518776079019, + "StackExchange": 0.1457329342638453, + "Pile-CC": 0.14315666920195022, + "USPTO_Backgrounds": 0.15874954553941886 + }, + "col_means": { + "Github": 0.13050790891051292, + "Wikipedia_en": 0.14201401859521867, + "StackExchange": 0.1259393160790205, + "Pile-CC": 0.14761354570587476, + "USPTO_Backgrounds": 0.14299470042188964 + }, + "n": 5 + }, + "per_prefix_seed_draw": [ + { + "fit_pct": 76.25362135985992, + "read_pct": 20.720708110695465, + "residual_pct": 3.0256705294446196, + "fit_span": 0.04553360372781755, + "read_span": 0.020313016325235356, + "row_means": { + "Github": 0.11447363302111625, + "Wikipedia_en": 0.12942224599421023, + "StackExchange": 0.146284611299634, + "Pile-CC": 0.14431209467351436, + "USPTO_Backgrounds": 0.1600072367489338 + }, + "col_means": { + "Github": 0.13079405024647714, + "Wikipedia_en": 0.14585725516080855, + "StackExchange": 0.12786675058305264, + "Pile-CC": 0.148179766908288, + "USPTO_Backgrounds": 0.1418019988387823 + }, + "n": 5 + }, + { + "fit_pct": 66.70604334540664, + "read_pct": 31.672852651040355, + "residual_pct": 1.621104003553009, + "fit_span": 0.047264108434319496, + "read_span": 0.032970283925533306, + "row_means": { + "Github": 0.10957772865891457, + "Wikipedia_en": 0.12641755901277066, + "StackExchange": 0.14348228890448808, + "Pile-CC": 0.14053768675774336, + "USPTO_Backgrounds": 0.15684183709323407 + }, + "col_means": { + "Github": 0.12860518023371698, + "Wikipedia_en": 0.138508844897151, + "StackExchange": 0.11861159011721611, + "Pile-CC": 0.15158187404274942, + "USPTO_Backgrounds": 0.13954961113631725 + }, + "n": 5 + }, + { + "fit_pct": 83.21905386270309, + "read_pct": 13.596775884069, + "residual_pct": 3.184170253227901, + "fit_span": 0.04631553515791893, + "read_span": 0.01629288375377655, + "row_means": { + "Github": 0.11308402761816978, + "Wikipedia_en": 0.13131582781672477, + "StackExchange": 0.1474319025874138, + "Pile-CC": 0.14462022617459297, + "USPTO_Backgrounds": 0.15939956277608872 + }, + "col_means": { + "Github": 0.13212449625134468, + "Wikipedia_en": 0.14167595572769642, + "StackExchange": 0.13133960753679275, + "Pile-CC": 0.14307899616658687, + "USPTO_Backgrounds": 0.1476324912905693 + }, + "n": 5 + } + ], + "fit_gt_read_in_every_draw": true, + "leave_one_out": { + "Github": { + "fit_pct": 56.42933319341921, + "read_pct": 41.62336189329218, + "residual_pct": 1.9473049132886122, + "fit_span": 0.029569615672032035, + "read_span": 0.023711827211081976, + "row_means": { + "Wikipedia_en": 0.1315493759699166, + "StackExchange": 0.14707623957656324, + "Pile-CC": 0.1446309056909134, + "USPTO_Backgrounds": 0.16111899164194862 + }, + "col_means": { + "Wikipedia_en": 0.14946394379561145, + "StackExchange": 0.13093208723391098, + "Pile-CC": 0.15464391444499295, + "USPTO_Backgrounds": 0.14933556740482648 + }, + "n": 4 + }, + "Wikipedia_en": { + "fit_pct": 79.18854147864499, + "read_pct": 19.430142517148237, + "residual_pct": 1.3813160042067691, + "fit_span": 0.04533088173096378, + "read_span": 0.02020883274575075, + "row_means": { + "Github": 0.1124194994258384, + "StackExchange": 0.14496242492459716, + "Pile-CC": 0.1415124223412325, + "USPTO_Backgrounds": 0.15775038115680218 + }, + "col_means": { + "Github": 0.13336941509818037, + "StackExchange": 0.12911907033994793, + "Pile-CC": 0.14932790308569868, + "USPTO_Backgrounds": 0.14482833932464323 + }, + "n": 4 + }, + "StackExchange": { + "fit_pct": 86.60209918862283, + "read_pct": 12.380274347518588, + "residual_pct": 1.017626463858588, + "fit_span": 0.04693148511772355, + "read_span": 0.017915232758969085, + "row_means": { + "Github": 0.11398102100938559, + "Wikipedia_en": 0.13300977225104968, + "Pile-CC": 0.14697571069312593, + "USPTO_Backgrounds": 0.16091250612710914 + }, + "col_means": { + "Github": 0.12804495788489778, + "Wikipedia_en": 0.14031378033881386, + "Pile-CC": 0.14596019064386687, + "USPTO_Backgrounds": 0.14056008121309183 + }, + "n": 4 + }, + "Pile-CC": { + "fit_pct": 83.93558317633963, + "read_pct": 14.094736101430035, + "residual_pct": 1.969680722230337, + "fit_span": 0.045237126356611645, + "read_span": 0.01636039004661144, + "row_means": { + "Github": 0.11060006118689975, + "Wikipedia_en": 0.1261258179632326, + "StackExchange": 0.14360942634133, + "USPTO_Backgrounds": 0.1558371875435114 + }, + "col_means": { + "Github": 0.12881995532661678, + "Wikipedia_en": 0.14008410908281804, + "StackExchange": 0.12545401928946376, + "USPTO_Backgrounds": 0.1418144093360752 + }, + "n": 4 + }, + "USPTO_Backgrounds": { + "fit_pct": 67.79068546215659, + "read_pct": 29.354636152082982, + "residual_pct": 2.8546783857604257, + "fit_span": 0.03291760276382169, + "read_span": 0.022017468449970073, + "row_means": { + "Github": 0.11106527075171471, + "Wikipedia_en": 0.12739981080715856, + "StackExchange": 0.1439828735155364, + "Pile-CC": 0.14201687031115096 + }, + "col_means": { + "Github": 0.1258169458558162, + "Wikipedia_en": 0.13683097247655193, + "StackExchange": 0.1198997193016112, + "Pile-CC": 0.14191718775158127 + }, + "n": 4 + } + }, + "n_loo_fit_gt_read": "5/5", + "same_five_corpora_from_E52_operators": { + "fit_pct": 80.4084027189265, + "read_pct": 17.706197361686215, + "residual_pct": 1.885399919387287, + "fit_span": 0.04464112856321864, + "read_span": 0.018712922111153607, + "row_means": { + "Github": 0.11258839343984922, + "Wikipedia_en": 0.1292333228761951, + "StackExchange": 0.14479038457903598, + "Pile-CC": 0.14296645053144957, + "USPTO_Backgrounds": 0.15722952200306786 + }, + "col_means": { + "Github": 0.12755622220949994, + "Wikipedia_en": 0.13813900969922543, + "StackExchange": 0.13111390790591637, + "Pile-CC": 0.14372978929430247, + "USPTO_Backgrounds": 0.14626914432065355 + }, + "n": 5 + }, + "E52_same_five_minus_github": { + "fit_pct": 70.21356558670237, + "read_pct": 27.601611839591417, + "residual_pct": 2.1848225737062053, + "fit_span": 0.028080853768106984, + "read_span": 0.01639572462687891, + "row_means": { + "Wikipedia_en": 0.13224921274619797, + "StackExchange": 0.14693437842652202, + "Pile-CC": 0.14556780970241462, + "USPTO_Backgrounds": 0.16033006651430495 + }, + "col_means": { + "Wikipedia_en": 0.14563602450200253, + "StackExchange": 0.13643952947006457, + "Pile-CC": 0.15017065932042897, + "USPTO_Backgrounds": 0.15283525409694348 + }, + "n": 4 + }, + "E36_same_five_minus_github": { + "fit_pct": 56.42933319341921, + "read_pct": 41.62336189329218, + "residual_pct": 1.9473049132886122, + "fit_span": 0.029569615672032035, + "read_span": 0.023711827211081976, + "row_means": { + "Wikipedia_en": 0.1315493759699166, + "StackExchange": 0.14707623957656324, + "Pile-CC": 0.1446309056909134, + "USPTO_Backgrounds": 0.16111899164194862 + }, + "col_means": { + "Wikipedia_en": 0.14946394379561145, + "StackExchange": 0.13093208723391098, + "Pile-CC": 0.15464391444499295, + "USPTO_Backgrounds": 0.14933556740482648 + }, + "n": 4 + }, + "offdiagonal_only": { + "why": "E36's five diagonal rungs carry 17.0% prefix/fit overlap (E58). Dropping them removes the contamination entirely.", + "E36_5x5": { + "fit_pct": 78.34448026853826, + "read_pct": 18.17638411230237, + "n_cells": 20, + "fit_gt_read": true + }, + "E52_5x5": { + "fit_pct": 78.0443150477168, + "read_pct": 9.891246145582377, + "n_cells": 20, + "fit_gt_read": true + }, + "E36_4x4_minus_github": { + "fit_pct": 63.120703254496384, + "read_pct": 45.25423171432657, + "n_cells": 12, + "fit_gt_read": true + }, + "E52_4x4_minus_github": { + "fit_pct": 68.73959281067297, + "read_pct": 25.9630489733978, + "n_cells": 12, + "fit_gt_read": true + } + } + }, + "VERDICT": "REPLICATES \u2014 fit_pct > read_pct on an independent operator population, under both aggregations and on each of the three prefix-seed draws separately" + }, + "controls": { + "C1_agrees_with_e54": { + "n_quantities_checked": 20, + "max_abs_diff": 1.4210854715202004e-14, + "tolerance": 1e-12, + "fires": true, + "reference": "results/e54_aggregation_audit.json" + }, + "C2_e36_diagonal_leakage": { + "measured": 0.1704, + "executed": "recomputed with the five contaminated diagonal cells DROPPED: E36 85.5/13.0 vs 85.0/13.3 with them. The leakage does not drive the replication.", + "fires": true, + "measured_in": "results/e58_algebra_audit.json (t58)", + "expected": "~17% \u2014 t36_qladder.py load_pool() draws prefixes from the whole 2400-doc file; its operators are e28 N=400 fits from documents 0..800 of that file", + "direction": "inflates one cell per row (the diagonal), so it lands mostly in the residual rather than the row main effect" + } + }, + "provenance": { + "schema_version": 1, + "experiment": "E55", + "utc": "2026-08-17T02:08:36+00:00", + "script": { + "path": "experiments/t55_matrix_robustness.py", + "sha256": "7503087c32f802f0177d5a61d5b365fe00323fdb5844ceca765f5f68935e0260", + "n_lines": 262 + }, + "git": { + "commit": "f8ef435b29365012c0492e023fb6b4dfece5f0f3", + "commit_short": "f8ef435b2936", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t53_ladder_summary.py", + "experiments/t55_matrix_robustness.py", + "experiments/t56_predictor_registry.py", + "experiments/t57_grid_variance_ci.py", + "experiments/t58_algebra_audit.py", + "experiments/t59_read_dose.py", + "experiments/t60_fitter_determinism.py", + "experiments/trainval.py", + "paper/paper.tex", + "repro/exp/e61_randomized_null.sh", + "repro/exp/e62_ladder1b_band.sh", + "results/e35_containment.json", + "results/e52_factorial_410m_rerun.json", + "results/e53_ladder_summary.json", + "results/e55_matrix_robustness.json", + "results/e56_predictor_registry.json", + "results/e57_factorial_cells_410m.json", + "results/e57_grid_variance_ci.json", + "results/e58_algebra_audit.json", + "results/e60_fitter_determinism.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/experiments/E57_grid_variance_ci.md", + "specs/experiments/E59_read_dose_response.md", + "specs/experiments/E61_randomized_network_null.md", + "specs/experiments/E62_ladder1b_corrected_band.md", + "tests/test_anchor_fidelity.py", + "xperiments/t52_factorial.py" + ], + "dirty_diff_sha256": "53b489d1c7f0dd3e4697e4cea37649f60eda522d75256f523599cebf411b56ab", + "describe": "f8ef435-dirty" + }, + "argv": [ + "experiments/t55_matrix_robustness.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e52_factorial_410m.json", + "sha256": "6e81f3cae37d0c1906f8912af4bf8e75db0a0f259b2869f209701b74b6682590", + "bytes": 41190, + "exists": true + }, + { + "path": "results/e36_qladder_410m.json", + "sha256": "d0cc7b6381ac515f6d53401705e246d8c20c67884cda25e4b1d35d5312f45153", + "bytes": 80656, + "exists": true + }, + { + "path": "results/e54_aggregation_audit.json", + "sha256": "886885b41660dd01808aa626d5f87dddf6d5402f2c9b2f8241ac4470df82276b", + "bytes": 69079, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "5159a21fe803846f8e5f9e3250b84fd76425e27920bea439f98b13187bccdf73" + } +} \ No newline at end of file diff --git a/results/e55_matrix_robustness_rstrip.json b/results/e55_matrix_robustness_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..d25126eeb8afd30e370fa57aab692d3dd2b8f2e3 --- /dev/null +++ b/results/e55_matrix_robustness_rstrip.json @@ -0,0 +1,2758 @@ +{ + "experiment": "E55 \u2014 leave-k-out robustness of the fit>read ordering, and its replication on an independent operator population", + "why": "the paper defends its headline with 8 leave-one-out splits; leave-two-out is stronger and free, the read axis has an unreported leverage point of its own, and an independent 5x5 on N=400 fastfit operators has been on disk unanalysed.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/e52_factorial_410m.json", + "results/e36_qladder_410m.json", + "results/e54_aggregation_audit.json" + ], + "note_on_the_decomposition": "one observation per cell, so fit+read+residual = 100 exactly and the residual is NOT an interaction estimate. See E57.", + "e52_8x8": { + "persist": { + "full": { + "fit_pct": 50.697184677169346, + "read_pct": 48.13211438274321, + "residual_pct": 1.1707009400874475, + "fit_span": 0.017171468827291383, + "read_span": 0.01664830898387462, + "row_means": { + "Wikipedia_en": 0.09961745545313332, + "USPTO_Backgrounds": 0.10963667937838989, + "Pile-CC": 0.10827355975796106, + "StackExchange": 0.10898624043006243, + "Github": 0.0924652105510985, + "OOD_News_2024": 0.09569122234566343, + "OOD_arXiv_2023": 0.09788490975592544, + "OOD_CommonPile": 0.0989707839093171 + }, + "col_means": { + "Wikipedia_en": 0.10531951454274047, + "USPTO_Backgrounds": 0.10890259583894578, + "Pile-CC": 0.10510752553333683, + "StackExchange": 0.09225428685507116, + "Github": 0.09226517163559846, + "OOD_News_2024": 0.10725807921618172, + "OOD_arXiv_2023": 0.09938795663215892, + "OOD_CommonPile": 0.10103093132751786 + }, + "n": 8 + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 52.373250427828836, + "read_pct": 46.61005338540753, + "residual_pct": 1.016696186763633, + "fit_span": 0.017303615567156122, + "read_span": 0.0165367584815988, + "row_means": { + "USPTO_Backgrounds": 0.10909241095170497, + "Pile-CC": 0.10771189345423841, + "StackExchange": 0.10846411770446196, + "Github": 0.09178879538454884, + "OOD_News_2024": 0.09525459575511161, + "OOD_arXiv_2023": 0.09740781846626233, + "OOD_CommonPile": 0.09862754898153187 + }, + "col_means": { + "USPTO_Backgrounds": 0.10901381163931792, + "Pile-CC": 0.10548784245352541, + "StackExchange": 0.0927244208217229, + "Github": 0.09247705315771912, + "OOD_News_2024": 0.10750825819965924, + "OOD_arXiv_2023": 0.09981114827022548, + "OOD_CommonPile": 0.10132464615568992 + }, + "n": 7 + }, + "USPTO_Backgrounds": { + "fit_pct": 48.90013785418412, + "read_pct": 49.70701457618453, + "residual_pct": 1.3928475696313505, + "fit_span": 0.015998433523308778, + "read_span": 0.015076022723778365, + "row_means": { + "Wikipedia_en": 0.09840222262696083, + "Pile-CC": 0.10732170162493333, + "StackExchange": 0.10768341840289178, + "Github": 0.091684984879583, + "OOD_News_2024": 0.0947177019240039, + "OOD_arXiv_2023": 0.09682840401461969, + "OOD_CommonPile": 0.0979013044764066 + }, + "col_means": { + "Wikipedia_en": 0.10415850828239133, + "Pile-CC": 0.103846665340594, + "StackExchange": 0.09104025552441027, + "Github": 0.09119253544439931, + "OOD_News_2024": 0.10611627824818863, + "OOD_arXiv_2023": 0.09826278827589242, + "OOD_CommonPile": 0.09992270683352318 + }, + "n": 7 + }, + "Pile-CC": { + "fit_pct": 47.4662599964337, + "read_pct": 51.23363164594214, + "residual_pct": 1.3001083576241557, + "fit_span": 0.017079299537184095, + "read_span": 0.016827839917118947, + "row_means": { + "Wikipedia_en": 0.09921347664757854, + "USPTO_Backgrounds": 0.10902284116351178, + "StackExchange": 0.10863718172934439, + "Github": 0.09194354162632769, + "OOD_News_2024": 0.09514248459008597, + "OOD_arXiv_2023": 0.09733690423299633, + "OOD_CommonPile": 0.09831029655049658 + }, + "col_means": { + "Wikipedia_en": 0.10433584177970058, + "USPTO_Backgrounds": 0.10804060000320158, + "StackExchange": 0.0913995043777432, + "Github": 0.09121276008608263, + "OOD_News_2024": 0.10615363395662003, + "OOD_arXiv_2023": 0.09832581776112968, + "OOD_CommonPile": 0.10013856857586356 + }, + "n": 7 + }, + "StackExchange": { + "fit_pct": 54.30293560630481, + "read_pct": 44.71750309530783, + "residual_pct": 0.9795612983873632, + "fit_span": 0.0169366810866262, + "read_span": 0.01671054580152803, + "row_means": { + "Wikipedia_en": 0.10113947064807964, + "USPTO_Backgrounds": 0.11090584697963167, + "Pile-CC": 0.1097072448381888, + "Github": 0.09396916589300547, + "OOD_News_2024": 0.0970360472040724, + "OOD_arXiv_2023": 0.09902981570404437, + "OOD_CommonPile": 0.10012423403575899 + }, + "col_means": { + "Wikipedia_en": 0.10427357383323685, + "USPTO_Backgrounds": 0.10758782458447275, + "Pile-CC": 0.10420436470451513, + "Github": 0.09087727878294471, + "OOD_News_2024": 0.10626538671250085, + "OOD_arXiv_2023": 0.09853761416586442, + "OOD_CommonPile": 0.10016578251924661 + }, + "n": 7 + }, + "Github": { + "fit_pct": 50.83451434240633, + "read_pct": 47.96562247603154, + "residual_pct": 1.199863181562129, + "fit_span": 0.014015912493720417, + "read_span": 0.016742457825291365, + "row_means": { + "Wikipedia_en": 0.1008796632349018, + "USPTO_Backgrounds": 0.11104568715044667, + "Pile-CC": 0.10950806079735417, + "StackExchange": 0.10998707169090353, + "OOD_News_2024": 0.09702977465672626, + "OOD_arXiv_2023": 0.09939169281657549, + "OOD_CommonPile": 0.10037892185639412 + }, + "col_means": { + "Wikipedia_en": 0.10647942851785393, + "USPTO_Backgrounds": 0.11047056806569416, + "Pile-CC": 0.10639190160602863, + "StackExchange": 0.0937281102404028, + "OOD_News_2024": 0.10860243229276781, + "OOD_arXiv_2023": 0.10042639302629386, + "OOD_CommonPile": 0.10212203845426085 + }, + "n": 7 + }, + "OOD_News_2024": { + "fit_pct": 50.49955454200501, + "read_pct": 48.12259484369172, + "residual_pct": 1.3778506143032678, + "fit_span": 0.017138381758039542, + "read_span": 0.016708293558645337, + "row_means": { + "Wikipedia_en": 0.09877611675617537, + "USPTO_Backgrounds": 0.1088346784335694, + "Pile-CC": 0.10731418314722499, + "StackExchange": 0.1082404280997931, + "Github": 0.09169629667552986, + "OOD_arXiv_2023": 0.09715664176056014, + "OOD_CommonPile": 0.09814794496965727 + }, + "col_means": { + "Wikipedia_en": 0.10625835826605677, + "USPTO_Backgrounds": 0.10981641448775513, + "Pile-CC": 0.10590397394742698, + "StackExchange": 0.09310812092910979, + "Github": 0.09311428813093771, + "OOD_arXiv_2023": 0.10007737871399888, + "OOD_CommonPile": 0.10188775536722489 + }, + "n": 7 + }, + "OOD_arXiv_2023": { + "fit_pct": 50.57242596342314, + "read_pct": 48.44478039653153, + "residual_pct": 0.9827936400453328, + "fit_span": 0.0174609310522173, + "read_span": 0.016825227149289163, + "row_means": { + "Wikipedia_en": 0.10007343263705334, + "USPTO_Backgrounds": 0.10997561427158496, + "Pile-CC": 0.10848079276204642, + "StackExchange": 0.10950708136346841, + "Github": 0.09251468321936766, + "OOD_News_2024": 0.09585253952943262, + "OOD_CommonPile": 0.09930630639927196 + }, + "col_means": { + "Wikipedia_en": 0.10590450965119377, + "USPTO_Backgrounds": 0.10942004525235721, + "Pile-CC": 0.10559132226432363, + "StackExchange": 0.09259481810306805, + "Github": 0.09296913496477323, + "OOD_News_2024": 0.10786883542942445, + "OOD_CommonPile": 0.10136178451708504 + }, + "n": 7 + }, + "OOD_CommonPile": { + "fit_pct": 50.85297785477784, + "read_pct": 48.11818900866957, + "residual_pct": 1.0288331365525942, + "fit_span": 0.01742520418188105, + "read_span": 0.016803709279361465, + "row_means": { + "Wikipedia_en": 0.09970924515639329, + "USPTO_Backgrounds": 0.10975784746309121, + "Pile-CC": 0.10841585821065579, + "StackExchange": 0.10925756435072612, + "Github": 0.09233264328121016, + "OOD_News_2024": 0.09578523081653412, + "OOD_arXiv_2023": 0.09776633129240797 + }, + "col_means": { + "Wikipedia_en": 0.10588324113401569, + "USPTO_Backgrounds": 0.10925194668169651, + "Pile-CC": 0.10532371554937628, + "StackExchange": 0.09244823740233504, + "Github": 0.09271536497214425, + "OOD_News_2024": 0.10761913960607397, + "OOD_arXiv_2023": 0.0997830752253769 + }, + "n": 7 + } + }, + "n_loo_fit_gt_read": "6/8", + "leave_two_out": { + "Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 52.30067209055684, + "read_pct": 46.47792330335211, + "residual_pct": 1.2214046060910648, + "fit_span": 0.01609134197847366, + "read_span": 0.01495689571257336, + "row_means": { + "Pile-CC": 0.10650778124841896, + "StackExchange": 0.10685713821849613, + "Github": 0.09076579624002247, + "OOD_News_2024": 0.09404605083141683, + "OOD_arXiv_2023": 0.0960957132197951, + "OOD_CommonPile": 0.09732261715517207 + }, + "col_means": { + "Pile-CC": 0.1040802250486902, + "StackExchange": 0.09138640659706046, + "Github": 0.09126095785500689, + "OOD_News_2024": 0.10621785356758025, + "OOD_arXiv_2023": 0.09856898379425898, + "OOD_CommonPile": 0.1000806700507248 + }, + "n": 6 + }, + "Wikipedia_en+Pile-CC": { + "fit_pct": 50.30922645008709, + "read_pct": 48.56307298661112, + "residual_pct": 1.127700563301794, + "fit_span": 0.017218109185341737, + "read_span": 0.01674213252737429, + "row_means": { + "USPTO_Backgrounds": 0.10828555496323301, + "StackExchange": 0.1079698620993575, + "Github": 0.09106744577789128, + "OOD_News_2024": 0.09454163060851258, + "OOD_arXiv_2023": 0.09668896347456785, + "OOD_CommonPile": 0.09779977457494371 + }, + "col_means": { + "USPTO_Backgrounds": 0.10802668579767838, + "StackExchange": 0.09180553025928223, + "Github": 0.09128455327030409, + "OOD_News_2024": 0.10626143522741686, + "OOD_arXiv_2023": 0.09864251819370246, + "OOD_CommonPile": 0.1003325087501219 + }, + "n": 6 + }, + "Wikipedia_en+StackExchange": { + "fit_pct": 55.489003458453865, + "read_pct": 43.88542177383025, + "residual_pct": 0.625574767715879, + "fit_span": 0.017051720993024197, + "read_span": 0.016605289392518235, + "row_means": { + "USPTO_Backgrounds": 0.11048239508203955, + "Pile-CC": 0.109290914997217, + "Github": 0.09343067408901536, + "OOD_News_2024": 0.09675078699149674, + "OOD_arXiv_2023": 0.09866402685745723, + "OOD_CommonPile": 0.0999160349744165 + }, + "col_means": { + "USPTO_Backgrounds": 0.10749844780916142, + "Pile-CC": 0.10449754097326486, + "Github": 0.09089315841664318, + "OOD_News_2024": 0.10639181344261117, + "OOD_arXiv_2023": 0.0988896139992263, + "OOD_CommonPile": 0.10036425835073547 + }, + "n": 6 + }, + "Wikipedia_en+Github": { + "fit_pct": 50.74751292554641, + "read_pct": 48.37431796644749, + "residual_pct": 0.8781691080060962, + "fit_span": 0.013902072928397471, + "read_span": 0.01633941143820132, + "row_means": { + "USPTO_Backgrounds": 0.11064554194799038, + "Pile-CC": 0.10905853361624328, + "StackExchange": 0.10954473372117651, + "OOD_News_2024": 0.09674346901959291, + "OOD_arXiv_2023": 0.09908621682207687, + "OOD_CommonPile": 0.10021317076515751 + }, + "col_means": { + "USPTO_Backgrounds": 0.11086164853725307, + "Pile-CC": 0.10704966735836394, + "StackExchange": 0.09452223709905175, + "OOD_News_2024": 0.10911836661958929, + "OOD_arXiv_2023": 0.101093189336394, + "OOD_CommonPile": 0.1026465569415854 + }, + "n": 6 + }, + "Wikipedia_en+OOD_News_2024": { + "fit_pct": 53.07363642980562, + "read_pct": 45.68002694795497, + "residual_pct": 1.246336622239413, + "fit_span": 0.01728703844300644, + "read_span": 0.016595466707022527, + "row_means": { + "USPTO_Backgrounds": 0.10806603177830025, + "Pile-CC": 0.10649900969109256, + "StackExchange": 0.107506982864881, + "Github": 0.09077899333529381, + "OOD_arXiv_2023": 0.09647865725672562, + "OOD_CommonPile": 0.09761036439729785 + }, + "col_means": { + "USPTO_Backgrounds": 0.11009846936299086, + "Pile-CC": 0.10648041842332868, + "StackExchange": 0.09379891623587658, + "Github": 0.09350300265596834, + "OOD_arXiv_2023": 0.10068600597204985, + "OOD_CommonPile": 0.10237322667337678 + }, + "n": 6 + }, + "Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 52.185386350617975, + "read_pct": 47.016680600124765, + "residual_pct": 0.7979330492572599, + "fit_span": 0.017663345952880466, + "read_span": 0.016435975649532072, + "row_means": { + "USPTO_Backgrounds": 0.10939712358931838, + "Pile-CC": 0.10786005424171755, + "StackExchange": 0.10898474500583553, + "Github": 0.09173377763643792, + "OOD_News_2024": 0.09537002803775034, + "OOD_CommonPile": 0.09896178606518165 + }, + "col_means": { + "USPTO_Backgrounds": 0.10963603858835995, + "Pile-CC": 0.10611565812637477, + "StackExchange": 0.09320006293882788, + "Github": 0.09333365729544311, + "OOD_News_2024": 0.10826250361235536, + "OOD_CommonPile": 0.10175959401488029 + }, + "n": 6 + }, + "Wikipedia_en+OOD_CommonPile": { + "fit_pct": 52.8489532185628, + "read_pct": 46.25686799605305, + "residual_pct": 0.894178785384148, + "fit_span": 0.01762166460415486, + "read_span": 0.016410871467949756, + "row_means": { + "USPTO_Backgrounds": 0.10914306231274234, + "Pile-CC": 0.10778429726509516, + "StackExchange": 0.10869364182430286, + "Github": 0.09152139770858748, + "OOD_News_2024": 0.09529150120603541, + "OOD_arXiv_2023": 0.0971899617105481 + }, + "col_means": { + "USPTO_Backgrounds": 0.1094399235892558, + "Pile-CC": 0.10580345029226952, + "StackExchange": 0.09302905212130605, + "Github": 0.09303759230404264, + "OOD_News_2024": 0.10797119181844647, + "OOD_arXiv_2023": 0.10034265190199086 + }, + "n": 6 + }, + "USPTO_Backgrounds+Pile-CC": { + "fit_pct": 42.08107123390732, + "read_pct": 56.08737719584901, + "residual_pct": 1.831551570243672, + "fit_span": 0.01611271272542783, + "read_span": 0.01485150934593682, + "row_means": { + "Wikipedia_en": 0.09772837521611816, + "StackExchange": 0.10705904624752563, + "Github": 0.0909463335220978, + "OOD_News_2024": 0.09391525447222025, + "OOD_arXiv_2023": 0.09601297994765143, + "OOD_CommonPile": 0.09695248931896425 + }, + "col_means": { + "Wikipedia_en": 0.10281738901545329, + "StackExchange": 0.08984067074575082, + "Github": 0.08978594927143099, + "OOD_News_2024": 0.10463745861736781, + "OOD_arXiv_2023": 0.09683609820031389, + "OOD_CommonPile": 0.09869691287426072 + }, + "n": 6 + }, + "USPTO_Backgrounds+StackExchange": { + "fit_pct": 46.59915642774981, + "read_pct": 52.21732878700266, + "residual_pct": 1.1835147852475376, + "fit_span": 0.015526129363139199, + "read_span": 0.015373282414792036, + "row_means": { + "Wikipedia_en": 0.09997536821670279, + "Pile-CC": 0.10883569119636108, + "Github": 0.09330956183322188, + "OOD_News_2024": 0.09612441085520441, + "OOD_arXiv_2023": 0.09798804333054081, + "OOD_CommonPile": 0.09906874971843704 + }, + "col_means": { + "Wikipedia_en": 0.10274474307791226, + "Pile-CC": 0.10258283434151154, + "Github": 0.08939455441777008, + "OOD_News_2024": 0.10476783683256212, + "OOD_arXiv_2023": 0.09708319400583773, + "OOD_CommonPile": 0.09872866247487427 + }, + "n": 6 + }, + "USPTO_Backgrounds+Github": { + "fit_pct": 49.077827715914886, + "read_pct": 49.432990888192485, + "residual_pct": 1.4891813958926392, + "fit_span": 0.012516824986044048, + "read_span": 0.014937012424019888, + "row_means": { + "Wikipedia_en": 0.09967225956799532, + "Pile-CC": 0.10860330981538735, + "StackExchange": 0.10863391786934463, + "OOD_News_2024": 0.09611709288330059, + "OOD_arXiv_2023": 0.09841023329516045, + "OOD_CommonPile": 0.09936588550917805 + }, + "col_means": { + "Wikipedia_en": 0.10531824020996552, + "Pile-CC": 0.10513496072661062, + "StackExchange": 0.09255737758552035, + "OOD_News_2024": 0.10749439000954024, + "OOD_arXiv_2023": 0.09928676934300543, + "OOD_CommonPile": 0.10101096106572421 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 52.3829095816429, + "read_pct": 45.86575505849258, + "residual_pct": 1.7513353598645232, + "fit_span": 0.01593828593354879, + "read_span": 0.013226268193856985, + "row_means": { + "Wikipedia_en": 0.09721812200948113, + "Pile-CC": 0.10604378589023664, + "StackExchange": 0.10659616701304912, + "Github": 0.09065788107950033, + "OOD_arXiv_2023": 0.09580267372980922, + "OOD_CommonPile": 0.09676307914131838 + }, + "col_means": { + "Wikipedia_en": 0.10506032491620217, + "Pile-CC": 0.10456571179157537, + "StackExchange": 0.09183405672234518, + "Github": 0.09200439865709524, + "OOD_arXiv_2023": 0.09887958597866128, + "OOD_CommonPile": 0.10073763079751559 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 49.03669211778151, + "read_pct": 49.84208112785427, + "residual_pct": 1.121226754364218, + "fit_span": 0.016461263773359214, + "read_span": 0.015403323577009825, + "row_means": { + "Wikipedia_en": 0.09873165720383878, + "Pile-CC": 0.10740483044086162, + "StackExchange": 0.10807392915400366, + "Github": 0.09161266538064444, + "OOD_News_2024": 0.094743651901458, + "OOD_CommonPile": 0.09811450080920217 + }, + "col_means": { + "Wikipedia_en": 0.10464750153219535, + "Pile-CC": 0.10420095149462145, + "StackExchange": 0.09123520342529648, + "Github": 0.09183505329657002, + "OOD_News_2024": 0.1066385270023063, + "OOD_CommonPile": 0.1001239981390191 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 49.564449013565095, + "read_pct": 49.26922160720974, + "residual_pct": 1.16632937922517, + "fit_span": 0.01638254051967697, + "read_span": 0.015283022600622781, + "row_means": { + "Wikipedia_en": 0.09830677180973538, + "Pile-CC": 0.10732907346423923, + "StackExchange": 0.10778282597247098, + "Github": 0.09140028545279401, + "OOD_News_2024": 0.09466512506974309, + "OOD_arXiv_2023": 0.09651397818363168 + }, + "col_means": { + "Wikipedia_en": 0.10462268826215425, + "Pile-CC": 0.1038887436605162, + "StackExchange": 0.09106419260777464, + "Github": 0.09153898830516954, + "OOD_News_2024": 0.10634721520839742, + "OOD_arXiv_2023": 0.09853623190860229 + }, + "n": 6 + }, + "Pile-CC+StackExchange": { + "fit_pct": 45.29050803527647, + "read_pct": 53.588997206697115, + "residual_pct": 1.1204947580264173, + "fit_span": 0.016790018958056832, + "read_span": 0.016944884400625054, + "row_means": { + "Wikipedia_en": 0.10092183124075678, + "USPTO_Backgrounds": 0.1104012303291475, + "Github": 0.09361121137109067, + "OOD_News_2024": 0.09661999063230016, + "OOD_arXiv_2023": 0.09858129358531356, + "OOD_CommonPile": 0.09954590713820868 + }, + "col_means": { + "Wikipedia_en": 0.1029516321581064, + "USPTO_Backgrounds": 0.10636303423369234, + "Github": 0.08941814983306728, + "OOD_News_2024": 0.10481141849239875, + "OOD_arXiv_2023": 0.09715672840528121, + "OOD_CommonPile": 0.09898050117427139 + }, + "n": 6 + }, + "Pile-CC+Github": { + "fit_pct": 47.510177227994554, + "read_pct": 51.26384953198994, + "residual_pct": 1.2259732400155092, + "fit_span": 0.013951704534702, + "read_span": 0.016749733714041884, + "row_means": { + "Wikipedia_en": 0.10061872259204932, + "USPTO_Backgrounds": 0.11056437719509833, + "StackExchange": 0.109746641750206, + "OOD_News_2024": 0.09661267266039633, + "OOD_arXiv_2023": 0.0990034835499332, + "OOD_CommonPile": 0.09984304292894969 + }, + "col_means": { + "Wikipedia_en": 0.10552512929015966, + "USPTO_Backgrounds": 0.10972623496178399, + "StackExchange": 0.09297650124774211, + "OOD_News_2024": 0.10753797166937687, + "OOD_arXiv_2023": 0.0993603037424489, + "OOD_CommonPile": 0.10126279976512133 + }, + "n": 6 + }, + "Pile-CC+OOD_News_2024": { + "fit_pct": 48.709639615780446, + "read_pct": 49.739598768691835, + "residual_pct": 1.550761615527722, + "fit_span": 0.017025336408039063, + "read_span": 0.01693506171512936, + "row_means": { + "Wikipedia_en": 0.09816458503353513, + "USPTO_Backgrounds": 0.1079848670254082, + "StackExchange": 0.1077088908939105, + "Github": 0.09095953061736914, + "OOD_arXiv_2023": 0.09639592398458195, + "OOD_CommonPile": 0.09724023656109003 + }, + "col_means": { + "Wikipedia_en": 0.1052672139963963, + "USPTO_Backgrounds": 0.1089630557875218, + "StackExchange": 0.09225318038456694, + "Github": 0.09202799407239244, + "OOD_arXiv_2023": 0.09895312037810476, + "OOD_CommonPile": 0.1009894694969127 + }, + "n": 6 + }, + "Pile-CC+OOD_arXiv_2023": { + "fit_pct": 48.06804052585788, + "read_pct": 50.89878687805069, + "residual_pct": 1.033172596091423, + "fit_span": 0.0174016439179131, + "read_span": 0.01684629792537265, + "row_means": { + "Wikipedia_en": 0.09967812022789278, + "USPTO_Backgrounds": 0.10931595883642634, + "StackExchange": 0.10918665303486502, + "Github": 0.09191431491851323, + "OOD_News_2024": 0.09523923167855375, + "OOD_CommonPile": 0.09859165822897382 + }, + "col_means": { + "Wikipedia_en": 0.10485439061238948, + "USPTO_Backgrounds": 0.10850062501289089, + "StackExchange": 0.09165432708751824, + "Github": 0.09185864871186722, + "OOD_News_2024": 0.10668210866214294, + "OOD_CommonPile": 0.10037583683841621 + }, + "n": 6 + }, + "Pile-CC+OOD_CommonPile": { + "fit_pct": 47.939009855314474, + "read_pct": 50.925820744916464, + "residual_pct": 1.1351693997690666, + "fit_span": 0.017359962569187495, + "read_span": 0.01682119374379032, + "row_means": { + "Wikipedia_en": 0.09925323483378937, + "USPTO_Backgrounds": 0.1090618975598503, + "StackExchange": 0.10889554985333234, + "Github": 0.0917019349906628, + "OOD_News_2024": 0.09516070484683883, + "OOD_arXiv_2023": 0.09710722843840443 + }, + "col_means": { + "Wikipedia_en": 0.10482957734234838, + "USPTO_Backgrounds": 0.10830451001378673, + "StackExchange": 0.09148331626999641, + "Github": 0.09156258372046674, + "OOD_News_2024": 0.10639079686823405, + "OOD_arXiv_2023": 0.09860976630804577 + }, + "n": 6 + }, + "StackExchange+Github": { + "fit_pct": 70.3751011932439, + "read_pct": 28.25276809650362, + "residual_pct": 1.3721307102524773, + "fit_span": 0.013939388270524378, + "read_span": 0.009590597425294278, + "row_means": { + "Wikipedia_en": 0.10286571559263394, + "USPTO_Backgrounds": 0.11276121731390487, + "Pile-CC": 0.1113864435641854, + "OOD_News_2024": 0.0988218290433805, + "OOD_arXiv_2023": 0.10097854693282259, + "OOD_CommonPile": 0.10195930332842248 + }, + "col_means": { + "Wikipedia_en": 0.10545248335261864, + "USPTO_Backgrounds": 0.10919799697326703, + "Pile-CC": 0.10555227665118527, + "OOD_News_2024": 0.10766834988457116, + "OOD_arXiv_2023": 0.09960739954797275, + "OOD_CommonPile": 0.10129454936573488 + }, + "n": 6 + }, + "StackExchange+OOD_News_2024": { + "fit_pct": 53.89267941625402, + "read_pct": 45.005943565729666, + "residual_pct": 1.1013770180163052, + "fit_span": 0.016858948215721523, + "read_span": 0.01679821858027329, + "row_means": { + "Wikipedia_en": 0.10041157803411975, + "USPTO_Backgrounds": 0.11018170714421474, + "Pile-CC": 0.10882691963903468, + "Github": 0.09332275892849322, + "OOD_arXiv_2023": 0.09837098736747134, + "OOD_CommonPile": 0.09935649696056283 + }, + "col_means": { + "Wikipedia_en": 0.10519456805885528, + "USPTO_Backgrounds": 0.10843481779900482, + "Pile-CC": 0.10498302771615002, + "Github": 0.09163659921873153, + "OOD_arXiv_2023": 0.0992002161836286, + "OOD_CommonPile": 0.10102121909752626 + }, + "n": 6 + }, + "StackExchange+OOD_arXiv_2023": { + "fit_pct": 55.79032934066861, + "read_pct": 43.382788884458606, + "residual_pct": 0.8268817748727869, + "fit_span": 0.017235255725595547, + "read_span": 0.016505133166167613, + "row_means": { + "Wikipedia_en": 0.1019251132284774, + "USPTO_Backgrounds": 0.11151279895523288, + "Pile-CC": 0.11018796418965966, + "Github": 0.09427754322963733, + "OOD_News_2024": 0.09744838806153792, + "OOD_CommonPile": 0.10070791862844662 + }, + "col_means": { + "Wikipedia_en": 0.10478174467484846, + "USPTO_Backgrounds": 0.10797238702437392, + "Pile-CC": 0.10461826741919611, + "Github": 0.09146725385820631, + "OOD_News_2024": 0.10681248687733724, + "OOD_CommonPile": 0.10040758643902976 + }, + "n": 6 + }, + "StackExchange+OOD_CommonPile": { + "fit_pct": 55.344126996122945, + "read_pct": 43.8330744033383, + "residual_pct": 0.822798600538758, + "fit_span": 0.01719357437686994, + "read_span": 0.01660508315846393, + "row_means": { + "Wikipedia_en": 0.10150022783437398, + "USPTO_Backgrounds": 0.11125873767865684, + "Pile-CC": 0.11011220721303727, + "Github": 0.0940651633017869, + "OOD_News_2024": 0.09736986122982301, + "OOD_arXiv_2023": 0.09908229182129381 + }, + "col_means": { + "Wikipedia_en": 0.10475693140480737, + "USPTO_Backgrounds": 0.10777627202526976, + "Pile-CC": 0.10430605958509086, + "Github": 0.09117118886680584, + "OOD_News_2024": 0.10652117508342834, + "OOD_arXiv_2023": 0.09885686211356962 + }, + "n": 6 + }, + "Github+OOD_News_2024": { + "fit_pct": 45.368746755934815, + "read_pct": 53.22972039891428, + "residual_pct": 1.4015328451509066, + "fit_span": 0.011551676678074593, + "read_span": 0.016828131302760008, + "row_means": { + "Wikipedia_en": 0.10010846938541228, + "USPTO_Backgrounds": 0.11034485401016557, + "Pile-CC": 0.10859453825806095, + "StackExchange": 0.1092837625157295, + "OOD_arXiv_2023": 0.09879317733209098, + "OOD_CommonPile": 0.09965363275130383 + }, + "col_means": { + "Wikipedia_en": 0.10776806519090854, + "USPTO_Backgrounds": 0.11179801852709648, + "Pile-CC": 0.1075351541012491, + "StackExchange": 0.09496988722433647, + "OOD_arXiv_2023": 0.1014037915207963, + "OOD_CommonPile": 0.10330351768837621 + }, + "n": 6 + }, + "Github+OOD_arXiv_2023": { + "fit_pct": 49.11347777791474, + "read_pct": 49.76969929872892, + "residual_pct": 1.1168229233563398, + "fit_span": 0.014234875731549607, + "read_span": 0.01696455382517781, + "row_means": { + "Wikipedia_en": 0.10162200457976993, + "USPTO_Backgrounds": 0.1116759458211837, + "Pile-CC": 0.10995558280868593, + "StackExchange": 0.11076152465668404, + "OOD_News_2024": 0.0974410700896341, + "OOD_CommonPile": 0.10100505441918761 + }, + "col_means": { + "Wikipedia_en": 0.10735524180690172, + "USPTO_Backgrounds": 0.11133558775246558, + "Pile-CC": 0.10717039380429519, + "StackExchange": 0.09437103392728777, + "OOD_News_2024": 0.10953904005431536, + "OOD_CommonPile": 0.1026898850298797 + }, + "n": 6 + }, + "Github+OOD_CommonPile": { + "fit_pct": 49.59262915478606, + "read_pct": 49.29390919821961, + "residual_pct": 1.1134616469943321, + "fit_span": 0.014059341286688481, + "read_span": 0.01693944964359549, + "row_means": { + "Wikipedia_en": 0.10119711918566651, + "USPTO_Backgrounds": 0.11142188454460766, + "Pile-CC": 0.10987982583206354, + "StackExchange": 0.11047042147515135, + "OOD_News_2024": 0.09736254325791918, + "OOD_arXiv_2023": 0.09950448178591345 + }, + "col_means": { + "Wikipedia_en": 0.10733042853686062, + "USPTO_Backgrounds": 0.11113947275336142, + "Pile-CC": 0.10685818597018995, + "StackExchange": 0.09420002310976593, + "OOD_News_2024": 0.10924772826040646, + "OOD_arXiv_2023": 0.10106043745073731 + }, + "n": 6 + }, + "OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 49.4010775978473, + "read_pct": 49.41528089410974, + "residual_pct": 1.1836415080429687, + "fit_span": 0.01747057317557779, + "read_span": 0.016924695514090773, + "row_means": { + "Wikipedia_en": 0.09916786702125574, + "USPTO_Backgrounds": 0.10909643565149357, + "Pile-CC": 0.10739605888353523, + "StackExchange": 0.10872377380038853, + "Github": 0.09162586247591578, + "OOD_CommonPile": 0.09840224805132797 + }, + "col_means": { + "Wikipedia_en": 0.10709732651313836, + "USPTO_Backgrounds": 0.11057240857820337, + "Pile-CC": 0.10660114486925994, + "StackExchange": 0.0936477130641126, + "Github": 0.09407709809753147, + "OOD_CommonPile": 0.10241655476167108 + }, + "n": 6 + }, + "OOD_News_2024+OOD_CommonPile": { + "fit_pct": 50.24811436118757, + "read_pct": 48.55664572481285, + "residual_pct": 1.1952399139995844, + "fit_span": 0.017428891826852186, + "read_span": 0.016899591332508457, + "row_means": { + "Wikipedia_en": 0.09874298162715234, + "USPTO_Backgrounds": 0.10884237437491753, + "Pile-CC": 0.10732030190691283, + "StackExchange": 0.10843267061885584, + "Github": 0.09141348254806535, + "OOD_arXiv_2023": 0.0968969222205622 + }, + "col_means": { + "Wikipedia_en": 0.10707251324309726, + "USPTO_Backgrounds": 0.11037629357909921, + "Pile-CC": 0.10628893703515468, + "StackExchange": 0.09347670224659076, + "Github": 0.09378103310613099, + "OOD_arXiv_2023": 0.10065325408639317 + }, + "n": 6 + }, + "OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 50.61140520804757, + "read_pct": 48.63035002217219, + "residual_pct": 0.7582447697802386, + "fit_span": 0.01780519933672621, + "read_span": 0.017036013854926244, + "row_means": { + "Wikipedia_en": 0.10025651682150999, + "USPTO_Backgrounds": 0.11017346618593567, + "Pile-CC": 0.10868134645753781, + "StackExchange": 0.10991043275981038, + "Github": 0.09236826684920946, + "OOD_News_2024": 0.0959891022760766 + }, + "col_means": { + "Wikipedia_en": 0.10665968985909044, + "USPTO_Backgrounds": 0.10991386280446831, + "Pile-CC": 0.10592417673820077, + "StackExchange": 0.09287784894954207, + "Github": 0.09361168774560577, + "OOD_News_2024": 0.10839186525317254 + }, + "n": 6 + } + }, + "n_l2o_fit_gt_read": "16/28", + "l2o_failures": [ + "USPTO_Backgrounds+Pile-CC", + "USPTO_Backgrounds+StackExchange", + "USPTO_Backgrounds+Github", + "USPTO_Backgrounds+OOD_arXiv_2023", + "Pile-CC+StackExchange", + "Pile-CC+Github", + "Pile-CC+OOD_News_2024", + "Pile-CC+OOD_arXiv_2023", + "Pile-CC+OOD_CommonPile", + "Github+OOD_News_2024", + "Github+OOD_arXiv_2023", + "OOD_News_2024+OOD_arXiv_2023" + ], + "l2o_worst_split": { + "dropped": "USPTO_Backgrounds+Pile-CC", + "fit_pct": 42.08107123390732, + "read_pct": 56.08737719584901, + "residual_pct": 1.831551570243672, + "fit_span": 0.01611271272542783, + "read_span": 0.01485150934593682, + "row_means": { + "Wikipedia_en": 0.09772837521611816, + "StackExchange": 0.10705904624752563, + "Github": 0.0909463335220978, + "OOD_News_2024": 0.09391525447222025, + "OOD_arXiv_2023": 0.09601297994765143, + "OOD_CommonPile": 0.09695248931896425 + }, + "col_means": { + "Wikipedia_en": 0.10281738901545329, + "StackExchange": 0.08984067074575082, + "Github": 0.08978594927143099, + "OOD_News_2024": 0.10463745861736781, + "OOD_arXiv_2023": 0.09683609820031389, + "OOD_CommonPile": 0.09869691287426072 + }, + "n": 6 + }, + "fit_axis_leverage_point": "Github", + "read_axis_leverage_point": "StackExchange", + "drop_fit_leverage_only": { + "fit_pct": 50.83451434240633, + "read_pct": 47.96562247603154, + "residual_pct": 1.199863181562129, + "fit_span": 0.014015912493720417, + "read_span": 0.016742457825291365, + "row_means": { + "Wikipedia_en": 0.1008796632349018, + "USPTO_Backgrounds": 0.11104568715044667, + "Pile-CC": 0.10950806079735417, + "StackExchange": 0.10998707169090353, + "OOD_News_2024": 0.09702977465672626, + "OOD_arXiv_2023": 0.09939169281657549, + "OOD_CommonPile": 0.10037892185639412 + }, + "col_means": { + "Wikipedia_en": 0.10647942851785393, + "USPTO_Backgrounds": 0.11047056806569416, + "Pile-CC": 0.10639190160602863, + "StackExchange": 0.0937281102404028, + "OOD_News_2024": 0.10860243229276781, + "OOD_arXiv_2023": 0.10042639302629386, + "OOD_CommonPile": 0.10212203845426085 + }, + "n": 7 + }, + "drop_read_leverage_only": { + "fit_pct": 54.30293560630481, + "read_pct": 44.71750309530783, + "residual_pct": 0.9795612983873632, + "fit_span": 0.0169366810866262, + "read_span": 0.01671054580152803, + "row_means": { + "Wikipedia_en": 0.10113947064807964, + "USPTO_Backgrounds": 0.11090584697963167, + "Pile-CC": 0.1097072448381888, + "Github": 0.09396916589300547, + "OOD_News_2024": 0.0970360472040724, + "OOD_arXiv_2023": 0.09902981570404437, + "OOD_CommonPile": 0.10012423403575899 + }, + "col_means": { + "Wikipedia_en": 0.10427357383323685, + "USPTO_Backgrounds": 0.10758782458447275, + "Pile-CC": 0.10420436470451513, + "Github": 0.09087727878294471, + "OOD_News_2024": 0.10626538671250085, + "OOD_arXiv_2023": 0.09853761416586442, + "OOD_CommonPile": 0.10016578251924661 + }, + "n": 7 + }, + "asymmetry_note": "dropping the fit axis's weakest member (Github) moves fit share to 50.8%; dropping the read axis's weakest member (StackExchange) moves it to 54.3%. The paper shows only the first." + }, + "min": { + "full": { + "fit_pct": 53.35344735234028, + "read_pct": 44.60936175509925, + "residual_pct": 2.0371908925604707, + "fit_span": 0.03226158868314491, + "read_span": 0.024829129639288627, + "row_means": { + "Wikipedia_en": 0.23152156053369657, + "USPTO_Backgrounds": 0.2555947363480098, + "Pile-CC": 0.2559463235213318, + "StackExchange": 0.26002017346521217, + "Github": 0.22775858478206726, + "OOD_News_2024": 0.24214933404388528, + "OOD_arXiv_2023": 0.2535410887380648, + "OOD_CommonPile": 0.2469727687123749 + }, + "col_means": { + "Wikipedia_en": 0.2535931211275359, + "USPTO_Backgrounds": 0.2598997434601188, + "Pile-CC": 0.25737932473162395, + "StackExchange": 0.23572498860303312, + "Github": 0.2396430952577955, + "OOD_News_2024": 0.25592221629857603, + "OOD_arXiv_2023": 0.2362714668451291, + "OOD_CommonPile": 0.23507061382083017 + }, + "n": 8 + }, + "leave_one_out": { + "Wikipedia_en": { + "fit_pct": 46.81110557693417, + "read_pct": 50.85837600168466, + "residual_pct": 2.330518421381178, + "fit_span": 0.032255948695635056, + "read_span": 0.025139714496594606, + "row_means": { + "USPTO_Backgrounds": 0.2546521757271082, + "Pile-CC": 0.2547390000066823, + "StackExchange": 0.2590056600256099, + "Github": 0.22674971132997482, + "OOD_News_2024": 0.24077020276278732, + "OOD_arXiv_2023": 0.25297702633376634, + "OOD_CommonPile": 0.24625951936320653 + }, + "col_means": { + "USPTO_Backgrounds": 0.26222252150967956, + "Pile-CC": 0.2597060517570566, + "StackExchange": 0.23781028040463015, + "Github": 0.2419379295456031, + "OOD_News_2024": 0.2583817249814433, + "OOD_arXiv_2023": 0.23801198033763776, + "OOD_CommonPile": 0.23708280701308496 + }, + "n": 7 + }, + "USPTO_Backgrounds": { + "fit_pct": 56.6792018376361, + "read_pct": 41.06419873316167, + "residual_pct": 2.256599429202232, + "fit_span": 0.03192390514803786, + "read_span": 0.02218238359819802, + "row_means": { + "Wikipedia_en": 0.22979031245091133, + "Pile-CC": 0.2537775315906084, + "StackExchange": 0.258255518555996, + "Github": 0.22633161340795813, + "OOD_News_2024": 0.24016587825876382, + "OOD_arXiv_2023": 0.2513412351705252, + "OOD_CommonPile": 0.24497417915198538 + }, + "col_means": { + "Wikipedia_en": 0.25236461547513805, + "Pile-CC": 0.25601254614986596, + "StackExchange": 0.23383016255166794, + "Github": 0.23826757981368, + "OOD_News_2024": 0.2547577401209209, + "OOD_arXiv_2023": 0.23524334202742292, + "OOD_CommonPile": 0.23416028244805243 + }, + "n": 7 + }, + "Pile-CC": { + "fit_pct": 56.50763481529161, + "read_pct": 41.42528085634039, + "residual_pct": 2.0670843283680003, + "fit_span": 0.03248407571975673, + "read_span": 0.024528523726714047, + "row_means": { + "Wikipedia_en": 0.23015432124513957, + "USPTO_Backgrounds": 0.2539730165685926, + "StackExchange": 0.2585246256626551, + "Github": 0.22604054994289838, + "OOD_News_2024": 0.24043375243624998, + "OOD_arXiv_2023": 0.25223973326621546, + "OOD_CommonPile": 0.24558859879357947 + }, + "col_means": { + "Wikipedia_en": 0.2520496258423442, + "USPTO_Backgrounds": 0.25829572580636495, + "StackExchange": 0.23522730644733186, + "Github": 0.23859686986321493, + "OOD_News_2024": 0.254126624783708, + "OOD_arXiv_2023": 0.23489124309271575, + "OOD_CommonPile": 0.2337672020796509 + }, + "n": 7 + }, + "StackExchange": { + "fit_pct": 53.53726007459882, + "read_pct": 45.32738406925986, + "residual_pct": 1.135355856141325, + "fit_span": 0.0298434966317718, + "read_span": 0.025165532340132996, + "row_means": { + "Wikipedia_en": 0.2330063626111027, + "USPTO_Backgrounds": 0.25653844568878414, + "Pile-CC": 0.25833740349681605, + "Github": 0.22849390686504426, + "OOD_News_2024": 0.24472149337331456, + "OOD_arXiv_2023": 0.25514061026984736, + "OOD_CommonPile": 0.24809296849878534 + }, + "col_means": { + "Wikipedia_en": 0.25166045735397985, + "USPTO_Backgrounds": 0.25811788426446064, + "Pile-CC": 0.25550651282426856, + "Github": 0.23749172571632599, + "OOD_News_2024": 0.25388230683488977, + "OOD_arXiv_2023": 0.23471995188544195, + "OOD_CommonPile": 0.23295235192432764 + }, + "n": 7 + }, + "Github": { + "fit_pct": 41.358656075065184, + "read_pct": 56.523659412915094, + "residual_pct": 2.117684512019725, + "fit_span": 0.028123639521026395, + "read_span": 0.02546599780815459, + "row_means": { + "Wikipedia_en": 0.23265617557520432, + "USPTO_Backgrounds": 0.25649802677392486, + "Pile-CC": 0.2572291307358278, + "StackExchange": 0.2607798150962307, + "OOD_News_2024": 0.24324451167550354, + "OOD_arXiv_2023": 0.25475290841053405, + "OOD_CommonPile": 0.24801946458598925 + }, + "col_means": { + "Wikipedia_en": 0.2562748957247961, + "USPTO_Backgrounds": 0.2630643661828741, + "Pile-CC": 0.25989282417096315, + "StackExchange": 0.2375983683747195, + "OOD_News_2024": 0.2589125070573082, + "OOD_arXiv_2023": 0.2395631939024916, + "OOD_CommonPile": 0.23787387744006183 + }, + "n": 7 + }, + "OOD_News_2024": { + "fit_pct": 56.05510540136367, + "read_pct": 42.24118054522516, + "residual_pct": 1.7037140534111703, + "fit_span": 0.03184018684403292, + "read_span": 0.024672448768147365, + "row_means": { + "Wikipedia_en": 0.2304952612501525, + "USPTO_Backgrounds": 0.2543834773202737, + "Pile-CC": 0.2541541758954288, + "StackExchange": 0.25856568645390254, + "Github": 0.22672549960986962, + "OOD_arXiv_2023": 0.25217718614176626, + "OOD_CommonPile": 0.24563762645222365 + }, + "col_means": { + "Wikipedia_en": 0.2538488165726738, + "USPTO_Backgrounds": 0.2604520604487449, + "Pile-CC": 0.2578394560793799, + "StackExchange": 0.23737938429805494, + "Github": 0.24038023877711523, + "OOD_arXiv_2023": 0.23645934526705079, + "OOD_CommonPile": 0.23577961168059755 + }, + "n": 7 + }, + "OOD_arXiv_2023": { + "fit_pct": 56.00547196651275, + "read_pct": 42.12050880191628, + "residual_pct": 1.874019231570966, + "fit_span": 0.032027145049401684, + "read_span": 0.024876730661425345, + "row_means": { + "Wikipedia_en": 0.23258351598171487, + "USPTO_Backgrounds": 0.2573270786021437, + "Pile-CC": 0.25737679357980453, + "StackExchange": 0.2618613308798226, + "Github": 0.22983418583042092, + "OOD_News_2024": 0.243176907779915, + "OOD_CommonPile": 0.24839009061633122 + }, + "col_means": { + "Wikipedia_en": 0.2530364919217333, + "USPTO_Backgrounds": 0.2586082691385869, + "Pile-CC": 0.25662628868742593, + "StackExchange": 0.2347793529726683, + "Github": 0.23886948729022628, + "OOD_News_2024": 0.2548984747823505, + "OOD_CommonPile": 0.23373153847716158 + }, + "n": 7 + }, + "OOD_CommonPile": { + "fit_pct": 57.55685418677557, + "read_pct": 40.24133072996441, + "residual_pct": 2.2018150832600214, + "fit_span": 0.031948861550717134, + "read_span": 0.024509501918440785, + "row_means": { + "Wikipedia_en": 0.23302674611350374, + "USPTO_Backgrounds": 0.2576164224791148, + "Pile-CC": 0.25762515602308134, + "StackExchange": 0.26146613437504995, + "Github": 0.22951727282433282, + "OOD_News_2024": 0.2438695776498034, + "OOD_arXiv_2023": 0.25484065266828687 + }, + "col_means": { + "Wikipedia_en": 0.2538256364091048, + "USPTO_Backgrounds": 0.2597478645779784, + "Pile-CC": 0.2574818056727213, + "StackExchange": 0.2352383626595376, + "Github": 0.23964269492361281, + "OOD_News_2024": 0.2558655665507392, + "OOD_arXiv_2023": 0.23616003133947888 + }, + "n": 7 + } + }, + "n_loo_fit_gt_read": "6/8", + "leave_two_out": { + "Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 52.32062604200177, + "read_pct": 45.031943464782344, + "residual_pct": 2.647430493215886, + "fit_span": 0.03186104457342512, + "read_span": 0.022552065937607363, + "row_means": { + "Pile-CC": 0.2520075221683968, + "StackExchange": 0.25677781039159053, + "Github": 0.22491676581816542, + "OOD_News_2024": 0.2382263157999626, + "OOD_arXiv_2023": 0.25031652010425376, + "OOD_CommonPile": 0.24380895665122404 + }, + "col_means": { + "Pile-CC": 0.25849926458257766, + "StackExchange": 0.2359471986449703, + "Github": 0.2407156339087696, + "OOD_News_2024": 0.2574330875546568, + "OOD_arXiv_2023": 0.237102586965732, + "OOD_CommonPile": 0.23635611927688674 + }, + "n": 6 + }, + "Wikipedia_en+Pile-CC": { + "fit_pct": 52.442755923912436, + "read_pct": 45.218004916593706, + "residual_pct": 2.33923915949386, + "fit_span": 0.03251457690709719, + "read_span": 0.0248407717414752, + "row_means": { + "USPTO_Backgrounds": 0.2526030758809712, + "StackExchange": 0.2570917686826929, + "Github": 0.2245771917755957, + "OOD_News_2024": 0.23853883567369646, + "OOD_arXiv_2023": 0.251364767882559, + "OOD_CommonPile": 0.24452577956641713 + }, + "col_means": { + "USPTO_Backgrounds": 0.26073829725522685, + "StackExchange": 0.2375771998565782, + "Github": 0.24109980563322703, + "OOD_News_2024": 0.25669678632790843, + "OOD_arXiv_2023": 0.23669180487524027, + "OOD_CommonPile": 0.23589752551375165 + }, + "n": 6 + }, + "Wikipedia_en+StackExchange": { + "fit_pct": 47.13001388878387, + "read_pct": 51.79992772308668, + "residual_pct": 1.070058388129444, + "fit_span": 0.029887931207539825, + "read_span": 0.025583948457130662, + "row_means": { + "USPTO_Backgrounds": 0.25559607652119465, + "Pile-CC": 0.25732737272563905, + "Github": 0.22743944151809922, + "OOD_News_2024": 0.2435412001002718, + "OOD_arXiv_2023": 0.25474912438679626, + "OOD_CommonPile": 0.24744754422249066 + }, + "col_means": { + "USPTO_Backgrounds": 0.2605308154563385, + "Pile-CC": 0.2579088923693807, + "Github": 0.23981047079518988, + "OOD_News_2024": 0.25641174872095385, + "OOD_arXiv_2023": 0.23649196513342086, + "OOD_CommonPile": 0.23494686699920783 + }, + "n": 6 + }, + "Wikipedia_en+Github": { + "fit_pct": 21.009005511431113, + "read_pct": 76.38980395789154, + "residual_pct": 2.601190530677337, + "fit_span": 0.017904768235705498, + "read_span": 0.02595827225595715, + "row_means": { + "USPTO_Backgrounds": 0.2555489211205255, + "Pile-CC": 0.25603438783781945, + "StackExchange": 0.25972282302186445, + "OOD_News_2024": 0.24181805478615895, + "OOD_arXiv_2023": 0.2542968055509307, + "OOD_CommonPile": 0.24736178965756186 + }, + "col_means": { + "USPTO_Backgrounds": 0.2663017110278209, + "Pile-CC": 0.26302625560719106, + "StackExchange": 0.24034343877186376, + "OOD_News_2024": 0.2622803156471087, + "OOD_arXiv_2023": 0.24214241415331209, + "OOD_CommonPile": 0.2406886467675644 + }, + "n": 6 + }, + "Wikipedia_en+OOD_News_2024": { + "fit_pct": 48.998113064740714, + "read_pct": 48.966136042934444, + "residual_pct": 2.03575089232484, + "fit_span": 0.031763373218752694, + "read_span": 0.025008684289814176, + "row_means": { + "USPTO_Backgrounds": 0.2530819467579325, + "Pile-CC": 0.25244694052402067, + "StackExchange": 0.2571396729391482, + "Github": 0.2253762997203955, + "OOD_arXiv_2023": 0.2512917962373683, + "OOD_CommonPile": 0.244582978501502 + }, + "col_means": { + "USPTO_Backgrounds": 0.2632540210046702, + "Pile-CC": 0.2606306595003439, + "StackExchange": 0.24008795734908844, + "Github": 0.24318040269944402, + "OOD_arXiv_2023": 0.2385212574119645, + "OOD_CommonPile": 0.23824533671485604 + }, + "n": 6 + }, + "Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 49.163994924145854, + "read_pct": 48.57778104549241, + "residual_pct": 2.2582240303617396, + "fit_span": 0.031981491125016304, + "read_span": 0.025247013165305032, + "row_means": { + "USPTO_Backgrounds": 0.2565161482534475, + "Pile-CC": 0.25620666115579227, + "StackExchange": 0.26098459143605496, + "Github": 0.22900310031103865, + "OOD_News_2024": 0.241739183574639, + "OOD_CommonPile": 0.24779418669296083 + }, + "col_means": { + "USPTO_Backgrounds": 0.26110293114281913, + "Pile-CC": 0.2592152975430643, + "StackExchange": 0.23705458746947072, + "Github": 0.24141785929807358, + "OOD_News_2024": 0.25759727799299137, + "OOD_CommonPile": 0.2358559179775141 + }, + "n": 6 + }, + "Wikipedia_en+OOD_CommonPile": { + "fit_pct": 50.1961522600238, + "read_pct": 47.2140468581699, + "residual_pct": 2.5898008818062905, + "fit_span": 0.03189016037655096, + "read_span": 0.02484236038462434, + "row_means": { + "USPTO_Backgrounds": 0.2568537161099138, + "Pile-CC": 0.25649641733961526, + "StackExchange": 0.2605235288471535, + "Github": 0.22863336847060256, + "OOD_News_2024": 0.24254729842284212, + "OOD_arXiv_2023": 0.25439917385164235 + }, + "col_means": { + "USPTO_Backgrounds": 0.26243245915544255, + "Pile-CC": 0.26021340069257554, + "StackExchange": 0.23759009877081821, + "Github": 0.24231993487035786, + "OOD_News_2024": 0.2587255517227782, + "OOD_arXiv_2023": 0.23817205782979728 + }, + "n": 6 + }, + "USPTO_Backgrounds+Pile-CC": { + "fit_pct": 62.81630399104917, + "read_pct": 34.610844679778396, + "residual_pct": 2.572851329172437, + "fit_span": 0.03212719276823381, + "read_span": 0.019980890469418644, + "row_means": { + "Wikipedia_en": 0.2279066586004639, + "StackExchange": 0.25621660363481, + "Github": 0.22408941086657622, + "OOD_News_2024": 0.23783379041900238, + "OOD_arXiv_2023": 0.24945634485877774, + "OOD_CommonPile": 0.24302621598665913 + }, + "col_means": { + "Wikipedia_en": 0.2503591200336814, + "StackExchange": 0.2329337290281223, + "Github": 0.2368177309459834, + "OOD_News_2024": 0.2524688039906323, + "OOD_arXiv_2023": 0.2334617268466563, + "OOD_CommonPile": 0.23248791352121367 + }, + "n": 6 + }, + "USPTO_Backgrounds+StackExchange": { + "fit_pct": 55.48827295667549, + "read_pct": 43.2032514036185, + "residual_pct": 1.3084756397060118, + "fit_span": 0.02925399896447306, + "read_span": 0.022062547487655154, + "row_means": { + "Wikipedia_en": 0.23123404019408755, + "Pile-CC": 0.2562056595735528, + "Github": 0.22695166060907973, + "OOD_News_2024": 0.24283615484557772, + "OOD_arXiv_2023": 0.25284070136301495, + "OOD_CommonPile": 0.24594798064273266 + }, + "col_means": { + "Wikipedia_en": 0.24990509013058965, + "Pile-CC": 0.253599802494325, + "Github": 0.2355283961079463, + "OOD_News_2024": 0.25218376638367773, + "OOD_arXiv_2023": 0.2332618871048369, + "OOD_CommonPile": 0.23153725500666986 + }, + "n": 6 + }, + "USPTO_Backgrounds+Github": { + "fit_pct": 45.23962835179833, + "read_pct": 52.609332663665406, + "residual_pct": 2.1510389845362563, + "fit_span": 0.028022169321775425, + "read_span": 0.023017197788727517, + "row_means": { + "Wikipedia_en": 0.23082548865220615, + "Pile-CC": 0.2549126746857332, + "StackExchange": 0.25884765797398157, + "OOD_News_2024": 0.24111300953146486, + "OOD_arXiv_2023": 0.2523883825271494, + "OOD_CommonPile": 0.24586222607780386 + }, + "col_means": { + "Wikipedia_en": 0.25528860156320865, + "Pile-CC": 0.2587171657321354, + "StackExchange": 0.23569996794340786, + "OOD_News_2024": 0.25805233330983257, + "OOD_arXiv_2023": 0.23891233612472812, + "OOD_CommonPile": 0.23727903477502643 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 62.89153318271039, + "read_pct": 35.25419459192547, + "residual_pct": 1.854272225364142, + "fit_span": 0.03137598907988931, + "read_span": 0.021485844902970147, + "row_means": { + "Wikipedia_en": 0.22830442193964565, + "Pile-CC": 0.2513252273719344, + "StackExchange": 0.2562645078912653, + "Github": 0.224888518811376, + "OOD_arXiv_2023": 0.249383373213587, + "OOD_CommonPile": 0.243083414921744 + }, + "col_means": { + "Wikipedia_en": 0.25245817588573255, + "Pile-CC": 0.25632156962528824, + "StackExchange": 0.23544448652063255, + "Github": 0.2388983280122004, + "OOD_arXiv_2023": 0.23529117938338054, + "OOD_CommonPile": 0.2348357247223181 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 57.2648652589358, + "read_pct": 40.747019920579476, + "residual_pct": 1.9881148204847274, + "fit_span": 0.03159410698615289, + "read_span": 0.022495091026993846, + "row_means": { + "Wikipedia_en": 0.23074071912646846, + "Pile-CC": 0.255084948003706, + "StackExchange": 0.2601094263881721, + "Github": 0.2285153194020192, + "OOD_News_2024": 0.2410341383199449, + "OOD_CommonPile": 0.24629462311320283 + }, + "col_means": { + "Wikipedia_en": 0.2515104637929687, + "Pile-CC": 0.25490620766800864, + "StackExchange": 0.2324111166410148, + "Github": 0.23713578461082996, + "OOD_News_2024": 0.25336929565571525, + "OOD_CommonPile": 0.23244630598497612 + }, + "n": 6 + }, + "USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 59.18141567642879, + "read_pct": 38.42474601345063, + "residual_pct": 2.393838310120584, + "fit_span": 0.031502776237687574, + "read_span": 0.022957682875157537, + "row_means": { + "Wikipedia_en": 0.2312578209468888, + "Pile-CC": 0.25537470418752894, + "StackExchange": 0.25964836379927064, + "Github": 0.22814558756158307, + "OOD_News_2024": 0.24184225316814803, + "OOD_arXiv_2023": 0.25249075082786104 + }, + "col_means": { + "Wikipedia_en": 0.2524311323615688, + "Pile-CC": 0.25590431081751985, + "StackExchange": 0.23294662794236232, + "Github": 0.23803786018311426, + "OOD_News_2024": 0.25449756938550205, + "OOD_arXiv_2023": 0.2349419798012133 + }, + "n": 6 + }, + "Pile-CC+StackExchange": { + "fit_pct": 53.016947757511794, + "read_pct": 45.766170925816546, + "residual_pct": 1.2168813166716623, + "fit_span": 0.02819163760308316, + "read_span": 0.024870892558936708, + "row_means": { + "Wikipedia_en": 0.23165871712068717, + "USPTO_Backgrounds": 0.2548037241695932, + "Github": 0.22661208656651002, + "OOD_News_2024": 0.2431486747193116, + "OOD_arXiv_2023": 0.25388894914132026, + "OOD_CommonPile": 0.24666480355792575 + }, + "col_means": { + "Wikipedia_en": 0.24953760222566349, + "USPTO_Backgrounds": 0.2559495538024715, + "Github": 0.23591256783240372, + "OOD_News_2024": 0.2514474651569294, + "OOD_arXiv_2023": 0.23285110501434517, + "OOD_CommonPile": 0.23107866124353477 + }, + "n": 6 + }, + "Pile-CC+Github": { + "fit_pct": 43.67099341643807, + "read_pct": 54.18028651100135, + "residual_pct": 2.148720072560576, + "fit_span": 0.027911450686278205, + "read_span": 0.024900008362062576, + "row_means": { + "Wikipedia_en": 0.23125016557880573, + "USPTO_Backgrounds": 0.25475656876892405, + "StackExchange": 0.25916161626508394, + "OOD_News_2024": 0.24142552940519874, + "OOD_arXiv_2023": 0.2534366303054547, + "OOD_CommonPile": 0.24657904899299699 + }, + "col_means": { + "Wikipedia_en": 0.2549211136582825, + "USPTO_Backgrounds": 0.2617204493739539, + "StackExchange": 0.23732996915501575, + "OOD_News_2024": 0.2573160320830842, + "OOD_arXiv_2023": 0.2385015540342364, + "OOD_CommonPile": 0.23682044101189131 + }, + "n": 6 + }, + "Pile-CC+OOD_News_2024": { + "fit_pct": 61.7879384237363, + "read_pct": 36.71273480001148, + "residual_pct": 1.4993267762522144, + "fit_span": 0.032029521413561385, + "read_span": 0.024295628391620167, + "row_means": { + "Wikipedia_en": 0.22872909886624526, + "USPTO_Backgrounds": 0.25228959440633103, + "StackExchange": 0.2565784661823677, + "Github": 0.2245489447688063, + "OOD_arXiv_2023": 0.2504316209918923, + "OOD_CommonPile": 0.2438002378369371 + }, + "col_means": { + "Wikipedia_en": 0.2520906879808064, + "USPTO_Backgrounds": 0.25867275935080314, + "StackExchange": 0.23707448773224044, + "Github": 0.23928249973665786, + "OOD_arXiv_2023": 0.23488039729288884, + "OOD_CommonPile": 0.23437713095918297 + }, + "n": 6 + }, + "Pile-CC+OOD_arXiv_2023": { + "fit_pct": 58.57043175120877, + "read_pct": 39.66601100408638, + "residual_pct": 1.7635572447048484, + "fit_span": 0.032247639319824994, + "read_span": 0.02453395726711108, + "row_means": { + "Wikipedia_en": 0.23116539605306807, + "USPTO_Backgrounds": 0.255723795901846, + "StackExchange": 0.26042338467927445, + "Github": 0.22817574535944946, + "OOD_News_2024": 0.2413466581936788, + "OOD_CommonPile": 0.24701144602839595 + }, + "col_means": { + "Wikipedia_en": 0.2511429758880425, + "USPTO_Backgrounds": 0.2565216694889521, + "StackExchange": 0.23404111785262272, + "Github": 0.23751995633528739, + "OOD_News_2024": 0.2526329944289669, + "OOD_CommonPile": 0.23198771222184103 + }, + "n": 6 + }, + "Pile-CC+OOD_CommonPile": { + "fit_pct": 60.53397420111428, + "read_pct": 37.200777927836896, + "residual_pct": 2.2652478710488317, + "fit_span": 0.03215630857135965, + "read_span": 0.02331999979085389, + "row_means": { + "Wikipedia_en": 0.2316824978734884, + "USPTO_Backgrounds": 0.25606136375831234, + "StackExchange": 0.259962322090373, + "Github": 0.22780601351901336, + "OOD_News_2024": 0.2421547730418819, + "OOD_arXiv_2023": 0.25353899860616635 + }, + "col_means": { + "Wikipedia_en": 0.25206364445664264, + "USPTO_Backgrounds": 0.2578511975015755, + "StackExchange": 0.2345766291539702, + "Github": 0.2384220319075717, + "OOD_News_2024": 0.2537612681587537, + "OOD_arXiv_2023": 0.2345311977107216 + }, + "n": 6 + }, + "StackExchange+Github": { + "fit_pct": 42.84227087213813, + "read_pct": 56.17386765237576, + "residual_pct": 0.9838614754861128, + "fit_span": 0.025654978070546064, + "read_span": 0.02564318507771804, + "row_means": { + "Wikipedia_en": 0.2345775471724294, + "USPTO_Backgrounds": 0.25774956940914745, + "Pile-CC": 0.26023252524297547, + "OOD_News_2024": 0.24642789383177405, + "OOD_arXiv_2023": 0.2568209868096919, + "OOD_CommonPile": 0.2495008136490705 + }, + "col_means": { + "Wikipedia_en": 0.2544670837551907, + "USPTO_Backgrounds": 0.26151296757506554, + "Pile-CC": 0.2581267935189384, + "OOD_News_2024": 0.25703099447612965, + "OOD_arXiv_2023": 0.23830171429241698, + "OOD_CommonPile": 0.2358697824973475 + }, + "n": 6 + }, + "StackExchange+OOD_News_2024": { + "fit_pct": 55.292155304881796, + "read_pct": 43.595433444021396, + "residual_pct": 1.1124112510968156, + "fit_span": 0.02923388341786687, + "read_span": 0.02503880510727563, + "row_means": { + "Wikipedia_en": 0.2320564804598689, + "USPTO_Backgrounds": 0.25528259504655443, + "Pile-CC": 0.2566450779291767, + "Github": 0.2274111945113098, + "OOD_arXiv_2023": 0.2538159774961295, + "OOD_CommonPile": 0.24672200249301063 + }, + "col_means": { + "Wikipedia_en": 0.2516366580777146, + "USPTO_Backgrounds": 0.2584652775519148, + "Pile-CC": 0.25573119741209127, + "Github": 0.2379931648986207, + "OOD_arXiv_2023": 0.2346805575510694, + "OOD_CommonPile": 0.23342647244463915 + }, + "n": 6 + }, + "StackExchange+OOD_arXiv_2023": { + "fit_pct": 56.67529835096962, + "read_pct": 42.53112995095901, + "residual_pct": 0.7935716980713718, + "fit_span": 0.029366803458995283, + "read_span": 0.025277133982766542, + "row_means": { + "Wikipedia_en": 0.2344927776466917, + "USPTO_Backgrounds": 0.2587167965420694, + "Pile-CC": 0.2604047985609483, + "Github": 0.231037995101953, + "OOD_News_2024": 0.2463490226202541, + "OOD_CommonPile": 0.24993321068446947 + }, + "col_means": { + "Wikipedia_en": 0.2506889459849508, + "USPTO_Backgrounds": 0.25631418769006376, + "Pile-CC": 0.2543158354548116, + "Github": 0.23623062149725027, + "OOD_News_2024": 0.25234795682201233, + "OOD_CommonPile": 0.2310370537072972 + }, + "n": 6 + }, + "StackExchange+OOD_CommonPile": { + "fit_pct": 61.972325663831406, + "read_pct": 36.781390055683254, + "residual_pct": 1.2462842804853407, + "fit_span": 0.030026291483254336, + "read_span": 0.023312357733784983, + "row_means": { + "Wikipedia_en": 0.23500987946711205, + "USPTO_Backgrounds": 0.25905436439853574, + "Pile-CC": 0.2606945547447712, + "Github": 0.23066826326151688, + "OOD_News_2024": 0.24715713746845724, + "OOD_arXiv_2023": 0.2569233551104036 + }, + "col_means": { + "Wikipedia_en": 0.2516096145535509, + "USPTO_Backgrounds": 0.2576437157026871, + "Pile-CC": 0.25531393860432283, + "Github": 0.23713269706953455, + "OOD_News_2024": 0.25347623055179913, + "OOD_arXiv_2023": 0.23433135796890214 + }, + "n": 6 + }, + "Github+OOD_News_2024": { + "fit_pct": 41.51713771708313, + "read_pct": 56.74484797143785, + "residual_pct": 1.7380143114790165, + "fit_span": 0.02756159160355176, + "read_span": 0.02506792091040147, + "row_means": { + "Wikipedia_en": 0.23164792891798747, + "USPTO_Backgrounds": 0.2552354396458853, + "Pile-CC": 0.2553520930413571, + "StackExchange": 0.2592095205215392, + "OOD_arXiv_2023": 0.25336365866026395, + "OOD_CommonPile": 0.24663624792808184 + }, + "col_means": { + "Wikipedia_en": 0.25702016951033363, + "USPTO_Backgrounds": 0.2642361731233972, + "Pile-CC": 0.26084856064990164, + "StackExchange": 0.23984072664752604, + "OOD_arXiv_2023": 0.24033100657096063, + "OOD_CommonPile": 0.23916825221299573 + }, + "n": 6 + }, + "Github+OOD_arXiv_2023": { + "fit_pct": 46.003190504868655, + "read_pct": 51.97958476412894, + "residual_pct": 2.0172247310024174, + "fit_span": 0.028970212913635718, + "read_span": 0.02530624978589241, + "row_means": { + "Wikipedia_en": 0.23408422610481028, + "USPTO_Backgrounds": 0.2586696411414003, + "Pile-CC": 0.2591118136731287, + "StackExchange": 0.263054439018446, + "OOD_News_2024": 0.24462587730614124, + "OOD_CommonPile": 0.24984745611954068 + }, + "col_means": { + "Wikipedia_en": 0.2560724574175698, + "USPTO_Backgrounds": 0.26208508326154617, + "Pile-CC": 0.259433198692622, + "StackExchange": 0.2368073567679083, + "OOD_News_2024": 0.25821652374816717, + "OOD_CommonPile": 0.23677883347565376 + }, + "n": 6 + }, + "Github+OOD_CommonPile": { + "fit_pct": 46.56314258467692, + "read_pct": 51.21295811286591, + "residual_pct": 2.2238993024571663, + "fit_span": 0.027992048504313943, + "read_span": 0.02607174320491376, + "row_means": { + "Wikipedia_en": 0.2346013279252306, + "USPTO_Backgrounds": 0.2590072089978666, + "Pile-CC": 0.2594015698569516, + "StackExchange": 0.26259337642954456, + "OOD_News_2024": 0.24543399215434436, + "OOD_arXiv_2023": 0.256471036274538 + }, + "col_means": { + "Wikipedia_en": 0.2569931259861699, + "USPTO_Backgrounds": 0.26341461127416954, + "Pile-CC": 0.2604313018421332, + "StackExchange": 0.23734286806925578, + "OOD_News_2024": 0.25934479747795397, + "OOD_arXiv_2023": 0.2399818069887934 + }, + "n": 6 + }, + "OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 58.85391680517784, + "read_pct": 39.71228389826037, + "residual_pct": 1.4337992965617947, + "fit_span": 0.03149643563148047, + "read_span": 0.024701869815450028, + "row_means": { + "Wikipedia_en": 0.2315631593922498, + "USPTO_Backgrounds": 0.25620266677880726, + "Pile-CC": 0.2555243663593299, + "StackExchange": 0.26047128893572974, + "Github": 0.22897485330424927, + "OOD_CommonPile": 0.2470686449634808 + }, + "col_means": { + "Wikipedia_en": 0.2532420317400937, + "USPTO_Backgrounds": 0.2590373932383955, + "Pile-CC": 0.2570376025857749, + "StackExchange": 0.23655187534513297, + "Github": 0.2396005534015044, + "OOD_CommonPile": 0.23433552342294545 + }, + "n": 6 + }, + "OOD_News_2024+OOD_CommonPile": { + "fit_pct": 59.80536612060682, + "read_pct": 38.411537342785614, + "residual_pct": 1.7830965366075628, + "fit_span": 0.03140510488301515, + "read_span": 0.024006271003573026, + "row_means": { + "Wikipedia_en": 0.23208026121267014, + "USPTO_Backgrounds": 0.2565402346352736, + "Pile-CC": 0.25581412254315283, + "StackExchange": 0.2600102263468283, + "Github": 0.22860512146381315, + "OOD_arXiv_2023": 0.2534660269609756 + }, + "col_means": { + "Wikipedia_en": 0.2541627003086938, + "USPTO_Backgrounds": 0.26036692125101885, + "Pile-CC": 0.25803570573528606, + "StackExchange": 0.23708738664648046, + "Github": 0.24050262897378868, + "OOD_arXiv_2023": 0.23636065024744582 + }, + "n": 6 + }, + "OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 63.40419715517061, + "read_pct": 34.532472053862335, + "residual_pct": 2.063330790967069, + "fit_span": 0.03162322278927873, + "read_span": 0.0241618146223051, + "row_means": { + "Wikipedia_en": 0.23451655839949295, + "USPTO_Backgrounds": 0.2599744361307886, + "Pile-CC": 0.25957384317492443, + "StackExchange": 0.26385514484373507, + "Github": 0.23223192205445634, + "OOD_News_2024": 0.2453551209428244 + }, + "col_means": { + "Wikipedia_en": 0.2532149882159299, + "USPTO_Backgrounds": 0.2582158313891678, + "Pile-CC": 0.25662034377800647, + "StackExchange": 0.23405401676686272, + "Github": 0.23874008557241824, + "OOD_News_2024": 0.25466175982383665 + }, + "n": 6 + } + }, + "n_l2o_fit_gt_read": "20/28", + "l2o_failures": [ + "Wikipedia_en+StackExchange", + "Wikipedia_en+Github", + "USPTO_Backgrounds+Github", + "Pile-CC+Github", + "StackExchange+Github", + "Github+OOD_News_2024", + "Github+OOD_arXiv_2023", + "Github+OOD_CommonPile" + ], + "l2o_worst_split": { + "dropped": "Wikipedia_en+Github", + "fit_pct": 21.009005511431113, + "read_pct": 76.38980395789154, + "residual_pct": 2.601190530677337, + "fit_span": 0.017904768235705498, + "read_span": 0.02595827225595715, + "row_means": { + "USPTO_Backgrounds": 0.2555489211205255, + "Pile-CC": 0.25603438783781945, + "StackExchange": 0.25972282302186445, + "OOD_News_2024": 0.24181805478615895, + "OOD_arXiv_2023": 0.2542968055509307, + "OOD_CommonPile": 0.24736178965756186 + }, + "col_means": { + "USPTO_Backgrounds": 0.2663017110278209, + "Pile-CC": 0.26302625560719106, + "StackExchange": 0.24034343877186376, + "OOD_News_2024": 0.2622803156471087, + "OOD_arXiv_2023": 0.24214241415331209, + "OOD_CommonPile": 0.2406886467675644 + }, + "n": 6 + }, + "fit_axis_leverage_point": "Github", + "read_axis_leverage_point": "OOD_CommonPile", + "drop_fit_leverage_only": { + "fit_pct": 41.358656075065184, + "read_pct": 56.523659412915094, + "residual_pct": 2.117684512019725, + "fit_span": 0.028123639521026395, + "read_span": 0.02546599780815459, + "row_means": { + "Wikipedia_en": 0.23265617557520432, + "USPTO_Backgrounds": 0.25649802677392486, + "Pile-CC": 0.2572291307358278, + "StackExchange": 0.2607798150962307, + "OOD_News_2024": 0.24324451167550354, + "OOD_arXiv_2023": 0.25475290841053405, + "OOD_CommonPile": 0.24801946458598925 + }, + "col_means": { + "Wikipedia_en": 0.2562748957247961, + "USPTO_Backgrounds": 0.2630643661828741, + "Pile-CC": 0.25989282417096315, + "StackExchange": 0.2375983683747195, + "OOD_News_2024": 0.2589125070573082, + "OOD_arXiv_2023": 0.2395631939024916, + "OOD_CommonPile": 0.23787387744006183 + }, + "n": 7 + }, + "drop_read_leverage_only": { + "fit_pct": 57.55685418677557, + "read_pct": 40.24133072996441, + "residual_pct": 2.2018150832600214, + "fit_span": 0.031948861550717134, + "read_span": 0.024509501918440785, + "row_means": { + "Wikipedia_en": 0.23302674611350374, + "USPTO_Backgrounds": 0.2576164224791148, + "Pile-CC": 0.25762515602308134, + "StackExchange": 0.26146613437504995, + "Github": 0.22951727282433282, + "OOD_News_2024": 0.2438695776498034, + "OOD_arXiv_2023": 0.25484065266828687 + }, + "col_means": { + "Wikipedia_en": 0.2538256364091048, + "USPTO_Backgrounds": 0.2597478645779784, + "Pile-CC": 0.2574818056727213, + "StackExchange": 0.2352383626595376, + "Github": 0.23964269492361281, + "OOD_News_2024": 0.2558655665507392, + "OOD_arXiv_2023": 0.23616003133947888 + }, + "n": 7 + }, + "asymmetry_note": "dropping the fit axis's weakest member (Github) moves fit share to 41.4%; dropping the read axis's weakest member (OOD_CommonPile) moves it to 57.6%. The paper shows only the first." + } + }, + "e36_5x5_independent": { + "persist": { + "operators": "e28__410m_n400_s0.pt \u2014 N=400, seed block 0, fitted by fastfit", + "replication_unit": "prefix seed (3 draws), ONE operator per corpus", + "seed_averaged": { + "fit_pct": 37.21839312931188, + "read_pct": 61.492033642909696, + "residual_pct": 1.28957322777843, + "fit_span": 0.019122830497411383, + "read_span": 0.02229281177278608, + "row_means": { + "Github": 0.09033581784460694, + "Wikipedia_en": 0.09887435988057405, + "StackExchange": 0.10945864834201832, + "Pile-CC": 0.10696604104091724, + "USPTO_Backgrounds": 0.10852870717799912 + }, + "col_means": { + "Github": 0.08857963426349064, + "Wikipedia_en": 0.10963698355636249, + "StackExchange": 0.09466255073435605, + "Pile-CC": 0.11087244603627672, + "USPTO_Backgrounds": 0.11041195969562978 + }, + "n": 5 + }, + "per_prefix_seed_draw": [ + { + "fit_pct": 38.01197956822691, + "read_pct": 60.39268416330113, + "residual_pct": 1.5953362684719516, + "fit_span": 0.020585043304599815, + "read_span": 0.024156512715853756, + "row_means": { + "Github": 0.0882502949750051, + "Wikipedia_en": 0.09885050568729639, + "StackExchange": 0.10883533827960491, + "Pile-CC": 0.1068751526158303, + "USPTO_Backgrounds": 0.10829123963601887 + }, + "col_means": { + "Github": 0.08609445887617767, + "Wikipedia_en": 0.11011854553595185, + "StackExchange": 0.09511935244314372, + "Pile-CC": 0.11025097159203143, + "USPTO_Backgrounds": 0.1095192027464509 + }, + "n": 5 + }, + { + "fit_pct": 39.65691740343702, + "read_pct": 57.765676562248295, + "residual_pct": 2.5774060343146847, + "fit_span": 0.01913450184743852, + "read_span": 0.02111001591663808, + "row_means": { + "Github": 0.09169643383938819, + "Wikipedia_en": 0.09923369170166552, + "StackExchange": 0.11083093568682671, + "Pile-CC": 0.10808868755120785, + "USPTO_Backgrounds": 0.10927517526317387 + }, + "col_means": { + "Github": 0.09330056495964527, + "Wikipedia_en": 0.10770068213809282, + "StackExchange": 0.09347095298580825, + "Pile-CC": 0.11441058087628335, + "USPTO_Backgrounds": 0.11024214308243245 + }, + "n": 5 + }, + { + "fit_pct": 30.870979385905095, + "read_pct": 67.29091366539807, + "residual_pct": 1.8381069486968467, + "fit_span": 0.01764894634019583, + "read_span": 0.02513065430335701, + "row_means": { + "Github": 0.09106072471942753, + "Wikipedia_en": 0.09853888225276024, + "StackExchange": 0.10870967105962336, + "Pile-CC": 0.10593428295571357, + "USPTO_Backgrounds": 0.1080197066348046 + }, + "col_means": { + "Github": 0.08634387895464897, + "Wikipedia_en": 0.1110917229950428, + "StackExchange": 0.09539734677411615, + "Pile-CC": 0.10795578564051539, + "USPTO_Backgrounds": 0.11147453325800598 + }, + "n": 5 + } + ], + "fit_gt_read_in_every_draw": false, + "leave_one_out": { + "Github": { + "fit_pct": 24.47658568329658, + "read_pct": 73.34361580930278, + "residual_pct": 2.1797985074006436, + "fit_span": 0.009686764593546587, + "read_span": 0.016567733588938913, + "row_means": { + "Wikipedia_en": 0.10267983986025987, + "StackExchange": 0.11233038291102275, + "Pile-CC": 0.11063493196076403, + "USPTO_Backgrounds": 0.11236660445380646 + }, + "col_means": { + "Wikipedia_en": 0.11275589839594129, + "StackExchange": 0.09753622518619523, + "Pile-CC": 0.11361567682858247, + "USPTO_Backgrounds": 0.11410395877513414 + }, + "n": 4 + }, + "Wikipedia_en": { + "fit_pct": 40.25293531506797, + "read_pct": 58.69767737741175, + "residual_pct": 1.049387307520277, + "fit_span": 0.01931806630260932, + "read_span": 0.022084370379646612, + "row_means": { + "Github": 0.08862944125624683, + "StackExchange": 0.10794750755885615, + "Pile-CC": 0.10490835212015857, + "USPTO_Backgrounds": 0.1074911618003777 + }, + "col_means": { + "Github": 0.08981143283890561, + "StackExchange": 0.09599809045903385, + "Pile-CC": 0.11189580321855222, + "USPTO_Backgrounds": 0.1112711362191476 + }, + "n": 4 + }, + "StackExchange": { + "fit_pct": 35.113773774154645, + "read_pct": 64.0035823204324, + "residual_pct": 0.8826439054129439, + "fit_span": 0.018071053778597468, + "read_span": 0.02356068749019566, + "row_means": { + "Github": 0.09212780907400883, + "Wikipedia_en": 0.10126285189180635, + "Pile-CC": 0.10919965063997855, + "USPTO_Backgrounds": 0.1101988628526063 + }, + "col_means": { + "Github": 0.08623161531286314, + "Wikipedia_en": 0.10817042657678637, + "Pile-CC": 0.1097923028030588, + "USPTO_Backgrounds": 0.10859482976569174 + }, + "n": 4 + }, + "Pile-CC": { + "fit_pct": 42.113530594399414, + "read_pct": 56.588844540798014, + "residual_pct": 1.297624864802567, + "fit_span": 0.02008016409624057, + "read_span": 0.021828246999454376, + "row_means": { + "Github": 0.08794489158899524, + "Wikipedia_en": 0.09689819552392388, + "StackExchange": 0.10802505568523581, + "USPTO_Backgrounds": 0.1063312692005032 + }, + "col_means": { + "Github": 0.08765192348898078, + "Wikipedia_en": 0.10824703026446514, + "StackExchange": 0.09382028775677706, + "USPTO_Backgrounds": 0.10948017048843516 + }, + "n": 4 + }, + "USPTO_Backgrounds": { + "fit_pct": 35.870047433276746, + "read_pct": 63.17635082059406, + "residual_pct": 0.9536017461291897, + "fit_span": 0.018394409112321833, + "read_span": 0.021830679462679345, + "row_means": { + "Github": 0.08900878146135559, + "Wikipedia_en": 0.09684913645032794, + "StackExchange": 0.10740319057367742, + "Pile-CC": 0.10517277217004448 + }, + "col_means": { + "Github": 0.08743026331067086, + "Wikipedia_en": 0.10887650727333191, + "StackExchange": 0.09286616729805246, + "Pile-CC": 0.1092609427733502 + }, + "n": 4 + } + }, + "n_loo_fit_gt_read": "0/5", + "same_five_corpora_from_E52_operators": { + "fit_pct": 48.595776344410005, + "read_pct": 50.571315567125104, + "residual_pct": 0.8329080884648928, + "fit_span": 0.01788562354166061, + "read_span": 0.017197533233815593, + "row_means": { + "Github": 0.09127239868303554, + "Wikipedia_en": 0.09920645491944419, + "StackExchange": 0.10905113396338291, + "Pile-CC": 0.10741977654242267, + "USPTO_Backgrounds": 0.10915802222469614 + }, + "col_means": { + "Github": 0.09506975406108217, + "Wikipedia_en": 0.10824210613500326, + "StackExchange": 0.09419792907209032, + "Pile-CC": 0.10720253475889978, + "USPTO_Backgrounds": 0.11139546230590591 + }, + "n": 5 + }, + "E52_same_five_minus_github": { + "fit_pct": 27.35199418077972, + "read_pct": 71.36692108262613, + "residual_pct": 1.2810847365941476, + "fit_span": 0.010191553133255274, + "read_span": 0.01749959976878017, + "row_means": { + "Wikipedia_en": 0.10131256840411677, + "StackExchange": 0.11081881205318496, + "Pile-CC": 0.10936670755747602, + "USPTO_Backgrounds": 0.11150412153737205 + }, + "col_means": { + "Wikipedia_en": 0.11100260348951754, + "StackExchange": 0.09726303055067546, + "Pile-CC": 0.10997394519250116, + "USPTO_Backgrounds": 0.11476263031945563 + }, + "n": 4 + }, + "E36_same_five_minus_github": { + "fit_pct": 24.47658568329658, + "read_pct": 73.34361580930278, + "residual_pct": 2.1797985074006436, + "fit_span": 0.009686764593546587, + "read_span": 0.016567733588938913, + "row_means": { + "Wikipedia_en": 0.10267983986025987, + "StackExchange": 0.11233038291102275, + "Pile-CC": 0.11063493196076403, + "USPTO_Backgrounds": 0.11236660445380646 + }, + "col_means": { + "Wikipedia_en": 0.11275589839594129, + "StackExchange": 0.09753622518619523, + "Pile-CC": 0.11361567682858247, + "USPTO_Backgrounds": 0.11410395877513414 + }, + "n": 4 + }, + "offdiagonal_only": { + "why": "E36's five diagonal rungs carry 17.0% prefix/fit overlap (E58). Dropping them removes the contamination entirely.", + "E36_5x5": { + "fit_pct": 34.32840203691243, + "read_pct": 57.867177732106, + "n_cells": 20, + "fit_gt_read": false + }, + "E52_5x5": { + "fit_pct": 47.11514219571127, + "read_pct": 49.32603799570835, + "n_cells": 20, + "fit_gt_read": false + }, + "E36_4x4_minus_github": { + "fit_pct": 40.24342646470222, + "read_pct": 76.00988055207495, + "n_cells": 12, + "fit_gt_read": false + }, + "E52_4x4_minus_github": { + "fit_pct": 39.14494851977934, + "read_pct": 74.85941643522156, + "n_cells": 12, + "fit_gt_read": false + } + } + }, + "min": { + "operators": "e28__410m_n400_s0.pt \u2014 N=400, seed block 0, fitted by fastfit", + "replication_unit": "prefix seed (3 draws), ONE operator per corpus", + "seed_averaged": { + "fit_pct": 67.15696317298493, + "read_pct": 30.854588513358074, + "residual_pct": 1.9884483136569922, + "fit_span": 0.03396187820782262, + "read_span": 0.023042783613006296, + "row_means": { + "Github": 0.23116528401772182, + "Wikipedia_en": 0.23424909750620523, + "StackExchange": 0.26512716222554444, + "Pile-CC": 0.25784623639037213, + "USPTO_Backgrounds": 0.2609189863254627 + }, + "col_means": { + "Github": 0.23923848668734232, + "Wikipedia_en": 0.2572329893708229, + "StackExchange": 0.23722810876866182, + "Pile-CC": 0.2602708923816681, + "USPTO_Backgrounds": 0.2553362892568112 + }, + "n": 5 + }, + "per_prefix_seed_draw": [ + { + "fit_pct": 63.59538818189268, + "read_pct": 32.42498301201463, + "residual_pct": 3.979628806092687, + "fit_span": 0.032948142960667615, + "read_span": 0.02480117298662665, + "row_means": { + "Github": 0.23086294218897818, + "Wikipedia_en": 0.23401705510914325, + "StackExchange": 0.2638110851496458, + "Pile-CC": 0.25773832939565183, + "USPTO_Backgrounds": 0.26035829171538355 + }, + "col_means": { + "Github": 0.23701887637376784, + "Wikipedia_en": 0.25646688163280484, + "StackExchange": 0.23818604521453382, + "Pile-CC": 0.2618200493603945, + "USPTO_Backgrounds": 0.2532958509773016 + }, + "n": 5 + }, + { + "fit_pct": 57.51096410968613, + "read_pct": 40.88351631907341, + "residual_pct": 1.6055195712404522, + "fit_span": 0.03339642722159622, + "read_span": 0.03181405916810037, + "row_means": { + "Github": 0.22935324043035507, + "Wikipedia_en": 0.2320427041500807, + "StackExchange": 0.2627496676519513, + "Pile-CC": 0.25537864323705434, + "USPTO_Backgrounds": 0.25934330992400645 + }, + "col_means": { + "Github": 0.23634482607245444, + "Wikipedia_en": 0.25489282198250296, + "StackExchange": 0.2315104167163372, + "Pile-CC": 0.2633244758844376, + "USPTO_Backgrounds": 0.2527950247377157 + }, + "n": 5 + }, + { + "fit_pct": 75.66125190125138, + "read_pct": 21.65706202825769, + "residual_pct": 2.681686070490932, + "fit_span": 0.035541064441204084, + "read_span": 0.0183514001220465, + "row_means": { + "Github": 0.23327966943383216, + "Wikipedia_en": 0.2366875332593918, + "StackExchange": 0.26882073387503624, + "Pile-CC": 0.26042173653841016, + "USPTO_Backgrounds": 0.263055357336998 + }, + "col_means": { + "Github": 0.2443517576158047, + "Wikipedia_en": 0.26033926449716094, + "StackExchange": 0.24198786437511444, + "Pile-CC": 0.2556681519001722, + "USPTO_Backgrounds": 0.2599179920554161 + }, + "n": 5 + } + ], + "fit_gt_read_in_every_draw": true, + "leave_one_out": { + "Github": { + "fit_pct": 58.30875926640823, + "read_pct": 39.70032311982488, + "residual_pct": 1.9909176137668885, + "fit_span": 0.03002737367836139, + "read_span": 0.025147321137289208, + "row_means": { + "Wikipedia_en": 0.23709889249876143, + "StackExchange": 0.2671262661771228, + "Pile-CC": 0.2600813600079467, + "USPTO_Backgrounds": 0.2638985842155913 + }, + "col_means": { + "Wikipedia_en": 0.26151906372979283, + "StackExchange": 0.24055501064285637, + "Pile-CC": 0.2657023317801456, + "USPTO_Backgrounds": 0.2604286967466275 + }, + "n": 4 + }, + "Wikipedia_en": { + "fit_pct": 63.67287286778519, + "read_pct": 33.49186505858148, + "residual_pct": 2.8352620736333316, + "fit_span": 0.034362228571747755, + "read_span": 0.0228650085007151, + "row_means": { + "Github": 0.22893443203841646, + "StackExchange": 0.2632966606101642, + "Pile-CC": 0.25588418849123024, + "USPTO_Backgrounds": 0.25984051016469795 + }, + "col_means": { + "Github": 0.2433356289751828, + "StackExchange": 0.24135947360967597, + "Pile-CC": 0.26422448211039107, + "USPTO_Backgrounds": 0.2590362066092591 + }, + "n": 4 + }, + "StackExchange": { + "fit_pct": 72.63055334443037, + "read_pct": 26.845480417114555, + "residual_pct": 0.5239662384550815, + "fit_span": 0.030036273691803228, + "read_span": 0.021623960789293017, + "row_means": { + "Github": 0.23297647970418134, + "Wikipedia_en": 0.23763570953160523, + "Pile-CC": 0.2626483671522389, + "USPTO_Backgrounds": 0.26301275339598457 + }, + "col_means": { + "Github": 0.23476542175437015, + "Wikipedia_en": 0.2534289445417623, + "Pile-CC": 0.25638938254366317, + "USPTO_Backgrounds": 0.25168956094421446 + }, + "n": 4 + }, + "Pile-CC": { + "fit_pct": 74.14105112695103, + "read_pct": 24.429114060080902, + "residual_pct": 1.4298348129680751, + "fit_span": 0.03313939852329592, + "read_span": 0.018295956837634242, + "row_means": { + "Github": 0.2293203213252127, + "Wikipedia_en": 0.2316972385160625, + "StackExchange": 0.26245971984850863, + "USPTO_Backgrounds": 0.25828782754639784 + }, + "col_means": { + "Github": 0.23682167287915945, + "Wikipedia_en": 0.2551176297167937, + "StackExchange": 0.23687570762510102, + "USPTO_Backgrounds": 0.25295009701512755 + }, + "n": 4 + }, + "USPTO_Backgrounds": { + "fit_pct": 63.585900265026346, + "read_pct": 34.44068962944996, + "residual_pct": 1.9734101055236857, + "fit_span": 0.03371321195736526, + "read_span": 0.024078553666671138, + "row_means": { + "Github": 0.23021494019776584, + "Wikipedia_en": 0.23267721692100168, + "StackExchange": 0.2639281521551311, + "Pile-CC": 0.25608753093207876 + }, + "col_means": { + "Github": 0.23679795966794093, + "Wikipedia_en": 0.2552330139713983, + "StackExchange": 0.23339915644998352, + "Pile-CC": 0.25747771011665466 + }, + "n": 4 + } + }, + "n_loo_fit_gt_read": "5/5", + "same_five_corpora_from_E52_operators": { + "fit_pct": 66.3617251369317, + "read_pct": 32.14495780109832, + "residual_pct": 1.4933170619699698, + "fit_span": 0.030905587035748666, + "read_span": 0.022616316386395025, + "row_means": { + "Github": 0.23168027026785745, + "Wikipedia_en": 0.23367873897569047, + "StackExchange": 0.2625858573036061, + "Pile-CC": 0.2577903404293789, + "USPTO_Backgrounds": 0.25915461344851387 + }, + "col_means": { + "Github": 0.23959148456239038, + "Wikipedia_en": 0.25349733525680174, + "StackExchange": 0.2360359763726592, + "Pile-CC": 0.25711273147414127, + "USPTO_Backgrounds": 0.2586522927590542 + }, + "n": 5 + }, + "E52_same_five_minus_github": { + "fit_pct": 57.2387064131325, + "read_pct": 41.203205398466594, + "residual_pct": 1.5580881884009137, + "fit_span": 0.02835304120865964, + "read_span": 0.02448638193309305, + "row_means": { + "Wikipedia_en": 0.23620360990882747, + "StackExchange": 0.2645566511174871, + "Pile-CC": 0.26049625728175874, + "USPTO_Backgrounds": 0.2616253409689913 + }, + "col_means": { + "Wikipedia_en": 0.25816649433432354, + "StackExchange": 0.2393921379155169, + "Pile-CC": 0.2614447071786142, + "USPTO_Backgrounds": 0.26387851984860994 + }, + "n": 4 + }, + "E36_same_five_minus_github": { + "fit_pct": 58.30875926640823, + "read_pct": 39.70032311982488, + "residual_pct": 1.9909176137668885, + "fit_span": 0.03002737367836139, + "read_span": 0.025147321137289208, + "row_means": { + "Wikipedia_en": 0.23709889249876143, + "StackExchange": 0.2671262661771228, + "Pile-CC": 0.2600813600079467, + "USPTO_Backgrounds": 0.2638985842155913 + }, + "col_means": { + "Wikipedia_en": 0.26151906372979283, + "StackExchange": 0.24055501064285637, + "Pile-CC": 0.2657023317801456, + "USPTO_Backgrounds": 0.2604286967466275 + }, + "n": 4 + }, + "offdiagonal_only": { + "why": "E36's five diagonal rungs carry 17.0% prefix/fit overlap (E58). Dropping them removes the contamination entirely.", + "E36_5x5": { + "fit_pct": 71.40296315585613, + "read_pct": 32.60235062281941, + "n_cells": 20, + "fit_gt_read": true + }, + "E52_5x5": { + "fit_pct": 68.29334220707781, + "read_pct": 32.71656463012795, + "n_cells": 20, + "fit_gt_read": true + }, + "E36_4x4_minus_github": { + "fit_pct": 69.43924932515404, + "read_pct": 50.77052198406748, + "n_cells": 12, + "fit_gt_read": true + }, + "E52_4x4_minus_github": { + "fit_pct": 64.5209454241777, + "read_pct": 51.20278577818856, + "n_cells": 12, + "fit_gt_read": true + } + } + }, + "VERDICT": "DOES NOT REPLICATE \u2014 the ordering does not hold on the independent population" + }, + "controls": { + "C1_agrees_with_e54": { + "n_quantities_checked": 20, + "max_abs_diff": 43.376650268376714, + "tolerance": 1e-12, + "fires": false, + "reference": "results/e54_aggregation_audit.json" + }, + "C2_e36_diagonal_leakage": { + "measured": 0.1704, + "executed": "recomputed with the five contaminated diagonal cells DROPPED: E36 34.3/57.9 vs 37.2/61.5 with them. The leakage does not drive the replication.", + "fires": false, + "measured_in": "results/e58_algebra_audit.json (t58)", + "expected": "~17% \u2014 t36_qladder.py load_pool() draws prefixes from the whole 2400-doc file; its operators are e28 N=400 fits from documents 0..800 of that file", + "direction": "inflates one cell per row (the diagonal), so it lands mostly in the residual rather than the row main effect" + } + }, + "provenance": { + "schema_version": 1, + "experiment": "E55", + "utc": "2026-08-31T23:20:10+00:00", + "script": { + "path": "experiments/t55_matrix_robustness.py", + "sha256": "55af3d7e65afa70f7323354a2a58616f4d473402fb72ccacbe8a81b5b47d4084", + "n_lines": 276 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/t55_matrix_robustness.py", + "--e52", + "results/e52_factorial_410m_rstrip.json", + "--e36", + "results/e36_qladder_410m_rstrip.json", + "--out", + "results/e55_matrix_robustness_rstrip.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e52_factorial_410m_rstrip.json", + "sha256": "e3e42850ce7f436eafb50fa2f6ba7f94559b321540075023badd6e1cd6ef7e19", + "bytes": 45765, + "exists": true + }, + { + "path": "results/e36_qladder_410m_rstrip.json", + "sha256": "540bf21811bb5f522c0e9b22965d0d289e5ade98cfcc06fa2bdf966dbff1c185", + "bytes": 111158, + "exists": true + }, + { + "path": "results/e54_aggregation_audit.json", + "sha256": "1a1d45be09127b311a5d7ab534a0a1a52f4ab2f78d73b198f12090b27b2bb445", + "bytes": 69086, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "cea3358343031a423976c82a9fad5cc8dcdd1dcf5a9e45afcdce2879b2fb5d87" + } +} \ No newline at end of file diff --git a/results/e56_predictor_registry.json b/results/e56_predictor_registry.json new file mode 100644 index 0000000000000000000000000000000000000000..2b640ad50ef52db10bff430efd7b0fc30aa2333d --- /dev/null +++ b/results/e56_predictor_registry.json @@ -0,0 +1,517 @@ +{ + "experiment": "E56 \u2014 the candidate-predictor registry", + "why": "the paper's 'we tested 21 candidates' appears in no results file; Figure 2 shows 11. This enumerates every candidate across the seven source files with the criterion each was actually tested against.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/e31_local_bakeoff_410m.json", + "results/e38_jgeometry.json", + "results/e34a_dact_vs_floor_410m.json", + "results/e34b_jeval_dact_410m.json", + "results/e44_dread_410m.json", + "results/e45_disagreement_geometry.json", + "results/e50_concept_energy.json" + ], + "registry": [ + { + "predictor": "dispersion_L0", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": -0.9242882654186496, + "worst_loo": -0.6602694430492779, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "dispersion_band_mean", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": -0.9531498843872271, + "worst_loo": -0.7671626905078005, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "dispersion_slope", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": 0.9445686578610754, + "worst_loo": 0.7187361535787741, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "stable_rank", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": 0.6217659208337967, + "worst_loo": 0.5911743338547131, + "worst_loo_dropped": "Pile-CC", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "top8_spectral_share", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": -0.7882566086770282, + "worst_loo": 0.4765579902877013, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "condition_number", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": -0.07692108108636014, + "worst_loo": 0.03784692108272241, + "worst_loo_dropped": "StackExchange", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "unembed_alignment", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": 0.6323756162160421, + "worst_loo": -0.27176406633776684, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "mean_prompt_tokens", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": -0.8817956434996898, + "worst_loo": -0.3478077213273393, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "type_token_ratio", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": 0.30182668187780554, + "worst_loo": 0.1543887990841397, + "worst_loo_dropped": "Pile-CC", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "eval_vocab_overlap", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": 0.11840865018179178, + "worst_loo": -0.10454375717470109, + "worst_loo_dropped": "Pile-CC", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "model_cross_entropy", + "source_file": "e31_local_bakeoff_410m.json", + "target_outcome": "per-corpus read asymptote (mean over SIX eval sets, N>=75)", + "replication_unit": "corpus", + "n": 5, + "r": 0.8335856422219892, + "worst_loo": 0.13389888988376378, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "rho_I", + "source_file": "e38_jgeometry.json (E42.corpus)", + "target_outcome": "per-corpus read asymptote", + "replication_unit": "corpus", + "n": 5, + "r": 0.3136416931265416, + "worst_loo": 0.29803495604418107, + "worst_loo_dropped": "Wikipedia_en", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "layer_cos", + "source_file": "e38_jgeometry.json (E42.corpus)", + "target_outcome": "per-corpus read asymptote", + "replication_unit": "corpus", + "n": 5, + "r": -0.33094121553738315, + "worst_loo": -0.14422605915130887, + "worst_loo_dropped": "USPTO_Backgrounds", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "rho_I (vs SCALE)", + "source_file": "e38_jgeometry.json (E42.scale)", + "target_outcome": "read gap ACROSS MODEL SIZES \u2014 not a corpus predictor", + "replication_unit": "model size", + "n": 3, + "r": 0.2655500096599166, + "worst_loo": null, + "worst_loo_dropped": null, + "criterion": "none \u2014 no LOO is defined at n=3", + "passes": null, + "kind": "different-outcome", + "note": "correlates a geometry summary against a scale-varying outcome over 3 models. This cannot answer 'is this corpus a good one to fit on'." + }, + { + "predictor": "layer_cos (vs SCALE)", + "source_file": "e38_jgeometry.json (E42.scale)", + "target_outcome": "read gap ACROSS MODEL SIZES \u2014 not a corpus predictor", + "replication_unit": "model size", + "n": 3, + "r": -0.6297857110956958, + "worst_loo": null, + "worst_loo_dropped": null, + "criterion": "none \u2014 no LOO is defined at n=3", + "passes": null, + "kind": "different-outcome", + "note": "correlates a geometry summary against a scale-varying outcome over 3 models. This cannot answer 'is this corpus a good one to fit on'." + }, + { + "predictor": "spread_gain (vs SCALE)", + "source_file": "e38_jgeometry.json (E42.scale)", + "target_outcome": "read gap ACROSS MODEL SIZES \u2014 not a corpus predictor", + "replication_unit": "model size", + "n": 3, + "r": -0.309772888112111, + "worst_loo": null, + "worst_loo_dropped": null, + "criterion": "none \u2014 no LOO is defined at n=3", + "passes": null, + "kind": "different-outcome", + "note": "correlates a geometry summary against a scale-varying outcome over 3 models. This cannot answer 'is this corpus a good one to fit on'." + }, + { + "predictor": "act_cos (vs SCALE)", + "source_file": "e38_jgeometry.json (E42.scale)", + "target_outcome": "read gap ACROSS MODEL SIZES \u2014 not a corpus predictor", + "replication_unit": "model size", + "n": 3, + "r": -0.49389183554479377, + "worst_loo": null, + "worst_loo_dropped": null, + "criterion": "none \u2014 no LOO is defined at n=3", + "passes": null, + "kind": "different-outcome", + "note": "correlates a geometry summary against a scale-varying outcome over 3 models. This cannot answer 'is this corpus a good one to fit on'." + }, + { + "predictor": "D_act(J^P, I)", + "source_file": "e34a_dact_vs_floor_410m.json", + "target_outcome": "per-corpus read asymptote (5 admitted sets)", + "replication_unit": "corpus", + "n": 5, + "r": 0.5312051665024815, + "worst_loo": 0.47431043317842825, + "worst_loo_dropped": "Pile-CC", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "reference operator = identity" + }, + { + "predictor": "D_act(J^P, J^eval)", + "source_file": "e34b_jeval_dact_410m.json", + "target_outcome": "per-corpus read asymptote (5 admitted sets)", + "replication_unit": "corpus", + "n": 5, + "r": -0.1303408555151824, + "worst_loo": -0.07137783470288665, + "worst_loo_dropped": "Wikipedia_en", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "reference operator = J^eval" + }, + { + "predictor": "dread_vs_I", + "source_file": "e44_dread_410m.json", + "target_outcome": "per-corpus read asymptote", + "replication_unit": "corpus", + "n": 5, + "r": 0.7316797681402211, + "worst_loo": 0.3042927219600453, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "dread_vs_Jeval", + "source_file": "e44_dread_410m.json", + "target_outcome": "per-corpus read asymptote", + "replication_unit": "corpus", + "n": 5, + "r": -0.5462143495794671, + "worst_loo": 0.2704874836823881, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "candidate", + "note": "" + }, + { + "predictor": "dact_vs_I", + "source_file": "e44_dread_410m.json", + "target_outcome": "per-corpus read asymptote", + "replication_unit": "corpus", + "n": 5, + "r": 0.7549528612491698, + "worst_loo": -0.5089565539965326, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "duplicate", + "note": "same quantity as E34a's D_act(J^P, I); counted once" + }, + { + "predictor": "dread_random_rows_vs_I", + "source_file": "e44_dread_410m.json", + "target_outcome": "per-corpus read asymptote", + "replication_unit": "corpus", + "n": 5, + "r": 0.48807592860992544, + "worst_loo": 0.47202068506420203, + "worst_loo_dropped": "Wikipedia_en", + "criterion": "|r| >= 0.8 under every leave-one-corpus-out", + "passes": false, + "kind": "control", + "note": "random-row arm \u2014 a null control, not a candidate" + }, + { + "predictor": "disagreement geometry s_c (pre_norm)", + "source_file": "e45_disagreement_geometry.json", + "target_outcome": "|z(readability)| per (corpus, eval set)", + "replication_unit": "corpus x eval-set cell", + "n": 25, + "r": 0.6169298718606291, + "worst_loo": 0.2024534246986471, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.5 AND survives leave-one-SET-out AND leave-one-CORPUS-out", + "passes": false, + "kind": "candidate", + "note": "DIFFERENT criterion (0.5, not 0.8) and a different unit (25 cells, not 5 corpora). The post_norm variant additionally carries a transpose error \u2014 see e58_algebra_audit.json." + }, + { + "predictor": "disagreement geometry s_c (post_norm)", + "source_file": "e45_disagreement_geometry.json", + "target_outcome": "|z(readability)| per (corpus, eval set)", + "replication_unit": "corpus x eval-set cell", + "n": 25, + "r": 0.49614222895095655, + "worst_loo": 0.1524035039262419, + "worst_loo_dropped": "Github", + "criterion": "|r| >= 0.5 AND survives leave-one-SET-out AND leave-one-CORPUS-out", + "passes": false, + "kind": "candidate", + "note": "DIFFERENT criterion (0.5, not 0.8) and a different unit (25 cells, not 5 corpora). The post_norm variant additionally carries a transpose error \u2014 see e58_algebra_audit.json." + }, + { + "predictor": "random matched-norm Delta", + "source_file": "e45_disagreement_geometry.json", + "target_outcome": "|z(readability)| per (corpus, eval set)", + "replication_unit": "corpus x eval-set cell", + "n": 25, + "r": 0.030950976174021295, + "worst_loo": null, + "worst_loo_dropped": null, + "criterion": "|r| < 0.3 to fire", + "passes": null, + "kind": "control", + "note": "cannot fail: one Delta is shared across all five corpora, so x has zero within-set variance while the real statistic's x does not. See e58." + }, + { + "predictor": "concept-conditional activation energy", + "source_file": "e50_concept_energy.json", + "target_outcome": "per-concept readability", + "replication_unit": "concept x cell", + "n": 5815, + "r": 0.09872587496579752, + "worst_loo": 0.09800211043223954, + "worst_loo_dropped": "worst concept-LOO", + "criterion": "|r| plus a REQUIRED SIGN REVERSAL across concept sets", + "passes": false, + "kind": "candidate", + "note": "required_pattern_sign_reversal_ok = False; n = 5815 cells, not 5 corpora" + } + ], + "counts": { + "total_rows": 27, + "candidates": 20, + "candidates_at_n5_corpora_vs_the_read_asymptote": 17, + "candidates_on_a_different_unit_or_n": 3, + "controls_not_candidates": 2, + "duplicates": 1, + "different_outcome_variable": 4, + "candidates_passing_their_own_criterion": 0, + "n_from_e31": 11 + }, + "HOW_THE_PAPERS_21_IS_ASSEMBLED": "11 (E31) + 4 (E38/E42.scale) + 2 (E34a, E34b) + 1 (E44 D_read) + 2 (E45) + 1 (E50) = 21, which is the arithmetic RESULTS_TAXONOMY.md:132 implies. Two problems. The 4 from E42.scale correlate geometry against a SCALE-varying outcome over 3 model sizes; they are not candidate predictors of which corpus to fit on, and no leave-one-corpus-out is even defined for them. And E42's two genuine CORPUS-level predictors (rho_I, layer_cos) are then not in the 21 at all.", + "DEFENSIBLE_COUNTS": { + "candidate predictors of the per-corpus read asymptote, n=5 corpora, tested against the pre-registered |r|>=0.8 leave-one-corpus-out rule": 17, + "additional candidates on a different unit or with a different criterion": 3, + "of these, how many met their own criterion": 0 + }, + "controls": { + "C1_e31_rows_reproduce_exactly": { + "mismatches": [], + "fires": true + }, + "C2_controls_tagged_and_excluded": [ + "dread_random_rows_vs_I", + "random matched-norm Delta" + ], + "C3_duplicates_tagged": [ + "dact_vs_I" + ] + }, + "HETEROGENEITY": "three outcome variables (6-set asymptote for E31, 5-set asymptote for E34/E44, per-cell |z readability| for E45, per-concept readability for E50); three replication units (corpus, corpus x set cell, concept); n from 3 to 5815; and two different |r| thresholds (0.8 and 0.5). The honest sentence names the n=5 group and reports the rest separately.", + "provenance": { + "schema_version": 1, + "experiment": "E56", + "utc": "2026-08-17T00:45:41+00:00", + "script": { + "path": "experiments/t56_predictor_registry.py", + "sha256": "6910d08ed6d0a9e440ced7f2e9b41a317a382548e94f563d329222ad8a03408a", + "n_lines": 244 + }, + "git": { + "commit": "f8ef435b29365012c0492e023fb6b4dfece5f0f3", + "commit_short": "f8ef435b2936", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t53_ladder_summary.py", + "experiments/t55_matrix_robustness.py", + "experiments/t56_predictor_registry.py", + "results/e35_containment.json", + "results/e53_ladder_summary.json", + "results/e55_matrix_robustness.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/experiments/E57_grid_variance_ci.md", + "xperiments/t52_factorial.py" + ], + "dirty_diff_sha256": "afc7a07f9036dd20688df568954065b1dc6c531b68598e663d0da07e7055fb6a", + "describe": "f8ef435-dirty" + }, + "argv": [ + "experiments/t56_predictor_registry.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e31_local_bakeoff_410m.json", + "sha256": "f3e25e412a47b91800e9449002902527f80bbd45f4d48df582f97096af8788d7", + "bytes": 29618, + "exists": true + }, + { + "path": "results/e38_jgeometry.json", + "sha256": "f1de05f2c3fc57350b0bec1e14cbae7220a95c1dcac81dd8d5c3700970094e65", + "bytes": 12626, + "exists": true + }, + { + "path": "results/e34a_dact_vs_floor_410m.json", + "sha256": "941eab8e5b4759ec292e16d8f68aed101523404ac9f5a767a18e7b59133fdaa2", + "bytes": 5113, + "exists": true + }, + { + "path": "results/e34b_jeval_dact_410m.json", + "sha256": "cfc18da9384bee3c5c4de07fa38049117c33570f15eca9beb3295a8dc19ffe0f", + "bytes": 4514, + "exists": true + }, + { + "path": "results/e44_dread_410m.json", + "sha256": "3ea05b7a931bf3f93a19689ad2f500fea992fa1ce2ba571c20f84fabc35964fa", + "bytes": 3140, + "exists": true + }, + { + "path": "results/e45_disagreement_geometry.json", + "sha256": "dfd1a336e310f2f0aaf8b966f0b4ae09395a4833cbd14b4481e516c043be8739", + "bytes": 4406, + "exists": true + }, + { + "path": "results/e50_concept_energy.json", + "sha256": "96bea599c1a779cc1953a399e08ec1011f29a373890907135fd7c29ddf443892", + "bytes": 909, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "4602484fcc78d6741dab8c2d9fba3b38d6aca9e3c15ff1f5e70a6bd4c82ea5b7" + } +} \ No newline at end of file diff --git a/results/e57_factorial_cells_410m.json b/results/e57_factorial_cells_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..01f405779d13fdd50679975ebf9b037e181ec9e8 --- /dev/null +++ b/results/e57_factorial_cells_410m.json @@ -0,0 +1,1572 @@ +{ + "experiment": "E57 \u2014 the per-cell draws E52 averaged away", + "why": "the paper's headline is a variance decomposition of 64 cell MEANS with one observation each, so its three shares sum to 100% by construction, its 'interaction' is a residual conflated with cell noise, and it carries no interval. Each cell is a mean over 9 draws (3 fit seeds x 3 prefix seeds). This emits those draws so the shares can be bootstrapped.", + "unit": "admitted-set mean read AUC for one (fit corpus, fit seed, read rung, prefix seed)", + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_draws_per_matrix_cell": 9, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "model": "EleutherAI/pythia-410m-deduped", + "draws": { + "persist": { + "Wikipedia_en|Wikipedia_en|s0|p0": 0.037687454279512166, + "Wikipedia_en|Wikipedia_en|s0|p1": 0.04360535021405667, + "Wikipedia_en|Wikipedia_en|s0|p2": 0.04031957141123712, + "Wikipedia_en|Wikipedia_en|s1|p0": 0.0381647433154285, + "Wikipedia_en|Wikipedia_en|s1|p1": 0.04469228822272271, + "Wikipedia_en|Wikipedia_en|s1|p2": 0.040404757997021076, + "Wikipedia_en|Wikipedia_en|s2|p0": 0.0381647433154285, + "Wikipedia_en|Wikipedia_en|s2|p1": 0.04377746337559074, + "Wikipedia_en|Wikipedia_en|s2|p2": 0.039532129233703014, + "Wikipedia_en|USPTO_Backgrounds|s0|p0": 0.04156353797297925, + "Wikipedia_en|USPTO_Backgrounds|s0|p1": 0.04243574268184602, + "Wikipedia_en|USPTO_Backgrounds|s0|p2": 0.04019909638445825, + "Wikipedia_en|USPTO_Backgrounds|s1|p0": 0.04212089765351266, + "Wikipedia_en|USPTO_Backgrounds|s1|p1": 0.04274603589437902, + "Wikipedia_en|USPTO_Backgrounds|s1|p2": 0.04171427905093879, + "Wikipedia_en|USPTO_Backgrounds|s2|p0": 0.040916012856177984, + "Wikipedia_en|USPTO_Backgrounds|s2|p1": 0.04191302112303674, + "Wikipedia_en|USPTO_Backgrounds|s2|p2": 0.04011903021018952, + "Wikipedia_en|Pile-CC|s0|p0": 0.03965791100636125, + "Wikipedia_en|Pile-CC|s0|p1": 0.03914248633664101, + "Wikipedia_en|Pile-CC|s0|p2": 0.041064153192564845, + "Wikipedia_en|Pile-CC|s1|p0": 0.03938737632706761, + "Wikipedia_en|Pile-CC|s1|p1": 0.04021501408424229, + "Wikipedia_en|Pile-CC|s1|p2": 0.0426172103267163, + "Wikipedia_en|Pile-CC|s2|p0": 0.03885024646297097, + "Wikipedia_en|Pile-CC|s2|p1": 0.038989900681190195, + "Wikipedia_en|Pile-CC|s2|p2": 0.042323902761563656, + "Wikipedia_en|StackExchange|s0|p0": 0.034256494510918854, + "Wikipedia_en|StackExchange|s0|p1": 0.03419739182572812, + "Wikipedia_en|StackExchange|s0|p2": 0.033631662791594866, + "Wikipedia_en|StackExchange|s1|p0": 0.033363637048751116, + "Wikipedia_en|StackExchange|s1|p1": 0.03486514564137906, + "Wikipedia_en|StackExchange|s1|p2": 0.03427594755776227, + "Wikipedia_en|StackExchange|s2|p0": 0.03371354602277279, + "Wikipedia_en|StackExchange|s2|p1": 0.03354230809491128, + "Wikipedia_en|StackExchange|s2|p2": 0.03414359060116112, + "Wikipedia_en|Github|s0|p0": 0.03332046750001609, + "Wikipedia_en|Github|s0|p1": 0.03417315983679146, + "Wikipedia_en|Github|s0|p2": 0.035972073185257615, + "Wikipedia_en|Github|s1|p0": 0.034259540308266875, + "Wikipedia_en|Github|s1|p1": 0.03458548698108643, + "Wikipedia_en|Github|s1|p2": 0.035356608661822976, + "Wikipedia_en|Github|s2|p0": 0.03410695465281606, + "Wikipedia_en|Github|s2|p1": 0.03554478748701513, + "Wikipedia_en|Github|s2|p2": 0.03624946873169392, + "Wikipedia_en|OOD_News_2024|s0|p0": 0.03938806052319706, + "Wikipedia_en|OOD_News_2024|s0|p1": 0.04128914359025657, + "Wikipedia_en|OOD_News_2024|s0|p2": 0.044148194743320344, + "Wikipedia_en|OOD_News_2024|s1|p0": 0.03972031692974269, + "Wikipedia_en|OOD_News_2024|s1|p1": 0.04140385123901069, + "Wikipedia_en|OOD_News_2024|s1|p2": 0.04455296820960939, + "Wikipedia_en|OOD_News_2024|s2|p0": 0.03901360961608589, + "Wikipedia_en|OOD_News_2024|s2|p1": 0.03991144481115043, + "Wikipedia_en|OOD_News_2024|s2|p2": 0.04369798894040287, + "Wikipedia_en|OOD_arXiv_2023|s0|p0": 0.035181528958491984, + "Wikipedia_en|OOD_arXiv_2023|s0|p1": 0.03635661648586393, + "Wikipedia_en|OOD_arXiv_2023|s0|p2": 0.03698380454443395, + "Wikipedia_en|OOD_arXiv_2023|s1|p0": 0.03579117895569652, + "Wikipedia_en|OOD_arXiv_2023|s1|p1": 0.03706049108877778, + "Wikipedia_en|OOD_arXiv_2023|s1|p2": 0.03666853434406221, + "Wikipedia_en|OOD_arXiv_2023|s2|p0": 0.03529192309360951, + "Wikipedia_en|OOD_arXiv_2023|s2|p1": 0.036793197598308326, + "Wikipedia_en|OOD_arXiv_2023|s2|p2": 0.03618368967436254, + "Wikipedia_en|OOD_CommonPile|s0|p0": 0.03625752651132643, + "Wikipedia_en|OOD_CommonPile|s0|p1": 0.03712513237260282, + "Wikipedia_en|OOD_CommonPile|s0|p2": 0.038436542451381686, + "Wikipedia_en|OOD_CommonPile|s1|p0": 0.03811251767911017, + "Wikipedia_en|OOD_CommonPile|s1|p1": 0.03827449041418731, + "Wikipedia_en|OOD_CommonPile|s1|p2": 0.03863062364980578, + "Wikipedia_en|OOD_CommonPile|s2|p0": 0.03696747389622033, + "Wikipedia_en|OOD_CommonPile|s2|p1": 0.03822946446016431, + "Wikipedia_en|OOD_CommonPile|s2|p2": 0.03863386325538158, + "USPTO_Backgrounds|Wikipedia_en|s0|p0": 0.04754846366122365, + "USPTO_Backgrounds|Wikipedia_en|s0|p1": 0.05034518730826676, + "USPTO_Backgrounds|Wikipedia_en|s0|p2": 0.049007836589589716, + "USPTO_Backgrounds|Wikipedia_en|s1|p0": 0.04670790103264153, + "USPTO_Backgrounds|Wikipedia_en|s1|p1": 0.05076761790551245, + "USPTO_Backgrounds|Wikipedia_en|s1|p2": 0.048252454260364175, + "USPTO_Backgrounds|Wikipedia_en|s2|p0": 0.04834426683373749, + "USPTO_Backgrounds|Wikipedia_en|s2|p1": 0.05159551003016531, + "USPTO_Backgrounds|Wikipedia_en|s2|p2": 0.04926326381973922, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p0": 0.05048472099006176, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p1": 0.04920110672246665, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p2": 0.050384720670990646, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p0": 0.050406130286864935, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p1": 0.04874482150189578, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p2": 0.048373900610022245, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p0": 0.05211932349484414, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p1": 0.049685940123163165, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p2": 0.049829933303408326, + "USPTO_Backgrounds|Pile-CC|s0|p0": 0.049157181894406675, + "USPTO_Backgrounds|Pile-CC|s0|p1": 0.04813587157987058, + "USPTO_Backgrounds|Pile-CC|s0|p2": 0.0486748862080276, + "USPTO_Backgrounds|Pile-CC|s1|p0": 0.050147095182910564, + "USPTO_Backgrounds|Pile-CC|s1|p1": 0.04749519950710237, + "USPTO_Backgrounds|Pile-CC|s1|p2": 0.04827266102656722, + "USPTO_Backgrounds|Pile-CC|s2|p0": 0.05055835195817053, + "USPTO_Backgrounds|Pile-CC|s2|p1": 0.04861640161834657, + "USPTO_Backgrounds|Pile-CC|s2|p2": 0.04919329257681966, + "USPTO_Backgrounds|StackExchange|s0|p0": 0.04283932016696781, + "USPTO_Backgrounds|StackExchange|s0|p1": 0.04483984797261655, + "USPTO_Backgrounds|StackExchange|s0|p2": 0.042810790380463, + "USPTO_Backgrounds|StackExchange|s1|p0": 0.04335448981728405, + "USPTO_Backgrounds|StackExchange|s1|p1": 0.04378685294650495, + "USPTO_Backgrounds|StackExchange|s1|p2": 0.04201067774556577, + "USPTO_Backgrounds|StackExchange|s2|p0": 0.044850140972994265, + "USPTO_Backgrounds|StackExchange|s2|p1": 0.045725149055942896, + "USPTO_Backgrounds|StackExchange|s2|p2": 0.043293894408270715, + "USPTO_Backgrounds|Github|s0|p0": 0.04491386362351477, + "USPTO_Backgrounds|Github|s0|p1": 0.04478121572174132, + "USPTO_Backgrounds|Github|s0|p2": 0.047151662735268475, + "USPTO_Backgrounds|Github|s1|p0": 0.044198105670511724, + "USPTO_Backgrounds|Github|s1|p1": 0.044680919079110024, + "USPTO_Backgrounds|Github|s1|p2": 0.04681359394453466, + "USPTO_Backgrounds|Github|s2|p0": 0.04526377264410257, + "USPTO_Backgrounds|Github|s2|p1": 0.04616901460103691, + "USPTO_Backgrounds|Github|s2|p2": 0.0476668321993202, + "USPTO_Backgrounds|OOD_News_2024|s0|p0": 0.04859429798088968, + "USPTO_Backgrounds|OOD_News_2024|s0|p1": 0.04858191790990531, + "USPTO_Backgrounds|OOD_News_2024|s0|p2": 0.04977939403615892, + "USPTO_Backgrounds|OOD_News_2024|s1|p0": 0.04850155687890947, + "USPTO_Backgrounds|OOD_News_2024|s1|p1": 0.04856102610938251, + "USPTO_Backgrounds|OOD_News_2024|s1|p2": 0.04962680540047586, + "USPTO_Backgrounds|OOD_News_2024|s2|p0": 0.04796268339268863, + "USPTO_Backgrounds|OOD_News_2024|s2|p1": 0.048564265063032505, + "USPTO_Backgrounds|OOD_News_2024|s2|p2": 0.049647033819928764, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p0": 0.04610311691649258, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p1": 0.047125090518966316, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p2": 0.046955524990335104, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p0": 0.0456529111135751, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p1": 0.04685927769169211, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p2": 0.04681049273349345, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p0": 0.04550788109190762, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p1": 0.04661828870885074, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p2": 0.04806593046523631, + "USPTO_Backgrounds|OOD_CommonPile|s0|p0": 0.04776679603382945, + "USPTO_Backgrounds|OOD_CommonPile|s0|p1": 0.047657199669629335, + "USPTO_Backgrounds|OOD_CommonPile|s0|p2": 0.045324627216905354, + "USPTO_Backgrounds|OOD_CommonPile|s1|p0": 0.04687393931671977, + "USPTO_Backgrounds|OOD_CommonPile|s1|p1": 0.04669182421639562, + "USPTO_Backgrounds|OOD_CommonPile|s1|p2": 0.045244558807462455, + "USPTO_Backgrounds|OOD_CommonPile|s2|p0": 0.04858713774010539, + "USPTO_Backgrounds|OOD_CommonPile|s2|p1": 0.04818315776064992, + "USPTO_Backgrounds|OOD_CommonPile|s2|p2": 0.046324641536921264, + "Pile-CC|Wikipedia_en|s0|p0": 0.04825152647681534, + "Pile-CC|Wikipedia_en|s0|p1": 0.05187863605096936, + "Pile-CC|Wikipedia_en|s0|p2": 0.04894631956703961, + "Pile-CC|Wikipedia_en|s1|p0": 0.04780132216401398, + "Pile-CC|Wikipedia_en|s1|p1": 0.05182634526863694, + "Pile-CC|Wikipedia_en|s1|p2": 0.050018849736079576, + "Pile-CC|Wikipedia_en|s2|p0": 0.04743118179030716, + "Pile-CC|Wikipedia_en|s2|p1": 0.05175382951274514, + "Pile-CC|Wikipedia_en|s2|p2": 0.049133542692288756, + "Pile-CC|USPTO_Backgrounds|s0|p0": 0.05070397583767772, + "Pile-CC|USPTO_Backgrounds|s0|p1": 0.050584534322842954, + "Pile-CC|USPTO_Backgrounds|s0|p2": 0.04704875312745571, + "Pile-CC|USPTO_Backgrounds|s1|p0": 0.05029164925217629, + "Pile-CC|USPTO_Backgrounds|s1|p1": 0.049512006482109426, + "Pile-CC|USPTO_Backgrounds|s1|p2": 0.04745934084057808, + "Pile-CC|USPTO_Backgrounds|s2|p0": 0.049288399051874876, + "Pile-CC|USPTO_Backgrounds|s2|p1": 0.05006936839781702, + "Pile-CC|USPTO_Backgrounds|s2|p2": 0.0463287090882659, + "Pile-CC|Pile-CC|s0|p0": 0.050284375762566925, + "Pile-CC|Pile-CC|s0|p1": 0.04634758508764207, + "Pile-CC|Pile-CC|s0|p2": 0.05170507738366723, + "Pile-CC|Pile-CC|s1|p0": 0.050179793825373055, + "Pile-CC|Pile-CC|s1|p1": 0.04870075914077461, + "Pile-CC|Pile-CC|s1|p2": 0.05166288511827588, + "Pile-CC|Pile-CC|s2|p0": 0.050529703265056014, + "Pile-CC|Pile-CC|s2|p1": 0.04677756312303245, + "Pile-CC|Pile-CC|s2|p2": 0.050659632030874494, + "Pile-CC|StackExchange|s0|p0": 0.042963291588239375, + "Pile-CC|StackExchange|s0|p1": 0.04423040165565908, + "Pile-CC|StackExchange|s0|p2": 0.041990711051039395, + "Pile-CC|StackExchange|s1|p0": 0.04325335908215493, + "Pile-CC|StackExchange|s1|p1": 0.04577079168520868, + "Pile-CC|StackExchange|s1|p2": 0.041932603972963986, + "Pile-CC|StackExchange|s2|p0": 0.04201059106271714, + "Pile-CC|StackExchange|s2|p1": 0.04429632450919598, + "Pile-CC|StackExchange|s2|p2": 0.04144090178888291, + "Pile-CC|Github|s0|p0": 0.04292848231270909, + "Pile-CC|Github|s0|p1": 0.04337828024290502, + "Pile-CC|Github|s0|p2": 0.046667027613148095, + "Pile-CC|Github|s1|p0": 0.04383575031533837, + "Pile-CC|Github|s1|p1": 0.04249149623792618, + "Pile-CC|Github|s1|p2": 0.0460651921806857, + "Pile-CC|Github|s2|p0": 0.04284329311922193, + "Pile-CC|Github|s2|p1": 0.04284288524650037, + "Pile-CC|Github|s2|p2": 0.046204152470454576, + "Pile-CC|OOD_News_2024|s0|p0": 0.048610453959554434, + "Pile-CC|OOD_News_2024|s0|p1": 0.050505760964006184, + "Pile-CC|OOD_News_2024|s0|p2": 0.05291714668273926, + "Pile-CC|OOD_News_2024|s1|p0": 0.04789041122421622, + "Pile-CC|OOD_News_2024|s1|p1": 0.04989029048010707, + "Pile-CC|OOD_News_2024|s1|p2": 0.05225546401925385, + "Pile-CC|OOD_News_2024|s2|p0": 0.04715013718232512, + "Pile-CC|OOD_News_2024|s2|p1": 0.048699818830937144, + "Pile-CC|OOD_News_2024|s2|p2": 0.05084743839688599, + "Pile-CC|OOD_arXiv_2023|s0|p0": 0.04511925592087209, + "Pile-CC|OOD_arXiv_2023|s0|p1": 0.046386273298412564, + "Pile-CC|OOD_arXiv_2023|s0|p2": 0.04554894873872399, + "Pile-CC|OOD_arXiv_2023|s1|p0": 0.046794575871899724, + "Pile-CC|OOD_arXiv_2023|s1|p1": 0.046816250775009394, + "Pile-CC|OOD_arXiv_2023|s1|p2": 0.04731443366035819, + "Pile-CC|OOD_arXiv_2023|s2|p0": 0.04516144883818925, + "Pile-CC|OOD_arXiv_2023|s2|p1": 0.045521193835884335, + "Pile-CC|OOD_arXiv_2023|s2|p2": 0.045238655712455514, + "Pile-CC|OOD_CommonPile|s0|p0": 0.04562021093443036, + "Pile-CC|OOD_CommonPile|s0|p1": 0.04583600861951709, + "Pile-CC|OOD_CommonPile|s0|p2": 0.04630436021834612, + "Pile-CC|OOD_CommonPile|s1|p0": 0.046350385248661044, + "Pile-CC|OOD_CommonPile|s1|p1": 0.046206143032759425, + "Pile-CC|OOD_CommonPile|s1|p2": 0.0470582596026361, + "Pile-CC|OOD_CommonPile|s2|p0": 0.045845314580947163, + "Pile-CC|OOD_CommonPile|s2|p1": 0.044683408737182614, + "Pile-CC|OOD_CommonPile|s2|p2": 0.04494176860898733, + "StackExchange|Wikipedia_en|s0|p0": 0.04534177854657173, + "StackExchange|Wikipedia_en|s0|p1": 0.04874965557828546, + "StackExchange|Wikipedia_en|s0|p2": 0.04678497305139899, + "StackExchange|Wikipedia_en|s1|p0": 0.04307121969759464, + "StackExchange|Wikipedia_en|s1|p1": 0.04646752080880105, + "StackExchange|Wikipedia_en|s1|p2": 0.043854215275496246, + "StackExchange|Wikipedia_en|s2|p0": 0.04273896366357803, + "StackExchange|Wikipedia_en|s2|p1": 0.04667239612899721, + "StackExchange|Wikipedia_en|s2|p2": 0.0449721741490066, + "StackExchange|USPTO_Backgrounds|s0|p0": 0.052270094072446226, + "StackExchange|USPTO_Backgrounds|s0|p1": 0.04895114982500672, + "StackExchange|USPTO_Backgrounds|s0|p2": 0.048038117168471214, + "StackExchange|USPTO_Backgrounds|s1|p0": 0.04903161874972284, + "StackExchange|USPTO_Backgrounds|s1|p1": 0.04758786121383309, + "StackExchange|USPTO_Backgrounds|s1|p2": 0.045149553148075935, + "StackExchange|USPTO_Backgrounds|s2|p0": 0.04932168475352228, + "StackExchange|USPTO_Backgrounds|s2|p1": 0.04709291616454721, + "StackExchange|USPTO_Backgrounds|s2|p2": 0.04685519519262016, + "StackExchange|Pile-CC|s0|p0": 0.048280903091654184, + "StackExchange|Pile-CC|s0|p1": 0.04533625990152359, + "StackExchange|Pile-CC|s0|p2": 0.04717964669689536, + "StackExchange|Pile-CC|s1|p0": 0.04575491934083402, + "StackExchange|Pile-CC|s1|p1": 0.043663771636784074, + "StackExchange|Pile-CC|s1|p2": 0.04510386427864432, + "StackExchange|Pile-CC|s2|p0": 0.04568240283988416, + "StackExchange|Pile-CC|s2|p1": 0.04459598809480667, + "StackExchange|Pile-CC|s2|p2": 0.04510387098416686, + "StackExchange|StackExchange|s0|p0": 0.04351462812628597, + "StackExchange|StackExchange|s0|p1": 0.0444192714523524, + "StackExchange|StackExchange|s0|p2": 0.04388158111833036, + "StackExchange|StackExchange|s1|p0": 0.04172284197993577, + "StackExchange|StackExchange|s1|p1": 0.04303147359751165, + "StackExchange|StackExchange|s1|p2": 0.04150211112573743, + "StackExchange|StackExchange|s2|p0": 0.04070369771216065, + "StackExchange|StackExchange|s2|p1": 0.04377174465917051, + "StackExchange|StackExchange|s2|p2": 0.04169257441535592, + "StackExchange|Github|s0|p0": 0.04486154709011316, + "StackExchange|Github|s0|p1": 0.04346416727639735, + "StackExchange|Github|s0|p2": 0.049115256313234566, + "StackExchange|Github|s1|p0": 0.04405631143599749, + "StackExchange|Github|s1|p1": 0.040596610866487026, + "StackExchange|Github|s1|p2": 0.04614662090316415, + "StackExchange|Github|s2|p0": 0.044796580262482164, + "StackExchange|Github|s2|p1": 0.04077952075749636, + "StackExchange|Github|s2|p2": 0.04658927181735635, + "StackExchange|OOD_News_2024|s0|p0": 0.04541729120537639, + "StackExchange|OOD_News_2024|s0|p1": 0.04774485994130373, + "StackExchange|OOD_News_2024|s0|p2": 0.04773056162521243, + "StackExchange|OOD_News_2024|s1|p0": 0.04275637110695243, + "StackExchange|OOD_News_2024|s1|p1": 0.04507384244352579, + "StackExchange|OOD_News_2024|s1|p2": 0.04539180090650916, + "StackExchange|OOD_News_2024|s2|p0": 0.043929196428507566, + "StackExchange|OOD_News_2024|s2|p1": 0.04492125455290079, + "StackExchange|OOD_News_2024|s2|p2": 0.04691777331754565, + "StackExchange|OOD_arXiv_2023|s0|p0": 0.04673875542357564, + "StackExchange|OOD_arXiv_2023|s0|p1": 0.045826251665130255, + "StackExchange|OOD_arXiv_2023|s0|p2": 0.04515592325478792, + "StackExchange|OOD_arXiv_2023|s1|p0": 0.043398248683661225, + "StackExchange|OOD_arXiv_2023|s1|p1": 0.043597887316718695, + "StackExchange|OOD_arXiv_2023|s1|p2": 0.04296543952077627, + "StackExchange|OOD_arXiv_2023|s2|p0": 0.043750730156898496, + "StackExchange|OOD_arXiv_2023|s2|p1": 0.043597887316718695, + "StackExchange|OOD_arXiv_2023|s2|p2": 0.04362711915746331, + "StackExchange|OOD_CommonPile|s0|p0": 0.04539163038134575, + "StackExchange|OOD_CommonPile|s0|p1": 0.0481088756583631, + "StackExchange|OOD_CommonPile|s0|p2": 0.04627327444031835, + "StackExchange|OOD_CommonPile|s1|p0": 0.04309074804186821, + "StackExchange|OOD_CommonPile|s1|p1": 0.04560997290536761, + "StackExchange|OOD_CommonPile|s1|p2": 0.044966329168528316, + "StackExchange|OOD_CommonPile|s2|p0": 0.04346088469028473, + "StackExchange|OOD_CommonPile|s2|p1": 0.04500787639990449, + "StackExchange|OOD_CommonPile|s2|p2": 0.04466037387028336, + "Github|Wikipedia_en|s0|p0": 0.02299818522296846, + "Github|Wikipedia_en|s0|p1": 0.026630240306258202, + "Github|Wikipedia_en|s0|p2": 0.02534515131264925, + "Github|Wikipedia_en|s1|p0": 0.025510543677955865, + "Github|Wikipedia_en|s1|p1": 0.030111473705619573, + "Github|Wikipedia_en|s1|p2": 0.02743710605427623, + "Github|Wikipedia_en|s2|p0": 0.024175034556537866, + "Github|Wikipedia_en|s2|p1": 0.02667497657239437, + "Github|Wikipedia_en|s2|p2": 0.0251579275354743, + "Github|USPTO_Backgrounds|s0|p0": 0.02587893744930625, + "Github|USPTO_Backgrounds|s0|p1": 0.026390179200097917, + "Github|USPTO_Backgrounds|s0|p2": 0.026752790808677672, + "Github|USPTO_Backgrounds|s1|p0": 0.028322279546409847, + "Github|USPTO_Backgrounds|s1|p1": 0.027972468826919793, + "Github|USPTO_Backgrounds|s1|p2": 0.029306527273729444, + "Github|USPTO_Backgrounds|s2|p0": 0.02593122785910964, + "Github|USPTO_Backgrounds|s2|p1": 0.02546954290010035, + "Github|USPTO_Backgrounds|s2|p2": 0.025859934091567994, + "Github|Pile-CC|s0|p0": 0.02515146811492741, + "Github|Pile-CC|s0|p1": 0.024389901757240297, + "Github|Pile-CC|s0|p2": 0.026489410549402237, + "Github|Pile-CC|s1|p0": 0.027265126677230002, + "Github|Pile-CC|s1|p1": 0.02809273321181536, + "Github|Pile-CC|s1|p2": 0.029003529250621794, + "Github|Pile-CC|s2|p0": 0.02417854224331677, + "Github|Pile-CC|s2|p1": 0.024684277921915056, + "Github|Pile-CC|s2|p2": 0.025059421779587865, + "Github|StackExchange|s0|p0": 0.021186233148910105, + "Github|StackExchange|s0|p1": 0.01899896454997361, + "Github|StackExchange|s0|p2": 0.021202125074341893, + "Github|StackExchange|s1|p0": 0.023487116605974735, + "Github|StackExchange|s1|p1": 0.022092406777665018, + "Github|StackExchange|s1|p2": 0.02490347814746201, + "Github|StackExchange|s2|p0": 0.021106163994409143, + "Github|StackExchange|s2|p1": 0.019731681840494276, + "Github|StackExchange|s2|p2": 0.02183454525656998, + "Github|Github|s0|p0": 0.02421602364629507, + "Github|Github|s0|p1": 0.022202111966907977, + "Github|Github|s0|p2": 0.02491505742073059, + "Github|Github|s1|p0": 0.02566192615777254, + "Github|Github|s1|p1": 0.024100766889750956, + "Github|Github|s1|p2": 0.02683824971318245, + "Github|Github|s2|p0": 0.023270875820890068, + "Github|Github|s2|p1": 0.022087405901402235, + "Github|Github|s2|p2": 0.024464850872755052, + "Github|OOD_News_2024|s0|p0": 0.02355434806086123, + "Github|OOD_News_2024|s0|p1": 0.024397747684270145, + "Github|OOD_News_2024|s0|p2": 0.02593636615201831, + "Github|OOD_News_2024|s1|p0": 0.026948650367558002, + "Github|OOD_News_2024|s1|p1": 0.0274395695887506, + "Github|OOD_News_2024|s1|p2": 0.03017957229167223, + "Github|OOD_News_2024|s2|p0": 0.023516468750312925, + "Github|OOD_News_2024|s2|p1": 0.024300693767145276, + "Github|OOD_News_2024|s2|p2": 0.027092987019568682, + "Github|OOD_arXiv_2023|s0|p0": 0.02449531713500619, + "Github|OOD_arXiv_2023|s0|p1": 0.026078676618635655, + "Github|OOD_arXiv_2023|s0|p2": 0.024994170852005483, + "Github|OOD_arXiv_2023|s1|p0": 0.028093830216675997, + "Github|OOD_arXiv_2023|s1|p1": 0.029185744281858206, + "Github|OOD_arXiv_2023|s1|p2": 0.02758511919528246, + "Github|OOD_arXiv_2023|s2|p0": 0.024785382393747567, + "Github|OOD_arXiv_2023|s2|p1": 0.026666359044611454, + "Github|OOD_arXiv_2023|s2|p2": 0.02447145152837038, + "Github|OOD_CommonPile|s0|p0": 0.025595012493431567, + "Github|OOD_CommonPile|s0|p1": 0.024993591755628587, + "Github|OOD_CommonPile|s0|p2": 0.024846938252449036, + "Github|OOD_CommonPile|s1|p0": 0.028891593776643278, + "Github|OOD_CommonPile|s1|p1": 0.02769991457462311, + "Github|OOD_CommonPile|s1|p2": 0.027905744034796953, + "Github|OOD_CommonPile|s2|p0": 0.026110181398689748, + "Github|OOD_CommonPile|s2|p1": 0.025600407272577286, + "Github|OOD_CommonPile|s2|p2": 0.02589750224724412, + "OOD_News_2024|Wikipedia_en|s0|p0": 0.03671670316252858, + "OOD_News_2024|Wikipedia_en|s0|p1": 0.04154766141436994, + "OOD_News_2024|Wikipedia_en|s0|p2": 0.03797612129710615, + "OOD_News_2024|Wikipedia_en|s1|p0": 0.03743067388422787, + "OOD_News_2024|Wikipedia_en|s1|p1": 0.04147514565847814, + "OOD_News_2024|Wikipedia_en|s1|p2": 0.038365787779912355, + "OOD_News_2024|Wikipedia_en|s2|p0": 0.03611391170416027, + "OOD_News_2024|Wikipedia_en|s2|p1": 0.040932194283232096, + "OOD_News_2024|Wikipedia_en|s2|p2": 0.03859844258986413, + "OOD_News_2024|USPTO_Backgrounds|s0|p0": 0.041230500862002376, + "OOD_News_2024|USPTO_Backgrounds|s0|p1": 0.04164239864330739, + "OOD_News_2024|USPTO_Backgrounds|s0|p2": 0.038126584514975546, + "OOD_News_2024|USPTO_Backgrounds|s1|p0": 0.04147001262754202, + "OOD_News_2024|USPTO_Backgrounds|s1|p1": 0.041337224352173504, + "OOD_News_2024|USPTO_Backgrounds|s1|p2": 0.037525272998027506, + "OOD_News_2024|USPTO_Backgrounds|s2|p0": 0.04158041067421436, + "OOD_News_2024|USPTO_Backgrounds|s2|p1": 0.04007423396687955, + "OOD_News_2024|USPTO_Backgrounds|s2|p2": 0.0375977887539193, + "OOD_News_2024|Pile-CC|s0|p0": 0.03771652071736753, + "OOD_News_2024|Pile-CC|s0|p1": 0.03725688378326595, + "OOD_News_2024|Pile-CC|s0|p2": 0.03985171332024038, + "OOD_News_2024|Pile-CC|s1|p0": 0.038661668775603175, + "OOD_News_2024|Pile-CC|s1|p1": 0.03742214529775083, + "OOD_News_2024|Pile-CC|s1|p2": 0.04073701440356672, + "OOD_News_2024|Pile-CC|s2|p0": 0.03813894647173584, + "OOD_News_2024|Pile-CC|s2|p1": 0.03623921959660947, + "OOD_News_2024|Pile-CC|s2|p2": 0.040924237994477156, + "OOD_News_2024|StackExchange|s0|p0": 0.03539642095565796, + "OOD_News_2024|StackExchange|s0|p1": 0.032507660496048627, + "OOD_News_2024|StackExchange|s0|p2": 0.032678435742855075, + "OOD_News_2024|StackExchange|s1|p0": 0.03390832245349884, + "OOD_News_2024|StackExchange|s1|p1": 0.0328431575326249, + "OOD_News_2024|StackExchange|s1|p2": 0.0331265916582197, + "OOD_News_2024|StackExchange|s2|p0": 0.034496009349823, + "OOD_News_2024|StackExchange|s2|p1": 0.03224792021792382, + "OOD_News_2024|StackExchange|s2|p2": 0.03320666453801095, + "OOD_News_2024|Github|s0|p0": 0.03426561239175498, + "OOD_News_2024|Github|s0|p1": 0.03422586037777364, + "OOD_News_2024|Github|s0|p2": 0.03395344740711152, + "OOD_News_2024|Github|s1|p0": 0.03421184557955712, + "OOD_News_2024|Github|s1|p1": 0.03424001345410943, + "OOD_News_2024|Github|s1|p2": 0.03305910739582032, + "OOD_News_2024|Github|s2|p0": 0.03475479034241289, + "OOD_News_2024|Github|s2|p1": 0.033200642978772524, + "OOD_News_2024|Github|s2|p2": 0.03224632095079869, + "OOD_News_2024|OOD_News_2024|s0|p0": 0.03907562275417149, + "OOD_News_2024|OOD_News_2024|s0|p1": 0.038037475850433114, + "OOD_News_2024|OOD_News_2024|s0|p2": 0.04267560779117048, + "OOD_News_2024|OOD_News_2024|s1|p0": 0.039749975129961965, + "OOD_News_2024|OOD_News_2024|s1|p1": 0.03851800737902522, + "OOD_News_2024|OOD_News_2024|s1|p2": 0.04386608614586294, + "OOD_News_2024|OOD_News_2024|s2|p0": 0.038864674419164656, + "OOD_News_2024|OOD_News_2024|s2|p1": 0.03814786998555064, + "OOD_News_2024|OOD_News_2024|s2|p2": 0.04297322570346296, + "OOD_News_2024|OOD_arXiv_2023|s0|p0": 0.036149079957976935, + "OOD_News_2024|OOD_arXiv_2023|s0|p1": 0.035061447555199264, + "OOD_News_2024|OOD_arXiv_2023|s0|p2": 0.03549142493866384, + "OOD_News_2024|OOD_arXiv_2023|s1|p0": 0.0364813367370516, + "OOD_News_2024|OOD_arXiv_2023|s1|p1": 0.03631688528694212, + "OOD_News_2024|OOD_arXiv_2023|s1|p2": 0.03591385050676763, + "OOD_News_2024|OOD_arXiv_2023|s2|p0": 0.036294112959876654, + "OOD_News_2024|OOD_arXiv_2023|s2|p1": 0.035278993332758544, + "OOD_News_2024|OOD_arXiv_2023|s2|p2": 0.036046210071071984, + "OOD_News_2024|OOD_CommonPile|s0|p0": 0.03237517392262816, + "OOD_News_2024|OOD_CommonPile|s0|p1": 0.033100989926606414, + "OOD_News_2024|OOD_CommonPile|s0|p2": 0.03596177669242025, + "OOD_News_2024|OOD_CommonPile|s1|p0": 0.03315008087083697, + "OOD_News_2024|OOD_CommonPile|s1|p1": 0.034893559757620096, + "OOD_News_2024|OOD_CommonPile|s1|p2": 0.03671362427994609, + "OOD_News_2024|OOD_CommonPile|s2|p0": 0.03398807412013412, + "OOD_News_2024|OOD_CommonPile|s2|p1": 0.03459594184532762, + "OOD_News_2024|OOD_CommonPile|s2|p2": 0.03665123647078872, + "OOD_arXiv_2023|Wikipedia_en|s0|p0": 0.038727887254208324, + "OOD_arXiv_2023|Wikipedia_en|s0|p1": 0.044071207335218784, + "OOD_arXiv_2023|Wikipedia_en|s0|p2": 0.03957072671037167, + "OOD_arXiv_2023|Wikipedia_en|s1|p0": 0.038357752840965986, + "OOD_arXiv_2023|Wikipedia_en|s1|p1": 0.04405098198913038, + "OOD_arXiv_2023|Wikipedia_en|s1|p2": 0.039490654575638474, + "OOD_arXiv_2023|Wikipedia_en|s2|p0": 0.03761748177930713, + "OOD_arXiv_2023|Wikipedia_en|s2|p1": 0.043291582423262295, + "OOD_arXiv_2023|Wikipedia_en|s2|p2": 0.03947798095177859, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p0": 0.043173919711261985, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p1": 0.03929868265986443, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p2": 0.03943431116640568, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p0": 0.042961490247398613, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p1": 0.039073579013347626, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p2": 0.040922409668564794, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p0": 0.04289652723819017, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p1": 0.03952378705143929, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p2": 0.04007174372673035, + "OOD_arXiv_2023|Pile-CC|s0|p0": 0.04011216927319765, + "OOD_arXiv_2023|Pile-CC|s0|p1": 0.039493899303488435, + "OOD_arXiv_2023|Pile-CC|s0|p2": 0.04293656102381647, + "OOD_arXiv_2023|Pile-CC|s1|p0": 0.040264754183590414, + "OOD_arXiv_2023|Pile-CC|s1|p1": 0.0400552851613611, + "OOD_arXiv_2023|Pile-CC|s1|p2": 0.04282185318879783, + "OOD_arXiv_2023|Pile-CC|s2|p0": 0.03959699887782335, + "OOD_arXiv_2023|Pile-CC|s2|p1": 0.039130359841510656, + "OOD_arXiv_2023|Pile-CC|s2|p2": 0.04367682985030115, + "OOD_arXiv_2023|StackExchange|s0|p0": 0.03755808379501104, + "OOD_arXiv_2023|StackExchange|s0|p1": 0.038457607361488046, + "OOD_arXiv_2023|StackExchange|s0|p2": 0.03699849676340818, + "OOD_arXiv_2023|StackExchange|s1|p0": 0.03673774134367704, + "OOD_arXiv_2023|StackExchange|s1|p1": 0.037767888489179315, + "OOD_arXiv_2023|StackExchange|s1|p2": 0.03663521781563759, + "OOD_arXiv_2023|StackExchange|s2|p0": 0.03733298312872648, + "OOD_arXiv_2023|StackExchange|s2|p1": 0.03916755344253033, + "OOD_arXiv_2023|StackExchange|s2|p2": 0.03687369003891945, + "OOD_arXiv_2023|Github|s0|p0": 0.03632147805765271, + "OOD_arXiv_2023|Github|s0|p1": 0.035743088275194165, + "OOD_arXiv_2023|Github|s0|p2": 0.03812470398843289, + "OOD_arXiv_2023|Github|s1|p0": 0.035248953942209484, + "OOD_arXiv_2023|Github|s1|p1": 0.03522036597132683, + "OOD_arXiv_2023|Github|s1|p2": 0.03782708495855332, + "OOD_arXiv_2023|Github|s2|p0": 0.035356107819825414, + "OOD_arXiv_2023|Github|s2|p1": 0.03530043512582779, + "OOD_arXiv_2023|Github|s2|p2": 0.03946021385490894, + "OOD_arXiv_2023|OOD_News_2024|s0|p0": 0.03954499904066324, + "OOD_arXiv_2023|OOD_News_2024|s0|p1": 0.0407484196824953, + "OOD_arXiv_2023|OOD_News_2024|s0|p2": 0.04118536699097604, + "OOD_arXiv_2023|OOD_News_2024|s1|p0": 0.03955333910416812, + "OOD_arXiv_2023|OOD_News_2024|s1|p1": 0.040153183857910335, + "OOD_arXiv_2023|OOD_News_2024|s1|p2": 0.04083719903137535, + "OOD_arXiv_2023|OOD_News_2024|s2|p0": 0.0388393696397543, + "OOD_arXiv_2023|OOD_News_2024|s2|p1": 0.03958424627780914, + "OOD_arXiv_2023|OOD_News_2024|s2|p2": 0.04086323890369385, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p0": 0.039236063580028714, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p1": 0.03838620453607291, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p2": 0.03940372753422707, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p0": 0.03856830827426165, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p1": 0.039550378127023576, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p2": 0.03884312759619206, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p0": 0.037973072449676695, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p1": 0.038096139277331534, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p2": 0.039559671422466636, + "OOD_arXiv_2023|OOD_CommonPile|s0|p0": 0.040812817169353366, + "OOD_arXiv_2023|OOD_CommonPile|s0|p1": 0.040232782950624824, + "OOD_arXiv_2023|OOD_CommonPile|s0|p2": 0.03951485650613904, + "OOD_arXiv_2023|OOD_CommonPile|s1|p0": 0.041182955307886, + "OOD_arXiv_2023|OOD_CommonPile|s1|p1": 0.0413102903869003, + "OOD_arXiv_2023|OOD_CommonPile|s1|p2": 0.0391573921777308, + "OOD_arXiv_2023|OOD_CommonPile|s2|p0": 0.039919960452243684, + "OOD_arXiv_2023|OOD_CommonPile|s2|p1": 0.040628124168142675, + "OOD_arXiv_2023|OOD_CommonPile|s2|p2": 0.03842467376962304, + "OOD_CommonPile|Wikipedia_en|s0|p0": 0.036579592060297726, + "OOD_CommonPile|Wikipedia_en|s0|p1": 0.03941391441039741, + "OOD_CommonPile|Wikipedia_en|s0|p2": 0.03770494412165135, + "OOD_CommonPile|Wikipedia_en|s1|p0": 0.03769755465909839, + "OOD_CommonPile|Wikipedia_en|s1|p1": 0.04058416350744665, + "OOD_CommonPile|Wikipedia_en|s1|p2": 0.0388575381366536, + "OOD_CommonPile|Wikipedia_en|s2|p0": 0.035091493558138606, + "OOD_CommonPile|Wikipedia_en|s2|p1": 0.03933127275668084, + "OOD_CommonPile|Wikipedia_en|s2|p2": 0.03607181336265057, + "OOD_CommonPile|USPTO_Backgrounds|s0|p0": 0.042003673408180474, + "OOD_CommonPile|USPTO_Backgrounds|s0|p1": 0.03816804885864258, + "OOD_CommonPile|USPTO_Backgrounds|s0|p2": 0.03981860044877976, + "OOD_CommonPile|USPTO_Backgrounds|s1|p0": 0.042526392824947835, + "OOD_CommonPile|USPTO_Backgrounds|s1|p1": 0.038485894352197646, + "OOD_CommonPile|USPTO_Backgrounds|s1|p2": 0.04015007205307484, + "OOD_CommonPile|USPTO_Backgrounds|s2|p0": 0.04085107194259763, + "OOD_CommonPile|USPTO_Backgrounds|s2|p1": 0.03684521391987801, + "OOD_CommonPile|USPTO_Backgrounds|s2|p2": 0.038853227230720225, + "OOD_CommonPile|Pile-CC|s0|p0": 0.039699149574153124, + "OOD_CommonPile|Pile-CC|s0|p1": 0.03928320545237511, + "OOD_CommonPile|Pile-CC|s0|p2": 0.04219371690414846, + "OOD_CommonPile|Pile-CC|s1|p0": 0.04052880499511957, + "OOD_CommonPile|Pile-CC|s1|p1": 0.04028645928483456, + "OOD_CommonPile|Pile-CC|s1|p2": 0.0413289200514555, + "OOD_CommonPile|Pile-CC|s2|p0": 0.038403282128274444, + "OOD_CommonPile|Pile-CC|s2|p1": 0.03797802354674786, + "OOD_CommonPile|Pile-CC|s2|p2": 0.03977988462429494, + "OOD_CommonPile|StackExchange|s0|p0": 0.0347943221218884, + "OOD_CommonPile|StackExchange|s0|p1": 0.0371971896616742, + "OOD_CommonPile|StackExchange|s0|p2": 0.03643182187806815, + "OOD_CommonPile|StackExchange|s1|p0": 0.035917400382459164, + "OOD_CommonPile|StackExchange|s1|p1": 0.03827295664232224, + "OOD_CommonPile|StackExchange|s1|p2": 0.03636685728561133, + "OOD_CommonPile|StackExchange|s2|p0": 0.03382139662280679, + "OOD_CommonPile|StackExchange|s2|p1": 0.03614917423110455, + "OOD_CommonPile|StackExchange|s2|p2": 0.035076087224297225, + "OOD_CommonPile|Github|s0|p0": 0.0341214191634208, + "OOD_CommonPile|Github|s0|p1": 0.03550464184954762, + "OOD_CommonPile|Github|s0|p2": 0.03786927685141563, + "OOD_CommonPile|Github|s1|p0": 0.03532954598776996, + "OOD_CommonPile|Github|s1|p1": 0.03550464184954762, + "OOD_CommonPile|Github|s1|p2": 0.037519368529319766, + "OOD_CommonPile|Github|s2|p0": 0.03267119978554547, + "OOD_CommonPile|Github|s2|p1": 0.033739156648516656, + "OOD_CommonPile|Github|s2|p2": 0.03747891690582037, + "OOD_CommonPile|OOD_News_2024|s0|p0": 0.03865266921930015, + "OOD_CommonPile|OOD_News_2024|s0|p1": 0.03853014975320548, + "OOD_CommonPile|OOD_News_2024|s0|p2": 0.040657664998434484, + "OOD_CommonPile|OOD_News_2024|s1|p0": 0.03896243553608656, + "OOD_CommonPile|OOD_News_2024|s1|p1": 0.03922930133994669, + "OOD_CommonPile|OOD_News_2024|s1|p2": 0.04086861656978726, + "OOD_CommonPile|OOD_News_2024|s2|p0": 0.038063503126613794, + "OOD_CommonPile|OOD_News_2024|s2|p1": 0.03704205721151084, + "OOD_CommonPile|OOD_News_2024|s2|p2": 0.0400338601320982, + "OOD_CommonPile|OOD_arXiv_2023|s0|p0": 0.03841572336386889, + "OOD_CommonPile|OOD_arXiv_2023|s0|p1": 0.03803856065496802, + "OOD_CommonPile|OOD_arXiv_2023|s0|p2": 0.03738281305413693, + "OOD_CommonPile|OOD_arXiv_2023|s1|p0": 0.040083489916287365, + "OOD_CommonPile|OOD_arXiv_2023|s1|p1": 0.03848728644661605, + "OOD_CommonPile|OOD_arXiv_2023|s1|p2": 0.0378676577238366, + "OOD_CommonPile|OOD_arXiv_2023|s2|p0": 0.0373154137050733, + "OOD_CommonPile|OOD_arXiv_2023|s2|p1": 0.03722429233603179, + "OOD_CommonPile|OOD_arXiv_2023|s2|p2": 0.036521761259064076, + "OOD_CommonPile|OOD_CommonPile|s0|p0": 0.03953715111128986, + "OOD_CommonPile|OOD_CommonPile|s0|p1": 0.0394535590428859, + "OOD_CommonPile|OOD_CommonPile|s0|p2": 0.038331931550055744, + "OOD_CommonPile|OOD_CommonPile|s1|p0": 0.04062733114697039, + "OOD_CommonPile|OOD_CommonPile|s1|p1": 0.03977826316840947, + "OOD_CommonPile|OOD_CommonPile|s1|p2": 0.03860758962109685, + "OOD_CommonPile|OOD_CommonPile|s2|p0": 0.039269858272746205, + "OOD_CommonPile|OOD_CommonPile|s2|p1": 0.03881288995034993, + "OOD_CommonPile|OOD_CommonPile|s2|p2": 0.0375967831350863, + "logit|Q0|p0": 0.028439503419212996, + "shuf|Q0|p0": 0.021268022060394288, + "logit|Q0|p1": 0.028439503419212996, + "shuf|Q0|p1": 0.021268022060394288, + "logit|Q0|p2": 0.028439503419212996, + "shuf|Q0|p2": 0.021268022060394288, + "logit|Wikipedia_en|p0": 0.028726140107028186, + "shuf|Wikipedia_en|p0": 0.02310783523134887, + "logit|Wikipedia_en|p1": 0.030837458628229796, + "shuf|Wikipedia_en|p1": 0.02413048017770052, + "logit|Wikipedia_en|p2": 0.029703613999299706, + "shuf|Wikipedia_en|p2": 0.024289118126034737, + "logit|USPTO_Backgrounds|p0": 0.02871693994384259, + "shuf|USPTO_Backgrounds|p0": 0.024401361122727394, + "logit|USPTO_Backgrounds|p1": 0.03058738212566823, + "shuf|USPTO_Backgrounds|p1": 0.025304315984249114, + "logit|USPTO_Backgrounds|p2": 0.031176864611916245, + "shuf|USPTO_Backgrounds|p2": 0.022762071900069715, + "logit|Pile-CC|p0": 0.035120575386099515, + "shuf|Pile-CC|p0": 0.022151755448430777, + "logit|Pile-CC|p1": 0.03508521046023816, + "shuf|Pile-CC|p1": 0.021709074266254902, + "logit|Pile-CC|p2": 0.03481215860228985, + "shuf|Pile-CC|p2": 0.023998354747891425, + "logit|StackExchange|p0": 0.02859928123652935, + "shuf|StackExchange|p0": 0.018458761926740407, + "logit|StackExchange|p1": 0.026670536398887633, + "shuf|StackExchange|p1": 0.01743514370173216, + "logit|StackExchange|p2": 0.027009129035286607, + "shuf|StackExchange|p2": 0.01849499884992838, + "logit|Github|p0": 0.027017780649475753, + "shuf|Github|p0": 0.019806107599288227, + "logit|Github|p1": 0.02629018302541226, + "shuf|Github|p1": 0.019280839525163174, + "logit|Github|p2": 0.02436053443234414, + "shuf|Github|p2": 0.019516041968017817, + "logit|OOD_News_2024|p0": 0.03550712231080979, + "shuf|OOD_News_2024|p0": 0.024123887810856105, + "logit|OOD_News_2024|p1": 0.03712005380075425, + "shuf|OOD_News_2024|p1": 0.022528637386858463, + "logit|OOD_News_2024|p2": 0.03788821462076157, + "shuf|OOD_News_2024|p2": 0.02383031537756324, + "logit|OOD_arXiv_2023|p0": 0.026592944376170637, + "shuf|OOD_arXiv_2023|p0": 0.022637404035776854, + "logit|OOD_arXiv_2023|p1": 0.02745935469865799, + "shuf|OOD_arXiv_2023|p1": 0.02077593160793185, + "logit|OOD_arXiv_2023|p2": 0.026104292529635132, + "shuf|OOD_arXiv_2023|p2": 0.02104389388114214, + "logit|OOD_CommonPile|p0": 0.03556173413526267, + "shuf|OOD_CommonPile|p0": 0.01958368895575404, + "logit|OOD_CommonPile|p1": 0.03545751124620437, + "shuf|OOD_CommonPile|p1": 0.020708365738391875, + "logit|OOD_CommonPile|p2": 0.03281065821647644, + "shuf|OOD_CommonPile|p2": 0.022601932287216187, + "logit|SHUFFLED_Pile-CC|p0": 0.019922320428304375, + "shuf|SHUFFLED_Pile-CC|p0": 0.015490125399082899, + "logit|SHUFFLED_Pile-CC|p1": 0.019162460300140082, + "shuf|SHUFFLED_Pile-CC|p1": 0.01434438480064273, + "logit|SHUFFLED_Pile-CC|p2": 0.017466827854514123, + "shuf|SHUFFLED_Pile-CC|p2": 0.015702555235475302 + }, + "min": { + "Wikipedia_en|Wikipedia_en|s0|p0": 0.1314952816814184, + "Wikipedia_en|Wikipedia_en|s0|p1": 0.13528487123548985, + "Wikipedia_en|Wikipedia_en|s0|p2": 0.13345829024910927, + "Wikipedia_en|Wikipedia_en|s1|p0": 0.13206410072743893, + "Wikipedia_en|Wikipedia_en|s1|p1": 0.13533404991030692, + "Wikipedia_en|Wikipedia_en|s1|p2": 0.13453038409352303, + "Wikipedia_en|Wikipedia_en|s2|p0": 0.12990677542984486, + "Wikipedia_en|Wikipedia_en|s2|p1": 0.133492299541831, + "Wikipedia_en|Wikipedia_en|s2|p2": 0.13399313166737556, + "Wikipedia_en|USPTO_Backgrounds|s0|p0": 0.14007385782897472, + "Wikipedia_en|USPTO_Backgrounds|s0|p1": 0.13845476023852826, + "Wikipedia_en|USPTO_Backgrounds|s0|p2": 0.13388987220823764, + "Wikipedia_en|USPTO_Backgrounds|s1|p0": 0.1416717667132616, + "Wikipedia_en|USPTO_Backgrounds|s1|p1": 0.14080993905663491, + "Wikipedia_en|USPTO_Backgrounds|s1|p2": 0.13559157438576222, + "Wikipedia_en|USPTO_Backgrounds|s2|p0": 0.138703703135252, + "Wikipedia_en|USPTO_Backgrounds|s2|p1": 0.13860451877117158, + "Wikipedia_en|USPTO_Backgrounds|s2|p2": 0.13279332891106604, + "Wikipedia_en|Pile-CC|s0|p0": 0.13716441504657267, + "Wikipedia_en|Pile-CC|s0|p1": 0.13544980734586715, + "Wikipedia_en|Pile-CC|s0|p2": 0.1406457681208849, + "Wikipedia_en|Pile-CC|s1|p0": 0.14019205719232558, + "Wikipedia_en|Pile-CC|s1|p1": 0.1367002371698618, + "Wikipedia_en|Pile-CC|s1|p2": 0.1434601716697216, + "Wikipedia_en|Pile-CC|s2|p0": 0.1371540330350399, + "Wikipedia_en|Pile-CC|s2|p1": 0.13469483256340026, + "Wikipedia_en|Pile-CC|s2|p2": 0.13953884541988373, + "Wikipedia_en|StackExchange|s0|p0": 0.11986354738473892, + "Wikipedia_en|StackExchange|s0|p1": 0.1174791244789958, + "Wikipedia_en|StackExchange|s0|p2": 0.12304020710289479, + "Wikipedia_en|StackExchange|s1|p0": 0.1194436714053154, + "Wikipedia_en|StackExchange|s1|p1": 0.11803836189210415, + "Wikipedia_en|StackExchange|s1|p2": 0.12330680526793003, + "Wikipedia_en|StackExchange|s2|p0": 0.11876500882208348, + "Wikipedia_en|StackExchange|s2|p1": 0.11583884693682193, + "Wikipedia_en|StackExchange|s2|p2": 0.1200434122234583, + "Wikipedia_en|Github|s0|p0": 0.11371806338429451, + "Wikipedia_en|Github|s0|p1": 0.11780917681753636, + "Wikipedia_en|Github|s0|p2": 0.11871678121387959, + "Wikipedia_en|Github|s1|p0": 0.11582040563225746, + "Wikipedia_en|Github|s1|p1": 0.12019886709749698, + "Wikipedia_en|Github|s1|p2": 0.1184202253818512, + "Wikipedia_en|Github|s2|p0": 0.11457479931414127, + "Wikipedia_en|Github|s2|p1": 0.11827919371426106, + "Wikipedia_en|Github|s2|p2": 0.11699035800993443, + "Wikipedia_en|OOD_News_2024|s0|p0": 0.13576465286314487, + "Wikipedia_en|OOD_News_2024|s0|p1": 0.1364069476723671, + "Wikipedia_en|OOD_News_2024|s0|p2": 0.14100285731256007, + "Wikipedia_en|OOD_News_2024|s1|p0": 0.13383496925234795, + "Wikipedia_en|OOD_News_2024|s1|p1": 0.13661750890314578, + "Wikipedia_en|OOD_News_2024|s1|p2": 0.14217874445021153, + "Wikipedia_en|OOD_News_2024|s2|p0": 0.13318124786019325, + "Wikipedia_en|OOD_News_2024|s2|p1": 0.1357308529317379, + "Wikipedia_en|OOD_News_2024|s2|p2": 0.1400008223950863, + "Wikipedia_en|OOD_arXiv_2023|s0|p0": 0.13495293743908404, + "Wikipedia_en|OOD_arXiv_2023|s0|p1": 0.12740938253700734, + "Wikipedia_en|OOD_arXiv_2023|s0|p2": 0.1316213935613632, + "Wikipedia_en|OOD_arXiv_2023|s1|p0": 0.13659278377890588, + "Wikipedia_en|OOD_arXiv_2023|s1|p1": 0.1305436495691538, + "Wikipedia_en|OOD_arXiv_2023|s1|p2": 0.1355338167399168, + "Wikipedia_en|OOD_arXiv_2023|s2|p0": 0.13350272178649902, + "Wikipedia_en|OOD_arXiv_2023|s2|p1": 0.12782547026872634, + "Wikipedia_en|OOD_arXiv_2023|s2|p2": 0.1323194708675146, + "Wikipedia_en|OOD_CommonPile|s0|p0": 0.1261197067797184, + "Wikipedia_en|OOD_CommonPile|s0|p1": 0.1212854977697134, + "Wikipedia_en|OOD_CommonPile|s0|p2": 0.12230483815073967, + "Wikipedia_en|OOD_CommonPile|s1|p0": 0.12772722244262696, + "Wikipedia_en|OOD_CommonPile|s1|p1": 0.12077664285898208, + "Wikipedia_en|OOD_CommonPile|s1|p2": 0.12424149252474308, + "Wikipedia_en|OOD_CommonPile|s2|p0": 0.1265941672027111, + "Wikipedia_en|OOD_CommonPile|s2|p1": 0.12024476863443852, + "Wikipedia_en|OOD_CommonPile|s2|p2": 0.12263073474168777, + "USPTO_Backgrounds|Wikipedia_en|s0|p0": 0.15865903981029988, + "USPTO_Backgrounds|Wikipedia_en|s0|p1": 0.1594368640333414, + "USPTO_Backgrounds|Wikipedia_en|s0|p2": 0.1609773922711611, + "USPTO_Backgrounds|Wikipedia_en|s1|p0": 0.15415786467492582, + "USPTO_Backgrounds|Wikipedia_en|s1|p1": 0.15766826085746288, + "USPTO_Backgrounds|Wikipedia_en|s1|p2": 0.15936810337007046, + "USPTO_Backgrounds|Wikipedia_en|s2|p0": 0.15782200656831263, + "USPTO_Backgrounds|Wikipedia_en|s2|p1": 0.15970130153000356, + "USPTO_Backgrounds|Wikipedia_en|s2|p2": 0.1623216535896063, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p0": 0.1671406153589487, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p1": 0.16851548999547958, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p2": 0.16106511391699313, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p0": 0.16180277429521084, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p1": 0.16597656160593033, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p2": 0.15730201900005342, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p0": 0.16720139421522617, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p1": 0.1691258128732443, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p2": 0.16187019012868403, + "USPTO_Backgrounds|Pile-CC|s0|p0": 0.16361045688390732, + "USPTO_Backgrounds|Pile-CC|s0|p1": 0.1640078630298376, + "USPTO_Backgrounds|Pile-CC|s0|p2": 0.16455203406512736, + "USPTO_Backgrounds|Pile-CC|s1|p0": 0.16142426915466784, + "USPTO_Backgrounds|Pile-CC|s1|p1": 0.1605743058025837, + "USPTO_Backgrounds|Pile-CC|s1|p2": 0.1647999119013548, + "USPTO_Backgrounds|Pile-CC|s2|p0": 0.16487436927855015, + "USPTO_Backgrounds|Pile-CC|s2|p1": 0.16551438048481942, + "USPTO_Backgrounds|Pile-CC|s2|p2": 0.16598202399909495, + "USPTO_Backgrounds|StackExchange|s0|p0": 0.15085273757576942, + "USPTO_Backgrounds|StackExchange|s0|p1": 0.15416517574340105, + "USPTO_Backgrounds|StackExchange|s0|p2": 0.1587714310735464, + "USPTO_Backgrounds|StackExchange|s1|p0": 0.1481450531631708, + "USPTO_Backgrounds|StackExchange|s1|p1": 0.15227268766611815, + "USPTO_Backgrounds|StackExchange|s1|p2": 0.15382490456104278, + "USPTO_Backgrounds|StackExchange|s2|p0": 0.15232894271612168, + "USPTO_Backgrounds|StackExchange|s2|p1": 0.15657964777201414, + "USPTO_Backgrounds|StackExchange|s2|p2": 0.15948974154889584, + "USPTO_Backgrounds|Github|s0|p0": 0.1421130258589983, + "USPTO_Backgrounds|Github|s0|p1": 0.147469431348145, + "USPTO_Backgrounds|Github|s0|p2": 0.146794119104743, + "USPTO_Backgrounds|Github|s1|p0": 0.13769526965916157, + "USPTO_Backgrounds|Github|s1|p1": 0.14530146084725856, + "USPTO_Backgrounds|Github|s1|p2": 0.14440454207360745, + "USPTO_Backgrounds|Github|s2|p0": 0.14245902597904206, + "USPTO_Backgrounds|Github|s2|p1": 0.1488735280930996, + "USPTO_Backgrounds|Github|s2|p2": 0.1483356926590204, + "USPTO_Backgrounds|OOD_News_2024|s0|p0": 0.15932236909866332, + "USPTO_Backgrounds|OOD_News_2024|s0|p1": 0.15983906127512454, + "USPTO_Backgrounds|OOD_News_2024|s0|p2": 0.16681697480380536, + "USPTO_Backgrounds|OOD_News_2024|s1|p0": 0.15753016993403435, + "USPTO_Backgrounds|OOD_News_2024|s1|p1": 0.15698093734681606, + "USPTO_Backgrounds|OOD_News_2024|s1|p2": 0.16459277532994748, + "USPTO_Backgrounds|OOD_News_2024|s2|p0": 0.16166811920702456, + "USPTO_Backgrounds|OOD_News_2024|s2|p1": 0.16125984862446785, + "USPTO_Backgrounds|OOD_News_2024|s2|p2": 0.16720112152397631, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p0": 0.16320642046630382, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p1": 0.15523914210498332, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p2": 0.155629500746727, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p0": 0.15997118018567563, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p1": 0.15534372441470623, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p2": 0.15296751111745835, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p0": 0.1637862890958786, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p1": 0.15691512636840343, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p2": 0.1559907216578722, + "USPTO_Backgrounds|OOD_CommonPile|s0|p0": 0.14672414623200894, + "USPTO_Backgrounds|OOD_CommonPile|s0|p1": 0.14162651263177395, + "USPTO_Backgrounds|OOD_CommonPile|s0|p2": 0.1442489769309759, + "USPTO_Backgrounds|OOD_CommonPile|s1|p0": 0.1448067259043455, + "USPTO_Backgrounds|OOD_CommonPile|s1|p1": 0.14138467870652677, + "USPTO_Backgrounds|OOD_CommonPile|s1|p2": 0.1413637984544039, + "USPTO_Backgrounds|OOD_CommonPile|s2|p0": 0.1485120017081499, + "USPTO_Backgrounds|OOD_CommonPile|s2|p1": 0.14268824607133865, + "USPTO_Backgrounds|OOD_CommonPile|s2|p2": 0.14339683316648005, + "Pile-CC|Wikipedia_en|s0|p0": 0.1437114614993334, + "Pile-CC|Wikipedia_en|s0|p1": 0.14558811001479627, + "Pile-CC|Wikipedia_en|s0|p2": 0.14785966239869594, + "Pile-CC|Wikipedia_en|s1|p0": 0.14410060048103332, + "Pile-CC|Wikipedia_en|s1|p1": 0.14739834032952787, + "Pile-CC|Wikipedia_en|s1|p2": 0.14800133295357226, + "Pile-CC|Wikipedia_en|s2|p0": 0.1427424516528845, + "Pile-CC|Wikipedia_en|s2|p1": 0.14626773111522198, + "Pile-CC|Wikipedia_en|s2|p2": 0.14542454518377781, + "Pile-CC|USPTO_Backgrounds|s0|p0": 0.15401250682771206, + "Pile-CC|USPTO_Backgrounds|s0|p1": 0.15758568942546844, + "Pile-CC|USPTO_Backgrounds|s0|p2": 0.1496773984283209, + "Pile-CC|USPTO_Backgrounds|s1|p0": 0.15556987784802914, + "Pile-CC|USPTO_Backgrounds|s1|p1": 0.15879758335649968, + "Pile-CC|USPTO_Backgrounds|s1|p2": 0.1502711545675993, + "Pile-CC|USPTO_Backgrounds|s2|p0": 0.15251148380339147, + "Pile-CC|USPTO_Backgrounds|s2|p1": 0.1557144198566675, + "Pile-CC|USPTO_Backgrounds|s2|p2": 0.14967280812561512, + "Pile-CC|Pile-CC|s0|p0": 0.1475800473242998, + "Pile-CC|Pile-CC|s0|p1": 0.14771839492022992, + "Pile-CC|Pile-CC|s0|p2": 0.14847222156822681, + "Pile-CC|Pile-CC|s1|p0": 0.14855674095451832, + "Pile-CC|Pile-CC|s1|p1": 0.15146477073431014, + "Pile-CC|Pile-CC|s1|p2": 0.14917959608137607, + "Pile-CC|Pile-CC|s2|p0": 0.1471522107720375, + "Pile-CC|Pile-CC|s2|p1": 0.1469940420240164, + "Pile-CC|Pile-CC|s2|p2": 0.14881067536771297, + "Pile-CC|StackExchange|s0|p0": 0.13556620739400388, + "Pile-CC|StackExchange|s0|p1": 0.13278575874865056, + "Pile-CC|StackExchange|s0|p2": 0.13652288839221, + "Pile-CC|StackExchange|s1|p0": 0.1346248684450984, + "Pile-CC|StackExchange|s1|p1": 0.13341133035719394, + "Pile-CC|StackExchange|s1|p2": 0.1373356692492962, + "Pile-CC|StackExchange|s2|p0": 0.13374655060470103, + "Pile-CC|StackExchange|s2|p1": 0.13112555183470248, + "Pile-CC|StackExchange|s2|p2": 0.13448646664619446, + "Pile-CC|Github|s0|p0": 0.1305080521851778, + "Pile-CC|Github|s0|p1": 0.13364531621336936, + "Pile-CC|Github|s0|p2": 0.13335430733859538, + "Pile-CC|Github|s1|p0": 0.13293832615017892, + "Pile-CC|Github|s1|p1": 0.13471500761806965, + "Pile-CC|Github|s1|p2": 0.1331347491592169, + "Pile-CC|Github|s2|p0": 0.12963986694812774, + "Pile-CC|Github|s2|p1": 0.13312407173216342, + "Pile-CC|Github|s2|p2": 0.13198942728340626, + "Pile-CC|OOD_News_2024|s0|p0": 0.14884910471737384, + "Pile-CC|OOD_News_2024|s0|p1": 0.15050788782536983, + "Pile-CC|OOD_News_2024|s0|p2": 0.15228801667690278, + "Pile-CC|OOD_News_2024|s1|p0": 0.15080181807279586, + "Pile-CC|OOD_News_2024|s1|p1": 0.1518170926719904, + "Pile-CC|OOD_News_2024|s1|p2": 0.15367178209125995, + "Pile-CC|OOD_News_2024|s2|p0": 0.1478659722954035, + "Pile-CC|OOD_News_2024|s2|p1": 0.14945964254438876, + "Pile-CC|OOD_News_2024|s2|p2": 0.1500181421637535, + "Pile-CC|OOD_arXiv_2023|s0|p0": 0.14775749184191228, + "Pile-CC|OOD_arXiv_2023|s0|p1": 0.14260655641555786, + "Pile-CC|OOD_arXiv_2023|s0|p2": 0.14545965380966663, + "Pile-CC|OOD_arXiv_2023|s1|p0": 0.14929868541657926, + "Pile-CC|OOD_arXiv_2023|s1|p1": 0.14477681517601013, + "Pile-CC|OOD_arXiv_2023|s1|p2": 0.14658121354877948, + "Pile-CC|OOD_arXiv_2023|s2|p0": 0.1471331473439932, + "Pile-CC|OOD_arXiv_2023|s2|p1": 0.14129521697759628, + "Pile-CC|OOD_arXiv_2023|s2|p2": 0.1434501815587282, + "Pile-CC|OOD_CommonPile|s0|p0": 0.1442362952977419, + "Pile-CC|OOD_CommonPile|s0|p1": 0.13856977000832557, + "Pile-CC|OOD_CommonPile|s0|p2": 0.14069531448185443, + "Pile-CC|OOD_CommonPile|s1|p0": 0.1466800395399332, + "Pile-CC|OOD_CommonPile|s1|p1": 0.14128039814531804, + "Pile-CC|OOD_CommonPile|s1|p2": 0.14194846339523792, + "Pile-CC|OOD_CommonPile|s2|p0": 0.14276791028678418, + "Pile-CC|OOD_CommonPile|s2|p1": 0.13801012188196182, + "Pile-CC|OOD_CommonPile|s2|p2": 0.13852601423859595, + "StackExchange|Wikipedia_en|s0|p0": 0.14449691511690615, + "StackExchange|Wikipedia_en|s0|p1": 0.1468203667551279, + "StackExchange|Wikipedia_en|s0|p2": 0.1488727930933237, + "StackExchange|Wikipedia_en|s1|p0": 0.1423150572925806, + "StackExchange|Wikipedia_en|s1|p1": 0.14355442933738233, + "StackExchange|Wikipedia_en|s1|p2": 0.14513694643974304, + "StackExchange|Wikipedia_en|s2|p0": 0.14236885644495487, + "StackExchange|Wikipedia_en|s2|p1": 0.1439858816564083, + "StackExchange|Wikipedia_en|s2|p2": 0.14457972906529903, + "StackExchange|USPTO_Backgrounds|s0|p0": 0.15522875748574733, + "StackExchange|USPTO_Backgrounds|s0|p1": 0.16507935300469398, + "StackExchange|USPTO_Backgrounds|s0|p2": 0.15315595604479312, + "StackExchange|USPTO_Backgrounds|s1|p0": 0.15408372245728968, + "StackExchange|USPTO_Backgrounds|s1|p1": 0.16125575564801692, + "StackExchange|USPTO_Backgrounds|s1|p2": 0.1479051813483238, + "StackExchange|USPTO_Backgrounds|s2|p0": 0.15220016706734896, + "StackExchange|USPTO_Backgrounds|s2|p1": 0.1606854759156704, + "StackExchange|USPTO_Backgrounds|s2|p2": 0.14806856364011764, + "StackExchange|Pile-CC|s0|p0": 0.15160226207226515, + "StackExchange|Pile-CC|s0|p1": 0.14857019782066344, + "StackExchange|Pile-CC|s0|p2": 0.15357961356639863, + "StackExchange|Pile-CC|s1|p0": 0.1496371330693364, + "StackExchange|Pile-CC|s1|p1": 0.14715286158025265, + "StackExchange|Pile-CC|s1|p2": 0.15260775089263917, + "StackExchange|Pile-CC|s2|p0": 0.14967973362654446, + "StackExchange|Pile-CC|s2|p1": 0.14599973596632482, + "StackExchange|Pile-CC|s2|p2": 0.15104596503078938, + "StackExchange|StackExchange|s0|p0": 0.13835120368748904, + "StackExchange|StackExchange|s0|p1": 0.13688320778310298, + "StackExchange|StackExchange|s0|p2": 0.1421434234827757, + "StackExchange|StackExchange|s1|p0": 0.1381283663213253, + "StackExchange|StackExchange|s1|p1": 0.1335654504597187, + "StackExchange|StackExchange|s1|p2": 0.13818975389003754, + "StackExchange|StackExchange|s2|p0": 0.13719860482960938, + "StackExchange|StackExchange|s2|p1": 0.13495246432721614, + "StackExchange|StackExchange|s2|p2": 0.14055598713457584, + "StackExchange|Github|s0|p0": 0.13823740370571613, + "StackExchange|Github|s0|p1": 0.13817918933928014, + "StackExchange|Github|s0|p2": 0.13669187240302563, + "StackExchange|Github|s1|p0": 0.13611015789210795, + "StackExchange|Github|s1|p1": 0.13541071433573962, + "StackExchange|Github|s1|p2": 0.1327636856585741, + "StackExchange|Github|s2|p0": 0.13709317557513714, + "StackExchange|Github|s2|p1": 0.13629858810454606, + "StackExchange|Github|s2|p2": 0.13514489568769933, + "StackExchange|OOD_News_2024|s0|p0": 0.14986481182277203, + "StackExchange|OOD_News_2024|s0|p1": 0.14892755225300788, + "StackExchange|OOD_News_2024|s0|p2": 0.15148025266826154, + "StackExchange|OOD_News_2024|s1|p0": 0.14734542705118656, + "StackExchange|OOD_News_2024|s1|p1": 0.1474456511437893, + "StackExchange|OOD_News_2024|s1|p2": 0.14920851327478885, + "StackExchange|OOD_News_2024|s2|p0": 0.1463893663138151, + "StackExchange|OOD_News_2024|s2|p1": 0.14626411013305188, + "StackExchange|OOD_News_2024|s2|p2": 0.1473880462348461, + "StackExchange|OOD_arXiv_2023|s0|p0": 0.15725914128124713, + "StackExchange|OOD_arXiv_2023|s0|p1": 0.15396856926381589, + "StackExchange|OOD_arXiv_2023|s0|p2": 0.15177230201661587, + "StackExchange|OOD_arXiv_2023|s1|p0": 0.15321973338723183, + "StackExchange|OOD_arXiv_2023|s1|p1": 0.14806342273950576, + "StackExchange|OOD_arXiv_2023|s1|p2": 0.1484540406614542, + "StackExchange|OOD_arXiv_2023|s2|p0": 0.15185540691018104, + "StackExchange|OOD_arXiv_2023|s2|p1": 0.15159291997551919, + "StackExchange|OOD_arXiv_2023|s2|p2": 0.14798282347619535, + "StackExchange|OOD_CommonPile|s0|p0": 0.15002845413982868, + "StackExchange|OOD_CommonPile|s0|p1": 0.14691657945513725, + "StackExchange|OOD_CommonPile|s0|p2": 0.14598479345440865, + "StackExchange|OOD_CommonPile|s1|p0": 0.14705484099686145, + "StackExchange|OOD_CommonPile|s1|p1": 0.14468607157468796, + "StackExchange|OOD_CommonPile|s1|p2": 0.14296981319785118, + "StackExchange|OOD_CommonPile|s2|p0": 0.14789607264101506, + "StackExchange|OOD_CommonPile|s2|p1": 0.14392626881599427, + "StackExchange|OOD_CommonPile|s2|p2": 0.143027351424098, + "Github|Wikipedia_en|s0|p0": 0.10472181923687458, + "Github|Wikipedia_en|s0|p1": 0.11021871082484722, + "Github|Wikipedia_en|s0|p2": 0.10458091050386428, + "Github|Wikipedia_en|s1|p0": 0.10628068111836911, + "Github|Wikipedia_en|s1|p1": 0.11017489619553089, + "Github|Wikipedia_en|s1|p2": 0.10822749547660351, + "Github|Wikipedia_en|s2|p0": 0.10841156356036663, + "Github|Wikipedia_en|s2|p1": 0.11178305633366108, + "Github|Wikipedia_en|s2|p2": 0.10895942114293575, + "Github|USPTO_Backgrounds|s0|p0": 0.12151174955070018, + "Github|USPTO_Backgrounds|s0|p1": 0.11832044459879398, + "Github|USPTO_Backgrounds|s0|p2": 0.11612880863249302, + "Github|USPTO_Backgrounds|s1|p0": 0.12393109649419784, + "Github|USPTO_Backgrounds|s1|p1": 0.11937098428606988, + "Github|USPTO_Backgrounds|s1|p2": 0.11847091130912304, + "Github|USPTO_Backgrounds|s2|p0": 0.12509081289172172, + "Github|USPTO_Backgrounds|s2|p1": 0.11961253806948662, + "Github|USPTO_Backgrounds|s2|p2": 0.11760500110685826, + "Github|Pile-CC|s0|p0": 0.11522790454328061, + "Github|Pile-CC|s0|p1": 0.11571162529289722, + "Github|Pile-CC|s0|p2": 0.11930152587592602, + "Github|Pile-CC|s1|p0": 0.1158499527722597, + "Github|Pile-CC|s1|p1": 0.11869329996407033, + "Github|Pile-CC|s1|p2": 0.11982139758765697, + "Github|Pile-CC|s2|p0": 0.11712507717311382, + "Github|Pile-CC|s2|p1": 0.11924890838563443, + "Github|Pile-CC|s2|p2": 0.12071709111332893, + "Github|StackExchange|s0|p0": 0.10925496369600296, + "Github|StackExchange|s0|p1": 0.10768789723515511, + "Github|StackExchange|s0|p2": 0.10849158652126789, + "Github|StackExchange|s1|p0": 0.1103823397308588, + "Github|StackExchange|s1|p1": 0.10917473770678043, + "Github|StackExchange|s1|p2": 0.1093454796820879, + "Github|StackExchange|s2|p0": 0.11207031235098838, + "Github|StackExchange|s2|p1": 0.1119774304330349, + "Github|StackExchange|s2|p2": 0.10991804748773575, + "Github|Github|s0|p0": 0.10624762140214443, + "Github|Github|s0|p1": 0.1059819731861353, + "Github|Github|s0|p2": 0.1047787208110094, + "Github|Github|s1|p0": 0.10747543163597584, + "Github|Github|s1|p1": 0.10910154320299625, + "Github|Github|s1|p2": 0.10684436745941639, + "Github|Github|s2|p0": 0.10742016844451427, + "Github|Github|s2|p1": 0.10812124125659466, + "Github|Github|s2|p2": 0.1071061585098505, + "Github|OOD_News_2024|s0|p0": 0.11219368688762188, + "Github|OOD_News_2024|s0|p1": 0.11500626541674137, + "Github|OOD_News_2024|s0|p2": 0.11685916595160961, + "Github|OOD_News_2024|s1|p0": 0.11386775523424149, + "Github|OOD_News_2024|s1|p1": 0.11642411835491658, + "Github|OOD_News_2024|s1|p2": 0.11719844676554203, + "Github|OOD_News_2024|s2|p0": 0.11440489366650582, + "Github|OOD_News_2024|s2|p1": 0.11527235694229603, + "Github|OOD_News_2024|s2|p2": 0.1176420260220766, + "Github|OOD_arXiv_2023|s0|p0": 0.1124246172606945, + "Github|OOD_arXiv_2023|s0|p1": 0.10736321471631527, + "Github|OOD_arXiv_2023|s0|p2": 0.11215240247547627, + "Github|OOD_arXiv_2023|s1|p0": 0.11539147906005383, + "Github|OOD_arXiv_2023|s1|p1": 0.11026903651654721, + "Github|OOD_arXiv_2023|s1|p2": 0.11371447890996933, + "Github|OOD_arXiv_2023|s2|p0": 0.11422904171049594, + "Github|OOD_arXiv_2023|s2|p1": 0.10990415550768376, + "Github|OOD_arXiv_2023|s2|p2": 0.11350584365427493, + "Github|OOD_CommonPile|s0|p0": 0.11138113737106323, + "Github|OOD_CommonPile|s0|p1": 0.11169484369456768, + "Github|OOD_CommonPile|s0|p2": 0.11102768294513225, + "Github|OOD_CommonPile|s1|p0": 0.1136026319116354, + "Github|OOD_CommonPile|s1|p1": 0.11297967918217182, + "Github|OOD_CommonPile|s1|p2": 0.11418475843966007, + "Github|OOD_CommonPile|s2|p0": 0.1131815429776907, + "Github|OOD_CommonPile|s2|p1": 0.11611624620854855, + "Github|OOD_CommonPile|s2|p2": 0.11451728083193302, + "OOD_News_2024|Wikipedia_en|s0|p0": 0.13486163765192033, + "OOD_News_2024|Wikipedia_en|s0|p1": 0.1364224761724472, + "OOD_News_2024|Wikipedia_en|s0|p2": 0.13764243498444556, + "OOD_News_2024|Wikipedia_en|s1|p0": 0.1336891043931246, + "OOD_News_2024|Wikipedia_en|s1|p1": 0.13594831079244613, + "OOD_News_2024|Wikipedia_en|s1|p2": 0.13840426802635192, + "OOD_News_2024|Wikipedia_en|s2|p0": 0.13516195863485336, + "OOD_News_2024|Wikipedia_en|s2|p1": 0.136974174156785, + "OOD_News_2024|Wikipedia_en|s2|p2": 0.13865821361541747, + "OOD_News_2024|USPTO_Backgrounds|s0|p0": 0.14506222382187844, + "OOD_News_2024|USPTO_Backgrounds|s0|p1": 0.14526550993323326, + "OOD_News_2024|USPTO_Backgrounds|s0|p2": 0.13911073319613934, + "OOD_News_2024|USPTO_Backgrounds|s1|p0": 0.146680273860693, + "OOD_News_2024|USPTO_Backgrounds|s1|p1": 0.14634087346494198, + "OOD_News_2024|USPTO_Backgrounds|s1|p2": 0.13987445943057536, + "OOD_News_2024|USPTO_Backgrounds|s2|p0": 0.14899541437625885, + "OOD_News_2024|USPTO_Backgrounds|s2|p1": 0.14674318358302116, + "OOD_News_2024|USPTO_Backgrounds|s2|p2": 0.1395602460950613, + "OOD_News_2024|Pile-CC|s0|p0": 0.14070868007838727, + "OOD_News_2024|Pile-CC|s0|p1": 0.14009961038827895, + "OOD_News_2024|Pile-CC|s0|p2": 0.1449163757264614, + "OOD_News_2024|Pile-CC|s1|p0": 0.14202719666063784, + "OOD_News_2024|Pile-CC|s1|p1": 0.14143850281834602, + "OOD_News_2024|Pile-CC|s1|p2": 0.14566070213913918, + "OOD_News_2024|Pile-CC|s2|p0": 0.14384278729557992, + "OOD_News_2024|Pile-CC|s2|p1": 0.14112119302153586, + "OOD_News_2024|Pile-CC|s2|p2": 0.14572295621037484, + "OOD_News_2024|StackExchange|s0|p0": 0.12112892381846904, + "OOD_News_2024|StackExchange|s0|p1": 0.11762128211557865, + "OOD_News_2024|StackExchange|s0|p2": 0.12067867033183574, + "OOD_News_2024|StackExchange|s1|p0": 0.12376918159425258, + "OOD_News_2024|StackExchange|s1|p1": 0.11939822919666768, + "OOD_News_2024|StackExchange|s1|p2": 0.12282479219138623, + "OOD_News_2024|StackExchange|s2|p0": 0.12256133146584033, + "OOD_News_2024|StackExchange|s2|p1": 0.11773126386106014, + "OOD_News_2024|StackExchange|s2|p2": 0.12148497365415097, + "OOD_News_2024|Github|s0|p0": 0.12191336899995804, + "OOD_News_2024|Github|s0|p1": 0.12709311842918397, + "OOD_News_2024|Github|s0|p2": 0.12368766739964485, + "OOD_News_2024|Github|s1|p0": 0.12076359763741493, + "OOD_News_2024|Github|s1|p1": 0.1275537062436342, + "OOD_News_2024|Github|s1|p2": 0.12567985206842422, + "OOD_News_2024|Github|s2|p0": 0.12257032841444016, + "OOD_News_2024|Github|s2|p1": 0.12789174802601339, + "OOD_News_2024|Github|s2|p2": 0.1258739374577999, + "OOD_News_2024|OOD_News_2024|s0|p0": 0.1387073550373316, + "OOD_News_2024|OOD_News_2024|s0|p1": 0.1434318669140339, + "OOD_News_2024|OOD_News_2024|s0|p2": 0.14456919133663176, + "OOD_News_2024|OOD_News_2024|s1|p0": 0.14054211862385274, + "OOD_News_2024|OOD_News_2024|s1|p1": 0.1423490047454834, + "OOD_News_2024|OOD_News_2024|s1|p2": 0.14308991767466067, + "OOD_News_2024|OOD_News_2024|s2|p0": 0.14106240570545198, + "OOD_News_2024|OOD_News_2024|s2|p1": 0.14487843662500383, + "OOD_News_2024|OOD_News_2024|s2|p2": 0.14517133496701717, + "OOD_News_2024|OOD_arXiv_2023|s0|p0": 0.1414489448070526, + "OOD_News_2024|OOD_arXiv_2023|s0|p1": 0.13550705835223198, + "OOD_News_2024|OOD_arXiv_2023|s0|p2": 0.13826472461223602, + "OOD_News_2024|OOD_arXiv_2023|s1|p0": 0.14253144674003124, + "OOD_News_2024|OOD_arXiv_2023|s1|p1": 0.1375824362039566, + "OOD_News_2024|OOD_arXiv_2023|s1|p2": 0.13953054323792458, + "OOD_News_2024|OOD_arXiv_2023|s2|p0": 0.14292933754622936, + "OOD_News_2024|OOD_arXiv_2023|s2|p1": 0.13888718336820602, + "OOD_News_2024|OOD_arXiv_2023|s2|p2": 0.13899042904376985, + "OOD_News_2024|OOD_CommonPile|s0|p0": 0.13256734609603882, + "OOD_News_2024|OOD_CommonPile|s0|p1": 0.1276464655995369, + "OOD_News_2024|OOD_CommonPile|s0|p2": 0.12678277269005775, + "OOD_News_2024|OOD_CommonPile|s1|p0": 0.1356629304587841, + "OOD_News_2024|OOD_CommonPile|s1|p1": 0.12904721051454543, + "OOD_News_2024|OOD_CommonPile|s1|p2": 0.12944178767502307, + "OOD_News_2024|OOD_CommonPile|s2|p0": 0.13436706215143204, + "OOD_News_2024|OOD_CommonPile|s2|p1": 0.12855805978178977, + "OOD_News_2024|OOD_CommonPile|s2|p2": 0.12856159768998623, + "OOD_arXiv_2023|Wikipedia_en|s0|p0": 0.1428018994629383, + "OOD_arXiv_2023|Wikipedia_en|s0|p1": 0.14040045514702798, + "OOD_arXiv_2023|Wikipedia_en|s0|p2": 0.1454609576612711, + "OOD_arXiv_2023|Wikipedia_en|s1|p0": 0.14358339309692383, + "OOD_arXiv_2023|Wikipedia_en|s1|p1": 0.14175953082740306, + "OOD_arXiv_2023|Wikipedia_en|s1|p2": 0.14634908474981784, + "OOD_arXiv_2023|Wikipedia_en|s2|p0": 0.14248228520154954, + "OOD_arXiv_2023|Wikipedia_en|s2|p1": 0.14210982508957387, + "OOD_arXiv_2023|Wikipedia_en|s2|p2": 0.14514244236052037, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p0": 0.15357542578130962, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p1": 0.1510006608441472, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p2": 0.14225321281701325, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p0": 0.15326165203005077, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p1": 0.15135192070156336, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p2": 0.14344881046563387, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p0": 0.15479327142238616, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p1": 0.15056663602590561, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p2": 0.14360985029488801, + "OOD_arXiv_2023|Pile-CC|s0|p0": 0.14826207980513573, + "OOD_arXiv_2023|Pile-CC|s0|p1": 0.14777991697192192, + "OOD_arXiv_2023|Pile-CC|s0|p2": 0.14704287927597762, + "OOD_arXiv_2023|Pile-CC|s1|p0": 0.14807742908596994, + "OOD_arXiv_2023|Pile-CC|s1|p1": 0.14693760648369789, + "OOD_arXiv_2023|Pile-CC|s1|p2": 0.14893653616309166, + "OOD_arXiv_2023|Pile-CC|s2|p0": 0.14762210324406624, + "OOD_arXiv_2023|Pile-CC|s2|p1": 0.14713412448763846, + "OOD_arXiv_2023|Pile-CC|s2|p2": 0.14867167100310325, + "OOD_arXiv_2023|StackExchange|s0|p0": 0.1350220860913396, + "OOD_arXiv_2023|StackExchange|s0|p1": 0.1340720908716321, + "OOD_arXiv_2023|StackExchange|s0|p2": 0.1402300363406539, + "OOD_arXiv_2023|StackExchange|s1|p0": 0.13420782294124364, + "OOD_arXiv_2023|StackExchange|s1|p1": 0.13373161274939777, + "OOD_arXiv_2023|StackExchange|s1|p2": 0.1396141702309251, + "OOD_arXiv_2023|StackExchange|s2|p0": 0.13533333148807286, + "OOD_arXiv_2023|StackExchange|s2|p1": 0.1333851983770728, + "OOD_arXiv_2023|StackExchange|s2|p2": 0.14080181028693914, + "OOD_arXiv_2023|Github|s0|p0": 0.12196633070707322, + "OOD_arXiv_2023|Github|s0|p1": 0.1307462839409709, + "OOD_arXiv_2023|Github|s0|p2": 0.129108851775527, + "OOD_arXiv_2023|Github|s1|p0": 0.12099339216947555, + "OOD_arXiv_2023|Github|s1|p1": 0.12947062086313962, + "OOD_arXiv_2023|Github|s1|p2": 0.12685625925660132, + "OOD_arXiv_2023|Github|s2|p0": 0.12173797562718391, + "OOD_arXiv_2023|Github|s2|p1": 0.13024121206253766, + "OOD_arXiv_2023|Github|s2|p2": 0.1303920727223158, + "OOD_arXiv_2023|OOD_News_2024|s0|p0": 0.1436644211411476, + "OOD_arXiv_2023|OOD_News_2024|s0|p1": 0.14571815356612206, + "OOD_arXiv_2023|OOD_News_2024|s0|p2": 0.1497436098754406, + "OOD_arXiv_2023|OOD_News_2024|s1|p0": 0.14410990849137306, + "OOD_arXiv_2023|OOD_News_2024|s1|p1": 0.1442332945764065, + "OOD_arXiv_2023|OOD_News_2024|s1|p2": 0.1497814826667309, + "OOD_arXiv_2023|OOD_News_2024|s2|p0": 0.14349039569497107, + "OOD_arXiv_2023|OOD_News_2024|s2|p1": 0.14413045570254326, + "OOD_arXiv_2023|OOD_News_2024|s2|p2": 0.15093232430517672, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p0": 0.14870578423142433, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p1": 0.14305446892976761, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p2": 0.14268340300768614, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p0": 0.14820004478096963, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p1": 0.1421566367149353, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p2": 0.1423545114696026, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p0": 0.14980459287762643, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p1": 0.14230584353208542, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p2": 0.14217254798859358, + "OOD_arXiv_2023|OOD_CommonPile|s0|p0": 0.12943503260612488, + "OOD_arXiv_2023|OOD_CommonPile|s0|p1": 0.13154563196003438, + "OOD_arXiv_2023|OOD_CommonPile|s0|p2": 0.12972614988684655, + "OOD_arXiv_2023|OOD_CommonPile|s1|p0": 0.12850606199353934, + "OOD_arXiv_2023|OOD_CommonPile|s1|p1": 0.130877897515893, + "OOD_arXiv_2023|OOD_CommonPile|s1|p2": 0.12842097394168378, + "OOD_arXiv_2023|OOD_CommonPile|s2|p0": 0.12821140550076962, + "OOD_arXiv_2023|OOD_CommonPile|s2|p1": 0.13017832338809968, + "OOD_arXiv_2023|OOD_CommonPile|s2|p2": 0.12638943903148175, + "OOD_CommonPile|Wikipedia_en|s0|p0": 0.139744308963418, + "OOD_CommonPile|Wikipedia_en|s0|p1": 0.14106009677052497, + "OOD_CommonPile|Wikipedia_en|s0|p2": 0.13940333388745785, + "OOD_CommonPile|Wikipedia_en|s1|p0": 0.14246975108981133, + "OOD_CommonPile|Wikipedia_en|s1|p1": 0.1426856704056263, + "OOD_CommonPile|Wikipedia_en|s1|p2": 0.1423924509435892, + "OOD_CommonPile|Wikipedia_en|s2|p0": 0.13969158455729486, + "OOD_CommonPile|Wikipedia_en|s2|p1": 0.14126862064003945, + "OOD_CommonPile|Wikipedia_en|s2|p2": 0.13894762732088567, + "OOD_CommonPile|USPTO_Backgrounds|s0|p0": 0.14983783718198537, + "OOD_CommonPile|USPTO_Backgrounds|s0|p1": 0.14970208685845138, + "OOD_CommonPile|USPTO_Backgrounds|s0|p2": 0.14276783652603625, + "OOD_CommonPile|USPTO_Backgrounds|s1|p0": 0.15254374835640191, + "OOD_CommonPile|USPTO_Backgrounds|s1|p1": 0.1527953764423728, + "OOD_CommonPile|USPTO_Backgrounds|s1|p2": 0.1465577518567443, + "OOD_CommonPile|USPTO_Backgrounds|s2|p0": 0.14851349797099828, + "OOD_CommonPile|USPTO_Backgrounds|s2|p1": 0.14697987604886292, + "OOD_CommonPile|USPTO_Backgrounds|s2|p2": 0.14291422087699174, + "OOD_CommonPile|Pile-CC|s0|p0": 0.14710925593972207, + "OOD_CommonPile|Pile-CC|s0|p1": 0.14605959840118884, + "OOD_CommonPile|Pile-CC|s0|p2": 0.1486803473904729, + "OOD_CommonPile|Pile-CC|s1|p0": 0.14828060362488032, + "OOD_CommonPile|Pile-CC|s1|p1": 0.14762085005640985, + "OOD_CommonPile|Pile-CC|s1|p2": 0.15118215847760438, + "OOD_CommonPile|Pile-CC|s2|p0": 0.146185109205544, + "OOD_CommonPile|Pile-CC|s2|p1": 0.14593285396695138, + "OOD_CommonPile|Pile-CC|s2|p2": 0.14749929625540972, + "OOD_CommonPile|StackExchange|s0|p0": 0.13769646547734737, + "OOD_CommonPile|StackExchange|s0|p1": 0.13425580598413944, + "OOD_CommonPile|StackExchange|s0|p2": 0.13896569330245256, + "OOD_CommonPile|StackExchange|s1|p0": 0.13924720752984285, + "OOD_CommonPile|StackExchange|s1|p1": 0.1362179521471262, + "OOD_CommonPile|StackExchange|s1|p2": 0.1419050382450223, + "OOD_CommonPile|StackExchange|s2|p0": 0.1358552198857069, + "OOD_CommonPile|StackExchange|s2|p1": 0.13534963615238665, + "OOD_CommonPile|StackExchange|s2|p2": 0.1377167023718357, + "OOD_CommonPile|Github|s0|p0": 0.12233414202928543, + "OOD_CommonPile|Github|s0|p1": 0.13115269783884287, + "OOD_CommonPile|Github|s0|p2": 0.12898012734949588, + "OOD_CommonPile|Github|s1|p0": 0.12387640923261642, + "OOD_CommonPile|Github|s1|p1": 0.1318666733801365, + "OOD_CommonPile|Github|s1|p2": 0.13118611834943295, + "OOD_CommonPile|Github|s2|p0": 0.12081060074269771, + "OOD_CommonPile|Github|s2|p1": 0.13013178948312998, + "OOD_CommonPile|Github|s2|p2": 0.12764853835105897, + "OOD_CommonPile|OOD_News_2024|s0|p0": 0.14499912038445473, + "OOD_CommonPile|OOD_News_2024|s0|p1": 0.14552821181714534, + "OOD_CommonPile|OOD_News_2024|s0|p2": 0.15087285302579403, + "OOD_CommonPile|OOD_News_2024|s1|p0": 0.1453522689640522, + "OOD_CommonPile|OOD_News_2024|s1|p1": 0.1469458170235157, + "OOD_CommonPile|OOD_News_2024|s1|p2": 0.15380928292870522, + "OOD_CommonPile|OOD_News_2024|s2|p0": 0.14182413201779126, + "OOD_CommonPile|OOD_News_2024|s2|p1": 0.14482717141509055, + "OOD_CommonPile|OOD_News_2024|s2|p2": 0.14878604114055632, + "OOD_CommonPile|OOD_arXiv_2023|s0|p0": 0.14463903680443763, + "OOD_CommonPile|OOD_arXiv_2023|s0|p1": 0.13837069198489188, + "OOD_CommonPile|OOD_arXiv_2023|s0|p2": 0.13659614697098732, + "OOD_CommonPile|OOD_arXiv_2023|s1|p0": 0.1479431388899684, + "OOD_CommonPile|OOD_arXiv_2023|s1|p1": 0.139229029789567, + "OOD_CommonPile|OOD_arXiv_2023|s1|p2": 0.13885941188782452, + "OOD_CommonPile|OOD_arXiv_2023|s2|p0": 0.1438038794323802, + "OOD_CommonPile|OOD_arXiv_2023|s2|p1": 0.13630836196243762, + "OOD_CommonPile|OOD_arXiv_2023|s2|p2": 0.13602422624826432, + "OOD_CommonPile|OOD_CommonPile|s0|p0": 0.12909803725779057, + "OOD_CommonPile|OOD_CommonPile|s0|p1": 0.1318016305565834, + "OOD_CommonPile|OOD_CommonPile|s0|p2": 0.12763514034450055, + "OOD_CommonPile|OOD_CommonPile|s1|p0": 0.13201464228332044, + "OOD_CommonPile|OOD_CommonPile|s1|p1": 0.1352139636874199, + "OOD_CommonPile|OOD_CommonPile|s1|p2": 0.12987643703818322, + "OOD_CommonPile|OOD_CommonPile|s2|p0": 0.1279674131423235, + "OOD_CommonPile|OOD_CommonPile|s2|p1": 0.13204800486564636, + "OOD_CommonPile|OOD_CommonPile|s2|p2": 0.1282028838992119, + "logit|Q0|p0": 0.1247396256774664, + "shuf|Q0|p0": 0.1422879509627819, + "logit|Q0|p1": 0.1247396256774664, + "shuf|Q0|p1": 0.1422879509627819, + "logit|Q0|p2": 0.1247396256774664, + "shuf|Q0|p2": 0.1422879509627819, + "logit|Wikipedia_en|p0": 0.12441348172724247, + "shuf|Wikipedia_en|p0": 0.13555061146616937, + "logit|Wikipedia_en|p1": 0.12603520192205905, + "shuf|Wikipedia_en|p1": 0.13675951808691025, + "logit|Wikipedia_en|p2": 0.12318613156676292, + "shuf|Wikipedia_en|p2": 0.13697959780693053, + "logit|USPTO_Backgrounds|p0": 0.12288859896361828, + "shuf|USPTO_Backgrounds|p0": 0.1432977557182312, + "logit|USPTO_Backgrounds|p1": 0.12467962317168713, + "shuf|USPTO_Backgrounds|p1": 0.14704293757677078, + "logit|USPTO_Backgrounds|p2": 0.12205531150102615, + "shuf|USPTO_Backgrounds|p2": 0.13966306820511817, + "logit|Pile-CC|p0": 0.1294180490076542, + "shuf|Pile-CC|p0": 0.14222755655646324, + "logit|Pile-CC|p1": 0.12919799163937568, + "shuf|Pile-CC|p1": 0.13949708864092827, + "logit|Pile-CC|p2": 0.12970054894685745, + "shuf|Pile-CC|p2": 0.13793092966079712, + "logit|StackExchange|p0": 0.12327122092247009, + "shuf|StackExchange|p0": 0.13950061425566673, + "logit|StackExchange|p1": 0.12136632464826107, + "shuf|StackExchange|p1": 0.13919788375496864, + "logit|StackExchange|p2": 0.12554268538951874, + "shuf|StackExchange|p2": 0.14503461569547654, + "logit|Github|p0": 0.12177085354924203, + "shuf|Github|p0": 0.12981337755918504, + "logit|Github|p1": 0.12349991276860237, + "shuf|Github|p1": 0.13674261644482613, + "logit|Github|p2": 0.11834828741848469, + "shuf|Github|p2": 0.13636190369725226, + "logit|OOD_News_2024|p0": 0.13004022762179374, + "shuf|OOD_News_2024|p0": 0.13637514486908914, + "logit|OOD_News_2024|p1": 0.13050479404628276, + "shuf|OOD_News_2024|p1": 0.13769909664988517, + "logit|OOD_News_2024|p2": 0.13118069656193257, + "shuf|OOD_News_2024|p2": 0.14654124304652213, + "logit|OOD_arXiv_2023|p0": 0.1273316565901041, + "shuf|OOD_arXiv_2023|p0": 0.13678908571600915, + "logit|OOD_arXiv_2023|p1": 0.11337111741304398, + "shuf|OOD_arXiv_2023|p1": 0.1261425122618675, + "logit|OOD_arXiv_2023|p2": 0.11835215277969838, + "shuf|OOD_arXiv_2023|p2": 0.13648709356784822, + "logit|OOD_CommonPile|p0": 0.12013521678745746, + "shuf|OOD_CommonPile|p0": 0.13690541982650756, + "logit|OOD_CommonPile|p1": 0.1152633611112833, + "shuf|OOD_CommonPile|p1": 0.13219560906291009, + "logit|OOD_CommonPile|p2": 0.12013627588748932, + "shuf|OOD_CommonPile|p2": 0.13602811992168426, + "logit|SHUFFLED_Pile-CC|p0": 0.08761658929288388, + "shuf|SHUFFLED_Pile-CC|p0": 0.10673588179051877, + "logit|SHUFFLED_Pile-CC|p1": 0.08926036544144153, + "shuf|SHUFFLED_Pile-CC|p1": 0.10876993574202061, + "logit|SHUFFLED_Pile-CC|p2": 0.0900193702429533, + "shuf|SHUFFLED_Pile-CC|p2": 0.10551640428602696 + } + }, + "control_C1_reproduces_stored_e52_matrix": { + "reference": "results/e52_factorial_410m.json", + "checked": true, + "n_cells_compared": 128, + "max_abs_diff": 0.0, + "worst_cell": null, + "tolerance": 1e-12, + "fires": true + }, + "provenance": { + "schema_version": 1, + "experiment": "E57", + "utc": "2026-08-17T01:31:15+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "38b436c06a057ed96391f6a9c386ceecf34d7a29a51780bd470e740aa39d923f", + "n_lines": 599 + }, + "git": { + "commit": "f8ef435b29365012c0492e023fb6b4dfece5f0f3", + "commit_short": "f8ef435b2936", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t53_ladder_summary.py", + "experiments/t55_matrix_robustness.py", + "experiments/t56_predictor_registry.py", + "experiments/t58_algebra_audit.py", + "experiments/t59_read_dose.py", + "experiments/t60_fitter_determinism.py", + "paper/paper.tex", + "results/e35_containment.json", + "results/e52_factorial_410m_rerun.json", + "results/e53_ladder_summary.json", + "results/e55_matrix_robustness.json", + "results/e56_predictor_registry.json", + "results/e58_algebra_audit.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/experiments/E57_grid_variance_ci.md", + "specs/experiments/E59_read_dose_response.md", + "tests/test_anchor_fidelity.py", + "xperiments/t52_factorial.py" + ], + "dirty_diff_sha256": "4a72c81a812589b5e476dff564e05d42f643a7dd4b402e3ce4f3d7b37238d070", + "describe": "f8ef435-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--out", + "results/e52_factorial_410m_rerun.json", + "--cells-out", + "results/e57_factorial_cells_410m.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e52_factorial_410m.json", + "sha256": "6e81f3cae37d0c1906f8912af4bf8e75db0a0f259b2869f209701b74b6682590", + "bytes": 41190, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "8d15b390fc341469cbc37ca5c9e715dbe6eece9e09a943858077ec06fee99c0c" + } +} \ No newline at end of file diff --git a/results/e57_factorial_cells_410m_pooled.json b/results/e57_factorial_cells_410m_pooled.json new file mode 100644 index 0000000000000000000000000000000000000000..26d0bf2bd8d0491b2ce2169e8bc5dc08a9ec1960 --- /dev/null +++ b/results/e57_factorial_cells_410m_pooled.json @@ -0,0 +1,1591 @@ +{ + "experiment": "E57 \u2014 the per-cell draws E52 averaged away", + "why": "the paper's headline is a variance decomposition of 64 cell MEANS with one observation each, so its three shares sum to 100% by construction, its 'interaction' is a residual conflated with cell noise, and it carries no interval. Each cell is a mean over 9 draws (3 fit seeds x 3 prefix seeds). This emits those draws so the shares can be bootstrapped.", + "unit": "admitted-set mean read AUC for one (fit corpus, fit seed, read rung, prefix seed)", + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_draws_per_matrix_cell": 9, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "model": "EleutherAI/pythia-410m-deduped", + "draws": { + "persist": { + "Wikipedia_en|Wikipedia_en|s0|p0": 0.037687454279512166, + "Wikipedia_en|Wikipedia_en|s0|p1": 0.04360535021405667, + "Wikipedia_en|Wikipedia_en|s0|p2": 0.04031957141123712, + "Wikipedia_en|Wikipedia_en|s1|p0": 0.0381647433154285, + "Wikipedia_en|Wikipedia_en|s1|p1": 0.04469228822272271, + "Wikipedia_en|Wikipedia_en|s1|p2": 0.040404757997021076, + "Wikipedia_en|Wikipedia_en|s2|p0": 0.0381647433154285, + "Wikipedia_en|Wikipedia_en|s2|p1": 0.04377746337559074, + "Wikipedia_en|Wikipedia_en|s2|p2": 0.039532129233703014, + "Wikipedia_en|USPTO_Backgrounds|s0|p0": 0.04156353797297925, + "Wikipedia_en|USPTO_Backgrounds|s0|p1": 0.04243574268184602, + "Wikipedia_en|USPTO_Backgrounds|s0|p2": 0.04019909638445825, + "Wikipedia_en|USPTO_Backgrounds|s1|p0": 0.04212089765351266, + "Wikipedia_en|USPTO_Backgrounds|s1|p1": 0.04274603589437902, + "Wikipedia_en|USPTO_Backgrounds|s1|p2": 0.04171427905093879, + "Wikipedia_en|USPTO_Backgrounds|s2|p0": 0.040916012856177984, + "Wikipedia_en|USPTO_Backgrounds|s2|p1": 0.04191302112303674, + "Wikipedia_en|USPTO_Backgrounds|s2|p2": 0.04011903021018952, + "Wikipedia_en|Pile-CC|s0|p0": 0.03965791100636125, + "Wikipedia_en|Pile-CC|s0|p1": 0.03914248633664101, + "Wikipedia_en|Pile-CC|s0|p2": 0.041064153192564845, + "Wikipedia_en|Pile-CC|s1|p0": 0.03938737632706761, + "Wikipedia_en|Pile-CC|s1|p1": 0.04021501408424229, + "Wikipedia_en|Pile-CC|s1|p2": 0.0426172103267163, + "Wikipedia_en|Pile-CC|s2|p0": 0.03885024646297097, + "Wikipedia_en|Pile-CC|s2|p1": 0.038989900681190195, + "Wikipedia_en|Pile-CC|s2|p2": 0.042323902761563656, + "Wikipedia_en|StackExchange|s0|p0": 0.034256494510918854, + "Wikipedia_en|StackExchange|s0|p1": 0.03419739182572812, + "Wikipedia_en|StackExchange|s0|p2": 0.033631662791594866, + "Wikipedia_en|StackExchange|s1|p0": 0.033363637048751116, + "Wikipedia_en|StackExchange|s1|p1": 0.03486514564137906, + "Wikipedia_en|StackExchange|s1|p2": 0.03427594755776227, + "Wikipedia_en|StackExchange|s2|p0": 0.03371354602277279, + "Wikipedia_en|StackExchange|s2|p1": 0.03354230809491128, + "Wikipedia_en|StackExchange|s2|p2": 0.03414359060116112, + "Wikipedia_en|Github|s0|p0": 0.03332046750001609, + "Wikipedia_en|Github|s0|p1": 0.03417315983679146, + "Wikipedia_en|Github|s0|p2": 0.035972073185257615, + "Wikipedia_en|Github|s1|p0": 0.034259540308266875, + "Wikipedia_en|Github|s1|p1": 0.03458548698108643, + "Wikipedia_en|Github|s1|p2": 0.035356608661822976, + "Wikipedia_en|Github|s2|p0": 0.03410695465281606, + "Wikipedia_en|Github|s2|p1": 0.03554478748701513, + "Wikipedia_en|Github|s2|p2": 0.03624946873169392, + "Wikipedia_en|OOD_News_2024|s0|p0": 0.03938806052319706, + "Wikipedia_en|OOD_News_2024|s0|p1": 0.04128914359025657, + "Wikipedia_en|OOD_News_2024|s0|p2": 0.044148194743320344, + "Wikipedia_en|OOD_News_2024|s1|p0": 0.03972031692974269, + "Wikipedia_en|OOD_News_2024|s1|p1": 0.04140385123901069, + "Wikipedia_en|OOD_News_2024|s1|p2": 0.04455296820960939, + "Wikipedia_en|OOD_News_2024|s2|p0": 0.03901360961608589, + "Wikipedia_en|OOD_News_2024|s2|p1": 0.03991144481115043, + "Wikipedia_en|OOD_News_2024|s2|p2": 0.04369798894040287, + "Wikipedia_en|OOD_arXiv_2023|s0|p0": 0.035181528958491984, + "Wikipedia_en|OOD_arXiv_2023|s0|p1": 0.03635661648586393, + "Wikipedia_en|OOD_arXiv_2023|s0|p2": 0.03698380454443395, + "Wikipedia_en|OOD_arXiv_2023|s1|p0": 0.03579117895569652, + "Wikipedia_en|OOD_arXiv_2023|s1|p1": 0.03706049108877778, + "Wikipedia_en|OOD_arXiv_2023|s1|p2": 0.03666853434406221, + "Wikipedia_en|OOD_arXiv_2023|s2|p0": 0.03529192309360951, + "Wikipedia_en|OOD_arXiv_2023|s2|p1": 0.036793197598308326, + "Wikipedia_en|OOD_arXiv_2023|s2|p2": 0.03618368967436254, + "Wikipedia_en|OOD_CommonPile|s0|p0": 0.03625752651132643, + "Wikipedia_en|OOD_CommonPile|s0|p1": 0.03712513237260282, + "Wikipedia_en|OOD_CommonPile|s0|p2": 0.038436542451381686, + "Wikipedia_en|OOD_CommonPile|s1|p0": 0.03811251767911017, + "Wikipedia_en|OOD_CommonPile|s1|p1": 0.03827449041418731, + "Wikipedia_en|OOD_CommonPile|s1|p2": 0.03863062364980578, + "Wikipedia_en|OOD_CommonPile|s2|p0": 0.03696747389622033, + "Wikipedia_en|OOD_CommonPile|s2|p1": 0.03822946446016431, + "Wikipedia_en|OOD_CommonPile|s2|p2": 0.03863386325538158, + "USPTO_Backgrounds|Wikipedia_en|s0|p0": 0.04754846366122365, + "USPTO_Backgrounds|Wikipedia_en|s0|p1": 0.05034518730826676, + "USPTO_Backgrounds|Wikipedia_en|s0|p2": 0.049007836589589716, + "USPTO_Backgrounds|Wikipedia_en|s1|p0": 0.04670790103264153, + "USPTO_Backgrounds|Wikipedia_en|s1|p1": 0.05076761790551245, + "USPTO_Backgrounds|Wikipedia_en|s1|p2": 0.048252454260364175, + "USPTO_Backgrounds|Wikipedia_en|s2|p0": 0.04834426683373749, + "USPTO_Backgrounds|Wikipedia_en|s2|p1": 0.05159551003016531, + "USPTO_Backgrounds|Wikipedia_en|s2|p2": 0.04926326381973922, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p0": 0.05048472099006176, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p1": 0.04920110672246665, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p2": 0.050384720670990646, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p0": 0.050406130286864935, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p1": 0.04874482150189578, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p2": 0.048373900610022245, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p0": 0.05211932349484414, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p1": 0.049685940123163165, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p2": 0.049829933303408326, + "USPTO_Backgrounds|Pile-CC|s0|p0": 0.049157181894406675, + "USPTO_Backgrounds|Pile-CC|s0|p1": 0.04813587157987058, + "USPTO_Backgrounds|Pile-CC|s0|p2": 0.0486748862080276, + "USPTO_Backgrounds|Pile-CC|s1|p0": 0.050147095182910564, + "USPTO_Backgrounds|Pile-CC|s1|p1": 0.04749519950710237, + "USPTO_Backgrounds|Pile-CC|s1|p2": 0.04827266102656722, + "USPTO_Backgrounds|Pile-CC|s2|p0": 0.05055835195817053, + "USPTO_Backgrounds|Pile-CC|s2|p1": 0.04861640161834657, + "USPTO_Backgrounds|Pile-CC|s2|p2": 0.04919329257681966, + "USPTO_Backgrounds|StackExchange|s0|p0": 0.04283932016696781, + "USPTO_Backgrounds|StackExchange|s0|p1": 0.04483984797261655, + "USPTO_Backgrounds|StackExchange|s0|p2": 0.042810790380463, + "USPTO_Backgrounds|StackExchange|s1|p0": 0.04335448981728405, + "USPTO_Backgrounds|StackExchange|s1|p1": 0.04378685294650495, + "USPTO_Backgrounds|StackExchange|s1|p2": 0.04201067774556577, + "USPTO_Backgrounds|StackExchange|s2|p0": 0.044850140972994265, + "USPTO_Backgrounds|StackExchange|s2|p1": 0.045725149055942896, + "USPTO_Backgrounds|StackExchange|s2|p2": 0.043293894408270715, + "USPTO_Backgrounds|Github|s0|p0": 0.04491386362351477, + "USPTO_Backgrounds|Github|s0|p1": 0.04478121572174132, + "USPTO_Backgrounds|Github|s0|p2": 0.047151662735268475, + "USPTO_Backgrounds|Github|s1|p0": 0.044198105670511724, + "USPTO_Backgrounds|Github|s1|p1": 0.044680919079110024, + "USPTO_Backgrounds|Github|s1|p2": 0.04681359394453466, + "USPTO_Backgrounds|Github|s2|p0": 0.04526377264410257, + "USPTO_Backgrounds|Github|s2|p1": 0.04616901460103691, + "USPTO_Backgrounds|Github|s2|p2": 0.0476668321993202, + "USPTO_Backgrounds|OOD_News_2024|s0|p0": 0.04859429798088968, + "USPTO_Backgrounds|OOD_News_2024|s0|p1": 0.04858191790990531, + "USPTO_Backgrounds|OOD_News_2024|s0|p2": 0.04977939403615892, + "USPTO_Backgrounds|OOD_News_2024|s1|p0": 0.04850155687890947, + "USPTO_Backgrounds|OOD_News_2024|s1|p1": 0.04856102610938251, + "USPTO_Backgrounds|OOD_News_2024|s1|p2": 0.04962680540047586, + "USPTO_Backgrounds|OOD_News_2024|s2|p0": 0.04796268339268863, + "USPTO_Backgrounds|OOD_News_2024|s2|p1": 0.048564265063032505, + "USPTO_Backgrounds|OOD_News_2024|s2|p2": 0.049647033819928764, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p0": 0.04610311691649258, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p1": 0.047125090518966316, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p2": 0.046955524990335104, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p0": 0.0456529111135751, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p1": 0.04685927769169211, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p2": 0.04681049273349345, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p0": 0.04550788109190762, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p1": 0.04661828870885074, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p2": 0.04806593046523631, + "USPTO_Backgrounds|OOD_CommonPile|s0|p0": 0.04776679603382945, + "USPTO_Backgrounds|OOD_CommonPile|s0|p1": 0.047657199669629335, + "USPTO_Backgrounds|OOD_CommonPile|s0|p2": 0.045324627216905354, + "USPTO_Backgrounds|OOD_CommonPile|s1|p0": 0.04687393931671977, + "USPTO_Backgrounds|OOD_CommonPile|s1|p1": 0.04669182421639562, + "USPTO_Backgrounds|OOD_CommonPile|s1|p2": 0.045244558807462455, + "USPTO_Backgrounds|OOD_CommonPile|s2|p0": 0.04858713774010539, + "USPTO_Backgrounds|OOD_CommonPile|s2|p1": 0.04818315776064992, + "USPTO_Backgrounds|OOD_CommonPile|s2|p2": 0.046324641536921264, + "Pile-CC|Wikipedia_en|s0|p0": 0.04825152647681534, + "Pile-CC|Wikipedia_en|s0|p1": 0.05187863605096936, + "Pile-CC|Wikipedia_en|s0|p2": 0.04894631956703961, + "Pile-CC|Wikipedia_en|s1|p0": 0.04780132216401398, + "Pile-CC|Wikipedia_en|s1|p1": 0.05182634526863694, + "Pile-CC|Wikipedia_en|s1|p2": 0.050018849736079576, + "Pile-CC|Wikipedia_en|s2|p0": 0.04743118179030716, + "Pile-CC|Wikipedia_en|s2|p1": 0.05175382951274514, + "Pile-CC|Wikipedia_en|s2|p2": 0.049133542692288756, + "Pile-CC|USPTO_Backgrounds|s0|p0": 0.05070397583767772, + "Pile-CC|USPTO_Backgrounds|s0|p1": 0.050584534322842954, + "Pile-CC|USPTO_Backgrounds|s0|p2": 0.04704875312745571, + "Pile-CC|USPTO_Backgrounds|s1|p0": 0.05029164925217629, + "Pile-CC|USPTO_Backgrounds|s1|p1": 0.049512006482109426, + "Pile-CC|USPTO_Backgrounds|s1|p2": 0.04745934084057808, + "Pile-CC|USPTO_Backgrounds|s2|p0": 0.049288399051874876, + "Pile-CC|USPTO_Backgrounds|s2|p1": 0.05006936839781702, + "Pile-CC|USPTO_Backgrounds|s2|p2": 0.0463287090882659, + "Pile-CC|Pile-CC|s0|p0": 0.050284375762566925, + "Pile-CC|Pile-CC|s0|p1": 0.04634758508764207, + "Pile-CC|Pile-CC|s0|p2": 0.05170507738366723, + "Pile-CC|Pile-CC|s1|p0": 0.050179793825373055, + "Pile-CC|Pile-CC|s1|p1": 0.04870075914077461, + "Pile-CC|Pile-CC|s1|p2": 0.05166288511827588, + "Pile-CC|Pile-CC|s2|p0": 0.050529703265056014, + "Pile-CC|Pile-CC|s2|p1": 0.04677756312303245, + "Pile-CC|Pile-CC|s2|p2": 0.050659632030874494, + "Pile-CC|StackExchange|s0|p0": 0.042963291588239375, + "Pile-CC|StackExchange|s0|p1": 0.04423040165565908, + "Pile-CC|StackExchange|s0|p2": 0.041990711051039395, + "Pile-CC|StackExchange|s1|p0": 0.04325335908215493, + "Pile-CC|StackExchange|s1|p1": 0.04577079168520868, + "Pile-CC|StackExchange|s1|p2": 0.041932603972963986, + "Pile-CC|StackExchange|s2|p0": 0.04201059106271714, + "Pile-CC|StackExchange|s2|p1": 0.04429632450919598, + "Pile-CC|StackExchange|s2|p2": 0.04144090178888291, + "Pile-CC|Github|s0|p0": 0.04292848231270909, + "Pile-CC|Github|s0|p1": 0.04337828024290502, + "Pile-CC|Github|s0|p2": 0.046667027613148095, + "Pile-CC|Github|s1|p0": 0.04383575031533837, + "Pile-CC|Github|s1|p1": 0.04249149623792618, + "Pile-CC|Github|s1|p2": 0.0460651921806857, + "Pile-CC|Github|s2|p0": 0.04284329311922193, + "Pile-CC|Github|s2|p1": 0.04284288524650037, + "Pile-CC|Github|s2|p2": 0.046204152470454576, + "Pile-CC|OOD_News_2024|s0|p0": 0.048610453959554434, + "Pile-CC|OOD_News_2024|s0|p1": 0.050505760964006184, + "Pile-CC|OOD_News_2024|s0|p2": 0.05291714668273926, + "Pile-CC|OOD_News_2024|s1|p0": 0.04789041122421622, + "Pile-CC|OOD_News_2024|s1|p1": 0.04989029048010707, + "Pile-CC|OOD_News_2024|s1|p2": 0.05225546401925385, + "Pile-CC|OOD_News_2024|s2|p0": 0.04715013718232512, + "Pile-CC|OOD_News_2024|s2|p1": 0.048699818830937144, + "Pile-CC|OOD_News_2024|s2|p2": 0.05084743839688599, + "Pile-CC|OOD_arXiv_2023|s0|p0": 0.04511925592087209, + "Pile-CC|OOD_arXiv_2023|s0|p1": 0.046386273298412564, + "Pile-CC|OOD_arXiv_2023|s0|p2": 0.04554894873872399, + "Pile-CC|OOD_arXiv_2023|s1|p0": 0.046794575871899724, + "Pile-CC|OOD_arXiv_2023|s1|p1": 0.046816250775009394, + "Pile-CC|OOD_arXiv_2023|s1|p2": 0.04731443366035819, + "Pile-CC|OOD_arXiv_2023|s2|p0": 0.04516144883818925, + "Pile-CC|OOD_arXiv_2023|s2|p1": 0.045521193835884335, + "Pile-CC|OOD_arXiv_2023|s2|p2": 0.045238655712455514, + "Pile-CC|OOD_CommonPile|s0|p0": 0.04562021093443036, + "Pile-CC|OOD_CommonPile|s0|p1": 0.04583600861951709, + "Pile-CC|OOD_CommonPile|s0|p2": 0.04630436021834612, + "Pile-CC|OOD_CommonPile|s1|p0": 0.046350385248661044, + "Pile-CC|OOD_CommonPile|s1|p1": 0.046206143032759425, + "Pile-CC|OOD_CommonPile|s1|p2": 0.0470582596026361, + "Pile-CC|OOD_CommonPile|s2|p0": 0.045845314580947163, + "Pile-CC|OOD_CommonPile|s2|p1": 0.044683408737182614, + "Pile-CC|OOD_CommonPile|s2|p2": 0.04494176860898733, + "StackExchange|Wikipedia_en|s0|p0": 0.04534177854657173, + "StackExchange|Wikipedia_en|s0|p1": 0.04874965557828546, + "StackExchange|Wikipedia_en|s0|p2": 0.04678497305139899, + "StackExchange|Wikipedia_en|s1|p0": 0.04307121969759464, + "StackExchange|Wikipedia_en|s1|p1": 0.04646752080880105, + "StackExchange|Wikipedia_en|s1|p2": 0.043854215275496246, + "StackExchange|Wikipedia_en|s2|p0": 0.04273896366357803, + "StackExchange|Wikipedia_en|s2|p1": 0.04667239612899721, + "StackExchange|Wikipedia_en|s2|p2": 0.0449721741490066, + "StackExchange|USPTO_Backgrounds|s0|p0": 0.052270094072446226, + "StackExchange|USPTO_Backgrounds|s0|p1": 0.04895114982500672, + "StackExchange|USPTO_Backgrounds|s0|p2": 0.048038117168471214, + "StackExchange|USPTO_Backgrounds|s1|p0": 0.04903161874972284, + "StackExchange|USPTO_Backgrounds|s1|p1": 0.04758786121383309, + "StackExchange|USPTO_Backgrounds|s1|p2": 0.045149553148075935, + "StackExchange|USPTO_Backgrounds|s2|p0": 0.04932168475352228, + "StackExchange|USPTO_Backgrounds|s2|p1": 0.04709291616454721, + "StackExchange|USPTO_Backgrounds|s2|p2": 0.04685519519262016, + "StackExchange|Pile-CC|s0|p0": 0.048280903091654184, + "StackExchange|Pile-CC|s0|p1": 0.04533625990152359, + "StackExchange|Pile-CC|s0|p2": 0.04717964669689536, + "StackExchange|Pile-CC|s1|p0": 0.04575491934083402, + "StackExchange|Pile-CC|s1|p1": 0.043663771636784074, + "StackExchange|Pile-CC|s1|p2": 0.04510386427864432, + "StackExchange|Pile-CC|s2|p0": 0.04568240283988416, + "StackExchange|Pile-CC|s2|p1": 0.04459598809480667, + "StackExchange|Pile-CC|s2|p2": 0.04510387098416686, + "StackExchange|StackExchange|s0|p0": 0.04351462812628597, + "StackExchange|StackExchange|s0|p1": 0.0444192714523524, + "StackExchange|StackExchange|s0|p2": 0.04388158111833036, + "StackExchange|StackExchange|s1|p0": 0.04172284197993577, + "StackExchange|StackExchange|s1|p1": 0.04303147359751165, + "StackExchange|StackExchange|s1|p2": 0.04150211112573743, + "StackExchange|StackExchange|s2|p0": 0.04070369771216065, + "StackExchange|StackExchange|s2|p1": 0.04377174465917051, + "StackExchange|StackExchange|s2|p2": 0.04169257441535592, + "StackExchange|Github|s0|p0": 0.04486154709011316, + "StackExchange|Github|s0|p1": 0.04346416727639735, + "StackExchange|Github|s0|p2": 0.049115256313234566, + "StackExchange|Github|s1|p0": 0.04405631143599749, + "StackExchange|Github|s1|p1": 0.040596610866487026, + "StackExchange|Github|s1|p2": 0.04614662090316415, + "StackExchange|Github|s2|p0": 0.044796580262482164, + "StackExchange|Github|s2|p1": 0.04077952075749636, + "StackExchange|Github|s2|p2": 0.04658927181735635, + "StackExchange|OOD_News_2024|s0|p0": 0.04541729120537639, + "StackExchange|OOD_News_2024|s0|p1": 0.04774485994130373, + "StackExchange|OOD_News_2024|s0|p2": 0.04773056162521243, + "StackExchange|OOD_News_2024|s1|p0": 0.04275637110695243, + "StackExchange|OOD_News_2024|s1|p1": 0.04507384244352579, + "StackExchange|OOD_News_2024|s1|p2": 0.04539180090650916, + "StackExchange|OOD_News_2024|s2|p0": 0.043929196428507566, + "StackExchange|OOD_News_2024|s2|p1": 0.04492125455290079, + "StackExchange|OOD_News_2024|s2|p2": 0.04691777331754565, + "StackExchange|OOD_arXiv_2023|s0|p0": 0.04673875542357564, + "StackExchange|OOD_arXiv_2023|s0|p1": 0.045826251665130255, + "StackExchange|OOD_arXiv_2023|s0|p2": 0.04515592325478792, + "StackExchange|OOD_arXiv_2023|s1|p0": 0.043398248683661225, + "StackExchange|OOD_arXiv_2023|s1|p1": 0.043597887316718695, + "StackExchange|OOD_arXiv_2023|s1|p2": 0.04296543952077627, + "StackExchange|OOD_arXiv_2023|s2|p0": 0.043750730156898496, + "StackExchange|OOD_arXiv_2023|s2|p1": 0.043597887316718695, + "StackExchange|OOD_arXiv_2023|s2|p2": 0.04362711915746331, + "StackExchange|OOD_CommonPile|s0|p0": 0.04539163038134575, + "StackExchange|OOD_CommonPile|s0|p1": 0.0481088756583631, + "StackExchange|OOD_CommonPile|s0|p2": 0.04627327444031835, + "StackExchange|OOD_CommonPile|s1|p0": 0.04309074804186821, + "StackExchange|OOD_CommonPile|s1|p1": 0.04560997290536761, + "StackExchange|OOD_CommonPile|s1|p2": 0.044966329168528316, + "StackExchange|OOD_CommonPile|s2|p0": 0.04346088469028473, + "StackExchange|OOD_CommonPile|s2|p1": 0.04500787639990449, + "StackExchange|OOD_CommonPile|s2|p2": 0.04466037387028336, + "Github|Wikipedia_en|s0|p0": 0.02299818522296846, + "Github|Wikipedia_en|s0|p1": 0.026630240306258202, + "Github|Wikipedia_en|s0|p2": 0.02534515131264925, + "Github|Wikipedia_en|s1|p0": 0.025510543677955865, + "Github|Wikipedia_en|s1|p1": 0.030111473705619573, + "Github|Wikipedia_en|s1|p2": 0.02743710605427623, + "Github|Wikipedia_en|s2|p0": 0.024175034556537866, + "Github|Wikipedia_en|s2|p1": 0.02667497657239437, + "Github|Wikipedia_en|s2|p2": 0.0251579275354743, + "Github|USPTO_Backgrounds|s0|p0": 0.02587893744930625, + "Github|USPTO_Backgrounds|s0|p1": 0.026390179200097917, + "Github|USPTO_Backgrounds|s0|p2": 0.026752790808677672, + "Github|USPTO_Backgrounds|s1|p0": 0.028322279546409847, + "Github|USPTO_Backgrounds|s1|p1": 0.027972468826919793, + "Github|USPTO_Backgrounds|s1|p2": 0.029306527273729444, + "Github|USPTO_Backgrounds|s2|p0": 0.02593122785910964, + "Github|USPTO_Backgrounds|s2|p1": 0.02546954290010035, + "Github|USPTO_Backgrounds|s2|p2": 0.025859934091567994, + "Github|Pile-CC|s0|p0": 0.02515146811492741, + "Github|Pile-CC|s0|p1": 0.024389901757240297, + "Github|Pile-CC|s0|p2": 0.026489410549402237, + "Github|Pile-CC|s1|p0": 0.027265126677230002, + "Github|Pile-CC|s1|p1": 0.02809273321181536, + "Github|Pile-CC|s1|p2": 0.029003529250621794, + "Github|Pile-CC|s2|p0": 0.02417854224331677, + "Github|Pile-CC|s2|p1": 0.024684277921915056, + "Github|Pile-CC|s2|p2": 0.025059421779587865, + "Github|StackExchange|s0|p0": 0.021186233148910105, + "Github|StackExchange|s0|p1": 0.01899896454997361, + "Github|StackExchange|s0|p2": 0.021202125074341893, + "Github|StackExchange|s1|p0": 0.023487116605974735, + "Github|StackExchange|s1|p1": 0.022092406777665018, + "Github|StackExchange|s1|p2": 0.02490347814746201, + "Github|StackExchange|s2|p0": 0.021106163994409143, + "Github|StackExchange|s2|p1": 0.019731681840494276, + "Github|StackExchange|s2|p2": 0.02183454525656998, + "Github|Github|s0|p0": 0.02421602364629507, + "Github|Github|s0|p1": 0.022202111966907977, + "Github|Github|s0|p2": 0.02491505742073059, + "Github|Github|s1|p0": 0.02566192615777254, + "Github|Github|s1|p1": 0.024100766889750956, + "Github|Github|s1|p2": 0.02683824971318245, + "Github|Github|s2|p0": 0.023270875820890068, + "Github|Github|s2|p1": 0.022087405901402235, + "Github|Github|s2|p2": 0.024464850872755052, + "Github|OOD_News_2024|s0|p0": 0.02355434806086123, + "Github|OOD_News_2024|s0|p1": 0.024397747684270145, + "Github|OOD_News_2024|s0|p2": 0.02593636615201831, + "Github|OOD_News_2024|s1|p0": 0.026948650367558002, + "Github|OOD_News_2024|s1|p1": 0.0274395695887506, + "Github|OOD_News_2024|s1|p2": 0.03017957229167223, + "Github|OOD_News_2024|s2|p0": 0.023516468750312925, + "Github|OOD_News_2024|s2|p1": 0.024300693767145276, + "Github|OOD_News_2024|s2|p2": 0.027092987019568682, + "Github|OOD_arXiv_2023|s0|p0": 0.02449531713500619, + "Github|OOD_arXiv_2023|s0|p1": 0.026078676618635655, + "Github|OOD_arXiv_2023|s0|p2": 0.024994170852005483, + "Github|OOD_arXiv_2023|s1|p0": 0.028093830216675997, + "Github|OOD_arXiv_2023|s1|p1": 0.029185744281858206, + "Github|OOD_arXiv_2023|s1|p2": 0.02758511919528246, + "Github|OOD_arXiv_2023|s2|p0": 0.024785382393747567, + "Github|OOD_arXiv_2023|s2|p1": 0.026666359044611454, + "Github|OOD_arXiv_2023|s2|p2": 0.02447145152837038, + "Github|OOD_CommonPile|s0|p0": 0.025595012493431567, + "Github|OOD_CommonPile|s0|p1": 0.024993591755628587, + "Github|OOD_CommonPile|s0|p2": 0.024846938252449036, + "Github|OOD_CommonPile|s1|p0": 0.028891593776643278, + "Github|OOD_CommonPile|s1|p1": 0.02769991457462311, + "Github|OOD_CommonPile|s1|p2": 0.027905744034796953, + "Github|OOD_CommonPile|s2|p0": 0.026110181398689748, + "Github|OOD_CommonPile|s2|p1": 0.025600407272577286, + "Github|OOD_CommonPile|s2|p2": 0.02589750224724412, + "OOD_News_2024|Wikipedia_en|s0|p0": 0.03671670316252858, + "OOD_News_2024|Wikipedia_en|s0|p1": 0.04154766141436994, + "OOD_News_2024|Wikipedia_en|s0|p2": 0.03797612129710615, + "OOD_News_2024|Wikipedia_en|s1|p0": 0.03743067388422787, + "OOD_News_2024|Wikipedia_en|s1|p1": 0.04147514565847814, + "OOD_News_2024|Wikipedia_en|s1|p2": 0.038365787779912355, + "OOD_News_2024|Wikipedia_en|s2|p0": 0.03611391170416027, + "OOD_News_2024|Wikipedia_en|s2|p1": 0.040932194283232096, + "OOD_News_2024|Wikipedia_en|s2|p2": 0.03859844258986413, + "OOD_News_2024|USPTO_Backgrounds|s0|p0": 0.041230500862002376, + "OOD_News_2024|USPTO_Backgrounds|s0|p1": 0.04164239864330739, + "OOD_News_2024|USPTO_Backgrounds|s0|p2": 0.038126584514975546, + "OOD_News_2024|USPTO_Backgrounds|s1|p0": 0.04147001262754202, + "OOD_News_2024|USPTO_Backgrounds|s1|p1": 0.041337224352173504, + "OOD_News_2024|USPTO_Backgrounds|s1|p2": 0.037525272998027506, + "OOD_News_2024|USPTO_Backgrounds|s2|p0": 0.04158041067421436, + "OOD_News_2024|USPTO_Backgrounds|s2|p1": 0.04007423396687955, + "OOD_News_2024|USPTO_Backgrounds|s2|p2": 0.0375977887539193, + "OOD_News_2024|Pile-CC|s0|p0": 0.03771652071736753, + "OOD_News_2024|Pile-CC|s0|p1": 0.03725688378326595, + "OOD_News_2024|Pile-CC|s0|p2": 0.03985171332024038, + "OOD_News_2024|Pile-CC|s1|p0": 0.038661668775603175, + "OOD_News_2024|Pile-CC|s1|p1": 0.03742214529775083, + "OOD_News_2024|Pile-CC|s1|p2": 0.04073701440356672, + "OOD_News_2024|Pile-CC|s2|p0": 0.03813894647173584, + "OOD_News_2024|Pile-CC|s2|p1": 0.03623921959660947, + "OOD_News_2024|Pile-CC|s2|p2": 0.040924237994477156, + "OOD_News_2024|StackExchange|s0|p0": 0.03539642095565796, + "OOD_News_2024|StackExchange|s0|p1": 0.032507660496048627, + "OOD_News_2024|StackExchange|s0|p2": 0.032678435742855075, + "OOD_News_2024|StackExchange|s1|p0": 0.03390832245349884, + "OOD_News_2024|StackExchange|s1|p1": 0.0328431575326249, + "OOD_News_2024|StackExchange|s1|p2": 0.0331265916582197, + "OOD_News_2024|StackExchange|s2|p0": 0.034496009349823, + "OOD_News_2024|StackExchange|s2|p1": 0.03224792021792382, + "OOD_News_2024|StackExchange|s2|p2": 0.03320666453801095, + "OOD_News_2024|Github|s0|p0": 0.03426561239175498, + "OOD_News_2024|Github|s0|p1": 0.03422586037777364, + "OOD_News_2024|Github|s0|p2": 0.03395344740711152, + "OOD_News_2024|Github|s1|p0": 0.03421184557955712, + "OOD_News_2024|Github|s1|p1": 0.03424001345410943, + "OOD_News_2024|Github|s1|p2": 0.03305910739582032, + "OOD_News_2024|Github|s2|p0": 0.03475479034241289, + "OOD_News_2024|Github|s2|p1": 0.033200642978772524, + "OOD_News_2024|Github|s2|p2": 0.03224632095079869, + "OOD_News_2024|OOD_News_2024|s0|p0": 0.03907562275417149, + "OOD_News_2024|OOD_News_2024|s0|p1": 0.038037475850433114, + "OOD_News_2024|OOD_News_2024|s0|p2": 0.04267560779117048, + "OOD_News_2024|OOD_News_2024|s1|p0": 0.039749975129961965, + "OOD_News_2024|OOD_News_2024|s1|p1": 0.03851800737902522, + "OOD_News_2024|OOD_News_2024|s1|p2": 0.04386608614586294, + "OOD_News_2024|OOD_News_2024|s2|p0": 0.038864674419164656, + "OOD_News_2024|OOD_News_2024|s2|p1": 0.03814786998555064, + "OOD_News_2024|OOD_News_2024|s2|p2": 0.04297322570346296, + "OOD_News_2024|OOD_arXiv_2023|s0|p0": 0.036149079957976935, + "OOD_News_2024|OOD_arXiv_2023|s0|p1": 0.035061447555199264, + "OOD_News_2024|OOD_arXiv_2023|s0|p2": 0.03549142493866384, + "OOD_News_2024|OOD_arXiv_2023|s1|p0": 0.0364813367370516, + "OOD_News_2024|OOD_arXiv_2023|s1|p1": 0.03631688528694212, + "OOD_News_2024|OOD_arXiv_2023|s1|p2": 0.03591385050676763, + "OOD_News_2024|OOD_arXiv_2023|s2|p0": 0.036294112959876654, + "OOD_News_2024|OOD_arXiv_2023|s2|p1": 0.035278993332758544, + "OOD_News_2024|OOD_arXiv_2023|s2|p2": 0.036046210071071984, + "OOD_News_2024|OOD_CommonPile|s0|p0": 0.03237517392262816, + "OOD_News_2024|OOD_CommonPile|s0|p1": 0.033100989926606414, + "OOD_News_2024|OOD_CommonPile|s0|p2": 0.03596177669242025, + "OOD_News_2024|OOD_CommonPile|s1|p0": 0.03315008087083697, + "OOD_News_2024|OOD_CommonPile|s1|p1": 0.034893559757620096, + "OOD_News_2024|OOD_CommonPile|s1|p2": 0.03671362427994609, + "OOD_News_2024|OOD_CommonPile|s2|p0": 0.03398807412013412, + "OOD_News_2024|OOD_CommonPile|s2|p1": 0.03459594184532762, + "OOD_News_2024|OOD_CommonPile|s2|p2": 0.03665123647078872, + "OOD_arXiv_2023|Wikipedia_en|s0|p0": 0.038727887254208324, + "OOD_arXiv_2023|Wikipedia_en|s0|p1": 0.044071207335218784, + "OOD_arXiv_2023|Wikipedia_en|s0|p2": 0.03957072671037167, + "OOD_arXiv_2023|Wikipedia_en|s1|p0": 0.038357752840965986, + "OOD_arXiv_2023|Wikipedia_en|s1|p1": 0.04405098198913038, + "OOD_arXiv_2023|Wikipedia_en|s1|p2": 0.039490654575638474, + "OOD_arXiv_2023|Wikipedia_en|s2|p0": 0.03761748177930713, + "OOD_arXiv_2023|Wikipedia_en|s2|p1": 0.043291582423262295, + "OOD_arXiv_2023|Wikipedia_en|s2|p2": 0.03947798095177859, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p0": 0.043173919711261985, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p1": 0.03929868265986443, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p2": 0.03943431116640568, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p0": 0.042961490247398613, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p1": 0.039073579013347626, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p2": 0.040922409668564794, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p0": 0.04289652723819017, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p1": 0.03952378705143929, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p2": 0.04007174372673035, + "OOD_arXiv_2023|Pile-CC|s0|p0": 0.04011216927319765, + "OOD_arXiv_2023|Pile-CC|s0|p1": 0.039493899303488435, + "OOD_arXiv_2023|Pile-CC|s0|p2": 0.04293656102381647, + "OOD_arXiv_2023|Pile-CC|s1|p0": 0.040264754183590414, + "OOD_arXiv_2023|Pile-CC|s1|p1": 0.0400552851613611, + "OOD_arXiv_2023|Pile-CC|s1|p2": 0.04282185318879783, + "OOD_arXiv_2023|Pile-CC|s2|p0": 0.03959699887782335, + "OOD_arXiv_2023|Pile-CC|s2|p1": 0.039130359841510656, + "OOD_arXiv_2023|Pile-CC|s2|p2": 0.04367682985030115, + "OOD_arXiv_2023|StackExchange|s0|p0": 0.03755808379501104, + "OOD_arXiv_2023|StackExchange|s0|p1": 0.038457607361488046, + "OOD_arXiv_2023|StackExchange|s0|p2": 0.03699849676340818, + "OOD_arXiv_2023|StackExchange|s1|p0": 0.03673774134367704, + "OOD_arXiv_2023|StackExchange|s1|p1": 0.037767888489179315, + "OOD_arXiv_2023|StackExchange|s1|p2": 0.03663521781563759, + "OOD_arXiv_2023|StackExchange|s2|p0": 0.03733298312872648, + "OOD_arXiv_2023|StackExchange|s2|p1": 0.03916755344253033, + "OOD_arXiv_2023|StackExchange|s2|p2": 0.03687369003891945, + "OOD_arXiv_2023|Github|s0|p0": 0.03632147805765271, + "OOD_arXiv_2023|Github|s0|p1": 0.035743088275194165, + "OOD_arXiv_2023|Github|s0|p2": 0.03812470398843289, + "OOD_arXiv_2023|Github|s1|p0": 0.035248953942209484, + "OOD_arXiv_2023|Github|s1|p1": 0.03522036597132683, + "OOD_arXiv_2023|Github|s1|p2": 0.03782708495855332, + "OOD_arXiv_2023|Github|s2|p0": 0.035356107819825414, + "OOD_arXiv_2023|Github|s2|p1": 0.03530043512582779, + "OOD_arXiv_2023|Github|s2|p2": 0.03946021385490894, + "OOD_arXiv_2023|OOD_News_2024|s0|p0": 0.03954499904066324, + "OOD_arXiv_2023|OOD_News_2024|s0|p1": 0.0407484196824953, + "OOD_arXiv_2023|OOD_News_2024|s0|p2": 0.04118536699097604, + "OOD_arXiv_2023|OOD_News_2024|s1|p0": 0.03955333910416812, + "OOD_arXiv_2023|OOD_News_2024|s1|p1": 0.040153183857910335, + "OOD_arXiv_2023|OOD_News_2024|s1|p2": 0.04083719903137535, + "OOD_arXiv_2023|OOD_News_2024|s2|p0": 0.0388393696397543, + "OOD_arXiv_2023|OOD_News_2024|s2|p1": 0.03958424627780914, + "OOD_arXiv_2023|OOD_News_2024|s2|p2": 0.04086323890369385, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p0": 0.039236063580028714, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p1": 0.03838620453607291, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p2": 0.03940372753422707, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p0": 0.03856830827426165, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p1": 0.039550378127023576, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p2": 0.03884312759619206, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p0": 0.037973072449676695, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p1": 0.038096139277331534, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p2": 0.039559671422466636, + "OOD_arXiv_2023|OOD_CommonPile|s0|p0": 0.040812817169353366, + "OOD_arXiv_2023|OOD_CommonPile|s0|p1": 0.040232782950624824, + "OOD_arXiv_2023|OOD_CommonPile|s0|p2": 0.03951485650613904, + "OOD_arXiv_2023|OOD_CommonPile|s1|p0": 0.041182955307886, + "OOD_arXiv_2023|OOD_CommonPile|s1|p1": 0.0413102903869003, + "OOD_arXiv_2023|OOD_CommonPile|s1|p2": 0.0391573921777308, + "OOD_arXiv_2023|OOD_CommonPile|s2|p0": 0.039919960452243684, + "OOD_arXiv_2023|OOD_CommonPile|s2|p1": 0.040628124168142675, + "OOD_arXiv_2023|OOD_CommonPile|s2|p2": 0.03842467376962304, + "OOD_CommonPile|Wikipedia_en|s0|p0": 0.036579592060297726, + "OOD_CommonPile|Wikipedia_en|s0|p1": 0.03941391441039741, + "OOD_CommonPile|Wikipedia_en|s0|p2": 0.03770494412165135, + "OOD_CommonPile|Wikipedia_en|s1|p0": 0.03769755465909839, + "OOD_CommonPile|Wikipedia_en|s1|p1": 0.04058416350744665, + "OOD_CommonPile|Wikipedia_en|s1|p2": 0.0388575381366536, + "OOD_CommonPile|Wikipedia_en|s2|p0": 0.035091493558138606, + "OOD_CommonPile|Wikipedia_en|s2|p1": 0.03933127275668084, + "OOD_CommonPile|Wikipedia_en|s2|p2": 0.03607181336265057, + "OOD_CommonPile|USPTO_Backgrounds|s0|p0": 0.042003673408180474, + "OOD_CommonPile|USPTO_Backgrounds|s0|p1": 0.03816804885864258, + "OOD_CommonPile|USPTO_Backgrounds|s0|p2": 0.03981860044877976, + "OOD_CommonPile|USPTO_Backgrounds|s1|p0": 0.042526392824947835, + "OOD_CommonPile|USPTO_Backgrounds|s1|p1": 0.038485894352197646, + "OOD_CommonPile|USPTO_Backgrounds|s1|p2": 0.04015007205307484, + "OOD_CommonPile|USPTO_Backgrounds|s2|p0": 0.04085107194259763, + "OOD_CommonPile|USPTO_Backgrounds|s2|p1": 0.03684521391987801, + "OOD_CommonPile|USPTO_Backgrounds|s2|p2": 0.038853227230720225, + "OOD_CommonPile|Pile-CC|s0|p0": 0.039699149574153124, + "OOD_CommonPile|Pile-CC|s0|p1": 0.03928320545237511, + "OOD_CommonPile|Pile-CC|s0|p2": 0.04219371690414846, + "OOD_CommonPile|Pile-CC|s1|p0": 0.04052880499511957, + "OOD_CommonPile|Pile-CC|s1|p1": 0.04028645928483456, + "OOD_CommonPile|Pile-CC|s1|p2": 0.0413289200514555, + "OOD_CommonPile|Pile-CC|s2|p0": 0.038403282128274444, + "OOD_CommonPile|Pile-CC|s2|p1": 0.03797802354674786, + "OOD_CommonPile|Pile-CC|s2|p2": 0.03977988462429494, + "OOD_CommonPile|StackExchange|s0|p0": 0.0347943221218884, + "OOD_CommonPile|StackExchange|s0|p1": 0.0371971896616742, + "OOD_CommonPile|StackExchange|s0|p2": 0.03643182187806815, + "OOD_CommonPile|StackExchange|s1|p0": 0.035917400382459164, + "OOD_CommonPile|StackExchange|s1|p1": 0.03827295664232224, + "OOD_CommonPile|StackExchange|s1|p2": 0.03636685728561133, + "OOD_CommonPile|StackExchange|s2|p0": 0.03382139662280679, + "OOD_CommonPile|StackExchange|s2|p1": 0.03614917423110455, + "OOD_CommonPile|StackExchange|s2|p2": 0.035076087224297225, + "OOD_CommonPile|Github|s0|p0": 0.0341214191634208, + "OOD_CommonPile|Github|s0|p1": 0.03550464184954762, + "OOD_CommonPile|Github|s0|p2": 0.03786927685141563, + "OOD_CommonPile|Github|s1|p0": 0.03532954598776996, + "OOD_CommonPile|Github|s1|p1": 0.03550464184954762, + "OOD_CommonPile|Github|s1|p2": 0.037519368529319766, + "OOD_CommonPile|Github|s2|p0": 0.03267119978554547, + "OOD_CommonPile|Github|s2|p1": 0.033739156648516656, + "OOD_CommonPile|Github|s2|p2": 0.03747891690582037, + "OOD_CommonPile|OOD_News_2024|s0|p0": 0.03865266921930015, + "OOD_CommonPile|OOD_News_2024|s0|p1": 0.03853014975320548, + "OOD_CommonPile|OOD_News_2024|s0|p2": 0.040657664998434484, + "OOD_CommonPile|OOD_News_2024|s1|p0": 0.03896243553608656, + "OOD_CommonPile|OOD_News_2024|s1|p1": 0.03922930133994669, + "OOD_CommonPile|OOD_News_2024|s1|p2": 0.04086861656978726, + "OOD_CommonPile|OOD_News_2024|s2|p0": 0.038063503126613794, + "OOD_CommonPile|OOD_News_2024|s2|p1": 0.03704205721151084, + "OOD_CommonPile|OOD_News_2024|s2|p2": 0.0400338601320982, + "OOD_CommonPile|OOD_arXiv_2023|s0|p0": 0.03841572336386889, + "OOD_CommonPile|OOD_arXiv_2023|s0|p1": 0.03803856065496802, + "OOD_CommonPile|OOD_arXiv_2023|s0|p2": 0.03738281305413693, + "OOD_CommonPile|OOD_arXiv_2023|s1|p0": 0.040083489916287365, + "OOD_CommonPile|OOD_arXiv_2023|s1|p1": 0.03848728644661605, + "OOD_CommonPile|OOD_arXiv_2023|s1|p2": 0.0378676577238366, + "OOD_CommonPile|OOD_arXiv_2023|s2|p0": 0.0373154137050733, + "OOD_CommonPile|OOD_arXiv_2023|s2|p1": 0.03722429233603179, + "OOD_CommonPile|OOD_arXiv_2023|s2|p2": 0.036521761259064076, + "OOD_CommonPile|OOD_CommonPile|s0|p0": 0.03953715111128986, + "OOD_CommonPile|OOD_CommonPile|s0|p1": 0.0394535590428859, + "OOD_CommonPile|OOD_CommonPile|s0|p2": 0.038331931550055744, + "OOD_CommonPile|OOD_CommonPile|s1|p0": 0.04062733114697039, + "OOD_CommonPile|OOD_CommonPile|s1|p1": 0.03977826316840947, + "OOD_CommonPile|OOD_CommonPile|s1|p2": 0.03860758962109685, + "OOD_CommonPile|OOD_CommonPile|s2|p0": 0.039269858272746205, + "OOD_CommonPile|OOD_CommonPile|s2|p1": 0.03881288995034993, + "OOD_CommonPile|OOD_CommonPile|s2|p2": 0.0375967831350863, + "logit|Q0|p0": 0.028439503419212996, + "shuf|Q0|p0": 0.021268022060394288, + "logit|Q0|p1": 0.028439503419212996, + "shuf|Q0|p1": 0.021268022060394288, + "logit|Q0|p2": 0.028439503419212996, + "shuf|Q0|p2": 0.021268022060394288, + "logit|Wikipedia_en|p0": 0.028726140107028186, + "shuf|Wikipedia_en|p0": 0.02310783523134887, + "logit|Wikipedia_en|p1": 0.030837458628229796, + "shuf|Wikipedia_en|p1": 0.02413048017770052, + "logit|Wikipedia_en|p2": 0.029703613999299706, + "shuf|Wikipedia_en|p2": 0.024289118126034737, + "logit|USPTO_Backgrounds|p0": 0.02871693994384259, + "shuf|USPTO_Backgrounds|p0": 0.024401361122727394, + "logit|USPTO_Backgrounds|p1": 0.03058738212566823, + "shuf|USPTO_Backgrounds|p1": 0.025304315984249114, + "logit|USPTO_Backgrounds|p2": 0.031176864611916245, + "shuf|USPTO_Backgrounds|p2": 0.022762071900069715, + "logit|Pile-CC|p0": 0.035120575386099515, + "shuf|Pile-CC|p0": 0.022151755448430777, + "logit|Pile-CC|p1": 0.03508521046023816, + "shuf|Pile-CC|p1": 0.021709074266254902, + "logit|Pile-CC|p2": 0.03481215860228985, + "shuf|Pile-CC|p2": 0.023998354747891425, + "logit|StackExchange|p0": 0.02859928123652935, + "shuf|StackExchange|p0": 0.018458761926740407, + "logit|StackExchange|p1": 0.026670536398887633, + "shuf|StackExchange|p1": 0.01743514370173216, + "logit|StackExchange|p2": 0.027009129035286607, + "shuf|StackExchange|p2": 0.01849499884992838, + "logit|Github|p0": 0.027017780649475753, + "shuf|Github|p0": 0.019806107599288227, + "logit|Github|p1": 0.02629018302541226, + "shuf|Github|p1": 0.019280839525163174, + "logit|Github|p2": 0.02436053443234414, + "shuf|Github|p2": 0.019516041968017817, + "logit|OOD_News_2024|p0": 0.03550712231080979, + "shuf|OOD_News_2024|p0": 0.024123887810856105, + "logit|OOD_News_2024|p1": 0.03712005380075425, + "shuf|OOD_News_2024|p1": 0.022528637386858463, + "logit|OOD_News_2024|p2": 0.03788821462076157, + "shuf|OOD_News_2024|p2": 0.02383031537756324, + "logit|OOD_arXiv_2023|p0": 0.026592944376170637, + "shuf|OOD_arXiv_2023|p0": 0.022637404035776854, + "logit|OOD_arXiv_2023|p1": 0.02745935469865799, + "shuf|OOD_arXiv_2023|p1": 0.02077593160793185, + "logit|OOD_arXiv_2023|p2": 0.026104292529635132, + "shuf|OOD_arXiv_2023|p2": 0.02104389388114214, + "logit|OOD_CommonPile|p0": 0.03556173413526267, + "shuf|OOD_CommonPile|p0": 0.01958368895575404, + "logit|OOD_CommonPile|p1": 0.03545751124620437, + "shuf|OOD_CommonPile|p1": 0.020708365738391875, + "logit|OOD_CommonPile|p2": 0.03281065821647644, + "shuf|OOD_CommonPile|p2": 0.022601932287216187, + "logit|SHUFFLED_Pile-CC|p0": 0.019922320428304375, + "shuf|SHUFFLED_Pile-CC|p0": 0.015490125399082899, + "logit|SHUFFLED_Pile-CC|p1": 0.019162460300140082, + "shuf|SHUFFLED_Pile-CC|p1": 0.01434438480064273, + "logit|SHUFFLED_Pile-CC|p2": 0.017466827854514123, + "shuf|SHUFFLED_Pile-CC|p2": 0.015702555235475302 + }, + "min": { + "Wikipedia_en|Wikipedia_en|s0|p0": 0.1314952816814184, + "Wikipedia_en|Wikipedia_en|s0|p1": 0.13528487123548985, + "Wikipedia_en|Wikipedia_en|s0|p2": 0.13345829024910927, + "Wikipedia_en|Wikipedia_en|s1|p0": 0.13206410072743893, + "Wikipedia_en|Wikipedia_en|s1|p1": 0.13533404991030692, + "Wikipedia_en|Wikipedia_en|s1|p2": 0.13453038409352303, + "Wikipedia_en|Wikipedia_en|s2|p0": 0.12990677542984486, + "Wikipedia_en|Wikipedia_en|s2|p1": 0.133492299541831, + "Wikipedia_en|Wikipedia_en|s2|p2": 0.13399313166737556, + "Wikipedia_en|USPTO_Backgrounds|s0|p0": 0.14007385782897472, + "Wikipedia_en|USPTO_Backgrounds|s0|p1": 0.13845476023852826, + "Wikipedia_en|USPTO_Backgrounds|s0|p2": 0.13388987220823764, + "Wikipedia_en|USPTO_Backgrounds|s1|p0": 0.1416717667132616, + "Wikipedia_en|USPTO_Backgrounds|s1|p1": 0.14080993905663491, + "Wikipedia_en|USPTO_Backgrounds|s1|p2": 0.13559157438576222, + "Wikipedia_en|USPTO_Backgrounds|s2|p0": 0.138703703135252, + "Wikipedia_en|USPTO_Backgrounds|s2|p1": 0.13860451877117158, + "Wikipedia_en|USPTO_Backgrounds|s2|p2": 0.13279332891106604, + "Wikipedia_en|Pile-CC|s0|p0": 0.13716441504657267, + "Wikipedia_en|Pile-CC|s0|p1": 0.13544980734586715, + "Wikipedia_en|Pile-CC|s0|p2": 0.1406457681208849, + "Wikipedia_en|Pile-CC|s1|p0": 0.14019205719232558, + "Wikipedia_en|Pile-CC|s1|p1": 0.1367002371698618, + "Wikipedia_en|Pile-CC|s1|p2": 0.1434601716697216, + "Wikipedia_en|Pile-CC|s2|p0": 0.1371540330350399, + "Wikipedia_en|Pile-CC|s2|p1": 0.13469483256340026, + "Wikipedia_en|Pile-CC|s2|p2": 0.13953884541988373, + "Wikipedia_en|StackExchange|s0|p0": 0.11986354738473892, + "Wikipedia_en|StackExchange|s0|p1": 0.1174791244789958, + "Wikipedia_en|StackExchange|s0|p2": 0.12304020710289479, + "Wikipedia_en|StackExchange|s1|p0": 0.1194436714053154, + "Wikipedia_en|StackExchange|s1|p1": 0.11803836189210415, + "Wikipedia_en|StackExchange|s1|p2": 0.12330680526793003, + "Wikipedia_en|StackExchange|s2|p0": 0.11876500882208348, + "Wikipedia_en|StackExchange|s2|p1": 0.11583884693682193, + "Wikipedia_en|StackExchange|s2|p2": 0.1200434122234583, + "Wikipedia_en|Github|s0|p0": 0.11371806338429451, + "Wikipedia_en|Github|s0|p1": 0.11780917681753636, + "Wikipedia_en|Github|s0|p2": 0.11871678121387959, + "Wikipedia_en|Github|s1|p0": 0.11582040563225746, + "Wikipedia_en|Github|s1|p1": 0.12019886709749698, + "Wikipedia_en|Github|s1|p2": 0.1184202253818512, + "Wikipedia_en|Github|s2|p0": 0.11457479931414127, + "Wikipedia_en|Github|s2|p1": 0.11827919371426106, + "Wikipedia_en|Github|s2|p2": 0.11699035800993443, + "Wikipedia_en|OOD_News_2024|s0|p0": 0.13576465286314487, + "Wikipedia_en|OOD_News_2024|s0|p1": 0.1364069476723671, + "Wikipedia_en|OOD_News_2024|s0|p2": 0.14100285731256007, + "Wikipedia_en|OOD_News_2024|s1|p0": 0.13383496925234795, + "Wikipedia_en|OOD_News_2024|s1|p1": 0.13661750890314578, + "Wikipedia_en|OOD_News_2024|s1|p2": 0.14217874445021153, + "Wikipedia_en|OOD_News_2024|s2|p0": 0.13318124786019325, + "Wikipedia_en|OOD_News_2024|s2|p1": 0.1357308529317379, + "Wikipedia_en|OOD_News_2024|s2|p2": 0.1400008223950863, + "Wikipedia_en|OOD_arXiv_2023|s0|p0": 0.13495293743908404, + "Wikipedia_en|OOD_arXiv_2023|s0|p1": 0.12740938253700734, + "Wikipedia_en|OOD_arXiv_2023|s0|p2": 0.1316213935613632, + "Wikipedia_en|OOD_arXiv_2023|s1|p0": 0.13659278377890588, + "Wikipedia_en|OOD_arXiv_2023|s1|p1": 0.1305436495691538, + "Wikipedia_en|OOD_arXiv_2023|s1|p2": 0.1355338167399168, + "Wikipedia_en|OOD_arXiv_2023|s2|p0": 0.13350272178649902, + "Wikipedia_en|OOD_arXiv_2023|s2|p1": 0.12782547026872634, + "Wikipedia_en|OOD_arXiv_2023|s2|p2": 0.1323194708675146, + "Wikipedia_en|OOD_CommonPile|s0|p0": 0.1261197067797184, + "Wikipedia_en|OOD_CommonPile|s0|p1": 0.1212854977697134, + "Wikipedia_en|OOD_CommonPile|s0|p2": 0.12230483815073967, + "Wikipedia_en|OOD_CommonPile|s1|p0": 0.12772722244262696, + "Wikipedia_en|OOD_CommonPile|s1|p1": 0.12077664285898208, + "Wikipedia_en|OOD_CommonPile|s1|p2": 0.12424149252474308, + "Wikipedia_en|OOD_CommonPile|s2|p0": 0.1265941672027111, + "Wikipedia_en|OOD_CommonPile|s2|p1": 0.12024476863443852, + "Wikipedia_en|OOD_CommonPile|s2|p2": 0.12263073474168777, + "USPTO_Backgrounds|Wikipedia_en|s0|p0": 0.15865903981029988, + "USPTO_Backgrounds|Wikipedia_en|s0|p1": 0.1594368640333414, + "USPTO_Backgrounds|Wikipedia_en|s0|p2": 0.1609773922711611, + "USPTO_Backgrounds|Wikipedia_en|s1|p0": 0.15415786467492582, + "USPTO_Backgrounds|Wikipedia_en|s1|p1": 0.15766826085746288, + "USPTO_Backgrounds|Wikipedia_en|s1|p2": 0.15936810337007046, + "USPTO_Backgrounds|Wikipedia_en|s2|p0": 0.15782200656831263, + "USPTO_Backgrounds|Wikipedia_en|s2|p1": 0.15970130153000356, + "USPTO_Backgrounds|Wikipedia_en|s2|p2": 0.1623216535896063, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p0": 0.1671406153589487, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p1": 0.16851548999547958, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p2": 0.16106511391699313, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p0": 0.16180277429521084, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p1": 0.16597656160593033, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p2": 0.15730201900005342, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p0": 0.16720139421522617, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p1": 0.1691258128732443, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p2": 0.16187019012868403, + "USPTO_Backgrounds|Pile-CC|s0|p0": 0.16361045688390732, + "USPTO_Backgrounds|Pile-CC|s0|p1": 0.1640078630298376, + "USPTO_Backgrounds|Pile-CC|s0|p2": 0.16455203406512736, + "USPTO_Backgrounds|Pile-CC|s1|p0": 0.16142426915466784, + "USPTO_Backgrounds|Pile-CC|s1|p1": 0.1605743058025837, + "USPTO_Backgrounds|Pile-CC|s1|p2": 0.1647999119013548, + "USPTO_Backgrounds|Pile-CC|s2|p0": 0.16487436927855015, + "USPTO_Backgrounds|Pile-CC|s2|p1": 0.16551438048481942, + "USPTO_Backgrounds|Pile-CC|s2|p2": 0.16598202399909495, + "USPTO_Backgrounds|StackExchange|s0|p0": 0.15085273757576942, + "USPTO_Backgrounds|StackExchange|s0|p1": 0.15416517574340105, + "USPTO_Backgrounds|StackExchange|s0|p2": 0.1587714310735464, + "USPTO_Backgrounds|StackExchange|s1|p0": 0.1481450531631708, + "USPTO_Backgrounds|StackExchange|s1|p1": 0.15227268766611815, + "USPTO_Backgrounds|StackExchange|s1|p2": 0.15382490456104278, + "USPTO_Backgrounds|StackExchange|s2|p0": 0.15232894271612168, + "USPTO_Backgrounds|StackExchange|s2|p1": 0.15657964777201414, + "USPTO_Backgrounds|StackExchange|s2|p2": 0.15948974154889584, + "USPTO_Backgrounds|Github|s0|p0": 0.1421130258589983, + "USPTO_Backgrounds|Github|s0|p1": 0.147469431348145, + "USPTO_Backgrounds|Github|s0|p2": 0.146794119104743, + "USPTO_Backgrounds|Github|s1|p0": 0.13769526965916157, + "USPTO_Backgrounds|Github|s1|p1": 0.14530146084725856, + "USPTO_Backgrounds|Github|s1|p2": 0.14440454207360745, + "USPTO_Backgrounds|Github|s2|p0": 0.14245902597904206, + "USPTO_Backgrounds|Github|s2|p1": 0.1488735280930996, + "USPTO_Backgrounds|Github|s2|p2": 0.1483356926590204, + "USPTO_Backgrounds|OOD_News_2024|s0|p0": 0.15932236909866332, + "USPTO_Backgrounds|OOD_News_2024|s0|p1": 0.15983906127512454, + "USPTO_Backgrounds|OOD_News_2024|s0|p2": 0.16681697480380536, + "USPTO_Backgrounds|OOD_News_2024|s1|p0": 0.15753016993403435, + "USPTO_Backgrounds|OOD_News_2024|s1|p1": 0.15698093734681606, + "USPTO_Backgrounds|OOD_News_2024|s1|p2": 0.16459277532994748, + "USPTO_Backgrounds|OOD_News_2024|s2|p0": 0.16166811920702456, + "USPTO_Backgrounds|OOD_News_2024|s2|p1": 0.16125984862446785, + "USPTO_Backgrounds|OOD_News_2024|s2|p2": 0.16720112152397631, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p0": 0.16320642046630382, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p1": 0.15523914210498332, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p2": 0.155629500746727, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p0": 0.15997118018567563, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p1": 0.15534372441470623, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p2": 0.15296751111745835, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p0": 0.1637862890958786, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p1": 0.15691512636840343, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p2": 0.1559907216578722, + "USPTO_Backgrounds|OOD_CommonPile|s0|p0": 0.14672414623200894, + "USPTO_Backgrounds|OOD_CommonPile|s0|p1": 0.14162651263177395, + "USPTO_Backgrounds|OOD_CommonPile|s0|p2": 0.1442489769309759, + "USPTO_Backgrounds|OOD_CommonPile|s1|p0": 0.1448067259043455, + "USPTO_Backgrounds|OOD_CommonPile|s1|p1": 0.14138467870652677, + "USPTO_Backgrounds|OOD_CommonPile|s1|p2": 0.1413637984544039, + "USPTO_Backgrounds|OOD_CommonPile|s2|p0": 0.1485120017081499, + "USPTO_Backgrounds|OOD_CommonPile|s2|p1": 0.14268824607133865, + "USPTO_Backgrounds|OOD_CommonPile|s2|p2": 0.14339683316648005, + "Pile-CC|Wikipedia_en|s0|p0": 0.1437114614993334, + "Pile-CC|Wikipedia_en|s0|p1": 0.14558811001479627, + "Pile-CC|Wikipedia_en|s0|p2": 0.14785966239869594, + "Pile-CC|Wikipedia_en|s1|p0": 0.14410060048103332, + "Pile-CC|Wikipedia_en|s1|p1": 0.14739834032952787, + "Pile-CC|Wikipedia_en|s1|p2": 0.14800133295357226, + "Pile-CC|Wikipedia_en|s2|p0": 0.1427424516528845, + "Pile-CC|Wikipedia_en|s2|p1": 0.14626773111522198, + "Pile-CC|Wikipedia_en|s2|p2": 0.14542454518377781, + "Pile-CC|USPTO_Backgrounds|s0|p0": 0.15401250682771206, + "Pile-CC|USPTO_Backgrounds|s0|p1": 0.15758568942546844, + "Pile-CC|USPTO_Backgrounds|s0|p2": 0.1496773984283209, + "Pile-CC|USPTO_Backgrounds|s1|p0": 0.15556987784802914, + "Pile-CC|USPTO_Backgrounds|s1|p1": 0.15879758335649968, + "Pile-CC|USPTO_Backgrounds|s1|p2": 0.1502711545675993, + "Pile-CC|USPTO_Backgrounds|s2|p0": 0.15251148380339147, + "Pile-CC|USPTO_Backgrounds|s2|p1": 0.1557144198566675, + "Pile-CC|USPTO_Backgrounds|s2|p2": 0.14967280812561512, + "Pile-CC|Pile-CC|s0|p0": 0.1475800473242998, + "Pile-CC|Pile-CC|s0|p1": 0.14771839492022992, + "Pile-CC|Pile-CC|s0|p2": 0.14847222156822681, + "Pile-CC|Pile-CC|s1|p0": 0.14855674095451832, + "Pile-CC|Pile-CC|s1|p1": 0.15146477073431014, + "Pile-CC|Pile-CC|s1|p2": 0.14917959608137607, + "Pile-CC|Pile-CC|s2|p0": 0.1471522107720375, + "Pile-CC|Pile-CC|s2|p1": 0.1469940420240164, + "Pile-CC|Pile-CC|s2|p2": 0.14881067536771297, + "Pile-CC|StackExchange|s0|p0": 0.13556620739400388, + "Pile-CC|StackExchange|s0|p1": 0.13278575874865056, + "Pile-CC|StackExchange|s0|p2": 0.13652288839221, + "Pile-CC|StackExchange|s1|p0": 0.1346248684450984, + "Pile-CC|StackExchange|s1|p1": 0.13341133035719394, + "Pile-CC|StackExchange|s1|p2": 0.1373356692492962, + "Pile-CC|StackExchange|s2|p0": 0.13374655060470103, + "Pile-CC|StackExchange|s2|p1": 0.13112555183470248, + "Pile-CC|StackExchange|s2|p2": 0.13448646664619446, + "Pile-CC|Github|s0|p0": 0.1305080521851778, + "Pile-CC|Github|s0|p1": 0.13364531621336936, + "Pile-CC|Github|s0|p2": 0.13335430733859538, + "Pile-CC|Github|s1|p0": 0.13293832615017892, + "Pile-CC|Github|s1|p1": 0.13471500761806965, + "Pile-CC|Github|s1|p2": 0.1331347491592169, + "Pile-CC|Github|s2|p0": 0.12963986694812774, + "Pile-CC|Github|s2|p1": 0.13312407173216342, + "Pile-CC|Github|s2|p2": 0.13198942728340626, + "Pile-CC|OOD_News_2024|s0|p0": 0.14884910471737384, + "Pile-CC|OOD_News_2024|s0|p1": 0.15050788782536983, + "Pile-CC|OOD_News_2024|s0|p2": 0.15228801667690278, + "Pile-CC|OOD_News_2024|s1|p0": 0.15080181807279586, + "Pile-CC|OOD_News_2024|s1|p1": 0.1518170926719904, + "Pile-CC|OOD_News_2024|s1|p2": 0.15367178209125995, + "Pile-CC|OOD_News_2024|s2|p0": 0.1478659722954035, + "Pile-CC|OOD_News_2024|s2|p1": 0.14945964254438876, + "Pile-CC|OOD_News_2024|s2|p2": 0.1500181421637535, + "Pile-CC|OOD_arXiv_2023|s0|p0": 0.14775749184191228, + "Pile-CC|OOD_arXiv_2023|s0|p1": 0.14260655641555786, + "Pile-CC|OOD_arXiv_2023|s0|p2": 0.14545965380966663, + "Pile-CC|OOD_arXiv_2023|s1|p0": 0.14929868541657926, + "Pile-CC|OOD_arXiv_2023|s1|p1": 0.14477681517601013, + "Pile-CC|OOD_arXiv_2023|s1|p2": 0.14658121354877948, + "Pile-CC|OOD_arXiv_2023|s2|p0": 0.1471331473439932, + "Pile-CC|OOD_arXiv_2023|s2|p1": 0.14129521697759628, + "Pile-CC|OOD_arXiv_2023|s2|p2": 0.1434501815587282, + "Pile-CC|OOD_CommonPile|s0|p0": 0.1442362952977419, + "Pile-CC|OOD_CommonPile|s0|p1": 0.13856977000832557, + "Pile-CC|OOD_CommonPile|s0|p2": 0.14069531448185443, + "Pile-CC|OOD_CommonPile|s1|p0": 0.1466800395399332, + "Pile-CC|OOD_CommonPile|s1|p1": 0.14128039814531804, + "Pile-CC|OOD_CommonPile|s1|p2": 0.14194846339523792, + "Pile-CC|OOD_CommonPile|s2|p0": 0.14276791028678418, + "Pile-CC|OOD_CommonPile|s2|p1": 0.13801012188196182, + "Pile-CC|OOD_CommonPile|s2|p2": 0.13852601423859595, + "StackExchange|Wikipedia_en|s0|p0": 0.14449691511690615, + "StackExchange|Wikipedia_en|s0|p1": 0.1468203667551279, + "StackExchange|Wikipedia_en|s0|p2": 0.1488727930933237, + "StackExchange|Wikipedia_en|s1|p0": 0.1423150572925806, + "StackExchange|Wikipedia_en|s1|p1": 0.14355442933738233, + "StackExchange|Wikipedia_en|s1|p2": 0.14513694643974304, + "StackExchange|Wikipedia_en|s2|p0": 0.14236885644495487, + "StackExchange|Wikipedia_en|s2|p1": 0.1439858816564083, + "StackExchange|Wikipedia_en|s2|p2": 0.14457972906529903, + "StackExchange|USPTO_Backgrounds|s0|p0": 0.15522875748574733, + "StackExchange|USPTO_Backgrounds|s0|p1": 0.16507935300469398, + "StackExchange|USPTO_Backgrounds|s0|p2": 0.15315595604479312, + "StackExchange|USPTO_Backgrounds|s1|p0": 0.15408372245728968, + "StackExchange|USPTO_Backgrounds|s1|p1": 0.16125575564801692, + "StackExchange|USPTO_Backgrounds|s1|p2": 0.1479051813483238, + "StackExchange|USPTO_Backgrounds|s2|p0": 0.15220016706734896, + "StackExchange|USPTO_Backgrounds|s2|p1": 0.1606854759156704, + "StackExchange|USPTO_Backgrounds|s2|p2": 0.14806856364011764, + "StackExchange|Pile-CC|s0|p0": 0.15160226207226515, + "StackExchange|Pile-CC|s0|p1": 0.14857019782066344, + "StackExchange|Pile-CC|s0|p2": 0.15357961356639863, + "StackExchange|Pile-CC|s1|p0": 0.1496371330693364, + "StackExchange|Pile-CC|s1|p1": 0.14715286158025265, + "StackExchange|Pile-CC|s1|p2": 0.15260775089263917, + "StackExchange|Pile-CC|s2|p0": 0.14967973362654446, + "StackExchange|Pile-CC|s2|p1": 0.14599973596632482, + "StackExchange|Pile-CC|s2|p2": 0.15104596503078938, + "StackExchange|StackExchange|s0|p0": 0.13835120368748904, + "StackExchange|StackExchange|s0|p1": 0.13688320778310298, + "StackExchange|StackExchange|s0|p2": 0.1421434234827757, + "StackExchange|StackExchange|s1|p0": 0.1381283663213253, + "StackExchange|StackExchange|s1|p1": 0.1335654504597187, + "StackExchange|StackExchange|s1|p2": 0.13818975389003754, + "StackExchange|StackExchange|s2|p0": 0.13719860482960938, + "StackExchange|StackExchange|s2|p1": 0.13495246432721614, + "StackExchange|StackExchange|s2|p2": 0.14055598713457584, + "StackExchange|Github|s0|p0": 0.13823740370571613, + "StackExchange|Github|s0|p1": 0.13817918933928014, + "StackExchange|Github|s0|p2": 0.13669187240302563, + "StackExchange|Github|s1|p0": 0.13611015789210795, + "StackExchange|Github|s1|p1": 0.13541071433573962, + "StackExchange|Github|s1|p2": 0.1327636856585741, + "StackExchange|Github|s2|p0": 0.13709317557513714, + "StackExchange|Github|s2|p1": 0.13629858810454606, + "StackExchange|Github|s2|p2": 0.13514489568769933, + "StackExchange|OOD_News_2024|s0|p0": 0.14986481182277203, + "StackExchange|OOD_News_2024|s0|p1": 0.14892755225300788, + "StackExchange|OOD_News_2024|s0|p2": 0.15148025266826154, + "StackExchange|OOD_News_2024|s1|p0": 0.14734542705118656, + "StackExchange|OOD_News_2024|s1|p1": 0.1474456511437893, + "StackExchange|OOD_News_2024|s1|p2": 0.14920851327478885, + "StackExchange|OOD_News_2024|s2|p0": 0.1463893663138151, + "StackExchange|OOD_News_2024|s2|p1": 0.14626411013305188, + "StackExchange|OOD_News_2024|s2|p2": 0.1473880462348461, + "StackExchange|OOD_arXiv_2023|s0|p0": 0.15725914128124713, + "StackExchange|OOD_arXiv_2023|s0|p1": 0.15396856926381589, + "StackExchange|OOD_arXiv_2023|s0|p2": 0.15177230201661587, + "StackExchange|OOD_arXiv_2023|s1|p0": 0.15321973338723183, + "StackExchange|OOD_arXiv_2023|s1|p1": 0.14806342273950576, + "StackExchange|OOD_arXiv_2023|s1|p2": 0.1484540406614542, + "StackExchange|OOD_arXiv_2023|s2|p0": 0.15185540691018104, + "StackExchange|OOD_arXiv_2023|s2|p1": 0.15159291997551919, + "StackExchange|OOD_arXiv_2023|s2|p2": 0.14798282347619535, + "StackExchange|OOD_CommonPile|s0|p0": 0.15002845413982868, + "StackExchange|OOD_CommonPile|s0|p1": 0.14691657945513725, + "StackExchange|OOD_CommonPile|s0|p2": 0.14598479345440865, + "StackExchange|OOD_CommonPile|s1|p0": 0.14705484099686145, + "StackExchange|OOD_CommonPile|s1|p1": 0.14468607157468796, + "StackExchange|OOD_CommonPile|s1|p2": 0.14296981319785118, + "StackExchange|OOD_CommonPile|s2|p0": 0.14789607264101506, + "StackExchange|OOD_CommonPile|s2|p1": 0.14392626881599427, + "StackExchange|OOD_CommonPile|s2|p2": 0.143027351424098, + "Github|Wikipedia_en|s0|p0": 0.10472181923687458, + "Github|Wikipedia_en|s0|p1": 0.11021871082484722, + "Github|Wikipedia_en|s0|p2": 0.10458091050386428, + "Github|Wikipedia_en|s1|p0": 0.10628068111836911, + "Github|Wikipedia_en|s1|p1": 0.11017489619553089, + "Github|Wikipedia_en|s1|p2": 0.10822749547660351, + "Github|Wikipedia_en|s2|p0": 0.10841156356036663, + "Github|Wikipedia_en|s2|p1": 0.11178305633366108, + "Github|Wikipedia_en|s2|p2": 0.10895942114293575, + "Github|USPTO_Backgrounds|s0|p0": 0.12151174955070018, + "Github|USPTO_Backgrounds|s0|p1": 0.11832044459879398, + "Github|USPTO_Backgrounds|s0|p2": 0.11612880863249302, + "Github|USPTO_Backgrounds|s1|p0": 0.12393109649419784, + "Github|USPTO_Backgrounds|s1|p1": 0.11937098428606988, + "Github|USPTO_Backgrounds|s1|p2": 0.11847091130912304, + "Github|USPTO_Backgrounds|s2|p0": 0.12509081289172172, + "Github|USPTO_Backgrounds|s2|p1": 0.11961253806948662, + "Github|USPTO_Backgrounds|s2|p2": 0.11760500110685826, + "Github|Pile-CC|s0|p0": 0.11522790454328061, + "Github|Pile-CC|s0|p1": 0.11571162529289722, + "Github|Pile-CC|s0|p2": 0.11930152587592602, + "Github|Pile-CC|s1|p0": 0.1158499527722597, + "Github|Pile-CC|s1|p1": 0.11869329996407033, + "Github|Pile-CC|s1|p2": 0.11982139758765697, + "Github|Pile-CC|s2|p0": 0.11712507717311382, + "Github|Pile-CC|s2|p1": 0.11924890838563443, + "Github|Pile-CC|s2|p2": 0.12071709111332893, + "Github|StackExchange|s0|p0": 0.10925496369600296, + "Github|StackExchange|s0|p1": 0.10768789723515511, + "Github|StackExchange|s0|p2": 0.10849158652126789, + "Github|StackExchange|s1|p0": 0.1103823397308588, + "Github|StackExchange|s1|p1": 0.10917473770678043, + "Github|StackExchange|s1|p2": 0.1093454796820879, + "Github|StackExchange|s2|p0": 0.11207031235098838, + "Github|StackExchange|s2|p1": 0.1119774304330349, + "Github|StackExchange|s2|p2": 0.10991804748773575, + "Github|Github|s0|p0": 0.10624762140214443, + "Github|Github|s0|p1": 0.1059819731861353, + "Github|Github|s0|p2": 0.1047787208110094, + "Github|Github|s1|p0": 0.10747543163597584, + "Github|Github|s1|p1": 0.10910154320299625, + "Github|Github|s1|p2": 0.10684436745941639, + "Github|Github|s2|p0": 0.10742016844451427, + "Github|Github|s2|p1": 0.10812124125659466, + "Github|Github|s2|p2": 0.1071061585098505, + "Github|OOD_News_2024|s0|p0": 0.11219368688762188, + "Github|OOD_News_2024|s0|p1": 0.11500626541674137, + "Github|OOD_News_2024|s0|p2": 0.11685916595160961, + "Github|OOD_News_2024|s1|p0": 0.11386775523424149, + "Github|OOD_News_2024|s1|p1": 0.11642411835491658, + "Github|OOD_News_2024|s1|p2": 0.11719844676554203, + "Github|OOD_News_2024|s2|p0": 0.11440489366650582, + "Github|OOD_News_2024|s2|p1": 0.11527235694229603, + "Github|OOD_News_2024|s2|p2": 0.1176420260220766, + "Github|OOD_arXiv_2023|s0|p0": 0.1124246172606945, + "Github|OOD_arXiv_2023|s0|p1": 0.10736321471631527, + "Github|OOD_arXiv_2023|s0|p2": 0.11215240247547627, + "Github|OOD_arXiv_2023|s1|p0": 0.11539147906005383, + "Github|OOD_arXiv_2023|s1|p1": 0.11026903651654721, + "Github|OOD_arXiv_2023|s1|p2": 0.11371447890996933, + "Github|OOD_arXiv_2023|s2|p0": 0.11422904171049594, + "Github|OOD_arXiv_2023|s2|p1": 0.10990415550768376, + "Github|OOD_arXiv_2023|s2|p2": 0.11350584365427493, + "Github|OOD_CommonPile|s0|p0": 0.11138113737106323, + "Github|OOD_CommonPile|s0|p1": 0.11169484369456768, + "Github|OOD_CommonPile|s0|p2": 0.11102768294513225, + "Github|OOD_CommonPile|s1|p0": 0.1136026319116354, + "Github|OOD_CommonPile|s1|p1": 0.11297967918217182, + "Github|OOD_CommonPile|s1|p2": 0.11418475843966007, + "Github|OOD_CommonPile|s2|p0": 0.1131815429776907, + "Github|OOD_CommonPile|s2|p1": 0.11611624620854855, + "Github|OOD_CommonPile|s2|p2": 0.11451728083193302, + "OOD_News_2024|Wikipedia_en|s0|p0": 0.13486163765192033, + "OOD_News_2024|Wikipedia_en|s0|p1": 0.1364224761724472, + "OOD_News_2024|Wikipedia_en|s0|p2": 0.13764243498444556, + "OOD_News_2024|Wikipedia_en|s1|p0": 0.1336891043931246, + "OOD_News_2024|Wikipedia_en|s1|p1": 0.13594831079244613, + "OOD_News_2024|Wikipedia_en|s1|p2": 0.13840426802635192, + "OOD_News_2024|Wikipedia_en|s2|p0": 0.13516195863485336, + "OOD_News_2024|Wikipedia_en|s2|p1": 0.136974174156785, + "OOD_News_2024|Wikipedia_en|s2|p2": 0.13865821361541747, + "OOD_News_2024|USPTO_Backgrounds|s0|p0": 0.14506222382187844, + "OOD_News_2024|USPTO_Backgrounds|s0|p1": 0.14526550993323326, + "OOD_News_2024|USPTO_Backgrounds|s0|p2": 0.13911073319613934, + "OOD_News_2024|USPTO_Backgrounds|s1|p0": 0.146680273860693, + "OOD_News_2024|USPTO_Backgrounds|s1|p1": 0.14634087346494198, + "OOD_News_2024|USPTO_Backgrounds|s1|p2": 0.13987445943057536, + "OOD_News_2024|USPTO_Backgrounds|s2|p0": 0.14899541437625885, + "OOD_News_2024|USPTO_Backgrounds|s2|p1": 0.14674318358302116, + "OOD_News_2024|USPTO_Backgrounds|s2|p2": 0.1395602460950613, + "OOD_News_2024|Pile-CC|s0|p0": 0.14070868007838727, + "OOD_News_2024|Pile-CC|s0|p1": 0.14009961038827895, + "OOD_News_2024|Pile-CC|s0|p2": 0.1449163757264614, + "OOD_News_2024|Pile-CC|s1|p0": 0.14202719666063784, + "OOD_News_2024|Pile-CC|s1|p1": 0.14143850281834602, + "OOD_News_2024|Pile-CC|s1|p2": 0.14566070213913918, + "OOD_News_2024|Pile-CC|s2|p0": 0.14384278729557992, + "OOD_News_2024|Pile-CC|s2|p1": 0.14112119302153586, + "OOD_News_2024|Pile-CC|s2|p2": 0.14572295621037484, + "OOD_News_2024|StackExchange|s0|p0": 0.12112892381846904, + "OOD_News_2024|StackExchange|s0|p1": 0.11762128211557865, + "OOD_News_2024|StackExchange|s0|p2": 0.12067867033183574, + "OOD_News_2024|StackExchange|s1|p0": 0.12376918159425258, + "OOD_News_2024|StackExchange|s1|p1": 0.11939822919666768, + "OOD_News_2024|StackExchange|s1|p2": 0.12282479219138623, + "OOD_News_2024|StackExchange|s2|p0": 0.12256133146584033, + "OOD_News_2024|StackExchange|s2|p1": 0.11773126386106014, + "OOD_News_2024|StackExchange|s2|p2": 0.12148497365415097, + "OOD_News_2024|Github|s0|p0": 0.12191336899995804, + "OOD_News_2024|Github|s0|p1": 0.12709311842918397, + "OOD_News_2024|Github|s0|p2": 0.12368766739964485, + "OOD_News_2024|Github|s1|p0": 0.12076359763741493, + "OOD_News_2024|Github|s1|p1": 0.1275537062436342, + "OOD_News_2024|Github|s1|p2": 0.12567985206842422, + "OOD_News_2024|Github|s2|p0": 0.12257032841444016, + "OOD_News_2024|Github|s2|p1": 0.12789174802601339, + "OOD_News_2024|Github|s2|p2": 0.1258739374577999, + "OOD_News_2024|OOD_News_2024|s0|p0": 0.1387073550373316, + "OOD_News_2024|OOD_News_2024|s0|p1": 0.1434318669140339, + "OOD_News_2024|OOD_News_2024|s0|p2": 0.14456919133663176, + "OOD_News_2024|OOD_News_2024|s1|p0": 0.14054211862385274, + "OOD_News_2024|OOD_News_2024|s1|p1": 0.1423490047454834, + "OOD_News_2024|OOD_News_2024|s1|p2": 0.14308991767466067, + "OOD_News_2024|OOD_News_2024|s2|p0": 0.14106240570545198, + "OOD_News_2024|OOD_News_2024|s2|p1": 0.14487843662500383, + "OOD_News_2024|OOD_News_2024|s2|p2": 0.14517133496701717, + "OOD_News_2024|OOD_arXiv_2023|s0|p0": 0.1414489448070526, + "OOD_News_2024|OOD_arXiv_2023|s0|p1": 0.13550705835223198, + "OOD_News_2024|OOD_arXiv_2023|s0|p2": 0.13826472461223602, + "OOD_News_2024|OOD_arXiv_2023|s1|p0": 0.14253144674003124, + "OOD_News_2024|OOD_arXiv_2023|s1|p1": 0.1375824362039566, + "OOD_News_2024|OOD_arXiv_2023|s1|p2": 0.13953054323792458, + "OOD_News_2024|OOD_arXiv_2023|s2|p0": 0.14292933754622936, + "OOD_News_2024|OOD_arXiv_2023|s2|p1": 0.13888718336820602, + "OOD_News_2024|OOD_arXiv_2023|s2|p2": 0.13899042904376985, + "OOD_News_2024|OOD_CommonPile|s0|p0": 0.13256734609603882, + "OOD_News_2024|OOD_CommonPile|s0|p1": 0.1276464655995369, + "OOD_News_2024|OOD_CommonPile|s0|p2": 0.12678277269005775, + "OOD_News_2024|OOD_CommonPile|s1|p0": 0.1356629304587841, + "OOD_News_2024|OOD_CommonPile|s1|p1": 0.12904721051454543, + "OOD_News_2024|OOD_CommonPile|s1|p2": 0.12944178767502307, + "OOD_News_2024|OOD_CommonPile|s2|p0": 0.13436706215143204, + "OOD_News_2024|OOD_CommonPile|s2|p1": 0.12855805978178977, + "OOD_News_2024|OOD_CommonPile|s2|p2": 0.12856159768998623, + "OOD_arXiv_2023|Wikipedia_en|s0|p0": 0.1428018994629383, + "OOD_arXiv_2023|Wikipedia_en|s0|p1": 0.14040045514702798, + "OOD_arXiv_2023|Wikipedia_en|s0|p2": 0.1454609576612711, + "OOD_arXiv_2023|Wikipedia_en|s1|p0": 0.14358339309692383, + "OOD_arXiv_2023|Wikipedia_en|s1|p1": 0.14175953082740306, + "OOD_arXiv_2023|Wikipedia_en|s1|p2": 0.14634908474981784, + "OOD_arXiv_2023|Wikipedia_en|s2|p0": 0.14248228520154954, + "OOD_arXiv_2023|Wikipedia_en|s2|p1": 0.14210982508957387, + "OOD_arXiv_2023|Wikipedia_en|s2|p2": 0.14514244236052037, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p0": 0.15357542578130962, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p1": 0.1510006608441472, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p2": 0.14225321281701325, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p0": 0.15326165203005077, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p1": 0.15135192070156336, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p2": 0.14344881046563387, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p0": 0.15479327142238616, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p1": 0.15056663602590561, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p2": 0.14360985029488801, + "OOD_arXiv_2023|Pile-CC|s0|p0": 0.14826207980513573, + "OOD_arXiv_2023|Pile-CC|s0|p1": 0.14777991697192192, + "OOD_arXiv_2023|Pile-CC|s0|p2": 0.14704287927597762, + "OOD_arXiv_2023|Pile-CC|s1|p0": 0.14807742908596994, + "OOD_arXiv_2023|Pile-CC|s1|p1": 0.14693760648369789, + "OOD_arXiv_2023|Pile-CC|s1|p2": 0.14893653616309166, + "OOD_arXiv_2023|Pile-CC|s2|p0": 0.14762210324406624, + "OOD_arXiv_2023|Pile-CC|s2|p1": 0.14713412448763846, + "OOD_arXiv_2023|Pile-CC|s2|p2": 0.14867167100310325, + "OOD_arXiv_2023|StackExchange|s0|p0": 0.1350220860913396, + "OOD_arXiv_2023|StackExchange|s0|p1": 0.1340720908716321, + "OOD_arXiv_2023|StackExchange|s0|p2": 0.1402300363406539, + "OOD_arXiv_2023|StackExchange|s1|p0": 0.13420782294124364, + "OOD_arXiv_2023|StackExchange|s1|p1": 0.13373161274939777, + "OOD_arXiv_2023|StackExchange|s1|p2": 0.1396141702309251, + "OOD_arXiv_2023|StackExchange|s2|p0": 0.13533333148807286, + "OOD_arXiv_2023|StackExchange|s2|p1": 0.1333851983770728, + "OOD_arXiv_2023|StackExchange|s2|p2": 0.14080181028693914, + "OOD_arXiv_2023|Github|s0|p0": 0.12196633070707322, + "OOD_arXiv_2023|Github|s0|p1": 0.1307462839409709, + "OOD_arXiv_2023|Github|s0|p2": 0.129108851775527, + "OOD_arXiv_2023|Github|s1|p0": 0.12099339216947555, + "OOD_arXiv_2023|Github|s1|p1": 0.12947062086313962, + "OOD_arXiv_2023|Github|s1|p2": 0.12685625925660132, + "OOD_arXiv_2023|Github|s2|p0": 0.12173797562718391, + "OOD_arXiv_2023|Github|s2|p1": 0.13024121206253766, + "OOD_arXiv_2023|Github|s2|p2": 0.1303920727223158, + "OOD_arXiv_2023|OOD_News_2024|s0|p0": 0.1436644211411476, + "OOD_arXiv_2023|OOD_News_2024|s0|p1": 0.14571815356612206, + "OOD_arXiv_2023|OOD_News_2024|s0|p2": 0.1497436098754406, + "OOD_arXiv_2023|OOD_News_2024|s1|p0": 0.14410990849137306, + "OOD_arXiv_2023|OOD_News_2024|s1|p1": 0.1442332945764065, + "OOD_arXiv_2023|OOD_News_2024|s1|p2": 0.1497814826667309, + "OOD_arXiv_2023|OOD_News_2024|s2|p0": 0.14349039569497107, + "OOD_arXiv_2023|OOD_News_2024|s2|p1": 0.14413045570254326, + "OOD_arXiv_2023|OOD_News_2024|s2|p2": 0.15093232430517672, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p0": 0.14870578423142433, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p1": 0.14305446892976761, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p2": 0.14268340300768614, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p0": 0.14820004478096963, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p1": 0.1421566367149353, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p2": 0.1423545114696026, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p0": 0.14980459287762643, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p1": 0.14230584353208542, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p2": 0.14217254798859358, + "OOD_arXiv_2023|OOD_CommonPile|s0|p0": 0.12943503260612488, + "OOD_arXiv_2023|OOD_CommonPile|s0|p1": 0.13154563196003438, + "OOD_arXiv_2023|OOD_CommonPile|s0|p2": 0.12972614988684655, + "OOD_arXiv_2023|OOD_CommonPile|s1|p0": 0.12850606199353934, + "OOD_arXiv_2023|OOD_CommonPile|s1|p1": 0.130877897515893, + "OOD_arXiv_2023|OOD_CommonPile|s1|p2": 0.12842097394168378, + "OOD_arXiv_2023|OOD_CommonPile|s2|p0": 0.12821140550076962, + "OOD_arXiv_2023|OOD_CommonPile|s2|p1": 0.13017832338809968, + "OOD_arXiv_2023|OOD_CommonPile|s2|p2": 0.12638943903148175, + "OOD_CommonPile|Wikipedia_en|s0|p0": 0.139744308963418, + "OOD_CommonPile|Wikipedia_en|s0|p1": 0.14106009677052497, + "OOD_CommonPile|Wikipedia_en|s0|p2": 0.13940333388745785, + "OOD_CommonPile|Wikipedia_en|s1|p0": 0.14246975108981133, + "OOD_CommonPile|Wikipedia_en|s1|p1": 0.1426856704056263, + "OOD_CommonPile|Wikipedia_en|s1|p2": 0.1423924509435892, + "OOD_CommonPile|Wikipedia_en|s2|p0": 0.13969158455729486, + "OOD_CommonPile|Wikipedia_en|s2|p1": 0.14126862064003945, + "OOD_CommonPile|Wikipedia_en|s2|p2": 0.13894762732088567, + "OOD_CommonPile|USPTO_Backgrounds|s0|p0": 0.14983783718198537, + "OOD_CommonPile|USPTO_Backgrounds|s0|p1": 0.14970208685845138, + "OOD_CommonPile|USPTO_Backgrounds|s0|p2": 0.14276783652603625, + "OOD_CommonPile|USPTO_Backgrounds|s1|p0": 0.15254374835640191, + "OOD_CommonPile|USPTO_Backgrounds|s1|p1": 0.1527953764423728, + "OOD_CommonPile|USPTO_Backgrounds|s1|p2": 0.1465577518567443, + "OOD_CommonPile|USPTO_Backgrounds|s2|p0": 0.14851349797099828, + "OOD_CommonPile|USPTO_Backgrounds|s2|p1": 0.14697987604886292, + "OOD_CommonPile|USPTO_Backgrounds|s2|p2": 0.14291422087699174, + "OOD_CommonPile|Pile-CC|s0|p0": 0.14710925593972207, + "OOD_CommonPile|Pile-CC|s0|p1": 0.14605959840118884, + "OOD_CommonPile|Pile-CC|s0|p2": 0.1486803473904729, + "OOD_CommonPile|Pile-CC|s1|p0": 0.14828060362488032, + "OOD_CommonPile|Pile-CC|s1|p1": 0.14762085005640985, + "OOD_CommonPile|Pile-CC|s1|p2": 0.15118215847760438, + "OOD_CommonPile|Pile-CC|s2|p0": 0.146185109205544, + "OOD_CommonPile|Pile-CC|s2|p1": 0.14593285396695138, + "OOD_CommonPile|Pile-CC|s2|p2": 0.14749929625540972, + "OOD_CommonPile|StackExchange|s0|p0": 0.13769646547734737, + "OOD_CommonPile|StackExchange|s0|p1": 0.13425580598413944, + "OOD_CommonPile|StackExchange|s0|p2": 0.13896569330245256, + "OOD_CommonPile|StackExchange|s1|p0": 0.13924720752984285, + "OOD_CommonPile|StackExchange|s1|p1": 0.1362179521471262, + "OOD_CommonPile|StackExchange|s1|p2": 0.1419050382450223, + "OOD_CommonPile|StackExchange|s2|p0": 0.1358552198857069, + "OOD_CommonPile|StackExchange|s2|p1": 0.13534963615238665, + "OOD_CommonPile|StackExchange|s2|p2": 0.1377167023718357, + "OOD_CommonPile|Github|s0|p0": 0.12233414202928543, + "OOD_CommonPile|Github|s0|p1": 0.13115269783884287, + "OOD_CommonPile|Github|s0|p2": 0.12898012734949588, + "OOD_CommonPile|Github|s1|p0": 0.12387640923261642, + "OOD_CommonPile|Github|s1|p1": 0.1318666733801365, + "OOD_CommonPile|Github|s1|p2": 0.13118611834943295, + "OOD_CommonPile|Github|s2|p0": 0.12081060074269771, + "OOD_CommonPile|Github|s2|p1": 0.13013178948312998, + "OOD_CommonPile|Github|s2|p2": 0.12764853835105897, + "OOD_CommonPile|OOD_News_2024|s0|p0": 0.14499912038445473, + "OOD_CommonPile|OOD_News_2024|s0|p1": 0.14552821181714534, + "OOD_CommonPile|OOD_News_2024|s0|p2": 0.15087285302579403, + "OOD_CommonPile|OOD_News_2024|s1|p0": 0.1453522689640522, + "OOD_CommonPile|OOD_News_2024|s1|p1": 0.1469458170235157, + "OOD_CommonPile|OOD_News_2024|s1|p2": 0.15380928292870522, + "OOD_CommonPile|OOD_News_2024|s2|p0": 0.14182413201779126, + "OOD_CommonPile|OOD_News_2024|s2|p1": 0.14482717141509055, + "OOD_CommonPile|OOD_News_2024|s2|p2": 0.14878604114055632, + "OOD_CommonPile|OOD_arXiv_2023|s0|p0": 0.14463903680443763, + "OOD_CommonPile|OOD_arXiv_2023|s0|p1": 0.13837069198489188, + "OOD_CommonPile|OOD_arXiv_2023|s0|p2": 0.13659614697098732, + "OOD_CommonPile|OOD_arXiv_2023|s1|p0": 0.1479431388899684, + "OOD_CommonPile|OOD_arXiv_2023|s1|p1": 0.139229029789567, + "OOD_CommonPile|OOD_arXiv_2023|s1|p2": 0.13885941188782452, + "OOD_CommonPile|OOD_arXiv_2023|s2|p0": 0.1438038794323802, + "OOD_CommonPile|OOD_arXiv_2023|s2|p1": 0.13630836196243762, + "OOD_CommonPile|OOD_arXiv_2023|s2|p2": 0.13602422624826432, + "OOD_CommonPile|OOD_CommonPile|s0|p0": 0.12909803725779057, + "OOD_CommonPile|OOD_CommonPile|s0|p1": 0.1318016305565834, + "OOD_CommonPile|OOD_CommonPile|s0|p2": 0.12763514034450055, + "OOD_CommonPile|OOD_CommonPile|s1|p0": 0.13201464228332044, + "OOD_CommonPile|OOD_CommonPile|s1|p1": 0.1352139636874199, + "OOD_CommonPile|OOD_CommonPile|s1|p2": 0.12987643703818322, + "OOD_CommonPile|OOD_CommonPile|s2|p0": 0.1279674131423235, + "OOD_CommonPile|OOD_CommonPile|s2|p1": 0.13204800486564636, + "OOD_CommonPile|OOD_CommonPile|s2|p2": 0.1282028838992119, + "logit|Q0|p0": 0.1247396256774664, + "shuf|Q0|p0": 0.1422879509627819, + "logit|Q0|p1": 0.1247396256774664, + "shuf|Q0|p1": 0.1422879509627819, + "logit|Q0|p2": 0.1247396256774664, + "shuf|Q0|p2": 0.1422879509627819, + "logit|Wikipedia_en|p0": 0.12441348172724247, + "shuf|Wikipedia_en|p0": 0.13555061146616937, + "logit|Wikipedia_en|p1": 0.12603520192205905, + "shuf|Wikipedia_en|p1": 0.13675951808691025, + "logit|Wikipedia_en|p2": 0.12318613156676292, + "shuf|Wikipedia_en|p2": 0.13697959780693053, + "logit|USPTO_Backgrounds|p0": 0.12288859896361828, + "shuf|USPTO_Backgrounds|p0": 0.1432977557182312, + "logit|USPTO_Backgrounds|p1": 0.12467962317168713, + "shuf|USPTO_Backgrounds|p1": 0.14704293757677078, + "logit|USPTO_Backgrounds|p2": 0.12205531150102615, + "shuf|USPTO_Backgrounds|p2": 0.13966306820511817, + "logit|Pile-CC|p0": 0.1294180490076542, + "shuf|Pile-CC|p0": 0.14222755655646324, + "logit|Pile-CC|p1": 0.12919799163937568, + "shuf|Pile-CC|p1": 0.13949708864092827, + "logit|Pile-CC|p2": 0.12970054894685745, + "shuf|Pile-CC|p2": 0.13793092966079712, + "logit|StackExchange|p0": 0.12327122092247009, + "shuf|StackExchange|p0": 0.13950061425566673, + "logit|StackExchange|p1": 0.12136632464826107, + "shuf|StackExchange|p1": 0.13919788375496864, + "logit|StackExchange|p2": 0.12554268538951874, + "shuf|StackExchange|p2": 0.14503461569547654, + "logit|Github|p0": 0.12177085354924203, + "shuf|Github|p0": 0.12981337755918504, + "logit|Github|p1": 0.12349991276860237, + "shuf|Github|p1": 0.13674261644482613, + "logit|Github|p2": 0.11834828741848469, + "shuf|Github|p2": 0.13636190369725226, + "logit|OOD_News_2024|p0": 0.13004022762179374, + "shuf|OOD_News_2024|p0": 0.13637514486908914, + "logit|OOD_News_2024|p1": 0.13050479404628276, + "shuf|OOD_News_2024|p1": 0.13769909664988517, + "logit|OOD_News_2024|p2": 0.13118069656193257, + "shuf|OOD_News_2024|p2": 0.14654124304652213, + "logit|OOD_arXiv_2023|p0": 0.1273316565901041, + "shuf|OOD_arXiv_2023|p0": 0.13678908571600915, + "logit|OOD_arXiv_2023|p1": 0.11337111741304398, + "shuf|OOD_arXiv_2023|p1": 0.1261425122618675, + "logit|OOD_arXiv_2023|p2": 0.11835215277969838, + "shuf|OOD_arXiv_2023|p2": 0.13648709356784822, + "logit|OOD_CommonPile|p0": 0.12013521678745746, + "shuf|OOD_CommonPile|p0": 0.13690541982650756, + "logit|OOD_CommonPile|p1": 0.1152633611112833, + "shuf|OOD_CommonPile|p1": 0.13219560906291009, + "logit|OOD_CommonPile|p2": 0.12013627588748932, + "shuf|OOD_CommonPile|p2": 0.13602811992168426, + "logit|SHUFFLED_Pile-CC|p0": 0.08761658929288388, + "shuf|SHUFFLED_Pile-CC|p0": 0.10673588179051877, + "logit|SHUFFLED_Pile-CC|p1": 0.08926036544144153, + "shuf|SHUFFLED_Pile-CC|p1": 0.10876993574202061, + "logit|SHUFFLED_Pile-CC|p2": 0.0900193702429533, + "shuf|SHUFFLED_Pile-CC|p2": 0.10551640428602696 + } + }, + "control_C1_reproduces_stored_e52_matrix": { + "reference": "results/e52_factorial_410m.json", + "checked": true, + "n_cells_compared": 128, + "max_abs_diff": 0.0, + "worst_cell": null, + "tolerance": 1e-12, + "fires": true + }, + "provenance": { + "schema_version": 1, + "experiment": "E57", + "utc": "2026-08-20T06:19:00+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "6d74ea7f4ef2a968b73923e3858cbafe48c5a8d735f3118ad31422b7d43a272d", + "n_lines": 851 + }, + "git": { + "commit": "64b28ecb5a5600eda58cf20e64c1d8ae680943e1", + "commit_short": "64b28ecb5a56", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/e52_factorial_410m_pooled.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/RESULTS_TAXONOMY.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "46c97419e7eecf171879be40caaf59516ff52e2d5c24a0721ce5c7f1fe41418c", + "describe": "64b28ec-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--workers", + "3", + "--out", + "results/e52_factorial_410m_pooled.json", + "--cells-out", + "results/e57_factorial_cells_410m_pooled.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e52_factorial_410m.json", + "sha256": "6e81f3cae37d0c1906f8912af4bf8e75db0a0f259b2869f209701b74b6682590", + "bytes": 41190, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "8d15b390fc341469cbc37ca5c9e715dbe6eece9e09a943858077ec06fee99c0c" + } +} \ No newline at end of file diff --git a/results/e57_factorial_cells_410m_rstrip.json b/results/e57_factorial_cells_410m_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..821bb2f528482d753f966e582d455e4dca65886f --- /dev/null +++ b/results/e57_factorial_cells_410m_rstrip.json @@ -0,0 +1,1626 @@ +{ + "experiment": "E57 \u2014 the per-cell draws E52 averaged away", + "why": "the paper's headline is a variance decomposition of 64 cell MEANS with one observation each, so its three shares sum to 100% by construction, its 'interaction' is a residual conflated with cell noise, and it carries no interval. Each cell is a mean over 9 draws (3 fit seeds x 3 prefix seeds). This emits those draws so the shares can be bootstrapped.", + "unit": "admitted-set mean read AUC for one (fit corpus, fit seed, read rung, prefix seed)", + "fit_corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ], + "read_rungs": [ + "Q0", + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile", + "SHUFFLED_Pile-CC" + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_draws_per_matrix_cell": 9, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "model": "EleutherAI/pythia-410m-deduped", + "draws": { + "persist": { + "Wikipedia_en|Wikipedia_en|s0|p0": 0.10287346355617047, + "Wikipedia_en|Wikipedia_en|s0|p1": 0.10705953489523382, + "Wikipedia_en|Wikipedia_en|s0|p2": 0.10752673945389687, + "Wikipedia_en|Wikipedia_en|s1|p0": 0.10310866497457027, + "Wikipedia_en|Wikipedia_en|s1|p1": 0.10558628912549466, + "Wikipedia_en|Wikipedia_en|s1|p2": 0.10705714016221464, + "Wikipedia_en|Wikipedia_en|s2|p0": 0.10397805888205766, + "Wikipedia_en|Wikipedia_en|s2|p1": 0.1065072926459834, + "Wikipedia_en|Wikipedia_en|s2|p2": 0.10776060610078275, + "Wikipedia_en|USPTO_Backgrounds|s0|p0": 0.10835215623956174, + "Wikipedia_en|USPTO_Backgrounds|s0|p1": 0.10936748213134706, + "Wikipedia_en|USPTO_Backgrounds|s0|p2": 0.10712199013214559, + "Wikipedia_en|USPTO_Backgrounds|s1|p0": 0.10757550995331258, + "Wikipedia_en|USPTO_Backgrounds|s1|p1": 0.10940038836561143, + "Wikipedia_en|USPTO_Backgrounds|s1|p2": 0.10810002710204572, + "Wikipedia_en|USPTO_Backgrounds|s2|p0": 0.10718514572363347, + "Wikipedia_en|USPTO_Backgrounds|s2|p1": 0.10891728079877794, + "Wikipedia_en|USPTO_Backgrounds|s2|p2": 0.10709678668063134, + "Wikipedia_en|Pile-CC|s0|p0": 0.10301695019006729, + "Wikipedia_en|Pile-CC|s0|p1": 0.10244665767531842, + "Wikipedia_en|Pile-CC|s0|p2": 0.10174250877462328, + "Wikipedia_en|Pile-CC|s1|p0": 0.10172510519623756, + "Wikipedia_en|Pile-CC|s1|p1": 0.10325945045333355, + "Wikipedia_en|Pile-CC|s1|p2": 0.10407478339038789, + "Wikipedia_en|Pile-CC|s2|p0": 0.10228180140256882, + "Wikipedia_en|Pile-CC|s2|p1": 0.10175694662611931, + "Wikipedia_en|Pile-CC|s2|p2": 0.1017035601194948, + "Wikipedia_en|StackExchange|s0|p0": 0.08948030322790146, + "Wikipedia_en|StackExchange|s0|p1": 0.08883110496681183, + "Wikipedia_en|StackExchange|s0|p2": 0.08885546787641943, + "Wikipedia_en|StackExchange|s1|p0": 0.08780822232365608, + "Wikipedia_en|StackExchange|s1|p1": 0.09055548075120896, + "Wikipedia_en|StackExchange|s1|p2": 0.08899792558513582, + "Wikipedia_en|StackExchange|s2|p0": 0.0892696101218462, + "Wikipedia_en|StackExchange|s2|p1": 0.08852593626361341, + "Wikipedia_en|StackExchange|s2|p2": 0.08834609067998826, + "Wikipedia_en|Github|s0|p0": 0.08922453611157835, + "Wikipedia_en|Github|s0|p1": 0.09481620851438492, + "Wikipedia_en|Github|s0|p2": 0.08858082394581288, + "Wikipedia_en|Github|s1|p0": 0.09077325025573373, + "Wikipedia_en|Github|s1|p1": 0.09396704870741815, + "Wikipedia_en|Github|s1|p2": 0.08699891928117723, + "Wikipedia_en|Github|s2|p0": 0.08937682555988431, + "Wikipedia_en|Github|s2|p1": 0.09461174658499658, + "Wikipedia_en|Github|s2|p2": 0.08868864986579865, + "Wikipedia_en|OOD_News_2024|s0|p0": 0.10322936563752591, + "Wikipedia_en|OOD_News_2024|s0|p1": 0.10387584171257913, + "Wikipedia_en|OOD_News_2024|s0|p2": 0.10859028375707566, + "Wikipedia_en|OOD_News_2024|s1|p0": 0.10311465910635889, + "Wikipedia_en|OOD_News_2024|s1|p1": 0.10413367911241948, + "Wikipedia_en|OOD_News_2024|s1|p2": 0.11006932607851923, + "Wikipedia_en|OOD_News_2024|s2|p0": 0.10344691178761423, + "Wikipedia_en|OOD_News_2024|s2|p1": 0.10392231694422663, + "Wikipedia_en|OOD_News_2024|s2|p2": 0.10917905285023152, + "Wikipedia_en|OOD_arXiv_2023|s0|p0": 0.10009014967363328, + "Wikipedia_en|OOD_arXiv_2023|s0|p1": 0.09289677338674665, + "Wikipedia_en|OOD_arXiv_2023|s0|p2": 0.09562335102818906, + "Wikipedia_en|OOD_arXiv_2023|s1|p0": 0.10027048911433667, + "Wikipedia_en|OOD_arXiv_2023|s1|p1": 0.09360064705833793, + "Wikipedia_en|OOD_arXiv_2023|s1|p2": 0.09504834054969251, + "Wikipedia_en|OOD_arXiv_2023|s2|p0": 0.1008278402267024, + "Wikipedia_en|OOD_arXiv_2023|s2|p1": 0.09411257794126868, + "Wikipedia_en|OOD_arXiv_2023|s2|p2": 0.09536036751233042, + "Wikipedia_en|OOD_CommonPile|s0|p0": 0.0989431984256953, + "Wikipedia_en|OOD_CommonPile|s0|p1": 0.09801026820205153, + "Wikipedia_en|OOD_CommonPile|s0|p2": 0.099261833447963, + "Wikipedia_en|OOD_CommonPile|s1|p0": 0.10001896391622722, + "Wikipedia_en|OOD_CommonPile|s1|p1": 0.0986754511948675, + "Wikipedia_en|OOD_CommonPile|s1|p2": 0.09912365758791566, + "Wikipedia_en|OOD_CommonPile|s2|p0": 0.09859652887098491, + "Wikipedia_en|OOD_CommonPile|s2|p1": 0.09861277537420392, + "Wikipedia_en|OOD_CommonPile|s2|p2": 0.09953167075291276, + "USPTO_Backgrounds|Wikipedia_en|s0|p0": 0.10981366685591638, + "USPTO_Backgrounds|Wikipedia_en|s0|p1": 0.11414047977887094, + "USPTO_Backgrounds|Wikipedia_en|s0|p2": 0.11450463808141649, + "USPTO_Backgrounds|Wikipedia_en|s1|p0": 0.10897310390137135, + "USPTO_Backgrounds|Wikipedia_en|s1|p1": 0.11560187987051904, + "USPTO_Backgrounds|Wikipedia_en|s1|p2": 0.1157282134052366, + "USPTO_Backgrounds|Wikipedia_en|s2|p0": 0.11190817966125906, + "USPTO_Backgrounds|Wikipedia_en|s2|p1": 0.11461158744059503, + "USPTO_Backgrounds|Wikipedia_en|s2|p2": 0.1157372762914747, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p0": 0.11757533177733422, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p1": 0.11750025253277271, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p2": 0.11985709469299763, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p0": 0.11808874036651104, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p1": 0.11713413796387613, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p2": 0.11782861899118871, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p0": 0.11796610082965345, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p1": 0.11781551565509289, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p2": 0.11718907977920026, + "USPTO_Backgrounds|Pile-CC|s0|p0": 0.11340135610662401, + "USPTO_Backgrounds|Pile-CC|s0|p1": 0.11403553984127938, + "USPTO_Backgrounds|Pile-CC|s0|p2": 0.11339056277647615, + "USPTO_Backgrounds|Pile-CC|s1|p0": 0.1140590054448694, + "USPTO_Backgrounds|Pile-CC|s1|p1": 0.11357520571909845, + "USPTO_Backgrounds|Pile-CC|s1|p2": 0.11428703451529146, + "USPTO_Backgrounds|Pile-CC|s2|p0": 0.11447026557289064, + "USPTO_Backgrounds|Pile-CC|s2|p1": 0.11401424161158502, + "USPTO_Backgrounds|Pile-CC|s2|p2": 0.11416871035471558, + "USPTO_Backgrounds|StackExchange|s0|p0": 0.09920990269165486, + "USPTO_Backgrounds|StackExchange|s0|p1": 0.10113982758484781, + "USPTO_Backgrounds|StackExchange|s0|p2": 0.09919712147675455, + "USPTO_Backgrounds|StackExchange|s1|p0": 0.10007498662453145, + "USPTO_Backgrounds|StackExchange|s1|p1": 0.10177074423991143, + "USPTO_Backgrounds|StackExchange|s1|p2": 0.09973101667128503, + "USPTO_Backgrounds|StackExchange|s2|p0": 0.10199995215516537, + "USPTO_Backgrounds|StackExchange|s2|p1": 0.10317190862260758, + "USPTO_Backgrounds|StackExchange|s2|p2": 0.1004770954605192, + "USPTO_Backgrounds|Github|s0|p0": 0.09990634275600314, + "USPTO_Backgrounds|Github|s0|p1": 0.09964822544716298, + "USPTO_Backgrounds|Github|s0|p2": 0.09964380240999163, + "USPTO_Backgrounds|Github|s1|p0": 0.09824179811403155, + "USPTO_Backgrounds|Github|s1|p1": 0.10013992837630212, + "USPTO_Backgrounds|Github|s1|p2": 0.09967329180799425, + "USPTO_Backgrounds|Github|s2|p0": 0.10006902804598213, + "USPTO_Backgrounds|Github|s2|p1": 0.1009458543267101, + "USPTO_Backgrounds|Github|s2|p2": 0.09969435348175466, + "USPTO_Backgrounds|OOD_News_2024|s0|p0": 0.11298159337602556, + "USPTO_Backgrounds|OOD_News_2024|s0|p1": 0.11538627943955362, + "USPTO_Backgrounds|OOD_News_2024|s0|p2": 0.11723750135861337, + "USPTO_Backgrounds|OOD_News_2024|s1|p0": 0.11325641400180757, + "USPTO_Backgrounds|OOD_News_2024|s1|p1": 0.11458616754971444, + "USPTO_Backgrounds|OOD_News_2024|s1|p2": 0.11798960617743433, + "USPTO_Backgrounds|OOD_News_2024|s2|p0": 0.11325467429123819, + "USPTO_Backgrounds|OOD_News_2024|s2|p1": 0.11538628018461168, + "USPTO_Backgrounds|OOD_News_2024|s2|p2": 0.11717765755020082, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p0": 0.10985615975223481, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p1": 0.10438272035680711, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p2": 0.1067752534057945, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p0": 0.10996073833666742, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p1": 0.10437664771452546, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p2": 0.10740943900309503, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p0": 0.1100754541810602, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p1": 0.104672791948542, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p2": 0.10786801143549382, + "USPTO_Backgrounds|OOD_CommonPile|s0|p0": 0.11021037744358182, + "USPTO_Backgrounds|OOD_CommonPile|s0|p1": 0.10951764369383454, + "USPTO_Backgrounds|OOD_CommonPile|s0|p2": 0.10590783571824432, + "USPTO_Backgrounds|OOD_CommonPile|s1|p0": 0.10983699904754758, + "USPTO_Backgrounds|OOD_CommonPile|s1|p1": 0.10934913912788033, + "USPTO_Backgrounds|OOD_CommonPile|s1|p2": 0.10549549823626876, + "USPTO_Backgrounds|OOD_CommonPile|s2|p0": 0.11129045831039548, + "USPTO_Backgrounds|OOD_CommonPile|s2|p1": 0.11110021555796265, + "USPTO_Backgrounds|OOD_CommonPile|s2|p2": 0.10638835793361068, + "Pile-CC|Wikipedia_en|s0|p0": 0.10977663840167225, + "Pile-CC|Wikipedia_en|s0|p1": 0.11226582238450647, + "Pile-CC|Wikipedia_en|s0|p2": 0.11307381144724786, + "Pile-CC|Wikipedia_en|s1|p0": 0.11117992154322565, + "Pile-CC|Wikipedia_en|s1|p1": 0.11297510052099824, + "Pile-CC|Wikipedia_en|s1|p2": 0.11544504365883768, + "Pile-CC|Wikipedia_en|s2|p0": 0.10913663334213197, + "Pile-CC|Wikipedia_en|s2|p1": 0.11214101472869516, + "Pile-CC|Wikipedia_en|s2|p2": 0.11385302892886102, + "Pile-CC|USPTO_Backgrounds|s0|p0": 0.11689561381936073, + "Pile-CC|USPTO_Backgrounds|s0|p1": 0.11545791826210916, + "Pile-CC|USPTO_Backgrounds|s0|p2": 0.1114997111260891, + "Pile-CC|USPTO_Backgrounds|s1|p0": 0.11865137442946434, + "Pile-CC|USPTO_Backgrounds|s1|p1": 0.11577425650320947, + "Pile-CC|USPTO_Backgrounds|s1|p2": 0.112876745313406, + "Pile-CC|USPTO_Backgrounds|s2|p0": 0.11605247035622597, + "Pile-CC|USPTO_Backgrounds|s2|p1": 0.11659135646186769, + "Pile-CC|USPTO_Backgrounds|s2|p2": 0.11062965393066407, + "Pile-CC|Pile-CC|s0|p0": 0.11368947527371347, + "Pile-CC|Pile-CC|s0|p1": 0.10877547687850893, + "Pile-CC|Pile-CC|s0|p2": 0.11249315971508622, + "Pile-CC|Pile-CC|s1|p0": 0.11442587501369417, + "Pile-CC|Pile-CC|s1|p1": 0.11172064519487321, + "Pile-CC|Pile-CC|s1|p2": 0.11297044763341546, + "Pile-CC|Pile-CC|s2|p0": 0.11456402367912233, + "Pile-CC|Pile-CC|s2|p1": 0.10940536237321793, + "Pile-CC|Pile-CC|s2|p2": 0.11074101338163019, + "Pile-CC|StackExchange|s0|p0": 0.09647673729341477, + "Pile-CC|StackExchange|s0|p1": 0.09985707155428827, + "Pile-CC|StackExchange|s0|p2": 0.09624616752844303, + "Pile-CC|StackExchange|s1|p0": 0.09964159529190511, + "Pile-CC|StackExchange|s1|p1": 0.1021943328436464, + "Pile-CC|StackExchange|s1|p2": 0.09618805970530957, + "Pile-CC|StackExchange|s2|p0": 0.09710012550931424, + "Pile-CC|StackExchange|s2|p1": 0.09988769392948597, + "Pile-CC|StackExchange|s2|p2": 0.09654809411149472, + "Pile-CC|Github|s0|p0": 0.09867565045133234, + "Pile-CC|Github|s0|p1": 0.10106712891720235, + "Pile-CC|Github|s0|p2": 0.09839950553141534, + "Pile-CC|Github|s1|p0": 0.10147361839190125, + "Pile-CC|Github|s1|p1": 0.10121930676978082, + "Pile-CC|Github|s1|p2": 0.09857688944321126, + "Pile-CC|Github|s2|p0": 0.0991824553348124, + "Pile-CC|Github|s2|p1": 0.10053173354826868, + "Pile-CC|Github|s2|p2": 0.09756218395195901, + "Pile-CC|OOD_News_2024|s0|p0": 0.11257225731387735, + "Pile-CC|OOD_News_2024|s0|p1": 0.11419705385342241, + "Pile-CC|OOD_News_2024|s0|p2": 0.11853634724393487, + "Pile-CC|OOD_News_2024|s1|p0": 0.11296368772163987, + "Pile-CC|OOD_News_2024|s1|p1": 0.1150851615704596, + "Pile-CC|OOD_News_2024|s1|p2": 0.1192458828445524, + "Pile-CC|OOD_News_2024|s2|p0": 0.11220575952902437, + "Pile-CC|OOD_News_2024|s2|p1": 0.1137446821667254, + "Pile-CC|OOD_News_2024|s2|p2": 0.11635193205438554, + "Pile-CC|OOD_arXiv_2023|s0|p0": 0.10853506824932993, + "Pile-CC|OOD_arXiv_2023|s0|p1": 0.10454859966412186, + "Pile-CC|OOD_arXiv_2023|s0|p2": 0.10581754418089986, + "Pile-CC|OOD_arXiv_2023|s1|p0": 0.11100725806318223, + "Pile-CC|OOD_arXiv_2023|s1|p1": 0.10551761770620942, + "Pile-CC|OOD_arXiv_2023|s1|p2": 0.10784276882186532, + "Pile-CC|OOD_arXiv_2023|s2|p0": 0.1089271632488817, + "Pile-CC|OOD_arXiv_2023|s2|p1": 0.10318359741941094, + "Pile-CC|OOD_arXiv_2023|s2|p2": 0.10602674121037126, + "Pile-CC|OOD_CommonPile|s0|p0": 0.10652299979701638, + "Pile-CC|OOD_CommonPile|s0|p1": 0.10688193077221513, + "Pile-CC|OOD_CommonPile|s0|p2": 0.10867929961532355, + "Pile-CC|OOD_CommonPile|s1|p0": 0.10805004993453622, + "Pile-CC|OOD_CommonPile|s1|p1": 0.10758432084694505, + "Pile-CC|OOD_CommonPile|s1|p2": 0.10993502987548709, + "Pile-CC|OOD_CommonPile|s2|p0": 0.10624627815559506, + "Pile-CC|OOD_CommonPile|s2|p1": 0.10528236338868738, + "Pile-CC|OOD_CommonPile|s2|p2": 0.10631496291607619, + "StackExchange|Wikipedia_en|s0|p0": 0.11112175965681673, + "StackExchange|Wikipedia_en|s0|p1": 0.11482277447357774, + "StackExchange|Wikipedia_en|s0|p2": 0.1154995913617313, + "StackExchange|Wikipedia_en|s1|p0": 0.11049981350079179, + "StackExchange|Wikipedia_en|s1|p1": 0.11311498708091676, + "StackExchange|Wikipedia_en|s1|p2": 0.1127384047023952, + "StackExchange|Wikipedia_en|s2|p0": 0.1093706869520247, + "StackExchange|Wikipedia_en|s2|p1": 0.11293273768387735, + "StackExchange|Wikipedia_en|s2|p2": 0.11366914017125965, + "StackExchange|USPTO_Backgrounds|s0|p0": 0.12343905693851412, + "StackExchange|USPTO_Backgrounds|s0|p1": 0.11932324254885315, + "StackExchange|USPTO_Backgrounds|s0|p2": 0.1148728062864393, + "StackExchange|USPTO_Backgrounds|s1|p0": 0.12070241621695459, + "StackExchange|USPTO_Backgrounds|s1|p1": 0.118587248865515, + "StackExchange|USPTO_Backgrounds|s1|p2": 0.11302319965325296, + "StackExchange|USPTO_Backgrounds|s2|p0": 0.12010543025098741, + "StackExchange|USPTO_Backgrounds|s2|p1": 0.11844910318031907, + "StackExchange|USPTO_Backgrounds|s2|p2": 0.114451447641477, + "StackExchange|Pile-CC|s0|p0": 0.1136845303233713, + "StackExchange|Pile-CC|s0|p1": 0.11343434480950236, + "StackExchange|Pile-CC|s0|p2": 0.11050147628411651, + "StackExchange|Pile-CC|s1|p0": 0.11131046093069016, + "StackExchange|Pile-CC|s1|p1": 0.11339281359687448, + "StackExchange|Pile-CC|s1|p2": 0.10842569386586547, + "StackExchange|Pile-CC|s2|p0": 0.11122029772959649, + "StackExchange|Pile-CC|s2|p1": 0.11302632940933108, + "StackExchange|Pile-CC|s2|p2": 0.10787091506645083, + "StackExchange|StackExchange|s0|p0": 0.10077225628774614, + "StackExchange|StackExchange|s0|p1": 0.10341567969880998, + "StackExchange|StackExchange|s0|p2": 0.10223113126121461, + "StackExchange|StackExchange|s1|p0": 0.09896282306872309, + "StackExchange|StackExchange|s1|p1": 0.10230527422390878, + "StackExchange|StackExchange|s1|p2": 0.10037113958969712, + "StackExchange|StackExchange|s2|p0": 0.0979436794994399, + "StackExchange|StackExchange|s2|p1": 0.10306319794617594, + "StackExchange|StackExchange|s2|p2": 0.10082134315744043, + "StackExchange|Github|s0|p0": 0.10423819115385413, + "StackExchange|Github|s0|p1": 0.10118143982253969, + "StackExchange|Github|s0|p2": 0.1039186098612845, + "StackExchange|Github|s1|p0": 0.10421217409893871, + "StackExchange|Github|s1|p1": 0.09886866183951497, + "StackExchange|Github|s1|p2": 0.10191642129793763, + "StackExchange|Github|s2|p0": 0.10365374563261867, + "StackExchange|Github|s2|p1": 0.09871931718662381, + "StackExchange|Github|s2|p2": 0.10111523354426026, + "StackExchange|OOD_News_2024|s0|p0": 0.11352613670751452, + "StackExchange|OOD_News_2024|s0|p1": 0.11554599236696958, + "StackExchange|OOD_News_2024|s0|p2": 0.11738898884505033, + "StackExchange|OOD_News_2024|s1|p0": 0.11162678627297282, + "StackExchange|OOD_News_2024|s1|p1": 0.11267009284347296, + "StackExchange|OOD_News_2024|s1|p2": 0.11508553270250559, + "StackExchange|OOD_News_2024|s2|p0": 0.11281726462766528, + "StackExchange|OOD_News_2024|s2|p1": 0.11199803259223699, + "StackExchange|OOD_News_2024|s2|p2": 0.11720351371914148, + "StackExchange|OOD_arXiv_2023|s0|p0": 0.11144447633996606, + "StackExchange|OOD_arXiv_2023|s0|p1": 0.10272708451375365, + "StackExchange|OOD_arXiv_2023|s0|p2": 0.1063292103819549, + "StackExchange|OOD_arXiv_2023|s1|p0": 0.10747666256502271, + "StackExchange|OOD_arXiv_2023|s1|p1": 0.10176212107762694, + "StackExchange|OOD_arXiv_2023|s1|p2": 0.10543743697926403, + "StackExchange|OOD_arXiv_2023|s2|p0": 0.10788210490718483, + "StackExchange|OOD_arXiv_2023|s2|p1": 0.10020368015393614, + "StackExchange|OOD_arXiv_2023|s2|p2": 0.10480040814727545, + "StackExchange|OOD_CommonPile|s0|p0": 0.10588276060298085, + "StackExchange|OOD_CommonPile|s0|p1": 0.11143070021644234, + "StackExchange|OOD_CommonPile|s0|p2": 0.10827568108215928, + "StackExchange|OOD_CommonPile|s1|p0": 0.10300944494083523, + "StackExchange|OOD_CommonPile|s1|p1": 0.10867205942049622, + "StackExchange|OOD_CommonPile|s1|p2": 0.10689621297642589, + "StackExchange|OOD_CommonPile|s2|p0": 0.10391671312972903, + "StackExchange|OOD_CommonPile|s2|p1": 0.10857178764417767, + "StackExchange|OOD_CommonPile|s2|p2": 0.10712739685550332, + "Github|Wikipedia_en|s0|p0": 0.0943807506468147, + "Github|Wikipedia_en|s0|p1": 0.09571044715121388, + "Github|Wikipedia_en|s0|p2": 0.09689728738740086, + "Github|Wikipedia_en|s1|p0": 0.09720580810680986, + "Github|Wikipedia_en|s1|p1": 0.10134021313861012, + "Github|Wikipedia_en|s1|p2": 0.09930194253101945, + "Github|Wikipedia_en|s2|p0": 0.09605752164497972, + "Github|Wikipedia_en|s2|p1": 0.09664223017171025, + "Github|Wikipedia_en|s2|p2": 0.09726484967395663, + "Github|USPTO_Backgrounds|s0|p0": 0.09803881645202636, + "Github|USPTO_Backgrounds|s0|p1": 0.096802489599213, + "Github|USPTO_Backgrounds|s0|p2": 0.09574282909743488, + "Github|USPTO_Backgrounds|s1|p0": 0.0994961578398943, + "Github|USPTO_Backgrounds|s1|p1": 0.10027356231585145, + "Github|USPTO_Backgrounds|s1|p2": 0.09973839945159853, + "Github|USPTO_Backgrounds|s2|p0": 0.09892328418791294, + "Github|USPTO_Backgrounds|s2|p1": 0.09771768650971353, + "Github|USPTO_Backgrounds|s2|p2": 0.09460788681171835, + "Github|Pile-CC|s0|p0": 0.0938994218595326, + "Github|Pile-CC|s0|p1": 0.09712222004309297, + "Github|Pile-CC|s0|p2": 0.09572154162451625, + "Github|Pile-CC|s1|p0": 0.09473202684894204, + "Github|Pile-CC|s1|p1": 0.10006348108872772, + "Github|Pile-CC|s1|p2": 0.100626275036484, + "Github|Pile-CC|s2|p0": 0.09244232410565019, + "Github|Pile-CC|s2|p1": 0.09587580608204008, + "Github|Pile-CC|s2|p2": 0.09456894053146243, + "Github|StackExchange|s0|p0": 0.079716055537574, + "Github|StackExchange|s0|p1": 0.07986645116470754, + "Github|StackExchange|s0|p2": 0.08210491738282144, + "Github|StackExchange|s1|p0": 0.08338625400792807, + "Github|StackExchange|s1|p1": 0.0811770138796419, + "Github|StackExchange|s1|p2": 0.08580627045594155, + "Github|StackExchange|s2|p0": 0.08124738119076938, + "Github|StackExchange|s2|p1": 0.0806168099399656, + "Github|StackExchange|s2|p2": 0.08351655486039818, + "Github|Github|s0|p0": 0.08441019980236888, + "Github|Github|s0|p1": 0.08204828975722193, + "Github|Github|s0|p2": 0.0816796594299376, + "Github|Github|s1|p0": 0.08772723777219653, + "Github|Github|s1|p1": 0.08448407845571637, + "Github|Github|s1|p2": 0.08310101525858045, + "Github|Github|s2|p0": 0.08351801251992583, + "Github|Github|s2|p1": 0.08169148741289974, + "Github|Github|s2|p2": 0.07996605196967721, + "Github|OOD_News_2024|s0|p0": 0.09599161995574831, + "Github|OOD_News_2024|s0|p1": 0.09451501192525029, + "Github|OOD_News_2024|s0|p2": 0.09908225508406758, + "Github|OOD_News_2024|s1|p0": 0.0999955783598125, + "Github|OOD_News_2024|s1|p1": 0.09992980221286415, + "Github|OOD_News_2024|s1|p2": 0.10202675415202975, + "Github|OOD_News_2024|s2|p0": 0.09439530475065112, + "Github|OOD_News_2024|s2|p1": 0.09497274542227388, + "Github|OOD_News_2024|s2|p2": 0.09971939725801349, + "Github|OOD_arXiv_2023|s0|p0": 0.09497318845242261, + "Github|OOD_arXiv_2023|s0|p1": 0.08588955113664269, + "Github|OOD_arXiv_2023|s0|p2": 0.09135149857029319, + "Github|OOD_arXiv_2023|s1|p0": 0.09864422287791967, + "Github|OOD_arXiv_2023|s1|p1": 0.0916469763033092, + "Github|OOD_arXiv_2023|s1|p2": 0.09290349027141928, + "Github|OOD_arXiv_2023|s2|p0": 0.09554064637050033, + "Github|OOD_arXiv_2023|s2|p1": 0.08807098111137748, + "Github|OOD_arXiv_2023|s2|p2": 0.09004956176504493, + "Github|OOD_CommonPile|s0|p0": 0.09318044185638427, + "Github|OOD_CommonPile|s0|p1": 0.09214780256152152, + "Github|OOD_CommonPile|s0|p2": 0.09007704397663474, + "Github|OOD_CommonPile|s1|p0": 0.09697884330525994, + "Github|OOD_CommonPile|s1|p1": 0.09514916148036719, + "Github|OOD_CommonPile|s1|p2": 0.09456002926453948, + "Github|OOD_CommonPile|s2|p0": 0.09371326407417654, + "Github|OOD_CommonPile|s2|p1": 0.09251251574605704, + "Github|OOD_CommonPile|s2|p2": 0.09221953069791197, + "OOD_News_2024|Wikipedia_en|s0|p0": 0.09431040424387901, + "OOD_News_2024|Wikipedia_en|s0|p1": 0.10099294013343751, + "OOD_News_2024|Wikipedia_en|s0|p2": 0.09926603636704386, + "OOD_News_2024|Wikipedia_en|s1|p0": 0.09500672086142004, + "OOD_News_2024|Wikipedia_en|s1|p1": 0.10247886455617845, + "OOD_News_2024|Wikipedia_en|s1|p2": 0.10088188177905977, + "OOD_News_2024|Wikipedia_en|s2|p0": 0.09398500535171479, + "OOD_News_2024|Wikipedia_en|s2|p1": 0.10087930983863772, + "OOD_News_2024|Wikipedia_en|s2|p2": 0.10092731318436562, + "OOD_News_2024|USPTO_Backgrounds|s0|p0": 0.10403475351631641, + "OOD_News_2024|USPTO_Backgrounds|s0|p1": 0.10329107146244496, + "OOD_News_2024|USPTO_Backgrounds|s0|p2": 0.10143266692757606, + "OOD_News_2024|USPTO_Backgrounds|s1|p0": 0.1046790361404419, + "OOD_News_2024|USPTO_Backgrounds|s1|p1": 0.10183911721687763, + "OOD_News_2024|USPTO_Backgrounds|s1|p2": 0.1006441331235692, + "OOD_News_2024|USPTO_Backgrounds|s2|p0": 0.10493446663022041, + "OOD_News_2024|USPTO_Backgrounds|s2|p1": 0.10150037731509656, + "OOD_News_2024|USPTO_Backgrounds|s2|p2": 0.10019716534297914, + "OOD_News_2024|Pile-CC|s0|p0": 0.09929459295235574, + "OOD_News_2024|Pile-CC|s0|p1": 0.09881729236803949, + "OOD_News_2024|Pile-CC|s0|p2": 0.09828447303734719, + "OOD_News_2024|Pile-CC|s1|p0": 0.10183348185382783, + "OOD_News_2024|Pile-CC|s1|p1": 0.09985194108448922, + "OOD_News_2024|Pile-CC|s1|p2": 0.10013622581027448, + "OOD_News_2024|Pile-CC|s2|p0": 0.10012178956530988, + "OOD_News_2024|Pile-CC|s2|p1": 0.09790745167993009, + "OOD_News_2024|Pile-CC|s2|p2": 0.09954423136077821, + "OOD_News_2024|StackExchange|s0|p0": 0.08735333383083344, + "OOD_News_2024|StackExchange|s0|p1": 0.0856907582608983, + "OOD_News_2024|StackExchange|s0|p2": 0.08639676049351692, + "OOD_News_2024|StackExchange|s1|p0": 0.08586523495614529, + "OOD_News_2024|StackExchange|s1|p1": 0.0863585128216073, + "OOD_News_2024|StackExchange|s1|p2": 0.08664004034362734, + "OOD_News_2024|StackExchange|s2|p0": 0.08645292222499848, + "OOD_News_2024|StackExchange|s2|p1": 0.08526144984643906, + "OOD_News_2024|StackExchange|s2|p2": 0.0864780222531408, + "OOD_News_2024|Github|s0|p0": 0.08590103215537966, + "OOD_News_2024|Github|s0|p1": 0.0892094851937145, + "OOD_News_2024|Github|s0|p2": 0.08405496566556395, + "OOD_News_2024|Github|s1|p0": 0.08847997586708516, + "OOD_News_2024|Github|s1|p1": 0.08941086689010262, + "OOD_News_2024|Github|s1|p2": 0.08356539669912308, + "OOD_News_2024|Github|s2|p0": 0.08698220762889833, + "OOD_News_2024|Github|s2|p1": 0.08764714165590703, + "OOD_News_2024|Github|s2|p2": 0.08164113375823945, + "OOD_News_2024|OOD_News_2024|s0|p0": 0.10120847611688077, + "OOD_News_2024|OOD_News_2024|s0|p1": 0.09802179792895913, + "OOD_News_2024|OOD_News_2024|s0|p2": 0.10591108477674424, + "OOD_News_2024|OOD_News_2024|s1|p0": 0.10267970571294427, + "OOD_News_2024|OOD_News_2024|s1|p1": 0.10035581709817051, + "OOD_News_2024|OOD_News_2024|s1|p2": 0.10925199701450765, + "OOD_News_2024|OOD_News_2024|s2|p0": 0.10079265208914875, + "OOD_News_2024|OOD_News_2024|s2|p1": 0.09761270890012383, + "OOD_News_2024|OOD_News_2024|s2|p2": 0.10711529902182519, + "OOD_News_2024|OOD_arXiv_2023|s0|p0": 0.09789862926118076, + "OOD_News_2024|OOD_arXiv_2023|s0|p1": 0.08996567069552838, + "OOD_News_2024|OOD_arXiv_2023|s0|p2": 0.09357428341172636, + "OOD_News_2024|OOD_arXiv_2023|s1|p0": 0.09982462688349188, + "OOD_News_2024|OOD_arXiv_2023|s1|p1": 0.09149850173853338, + "OOD_News_2024|OOD_arXiv_2023|s1|p2": 0.09505331865511835, + "OOD_News_2024|OOD_arXiv_2023|s2|p0": 0.09910027249716222, + "OOD_News_2024|OOD_arXiv_2023|s2|p1": 0.08940399973653257, + "OOD_News_2024|OOD_arXiv_2023|s2|p2": 0.09473871565423905, + "OOD_News_2024|OOD_CommonPile|s0|p0": 0.09377979235723614, + "OOD_News_2024|OOD_CommonPile|s0|p1": 0.09277759054675698, + "OOD_News_2024|OOD_CommonPile|s0|p2": 0.09559618728235364, + "OOD_News_2024|OOD_CommonPile|s1|p0": 0.09457235177978873, + "OOD_News_2024|OOD_CommonPile|s1|p1": 0.09481224501505495, + "OOD_News_2024|OOD_CommonPile|s1|p2": 0.09694003118202091, + "OOD_News_2024|OOD_CommonPile|s2|p0": 0.09513295320793987, + "OOD_News_2024|OOD_CommonPile|s2|p1": 0.09506941260769963, + "OOD_News_2024|OOD_CommonPile|s2|p2": 0.09661790346726776, + "OOD_arXiv_2023|Wikipedia_en|s0|p0": 0.09832508899271489, + "OOD_arXiv_2023|Wikipedia_en|s0|p1": 0.10218057534657418, + "OOD_arXiv_2023|Wikipedia_en|s0|p2": 0.10387855533044785, + "OOD_arXiv_2023|Wikipedia_en|s1|p0": 0.09819703958928586, + "OOD_arXiv_2023|Wikipedia_en|s1|p1": 0.10323461941443383, + "OOD_arXiv_2023|Wikipedia_en|s1|p2": 0.10327899891417473, + "OOD_arXiv_2023|Wikipedia_en|s2|p0": 0.09632763583213091, + "OOD_arXiv_2023|Wikipedia_en|s2|p1": 0.10164304028730839, + "OOD_arXiv_2023|Wikipedia_en|s2|p2": 0.1039553853450343, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p0": 0.1079747885465622, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p1": 0.10563842877745629, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p2": 0.10221900641918183, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p0": 0.10753792077302933, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p1": 0.10589559301733971, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p2": 0.10418937355279922, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p0": 0.10603113174438476, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p1": 0.10579101666808129, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p2": 0.10224679000675678, + "OOD_arXiv_2023|Pile-CC|s0|p0": 0.10305953919887542, + "OOD_arXiv_2023|Pile-CC|s0|p1": 0.10209137375932187, + "OOD_arXiv_2023|Pile-CC|s0|p2": 0.09996090023778378, + "OOD_arXiv_2023|Pile-CC|s1|p0": 0.10287986919283867, + "OOD_arXiv_2023|Pile-CC|s1|p1": 0.10258024237118661, + "OOD_arXiv_2023|Pile-CC|s1|p2": 0.10064306422136723, + "OOD_arXiv_2023|Pile-CC|s2|p0": 0.10193472057580948, + "OOD_arXiv_2023|Pile-CC|s2|p1": 0.10058104707859457, + "OOD_arXiv_2023|Pile-CC|s2|p2": 0.1017577791120857, + "OOD_arXiv_2023|StackExchange|s0|p0": 0.08897595778107643, + "OOD_arXiv_2023|StackExchange|s0|p1": 0.09096930536907166, + "OOD_arXiv_2023|StackExchange|s0|p2": 0.09028750881552697, + "OOD_arXiv_2023|StackExchange|s1|p0": 0.08913971148431302, + "OOD_arXiv_2023|StackExchange|s1|p1": 0.09107646520715207, + "OOD_arXiv_2023|StackExchange|s1|p2": 0.08940475061535835, + "OOD_arXiv_2023|StackExchange|s2|p0": 0.08793633431196213, + "OOD_arXiv_2023|StackExchange|s2|p1": 0.0903628985164687, + "OOD_arXiv_2023|StackExchange|s2|p2": 0.09068218097090722, + "OOD_arXiv_2023|Github|s0|p0": 0.08716002702713013, + "OOD_arXiv_2023|Github|s0|p1": 0.08788722157478332, + "OOD_arXiv_2023|Github|s0|p2": 0.08809386231005192, + "OOD_arXiv_2023|Github|s1|p0": 0.08682951107621192, + "OOD_arXiv_2023|Github|s1|p1": 0.0868273675441742, + "OOD_arXiv_2023|Github|s1|p2": 0.08746398277580739, + "OOD_arXiv_2023|Github|s2|p0": 0.08608493134379387, + "OOD_arXiv_2023|Github|s2|p1": 0.08664769753813743, + "OOD_arXiv_2023|Github|s2|p2": 0.08904225379228592, + "OOD_arXiv_2023|OOD_News_2024|s0|p0": 0.10076245255768299, + "OOD_arXiv_2023|OOD_News_2024|s0|p1": 0.10587077687960118, + "OOD_arXiv_2023|OOD_News_2024|s0|p2": 0.10441893029492348, + "OOD_arXiv_2023|OOD_News_2024|s1|p0": 0.1004561899928376, + "OOD_arXiv_2023|OOD_News_2024|s1|p1": 0.10660954366903752, + "OOD_arXiv_2023|OOD_News_2024|s1|p2": 0.10381102177780122, + "OOD_arXiv_2023|OOD_News_2024|s2|p0": 0.09778091609477997, + "OOD_arXiv_2023|OOD_News_2024|s2|p1": 0.10442730225622654, + "OOD_arXiv_2023|OOD_News_2024|s2|p2": 0.10270793798845261, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p0": 0.10033874853979796, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p1": 0.09228677104692906, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p2": 0.09679753209929913, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p0": 0.09959847673308105, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p1": 0.0929314663168043, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p2": 0.0957174527226016, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p0": 0.09900324686896056, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p1": 0.09173696737270802, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p2": 0.0951176458504051, + "OOD_arXiv_2023|OOD_CommonPile|s0|p0": 0.09885846241377294, + "OOD_arXiv_2023|OOD_CommonPile|s0|p1": 0.09836859456263483, + "OOD_arXiv_2023|OOD_CommonPile|s0|p2": 0.09852006072178483, + "OOD_arXiv_2023|OOD_CommonPile|s1|p0": 0.0994883397128433, + "OOD_arXiv_2023|OOD_CommonPile|s1|p1": 0.10098498067818582, + "OOD_arXiv_2023|OOD_CommonPile|s1|p2": 0.098682074341923, + "OOD_arXiv_2023|OOD_CommonPile|s2|p0": 0.09796560569666327, + "OOD_arXiv_2023|OOD_CommonPile|s2|p1": 0.09867376736365259, + "OOD_arXiv_2023|OOD_CommonPile|s2|p2": 0.09689274551346898, + "OOD_CommonPile|Wikipedia_en|s0|p0": 0.10059046819806099, + "OOD_CommonPile|Wikipedia_en|s0|p1": 0.10382766402326524, + "OOD_CommonPile|Wikipedia_en|s0|p2": 0.10481505158822983, + "OOD_CommonPile|Wikipedia_en|s1|p0": 0.0980191096663475, + "OOD_CommonPile|Wikipedia_en|s1|p1": 0.10130858621560038, + "OOD_CommonPile|Wikipedia_en|s1|p2": 0.10309284951072187, + "OOD_CommonPile|Wikipedia_en|s2|p0": 0.09676471129059791, + "OOD_CommonPile|Wikipedia_en|s2|p1": 0.10211405879817903, + "OOD_CommonPile|Wikipedia_en|s2|p2": 0.1018283563433215, + "OOD_CommonPile|USPTO_Backgrounds|s0|p0": 0.10993716269731521, + "OOD_CommonPile|USPTO_Backgrounds|s0|p1": 0.10755025148391724, + "OOD_CommonPile|USPTO_Backgrounds|s0|p2": 0.10623774777632207, + "OOD_CommonPile|USPTO_Backgrounds|s1|p0": 0.10858874842524528, + "OOD_CommonPile|USPTO_Backgrounds|s1|p1": 0.10577252581715584, + "OOD_CommonPile|USPTO_Backgrounds|s1|p2": 0.10499313101172447, + "OOD_CommonPile|USPTO_Backgrounds|s2|p0": 0.10774560272693634, + "OOD_CommonPile|USPTO_Backgrounds|s2|p1": 0.1044268898665905, + "OOD_CommonPile|USPTO_Backgrounds|s2|p2": 0.10286219965200871, + "OOD_CommonPile|Pile-CC|s0|p0": 0.10546645920258016, + "OOD_CommonPile|Pile-CC|s0|p1": 0.10488782774191349, + "OOD_CommonPile|Pile-CC|s0|p2": 0.10519207580946385, + "OOD_CommonPile|Pile-CC|s1|p0": 0.10427305549383163, + "OOD_CommonPile|Pile-CC|s1|p1": 0.10411011825781316, + "OOD_CommonPile|Pile-CC|s1|p2": 0.10122805275022984, + "OOD_CommonPile|Pile-CC|s2|p0": 0.10340902581810951, + "OOD_CommonPile|Pile-CC|s2|p1": 0.10254368435125798, + "OOD_CommonPile|Pile-CC|s2|p2": 0.10123745936434717, + "OOD_CommonPile|StackExchange|s0|p0": 0.09133448041975498, + "OOD_CommonPile|StackExchange|s0|p1": 0.09332759890239686, + "OOD_CommonPile|StackExchange|s0|p2": 0.09280050497036427, + "OOD_CommonPile|StackExchange|s1|p0": 0.089323028922081, + "OOD_CommonPile|StackExchange|s1|p1": 0.09336440439801663, + "OOD_CommonPile|StackExchange|s1|p2": 0.08880413200240582, + "OOD_CommonPile|StackExchange|s2|p0": 0.08906285911798477, + "OOD_CommonPile|StackExchange|s2|p1": 0.09172479982953519, + "OOD_CommonPile|StackExchange|s2|p2": 0.08832788865547628, + "OOD_CommonPile|Github|s0|p0": 0.08980295225046575, + "OOD_CommonPile|Github|s0|p1": 0.09031869731843471, + "OOD_CommonPile|Github|s0|p2": 0.0921266384422779, + "OOD_CommonPile|Github|s1|p0": 0.08756385124288499, + "OOD_CommonPile|Github|s1|p1": 0.08822311870753766, + "OOD_CommonPile|Github|s1|p2": 0.08942141234874726, + "OOD_CommonPile|Github|s2|p0": 0.08700107759796083, + "OOD_CommonPile|Github|s2|p1": 0.08827582001686096, + "OOD_CommonPile|Github|s2|p2": 0.08929079659283161, + "OOD_CommonPile|OOD_News_2024|s0|p0": 0.10261754072271287, + "OOD_CommonPile|OOD_News_2024|s0|p1": 0.10728695902507752, + "OOD_CommonPile|OOD_News_2024|s0|p2": 0.10852934184949845, + "OOD_CommonPile|OOD_News_2024|s1|p0": 0.10051904171705246, + "OOD_CommonPile|OOD_News_2024|s1|p1": 0.10551608938258142, + "OOD_CommonPile|OOD_News_2024|s1|p2": 0.10540087521076202, + "OOD_CommonPile|OOD_News_2024|s2|p0": 0.10204602803569288, + "OOD_CommonPile|OOD_News_2024|s2|p1": 0.10457268122117966, + "OOD_CommonPile|OOD_News_2024|s2|p2": 0.10608735121786594, + "OOD_CommonPile|OOD_arXiv_2023|s0|p0": 0.10200607988517732, + "OOD_CommonPile|OOD_arXiv_2023|s0|p1": 0.09373965905979276, + "OOD_CommonPile|OOD_arXiv_2023|s0|p2": 0.09705941181164235, + "OOD_CommonPile|OOD_arXiv_2023|s1|p0": 0.1018203642917797, + "OOD_CommonPile|OOD_arXiv_2023|s1|p1": 0.0934268148150295, + "OOD_CommonPile|OOD_arXiv_2023|s1|p2": 0.09543103638570756, + "OOD_CommonPile|OOD_arXiv_2023|s2|p0": 0.09906993869226426, + "OOD_CommonPile|OOD_arXiv_2023|s2|p1": 0.09162668972276151, + "OOD_CommonPile|OOD_arXiv_2023|s2|p2": 0.09541914365254342, + "OOD_CommonPile|OOD_CommonPile|s0|p0": 0.09886385067366063, + "OOD_CommonPile|OOD_CommonPile|s0|p1": 0.1011885245796293, + "OOD_CommonPile|OOD_CommonPile|s0|p2": 0.09972966322675347, + "OOD_CommonPile|OOD_CommonPile|s1|p0": 0.09919246076606214, + "OOD_CommonPile|OOD_CommonPile|s1|p1": 0.09914025641046464, + "OOD_CommonPile|OOD_CommonPile|s1|p2": 0.09790974976494908, + "OOD_CommonPile|OOD_CommonPile|s2|p0": 0.0978173294570297, + "OOD_CommonPile|OOD_CommonPile|s2|p1": 0.09950889772735536, + "OOD_CommonPile|OOD_CommonPile|s2|p2": 0.09745372654870152, + "logit|Q0|p0": 0.08317913140635938, + "shuf|Q0|p0": 0.0813209842890501, + "logit|Q0|p1": 0.08317913140635938, + "shuf|Q0|p1": 0.0813209842890501, + "logit|Q0|p2": 0.08317913140635938, + "shuf|Q0|p2": 0.0813209842890501, + "logit|Wikipedia_en|p0": 0.07340146719943733, + "shuf|Wikipedia_en|p0": 0.07935486109927296, + "logit|Wikipedia_en|p1": 0.07250181648414582, + "shuf|Wikipedia_en|p1": 0.07509253025054932, + "logit|Wikipedia_en|p2": 0.07630303453188389, + "shuf|Wikipedia_en|p2": 0.08124284911900759, + "logit|USPTO_Backgrounds|p0": 0.0820519546745345, + "shuf|USPTO_Backgrounds|p0": 0.08504631593823433, + "logit|USPTO_Backgrounds|p1": 0.08030368338804691, + "shuf|USPTO_Backgrounds|p1": 0.08468587063252926, + "logit|USPTO_Backgrounds|p2": 0.07827811490278691, + "shuf|USPTO_Backgrounds|p2": 0.08340702876448632, + "logit|Pile-CC|p0": 0.07749354566913098, + "shuf|Pile-CC|p0": 0.07952790632843972, + "logit|Pile-CC|p1": 0.07532729792874307, + "shuf|Pile-CC|p1": 0.07693478614091873, + "logit|Pile-CC|p2": 0.07531398746650667, + "shuf|Pile-CC|p2": 0.07729805875569581, + "logit|StackExchange|p0": 0.06309177875518798, + "shuf|StackExchange|p0": 0.06341148084029555, + "logit|StackExchange|p1": 0.0632233064621687, + "shuf|StackExchange|p1": 0.06579978726804256, + "logit|StackExchange|p2": 0.06550364296417684, + "shuf|StackExchange|p2": 0.06608042120933533, + "logit|Github|p0": 0.06397150668781251, + "shuf|Github|p0": 0.06534891556948423, + "logit|Github|p1": 0.06868080224376172, + "shuf|Github|p1": 0.0697057569399476, + "logit|Github|p2": 0.0658004532335326, + "shuf|Github|p2": 0.06398458331823349, + "logit|OOD_News_2024|p0": 0.0799933175323531, + "shuf|OOD_News_2024|p0": 0.08315554149448871, + "logit|OOD_News_2024|p1": 0.07694857630413025, + "shuf|OOD_News_2024|p1": 0.07979506626725197, + "logit|OOD_News_2024|p2": 0.07920456628780811, + "shuf|OOD_News_2024|p2": 0.08362352550029754, + "logit|OOD_arXiv_2023|p0": 0.06991660855710506, + "shuf|OOD_arXiv_2023|p0": 0.0785893840715289, + "logit|OOD_arXiv_2023|p1": 0.06607239060103894, + "shuf|OOD_arXiv_2023|p1": 0.06938266362994909, + "logit|OOD_arXiv_2023|p2": 0.07230275857727975, + "shuf|OOD_arXiv_2023|p2": 0.07276751305907965, + "logit|OOD_CommonPile|p0": 0.07296433269511908, + "shuf|OOD_CommonPile|p0": 0.07364496747031808, + "logit|OOD_CommonPile|p1": 0.0726003646850586, + "shuf|OOD_CommonPile|p1": 0.07506469637155533, + "logit|OOD_CommonPile|p2": 0.07151196151971817, + "shuf|OOD_CommonPile|p2": 0.07718269526958466, + "logit|SHUFFLED_Pile-CC|p0": 0.07090201887767762, + "shuf|SHUFFLED_Pile-CC|p0": 0.07396699357777833, + "logit|SHUFFLED_Pile-CC|p1": 0.06621075260918588, + "shuf|SHUFFLED_Pile-CC|p1": 0.06998175755143166, + "logit|SHUFFLED_Pile-CC|p2": 0.06560704000294208, + "shuf|SHUFFLED_Pile-CC|p2": 0.07315811403095722 + }, + "min": { + "Wikipedia_en|Wikipedia_en|s0|p0": 0.23559173233807087, + "Wikipedia_en|Wikipedia_en|s0|p1": 0.2419768188148737, + "Wikipedia_en|Wikipedia_en|s0|p2": 0.23857605531811715, + "Wikipedia_en|Wikipedia_en|s1|p0": 0.2355704691261053, + "Wikipedia_en|Wikipedia_en|s1|p1": 0.2424818158149719, + "Wikipedia_en|Wikipedia_en|s1|p2": 0.23949622586369515, + "Wikipedia_en|Wikipedia_en|s2|p0": 0.23400513865053654, + "Wikipedia_en|Wikipedia_en|s2|p1": 0.24223571456968784, + "Wikipedia_en|Wikipedia_en|s2|p2": 0.2406516782939434, + "Wikipedia_en|USPTO_Backgrounds|s0|p0": 0.24651608429849148, + "Wikipedia_en|USPTO_Backgrounds|s0|p1": 0.24143020175397395, + "Wikipedia_en|USPTO_Backgrounds|s0|p2": 0.2419296596199274, + "Wikipedia_en|USPTO_Backgrounds|s1|p0": 0.24809634126722813, + "Wikipedia_en|USPTO_Backgrounds|s1|p1": 0.24321485087275505, + "Wikipedia_en|USPTO_Backgrounds|s1|p2": 0.2439087603241205, + "Wikipedia_en|USPTO_Backgrounds|s2|p0": 0.24572716280817986, + "Wikipedia_en|USPTO_Backgrounds|s2|p1": 0.23986265063285828, + "Wikipedia_en|USPTO_Backgrounds|s2|p2": 0.24207696244120597, + "Wikipedia_en|Pile-CC|s0|p0": 0.23911351077258586, + "Wikipedia_en|Pile-CC|s0|p1": 0.23813782930374144, + "Wikipedia_en|Pile-CC|s0|p2": 0.24229598976671696, + "Wikipedia_en|Pile-CC|s1|p0": 0.24293801784515381, + "Wikipedia_en|Pile-CC|s1|p1": 0.23888144828379154, + "Wikipedia_en|Pile-CC|s1|p2": 0.24640221372246743, + "Wikipedia_en|Pile-CC|s2|p0": 0.24099382385611534, + "Wikipedia_en|Pile-CC|s2|p1": 0.23742006570100785, + "Wikipedia_en|Pile-CC|s2|p2": 0.24364722073078154, + "Wikipedia_en|StackExchange|s0|p0": 0.21854499280452727, + "Wikipedia_en|StackExchange|s0|p1": 0.21670846473425626, + "Wikipedia_en|StackExchange|s0|p2": 0.2287700030952692, + "Wikipedia_en|StackExchange|s1|p0": 0.2188444882631302, + "Wikipedia_en|StackExchange|s1|p1": 0.21711578778922558, + "Wikipedia_en|StackExchange|s1|p2": 0.22872391082346438, + "Wikipedia_en|StackExchange|s2|p0": 0.2185226310044527, + "Wikipedia_en|StackExchange|s2|p1": 0.21671680100262164, + "Wikipedia_en|StackExchange|s2|p2": 0.2262044344097376, + "Wikipedia_en|Github|s0|p0": 0.22048368602991103, + "Wikipedia_en|Github|s0|p1": 0.22245462723076342, + "Wikipedia_en|Github|s0|p2": 0.22570112459361552, + "Wikipedia_en|Github|s1|p0": 0.22321333214640618, + "Wikipedia_en|Github|s1|p1": 0.2239327307790518, + "Wikipedia_en|Github|s1|p2": 0.2257034182548523, + "Wikipedia_en|Github|s2|p0": 0.22219215966761113, + "Wikipedia_en|Github|s2|p1": 0.22358915843069554, + "Wikipedia_en|Github|s2|p2": 0.2249430600553751, + "Wikipedia_en|OOD_News_2024|s0|p0": 0.23720385767519475, + "Wikipedia_en|OOD_News_2024|s0|p1": 0.23854632824659347, + "Wikipedia_en|OOD_News_2024|s0|p2": 0.2381707113236189, + "Wikipedia_en|OOD_News_2024|s1|p0": 0.23822148963809014, + "Wikipedia_en|OOD_News_2024|s1|p1": 0.2380030196160078, + "Wikipedia_en|OOD_News_2024|s1|p2": 0.24264381565153598, + "Wikipedia_en|OOD_News_2024|s2|p0": 0.23782750591635704, + "Wikipedia_en|OOD_News_2024|s2|p1": 0.23773289620876312, + "Wikipedia_en|OOD_News_2024|s2|p2": 0.24000127539038657, + "Wikipedia_en|OOD_arXiv_2023|s0|p0": 0.2287459883838892, + "Wikipedia_en|OOD_arXiv_2023|s0|p1": 0.21680260710418225, + "Wikipedia_en|OOD_arXiv_2023|s0|p2": 0.22441966384649276, + "Wikipedia_en|OOD_arXiv_2023|s1|p0": 0.23060149177908898, + "Wikipedia_en|OOD_arXiv_2023|s1|p1": 0.2180108856409788, + "Wikipedia_en|OOD_arXiv_2023|s1|p2": 0.22784982137382032, + "Wikipedia_en|OOD_arXiv_2023|s2|p0": 0.22785247564315797, + "Wikipedia_en|OOD_arXiv_2023|s2|p1": 0.2178528845310211, + "Wikipedia_en|OOD_arXiv_2023|s2|p2": 0.22465503327548503, + "Wikipedia_en|OOD_CommonPile|s0|p0": 0.2243207275867462, + "Wikipedia_en|OOD_CommonPile|s0|p1": 0.21940520443022252, + "Wikipedia_en|OOD_CommonPile|s0|p2": 0.21518170163035394, + "Wikipedia_en|OOD_CommonPile|s1|p0": 0.22572337612509727, + "Wikipedia_en|OOD_CommonPile|s1|p1": 0.22214061841368676, + "Wikipedia_en|OOD_CommonPile|s1|p2": 0.21864149309694766, + "Wikipedia_en|OOD_CommonPile|s2|p0": 0.2260938860476017, + "Wikipedia_en|OOD_CommonPile|s2|p1": 0.21988762430846692, + "Wikipedia_en|OOD_CommonPile|s2|p2": 0.21747272163629533, + "USPTO_Backgrounds|Wikipedia_en|s0|p0": 0.26079800315201285, + "USPTO_Backgrounds|Wikipedia_en|s0|p1": 0.26023799665272235, + "USPTO_Backgrounds|Wikipedia_en|s0|p2": 0.26369450725615023, + "USPTO_Backgrounds|Wikipedia_en|s1|p0": 0.25861492566764355, + "USPTO_Backgrounds|Wikipedia_en|s1|p1": 0.2594308596104383, + "USPTO_Backgrounds|Wikipedia_en|s1|p2": 0.26490016989409926, + "USPTO_Backgrounds|Wikipedia_en|s2|p0": 0.2635689791291952, + "USPTO_Backgrounds|Wikipedia_en|s2|p1": 0.2620432171970606, + "USPTO_Backgrounds|Wikipedia_en|s2|p2": 0.2664452876895666, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p0": 0.2710931975394487, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p1": 0.26996083110570906, + "USPTO_Backgrounds|USPTO_Backgrounds|s0|p2": 0.26308480836451054, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p0": 0.2682929139584303, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p1": 0.26737085431814195, + "USPTO_Backgrounds|USPTO_Backgrounds|s1|p2": 0.26264346241950987, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p0": 0.2742444146424532, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p1": 0.27119845561683176, + "USPTO_Backgrounds|USPTO_Backgrounds|s2|p2": 0.2674694787710905, + "USPTO_Backgrounds|Pile-CC|s0|p0": 0.26990832686424254, + "USPTO_Backgrounds|Pile-CC|s0|p1": 0.2649305921047926, + "USPTO_Backgrounds|Pile-CC|s0|p2": 0.26439669243991376, + "USPTO_Backgrounds|Pile-CC|s1|p0": 0.268923794105649, + "USPTO_Backgrounds|Pile-CC|s1|p1": 0.2619213663041592, + "USPTO_Backgrounds|Pile-CC|s1|p2": 0.2662314247339964, + "USPTO_Backgrounds|Pile-CC|s2|p0": 0.272131796553731, + "USPTO_Backgrounds|Pile-CC|s2|p1": 0.2684003166854382, + "USPTO_Backgrounds|Pile-CC|s2|p2": 0.26567666344344615, + "USPTO_Backgrounds|StackExchange|s0|p0": 0.2445195846259594, + "USPTO_Backgrounds|StackExchange|s0|p1": 0.24857791382819414, + "USPTO_Backgrounds|StackExchange|s0|p2": 0.25245599560439586, + "USPTO_Backgrounds|StackExchange|s1|p0": 0.24438476003706455, + "USPTO_Backgrounds|StackExchange|s1|p1": 0.24861333500593902, + "USPTO_Backgrounds|StackExchange|s1|p2": 0.24828180968761443, + "USPTO_Backgrounds|StackExchange|s2|p0": 0.2484942317008972, + "USPTO_Backgrounds|StackExchange|s2|p1": 0.2515471769496799, + "USPTO_Backgrounds|StackExchange|s2|p2": 0.2540241312235594, + "USPTO_Backgrounds|Github|s0|p0": 0.24965590573847293, + "USPTO_Backgrounds|Github|s0|p1": 0.24742183703929185, + "USPTO_Backgrounds|Github|s0|p2": 0.24920591227710248, + "USPTO_Backgrounds|Github|s1|p0": 0.24828251339495183, + "USPTO_Backgrounds|Github|s1|p1": 0.24532638303935528, + "USPTO_Backgrounds|Github|s1|p2": 0.24791017286479472, + "USPTO_Backgrounds|Github|s2|p0": 0.2527140185236931, + "USPTO_Backgrounds|Github|s2|p1": 0.2507519446313381, + "USPTO_Backgrounds|Github|s2|p2": 0.2521766427904367, + "USPTO_Backgrounds|OOD_News_2024|s0|p0": 0.26018218100070956, + "USPTO_Backgrounds|OOD_News_2024|s0|p1": 0.26095977909862994, + "USPTO_Backgrounds|OOD_News_2024|s0|p2": 0.26718683503568175, + "USPTO_Backgrounds|OOD_News_2024|s1|p0": 0.2602081723511219, + "USPTO_Backgrounds|OOD_News_2024|s1|p1": 0.2603794734925032, + "USPTO_Backgrounds|OOD_News_2024|s1|p2": 0.26742078773677347, + "USPTO_Backgrounds|OOD_News_2024|s2|p0": 0.26465881802141666, + "USPTO_Backgrounds|OOD_News_2024|s2|p1": 0.26454676017165185, + "USPTO_Backgrounds|OOD_News_2024|s2|p2": 0.2711191389709711, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p0": 0.24896901212632655, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p1": 0.23700966648757457, + "USPTO_Backgrounds|OOD_arXiv_2023|s0|p2": 0.24048738181591034, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p0": 0.2503365736454725, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p1": 0.23678390495479107, + "USPTO_Backgrounds|OOD_arXiv_2023|s1|p2": 0.24144411981105804, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p0": 0.25340586006641386, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p1": 0.23959226422011853, + "USPTO_Backgrounds|OOD_arXiv_2023|s2|p2": 0.24318628199398518, + "USPTO_Backgrounds|OOD_CommonPile|s0|p0": 0.24241479746997358, + "USPTO_Backgrounds|OOD_CommonPile|s0|p1": 0.2418103117495775, + "USPTO_Backgrounds|OOD_CommonPile|s0|p2": 0.23755589686334133, + "USPTO_Backgrounds|OOD_CommonPile|s1|p0": 0.24074137322604655, + "USPTO_Backgrounds|OOD_CommonPile|s1|p1": 0.24397865645587444, + "USPTO_Backgrounds|OOD_CommonPile|s1|p2": 0.23760038055479527, + "USPTO_Backgrounds|OOD_CommonPile|s2|p0": 0.24500143714249134, + "USPTO_Backgrounds|OOD_CommonPile|s2|p1": 0.24555960893630982, + "USPTO_Backgrounds|OOD_CommonPile|s2|p2": 0.2383239384740591, + "Pile-CC|Wikipedia_en|s0|p0": 0.26227477230131624, + "Pile-CC|Wikipedia_en|s0|p1": 0.2647818010300398, + "Pile-CC|Wikipedia_en|s0|p2": 0.2654181521385908, + "Pile-CC|Wikipedia_en|s1|p0": 0.2640880897641182, + "Pile-CC|Wikipedia_en|s1|p1": 0.2681259285658598, + "Pile-CC|Wikipedia_en|s1|p2": 0.26783572472631934, + "Pile-CC|Wikipedia_en|s2|p0": 0.2593778442591429, + "Pile-CC|Wikipedia_en|s2|p1": 0.2631962638348341, + "Pile-CC|Wikipedia_en|s2|p2": 0.26447971649467944, + "Pile-CC|USPTO_Backgrounds|s0|p0": 0.2733915727585554, + "Pile-CC|USPTO_Backgrounds|s0|p1": 0.26858933866024015, + "Pile-CC|USPTO_Backgrounds|s0|p2": 0.26778230629861355, + "Pile-CC|USPTO_Backgrounds|s1|p0": 0.2767828684300184, + "Pile-CC|USPTO_Backgrounds|s1|p1": 0.27417271621525285, + "Pile-CC|USPTO_Backgrounds|s1|p2": 0.2716909270733595, + "Pile-CC|USPTO_Backgrounds|s2|p0": 0.2717436093837023, + "Pile-CC|USPTO_Backgrounds|s2|p1": 0.2678472075611353, + "Pile-CC|USPTO_Backgrounds|s2|p2": 0.26815025694668293, + "Pile-CC|Pile-CC|s0|p0": 0.26710597313940526, + "Pile-CC|Pile-CC|s0|p1": 0.26512613110244276, + "Pile-CC|Pile-CC|s0|p2": 0.26775723583996297, + "Pile-CC|Pile-CC|s1|p0": 0.26896970756351946, + "Pile-CC|Pile-CC|s1|p1": 0.27056138813495634, + "Pile-CC|Pile-CC|s1|p2": 0.26964860446751115, + "Pile-CC|Pile-CC|s2|p0": 0.2642395541071892, + "Pile-CC|Pile-CC|s2|p1": 0.26475665979087354, + "Pile-CC|Pile-CC|s2|p2": 0.2670920383185148, + "Pile-CC|StackExchange|s0|p0": 0.23599604479968547, + "Pile-CC|StackExchange|s0|p1": 0.23586784787476062, + "Pile-CC|StackExchange|s0|p2": 0.24471848532557489, + "Pile-CC|StackExchange|s1|p0": 0.23879887890070678, + "Pile-CC|StackExchange|s1|p1": 0.2383223671466112, + "Pile-CC|StackExchange|s1|p2": 0.24606648907065393, + "Pile-CC|StackExchange|s2|p0": 0.23616222850978374, + "Pile-CC|StackExchange|s2|p1": 0.23557006306946277, + "Pile-CC|StackExchange|s2|p2": 0.24137646853923797, + "Pile-CC|Github|s0|p0": 0.24502260647714139, + "Pile-CC|Github|s0|p1": 0.24520749673247338, + "Pile-CC|Github|s0|p2": 0.24892854131758213, + "Pile-CC|Github|s1|p0": 0.24867025539278984, + "Pile-CC|Github|s1|p1": 0.2477807763963938, + "Pile-CC|Github|s1|p2": 0.24930098988115787, + "Pile-CC|Github|s2|p0": 0.2431879721581936, + "Pile-CC|Github|s2|p1": 0.24630263186991214, + "Pile-CC|Github|s2|p2": 0.24829878695309163, + "Pile-CC|OOD_News_2024|s0|p0": 0.2650149080902338, + "Pile-CC|OOD_News_2024|s0|p1": 0.2684627126902342, + "Pile-CC|OOD_News_2024|s0|p2": 0.2696282148361206, + "Pile-CC|OOD_News_2024|s1|p0": 0.26954546868801116, + "Pile-CC|OOD_News_2024|s1|p1": 0.2719507772475481, + "Pile-CC|OOD_News_2024|s1|p2": 0.2731183160096407, + "Pile-CC|OOD_News_2024|s2|p0": 0.2642189998179674, + "Pile-CC|OOD_News_2024|s2|p1": 0.26735082156956197, + "Pile-CC|OOD_News_2024|s2|p2": 0.26713199317455294, + "Pile-CC|OOD_arXiv_2023|s0|p0": 0.25181057788431643, + "Pile-CC|OOD_arXiv_2023|s0|p1": 0.23790208846330643, + "Pile-CC|OOD_arXiv_2023|s0|p2": 0.24742024578154087, + "Pile-CC|OOD_arXiv_2023|s1|p0": 0.2543182235211134, + "Pile-CC|OOD_arXiv_2023|s1|p1": 0.24188863188028337, + "Pile-CC|OOD_arXiv_2023|s1|p2": 0.24866918958723544, + "Pile-CC|OOD_arXiv_2023|s2|p0": 0.2503766890615225, + "Pile-CC|OOD_arXiv_2023|s2|p1": 0.236076258122921, + "Pile-CC|OOD_arXiv_2023|s2|p2": 0.2449353937059641, + "Pile-CC|OOD_CommonPile|s0|p0": 0.24654796533286572, + "Pile-CC|OOD_CommonPile|s0|p1": 0.24658583477139473, + "Pile-CC|OOD_CommonPile|s0|p2": 0.23822186104953288, + "Pile-CC|OOD_CommonPile|s1|p0": 0.24898979477584363, + "Pile-CC|OOD_CommonPile|s1|p1": 0.25016585402190683, + "Pile-CC|OOD_CommonPile|s1|p2": 0.24091683961451055, + "Pile-CC|OOD_CommonPile|s2|p0": 0.24466599859297275, + "Pile-CC|OOD_CommonPile|s2|p1": 0.24575376510620117, + "Pile-CC|OOD_CommonPile|s2|p2": 0.23590255081653594, + "StackExchange|Wikipedia_en|s0|p0": 0.26708487756550314, + "StackExchange|Wikipedia_en|s0|p1": 0.26702655889093874, + "StackExchange|Wikipedia_en|s0|p2": 0.2713322076946497, + "StackExchange|Wikipedia_en|s1|p0": 0.26612422950565817, + "StackExchange|Wikipedia_en|s1|p1": 0.26468678526580336, + "StackExchange|Wikipedia_en|s1|p2": 0.2703015774488449, + "StackExchange|Wikipedia_en|s2|p0": 0.264142282679677, + "StackExchange|Wikipedia_en|s2|p1": 0.265266328305006, + "StackExchange|Wikipedia_en|s2|p2": 0.26813106052577496, + "StackExchange|USPTO_Backgrounds|s0|p0": 0.2739667546004057, + "StackExchange|USPTO_Backgrounds|s0|p1": 0.27554777786135676, + "StackExchange|USPTO_Backgrounds|s0|p2": 0.2730525989085436, + "StackExchange|USPTO_Backgrounds|s1|p0": 0.27432528547942636, + "StackExchange|USPTO_Backgrounds|s1|p1": 0.27360799722373486, + "StackExchange|USPTO_Backgrounds|s1|p2": 0.26837617307901385, + "StackExchange|USPTO_Backgrounds|s2|p0": 0.27206729669123886, + "StackExchange|USPTO_Backgrounds|s2|p1": 0.27351998910307884, + "StackExchange|USPTO_Backgrounds|s2|p2": 0.2668909475207329, + "StackExchange|Pile-CC|s0|p0": 0.2707675201818347, + "StackExchange|Pile-CC|s0|p1": 0.2683842495083809, + "StackExchange|Pile-CC|s0|p2": 0.2744338378310204, + "StackExchange|Pile-CC|s1|p0": 0.2712674351409078, + "StackExchange|Pile-CC|s1|p1": 0.2681666549295187, + "StackExchange|Pile-CC|s1|p2": 0.272859214246273, + "StackExchange|Pile-CC|s2|p0": 0.27037889789789915, + "StackExchange|Pile-CC|s2|p1": 0.2656990561634302, + "StackExchange|Pile-CC|s2|p2": 0.2724442068487406, + "StackExchange|StackExchange|s0|p0": 0.243925947137177, + "StackExchange|StackExchange|s0|p1": 0.24676271490752696, + "StackExchange|StackExchange|s0|p2": 0.25753657557070253, + "StackExchange|StackExchange|s1|p0": 0.24298066422343254, + "StackExchange|StackExchange|s1|p1": 0.24476130828261375, + "StackExchange|StackExchange|s1|p2": 0.2551237031817436, + "StackExchange|StackExchange|s2|p0": 0.24296057540923358, + "StackExchange|StackExchange|s2|p1": 0.24529658667743207, + "StackExchange|StackExchange|s2|p2": 0.25483956374228, + "StackExchange|Github|s0|p0": 0.2550347451120615, + "StackExchange|Github|s0|p1": 0.252735860273242, + "StackExchange|Github|s0|p2": 0.2574697744101286, + "StackExchange|Github|s1|p0": 0.2547786455601454, + "StackExchange|Github|s1|p1": 0.2550808789208531, + "StackExchange|Github|s1|p2": 0.25543036572635175, + "StackExchange|Github|s2|p0": 0.25405816845595836, + "StackExchange|Github|s2|p1": 0.25150021333247424, + "StackExchange|Github|s2|p2": 0.25623548664152623, + "StackExchange|OOD_News_2024|s0|p0": 0.2689829017966986, + "StackExchange|OOD_News_2024|s0|p1": 0.27478049620985984, + "StackExchange|OOD_News_2024|s0|p2": 0.272826286777854, + "StackExchange|OOD_News_2024|s1|p0": 0.2665801320225, + "StackExchange|OOD_News_2024|s1|p1": 0.2732456438243389, + "StackExchange|OOD_News_2024|s1|p2": 0.2693655643612146, + "StackExchange|OOD_News_2024|s2|p0": 0.2652173902839422, + "StackExchange|OOD_News_2024|s2|p1": 0.2718945246189833, + "StackExchange|OOD_News_2024|s2|p2": 0.2689213030040264, + "StackExchange|OOD_arXiv_2023|s0|p0": 0.24951795376837255, + "StackExchange|OOD_arXiv_2023|s0|p1": 0.24512783400714397, + "StackExchange|OOD_arXiv_2023|s0|p2": 0.2519645940512419, + "StackExchange|OOD_arXiv_2023|s1|p0": 0.24825628623366355, + "StackExchange|OOD_arXiv_2023|s1|p1": 0.24062920212745667, + "StackExchange|OOD_arXiv_2023|s1|p2": 0.2511869687587023, + "StackExchange|OOD_arXiv_2023|s2|p0": 0.24707536920905113, + "StackExchange|OOD_arXiv_2023|s2|p1": 0.24241991564631463, + "StackExchange|OOD_arXiv_2023|s2|p2": 0.24801052026450635, + "StackExchange|OOD_CommonPile|s0|p0": 0.25459455959498883, + "StackExchange|OOD_CommonPile|s0|p1": 0.2510111853480339, + "StackExchange|OOD_CommonPile|s0|p2": 0.2478191874921322, + "StackExchange|OOD_CommonPile|s1|p0": 0.2541017424315214, + "StackExchange|OOD_CommonPile|s1|p1": 0.2495775490999222, + "StackExchange|OOD_CommonPile|s1|p2": 0.24539121836423874, + "StackExchange|OOD_CommonPile|s2|p0": 0.2532394926995039, + "StackExchange|OOD_CommonPile|s2|p1": 0.24823651015758513, + "StackExchange|OOD_CommonPile|s2|p2": 0.245114578679204, + "Github|Wikipedia_en|s0|p0": 0.23403171263635159, + "Github|Wikipedia_en|s0|p1": 0.231602119281888, + "Github|Wikipedia_en|s0|p2": 0.23009944185614586, + "Github|Wikipedia_en|s1|p0": 0.23776253648102283, + "Github|Wikipedia_en|s1|p1": 0.2353681180626154, + "Github|Wikipedia_en|s1|p2": 0.23577983491122723, + "Github|Wikipedia_en|s2|p0": 0.23774986602365972, + "Github|Wikipedia_en|s2|p1": 0.23623924069106578, + "Github|Wikipedia_en|s2|p2": 0.2347534205764532, + "Github|USPTO_Backgrounds|s0|p0": 0.23857859708368778, + "Github|USPTO_Backgrounds|s0|p1": 0.23510799445211888, + "Github|USPTO_Backgrounds|s0|p2": 0.2319460991770029, + "Github|USPTO_Backgrounds|s1|p0": 0.24336212277412414, + "Github|USPTO_Backgrounds|s1|p1": 0.23917836621403693, + "Github|USPTO_Backgrounds|s1|p2": 0.23926933594048022, + "Github|USPTO_Backgrounds|s2|p0": 0.24167158380150794, + "Github|USPTO_Backgrounds|s2|p1": 0.23581885173916817, + "Github|USPTO_Backgrounds|s2|p2": 0.23479350842535496, + "Github|Pile-CC|s0|p0": 0.23414437063038349, + "Github|Pile-CC|s0|p1": 0.23919060714542867, + "Github|Pile-CC|s0|p2": 0.23655942492187024, + "Github|Pile-CC|s1|p0": 0.23897907845675945, + "Github|Pile-CC|s1|p1": 0.24522052295506, + "Github|Pile-CC|s1|p2": 0.2433925185352564, + "Github|Pile-CC|s2|p0": 0.23792345337569715, + "Github|Pile-CC|s2|p1": 0.24138508550822735, + "Github|Pile-CC|s2|p2": 0.24126839637756348, + "Github|StackExchange|s0|p0": 0.21985592618584632, + "Github|StackExchange|s0|p1": 0.2205735608935356, + "Github|StackExchange|s0|p2": 0.22019133456051349, + "Github|StackExchange|s1|p0": 0.22401885576546193, + "Github|StackExchange|s1|p1": 0.22281317599117756, + "Github|StackExchange|s1|p2": 0.22326628901064396, + "Github|StackExchange|s2|p0": 0.22387790754437448, + "Github|StackExchange|s2|p1": 0.22567571327090263, + "Github|StackExchange|s2|p2": 0.22322920858860015, + "Github|Github|s0|p0": 0.22176789529621602, + "Github|Github|s0|p1": 0.21917909272015096, + "Github|Github|s0|p2": 0.22362769283354283, + "Github|Github|s1|p0": 0.22527159787714482, + "Github|Github|s1|p1": 0.22162727303802968, + "Github|Github|s1|p2": 0.2286387462168932, + "Github|Github|s2|p0": 0.22380982972681523, + "Github|Github|s2|p1": 0.22135366909205914, + "Github|Github|s2|p2": 0.22565818540751933, + "Github|OOD_News_2024|s0|p0": 0.23022889532148838, + "Github|OOD_News_2024|s0|p1": 0.23455003090202808, + "Github|OOD_News_2024|s0|p2": 0.23558532409369945, + "Github|OOD_News_2024|s1|p0": 0.23291241228580475, + "Github|OOD_News_2024|s1|p1": 0.2383515987545252, + "Github|OOD_News_2024|s1|p2": 0.2380946885794401, + "Github|OOD_News_2024|s2|p0": 0.23282034173607827, + "Github|OOD_News_2024|s2|p1": 0.23525427617132663, + "Github|OOD_News_2024|s2|p2": 0.23711406104266644, + "Github|OOD_arXiv_2023|s0|p0": 0.213456891477108, + "Github|OOD_arXiv_2023|s0|p1": 0.20717236883938311, + "Github|OOD_arXiv_2023|s0|p2": 0.21279067359864712, + "Github|OOD_arXiv_2023|s1|p0": 0.2169382456690073, + "Github|OOD_arXiv_2023|s1|p1": 0.2109230447560549, + "Github|OOD_arXiv_2023|s1|p2": 0.2166394203901291, + "Github|OOD_arXiv_2023|s2|p0": 0.21511130221188068, + "Github|OOD_arXiv_2023|s2|p1": 0.20987408347427844, + "Github|OOD_arXiv_2023|s2|p2": 0.21615836657583715, + "Github|OOD_CommonPile|s0|p0": 0.21131003871560097, + "Github|OOD_CommonPile|s0|p1": 0.21548460833728314, + "Github|OOD_CommonPile|s0|p2": 0.20897954739630223, + "Github|OOD_CommonPile|s1|p0": 0.21728649027645588, + "Github|OOD_CommonPile|s1|p1": 0.22245729751884938, + "Github|OOD_CommonPile|s1|p2": 0.213410772010684, + "Github|OOD_CommonPile|s2|p0": 0.21534915305674077, + "Github|OOD_CommonPile|s2|p1": 0.22149287573993207, + "Github|OOD_CommonPile|s2|p2": 0.21325913332402707, + "OOD_News_2024|Wikipedia_en|s0|p0": 0.2482516035437584, + "OOD_News_2024|Wikipedia_en|s0|p1": 0.25266767889261244, + "OOD_News_2024|Wikipedia_en|s0|p2": 0.25385730937123296, + "OOD_News_2024|Wikipedia_en|s1|p0": 0.24838275648653507, + "OOD_News_2024|Wikipedia_en|s1|p1": 0.25154665634036066, + "OOD_News_2024|Wikipedia_en|s1|p2": 0.25474652349948884, + "OOD_News_2024|Wikipedia_en|s2|p0": 0.24873915910720826, + "OOD_News_2024|Wikipedia_en|s2|p1": 0.252572512999177, + "OOD_News_2024|Wikipedia_en|s2|p2": 0.25546507686376574, + "OOD_News_2024|USPTO_Backgrounds|s0|p0": 0.2558609418570995, + "OOD_News_2024|USPTO_Backgrounds|s0|p1": 0.2527121387422085, + "OOD_News_2024|USPTO_Backgrounds|s0|p2": 0.25498764030635357, + "OOD_News_2024|USPTO_Backgrounds|s1|p0": 0.2600372664630413, + "OOD_News_2024|USPTO_Backgrounds|s1|p1": 0.25529108457267286, + "OOD_News_2024|USPTO_Backgrounds|s1|p2": 0.25732936821877955, + "OOD_News_2024|USPTO_Backgrounds|s2|p0": 0.26056455373764037, + "OOD_News_2024|USPTO_Backgrounds|s2|p1": 0.2522442601621151, + "OOD_News_2024|USPTO_Backgrounds|s2|p2": 0.2552744667977095, + "OOD_News_2024|Pile-CC|s0|p0": 0.25393903665244577, + "OOD_News_2024|Pile-CC|s0|p1": 0.25207536816596987, + "OOD_News_2024|Pile-CC|s0|p2": 0.2518101193010807, + "OOD_News_2024|Pile-CC|s1|p0": 0.25880682207643985, + "OOD_News_2024|Pile-CC|s1|p1": 0.2514157347381115, + "OOD_News_2024|Pile-CC|s1|p2": 0.2540384642779827, + "OOD_News_2024|Pile-CC|s2|p0": 0.2592462085187435, + "OOD_News_2024|Pile-CC|s2|p1": 0.25149822160601615, + "OOD_News_2024|Pile-CC|s2|p2": 0.254595672339201, + "OOD_News_2024|StackExchange|s0|p0": 0.22071772180497645, + "OOD_News_2024|StackExchange|s0|p1": 0.2222265165299177, + "OOD_News_2024|StackExchange|s0|p2": 0.22673949263989926, + "OOD_News_2024|StackExchange|s1|p0": 0.22314042113721372, + "OOD_News_2024|StackExchange|s1|p1": 0.2225597385317087, + "OOD_News_2024|StackExchange|s1|p2": 0.2293855484575033, + "OOD_News_2024|StackExchange|s2|p0": 0.2224451694637537, + "OOD_News_2024|StackExchange|s2|p1": 0.22311381734907626, + "OOD_News_2024|StackExchange|s2|p2": 0.22696954272687436, + "OOD_News_2024|Github|s0|p0": 0.23275815024971963, + "OOD_News_2024|Github|s0|p1": 0.23334380686283113, + "OOD_News_2024|Github|s0|p2": 0.23459918424487114, + "OOD_News_2024|Github|s1|p0": 0.23409605473279954, + "OOD_News_2024|Github|s1|p1": 0.23519825525581836, + "OOD_News_2024|Github|s1|p2": 0.23633661419153212, + "OOD_News_2024|Github|s2|p0": 0.2336925148963928, + "OOD_News_2024|Github|s2|p1": 0.23411213047802448, + "OOD_News_2024|Github|s2|p2": 0.2362111046910286, + "OOD_News_2024|OOD_News_2024|s0|p0": 0.250081143155694, + "OOD_News_2024|OOD_News_2024|s0|p1": 0.25028723999857905, + "OOD_News_2024|OOD_News_2024|s0|p2": 0.25075814574956895, + "OOD_News_2024|OOD_News_2024|s1|p0": 0.2532215092331171, + "OOD_News_2024|OOD_News_2024|s1|p1": 0.25123438239097595, + "OOD_News_2024|OOD_News_2024|s1|p2": 0.2503373879939318, + "OOD_News_2024|OOD_News_2024|s2|p0": 0.25495421290397646, + "OOD_News_2024|OOD_News_2024|s2|p1": 0.24974605292081833, + "OOD_News_2024|OOD_News_2024|s2|p2": 0.252954026684165, + "OOD_News_2024|OOD_arXiv_2023|s0|p0": 0.23898195773363112, + "OOD_News_2024|OOD_arXiv_2023|s0|p1": 0.22419425919651986, + "OOD_News_2024|OOD_arXiv_2023|s0|p2": 0.23678381443023683, + "OOD_News_2024|OOD_arXiv_2023|s1|p0": 0.2416083302348852, + "OOD_News_2024|OOD_arXiv_2023|s1|p1": 0.2278545930981636, + "OOD_News_2024|OOD_arXiv_2023|s1|p2": 0.238906342536211, + "OOD_News_2024|OOD_arXiv_2023|s2|p0": 0.24129452668130397, + "OOD_News_2024|OOD_arXiv_2023|s2|p1": 0.22643646001815795, + "OOD_News_2024|OOD_arXiv_2023|s2|p2": 0.23854657709598542, + "OOD_News_2024|OOD_CommonPile|s0|p0": 0.23288928493857383, + "OOD_News_2024|OOD_CommonPile|s0|p1": 0.23041776716709136, + "OOD_News_2024|OOD_CommonPile|s0|p2": 0.22299868911504744, + "OOD_News_2024|OOD_CommonPile|s1|p0": 0.23598985597491265, + "OOD_News_2024|OOD_CommonPile|s1|p1": 0.23161862418055534, + "OOD_News_2024|OOD_CommonPile|s1|p2": 0.22560284473001957, + "OOD_News_2024|OOD_CommonPile|s2|p0": 0.2339274302124977, + "OOD_News_2024|OOD_CommonPile|s2|p1": 0.2321193426847458, + "OOD_News_2024|OOD_CommonPile|s2|p2": 0.2254048202186823, + "OOD_arXiv_2023|Wikipedia_en|s0|p0": 0.25952857062220575, + "OOD_arXiv_2023|Wikipedia_en|s0|p1": 0.25577165707945826, + "OOD_arXiv_2023|Wikipedia_en|s0|p2": 0.2595399830490351, + "OOD_arXiv_2023|Wikipedia_en|s1|p0": 0.25761561989784243, + "OOD_arXiv_2023|Wikipedia_en|s1|p1": 0.25330714993178843, + "OOD_arXiv_2023|Wikipedia_en|s1|p2": 0.25800027437508105, + "OOD_arXiv_2023|Wikipedia_en|s2|p0": 0.2568712949752808, + "OOD_arXiv_2023|Wikipedia_en|s2|p1": 0.25636265315115453, + "OOD_arXiv_2023|Wikipedia_en|s2|p2": 0.2604085270315409, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p0": 0.27410735581070184, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p1": 0.26812446732074025, + "OOD_arXiv_2023|USPTO_Backgrounds|s0|p2": 0.26384863872081044, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p0": 0.2728977421298623, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p1": 0.2664399990811944, + "OOD_arXiv_2023|USPTO_Backgrounds|s1|p2": 0.26349656004458666, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p0": 0.276577889919281, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p1": 0.2711338669061661, + "OOD_arXiv_2023|USPTO_Backgrounds|s2|p2": 0.2638340534642339, + "OOD_arXiv_2023|Pile-CC|s0|p0": 0.2632638059556484, + "OOD_arXiv_2023|Pile-CC|s0|p1": 0.2636157266795635, + "OOD_arXiv_2023|Pile-CC|s0|p2": 0.26379723381251097, + "OOD_arXiv_2023|Pile-CC|s1|p0": 0.26255967244505884, + "OOD_arXiv_2023|Pile-CC|s1|p1": 0.25834017619490623, + "OOD_arXiv_2023|Pile-CC|s1|p2": 0.26068329736590384, + "OOD_arXiv_2023|Pile-CC|s2|p0": 0.26281105503439905, + "OOD_arXiv_2023|Pile-CC|s2|p1": 0.2633179299533367, + "OOD_arXiv_2023|Pile-CC|s2|p2": 0.265466295927763, + "OOD_arXiv_2023|StackExchange|s0|p0": 0.24023806471377612, + "OOD_arXiv_2023|StackExchange|s0|p1": 0.2395213546231389, + "OOD_arXiv_2023|StackExchange|s0|p2": 0.2497177978977561, + "OOD_arXiv_2023|StackExchange|s1|p0": 0.23576978053897618, + "OOD_arXiv_2023|StackExchange|s1|p1": 0.23841930646449327, + "OOD_arXiv_2023|StackExchange|s1|p2": 0.24615461919456721, + "OOD_arXiv_2023|StackExchange|s2|p0": 0.24163931515067816, + "OOD_arXiv_2023|StackExchange|s2|p1": 0.23994213249534369, + "OOD_arXiv_2023|StackExchange|s2|p2": 0.24969757106155158, + "OOD_arXiv_2023|Github|s0|p0": 0.24125250652432442, + "OOD_arXiv_2023|Github|s0|p1": 0.2459969801828265, + "OOD_arXiv_2023|Github|s0|p2": 0.2474335614591837, + "OOD_arXiv_2023|Github|s1|p0": 0.24009924307465552, + "OOD_arXiv_2023|Github|s1|p1": 0.24447923433035612, + "OOD_arXiv_2023|Github|s1|p2": 0.24429393857717513, + "OOD_arXiv_2023|Github|s2|p0": 0.24079972878098488, + "OOD_arXiv_2023|Github|s2|p1": 0.24815802108496426, + "OOD_arXiv_2023|Github|s2|p2": 0.25301194526255133, + "OOD_arXiv_2023|OOD_News_2024|s0|p0": 0.260426390171051, + "OOD_arXiv_2023|OOD_News_2024|s0|p1": 0.26494633182883265, + "OOD_arXiv_2023|OOD_News_2024|s0|p2": 0.26593123003840446, + "OOD_arXiv_2023|OOD_News_2024|s1|p0": 0.2584548287093639, + "OOD_arXiv_2023|OOD_News_2024|s1|p1": 0.26116600558161734, + "OOD_arXiv_2023|OOD_News_2024|s1|p2": 0.26644947454333306, + "OOD_arXiv_2023|OOD_News_2024|s2|p0": 0.25805783495306966, + "OOD_arXiv_2023|OOD_News_2024|s2|p1": 0.2639310799539089, + "OOD_arXiv_2023|OOD_News_2024|s2|p2": 0.26843248642981055, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p0": 0.250489117950201, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p1": 0.23644275814294816, + "OOD_arXiv_2023|OOD_arXiv_2023|s0|p2": 0.24719151090830566, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p0": 0.24829945489764213, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p1": 0.23504118770360946, + "OOD_arXiv_2023|OOD_arXiv_2023|s1|p2": 0.2447248689830303, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p0": 0.25208783373236654, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p1": 0.23738111406564713, + "OOD_arXiv_2023|OOD_arXiv_2023|s2|p2": 0.24716101121157408, + "OOD_arXiv_2023|OOD_CommonPile|s0|p0": 0.2470088295638561, + "OOD_arXiv_2023|OOD_CommonPile|s0|p1": 0.24812843091785908, + "OOD_arXiv_2023|OOD_CommonPile|s0|p2": 0.24164734184741973, + "OOD_arXiv_2023|OOD_CommonPile|s1|p0": 0.24504089187830685, + "OOD_arXiv_2023|OOD_CommonPile|s1|p1": 0.24695885889232158, + "OOD_arXiv_2023|OOD_CommonPile|s1|p2": 0.23821127153933047, + "OOD_arXiv_2023|OOD_CommonPile|s2|p0": 0.24550589360296726, + "OOD_arXiv_2023|OOD_CommonPile|s2|p1": 0.24746591597795486, + "OOD_arXiv_2023|OOD_CommonPile|s2|p2": 0.24002983681857587, + "OOD_CommonPile|Wikipedia_en|s0|p0": 0.2527482692152262, + "OOD_CommonPile|Wikipedia_en|s0|p1": 0.25305079743266107, + "OOD_CommonPile|Wikipedia_en|s0|p2": 0.2529677849262953, + "OOD_CommonPile|Wikipedia_en|s1|p0": 0.2523725740611553, + "OOD_CommonPile|Wikipedia_en|s1|p1": 0.2535226963460445, + "OOD_CommonPile|Wikipedia_en|s1|p2": 0.253370276466012, + "OOD_CommonPile|Wikipedia_en|s2|p0": 0.24978544041514397, + "OOD_CommonPile|Wikipedia_en|s2|p1": 0.25103137716650964, + "OOD_CommonPile|Wikipedia_en|s2|p2": 0.24884041137993335, + "OOD_CommonPile|USPTO_Backgrounds|s0|p0": 0.2632263233885169, + "OOD_CommonPile|USPTO_Backgrounds|s0|p1": 0.26493328642100095, + "OOD_CommonPile|USPTO_Backgrounds|s0|p2": 0.26009960360825063, + "OOD_CommonPile|USPTO_Backgrounds|s1|p0": 0.2648737171664834, + "OOD_CommonPile|USPTO_Backgrounds|s1|p1": 0.26392368879169226, + "OOD_CommonPile|USPTO_Backgrounds|s1|p2": 0.2603767177090049, + "OOD_CommonPile|USPTO_Backgrounds|s2|p0": 0.25935944225639107, + "OOD_CommonPile|USPTO_Backgrounds|s2|p1": 0.2567653330042958, + "OOD_CommonPile|USPTO_Backgrounds|s2|p2": 0.2551079483702779, + "OOD_CommonPile|Pile-CC|s0|p0": 0.25972226187586783, + "OOD_CommonPile|Pile-CC|s0|p1": 0.2562075648456812, + "OOD_CommonPile|Pile-CC|s0|p2": 0.25851371306926013, + "OOD_CommonPile|Pile-CC|s1|p0": 0.2606124201789498, + "OOD_CommonPile|Pile-CC|s1|p1": 0.25360609516501426, + "OOD_CommonPile|Pile-CC|s1|p2": 0.25796925742179155, + "OOD_CommonPile|Pile-CC|s2|p0": 0.2564075054600835, + "OOD_CommonPile|Pile-CC|s2|p1": 0.2527011208236217, + "OOD_CommonPile|Pile-CC|s2|p2": 0.2542176844552159, + "OOD_CommonPile|StackExchange|s0|p0": 0.2363905232399702, + "OOD_CommonPile|StackExchange|s0|p1": 0.239471772685647, + "OOD_CommonPile|StackExchange|s0|p2": 0.24863372463732958, + "OOD_CommonPile|StackExchange|s1|p0": 0.23617794532328845, + "OOD_CommonPile|StackExchange|s1|p1": 0.2343326810747385, + "OOD_CommonPile|StackExchange|s1|p2": 0.2486434133723378, + "OOD_CommonPile|StackExchange|s2|p0": 0.23299273960292338, + "OOD_CommonPile|StackExchange|s2|p1": 0.23436020649969577, + "OOD_CommonPile|StackExchange|s2|p2": 0.24117932543158532, + "OOD_CommonPile|Github|s0|p0": 0.23757223188877105, + "OOD_CommonPile|Github|s0|p1": 0.24194753561168908, + "OOD_CommonPile|Github|s0|p2": 0.24505538828670978, + "OOD_CommonPile|Github|s1|p0": 0.23725720643997192, + "OOD_CommonPile|Github|s1|p1": 0.24136778935790063, + "OOD_CommonPile|Github|s1|p2": 0.24443952552974224, + "OOD_CommonPile|Github|s2|p0": 0.23209963254630567, + "OOD_CommonPile|Github|s2|p1": 0.23724418375641107, + "OOD_CommonPile|Github|s2|p2": 0.23982958495616913, + "OOD_CommonPile|OOD_News_2024|s0|p0": 0.2549568019807339, + "OOD_CommonPile|OOD_News_2024|s0|p1": 0.2576639335602522, + "OOD_CommonPile|OOD_News_2024|s0|p2": 0.2609452348202467, + "OOD_CommonPile|OOD_News_2024|s1|p0": 0.25358192548155783, + "OOD_CommonPile|OOD_News_2024|s1|p1": 0.2604420565068722, + "OOD_CommonPile|OOD_News_2024|s1|p2": 0.2604785941541195, + "OOD_CommonPile|OOD_News_2024|s2|p0": 0.24946372043341397, + "OOD_CommonPile|OOD_News_2024|s2|p1": 0.25381071493029594, + "OOD_CommonPile|OOD_News_2024|s2|p2": 0.25552589893341066, + "OOD_CommonPile|OOD_arXiv_2023|s0|p0": 0.24695753082633018, + "OOD_CommonPile|OOD_arXiv_2023|s0|p1": 0.23362126871943473, + "OOD_CommonPile|OOD_arXiv_2023|s0|p2": 0.2358368717133999, + "OOD_CommonPile|OOD_arXiv_2023|s1|p0": 0.24771717023104428, + "OOD_CommonPile|OOD_arXiv_2023|s1|p1": 0.23119124509394168, + "OOD_CommonPile|OOD_arXiv_2023|s1|p2": 0.23599768113344907, + "OOD_CommonPile|OOD_arXiv_2023|s2|p0": 0.24254967402666808, + "OOD_CommonPile|OOD_arXiv_2023|s2|p1": 0.22768239714205266, + "OOD_CommonPile|OOD_arXiv_2023|s2|p2": 0.23190979957580565, + "OOD_CommonPile|OOD_CommonPile|s0|p0": 0.23246933408081533, + "OOD_CommonPile|OOD_CommonPile|s0|p1": 0.23837118372321128, + "OOD_CommonPile|OOD_CommonPile|s0|p2": 0.23252568058669568, + "OOD_CommonPile|OOD_CommonPile|s1|p0": 0.23641609959304333, + "OOD_CommonPile|OOD_CommonPile|s1|p1": 0.23985061869025232, + "OOD_CommonPile|OOD_CommonPile|s1|p2": 0.23229695856571198, + "OOD_CommonPile|OOD_CommonPile|s2|p0": 0.2299693901091814, + "OOD_CommonPile|OOD_CommonPile|s2|p1": 0.23604161366820336, + "OOD_CommonPile|OOD_CommonPile|s2|p2": 0.2284572273492813, + "logit|Q0|p0": 0.19810852520167826, + "shuf|Q0|p0": 0.2481349878013134, + "logit|Q0|p1": 0.19810852520167826, + "shuf|Q0|p1": 0.2481349878013134, + "logit|Q0|p2": 0.19810852520167826, + "shuf|Q0|p2": 0.2481349878013134, + "logit|Wikipedia_en|p0": 0.2033720064908266, + "shuf|Wikipedia_en|p0": 0.24760952070355416, + "logit|Wikipedia_en|p1": 0.2047900039702654, + "shuf|Wikipedia_en|p1": 0.24703365340828895, + "logit|Wikipedia_en|p2": 0.19886889979243277, + "shuf|Wikipedia_en|p2": 0.243387883156538, + "logit|USPTO_Backgrounds|p0": 0.2122203078120947, + "shuf|USPTO_Backgrounds|p0": 0.25528149902820585, + "logit|USPTO_Backgrounds|p1": 0.2081274013966322, + "shuf|USPTO_Backgrounds|p1": 0.2531838655471802, + "logit|USPTO_Backgrounds|p2": 0.206907719373703, + "shuf|USPTO_Backgrounds|p2": 0.24857981875538826, + "logit|Pile-CC|p0": 0.20878250971436502, + "shuf|Pile-CC|p0": 0.24865657091140747, + "logit|Pile-CC|p1": 0.20850949063897134, + "shuf|Pile-CC|p1": 0.2438236244022846, + "logit|Pile-CC|p2": 0.20317537039518357, + "shuf|Pile-CC|p2": 0.24651725962758064, + "logit|StackExchange|p0": 0.1900142565369606, + "shuf|StackExchange|p0": 0.23016793355345727, + "logit|StackExchange|p1": 0.18867491744458675, + "shuf|StackExchange|p1": 0.23704585805535316, + "logit|StackExchange|p2": 0.19668752700090408, + "shuf|StackExchange|p2": 0.241612309217453, + "logit|Github|p0": 0.1975695587694645, + "shuf|Github|p0": 0.2303194060921669, + "logit|Github|p1": 0.19453195556998254, + "shuf|Github|p1": 0.2302835650742054, + "logit|Github|p2": 0.19555468298494816, + "shuf|Github|p2": 0.23580559268593787, + "logit|OOD_News_2024|p0": 0.20018524453043937, + "shuf|OOD_News_2024|p0": 0.24373726025223733, + "logit|OOD_News_2024|p1": 0.20086353681981564, + "shuf|OOD_News_2024|p1": 0.24059574007987977, + "logit|OOD_News_2024|p2": 0.19859205968677998, + "shuf|OOD_News_2024|p2": 0.2469678081572056, + "logit|OOD_arXiv_2023|p0": 0.19922805316746234, + "shuf|OOD_arXiv_2023|p0": 0.23877212554216384, + "logit|OOD_arXiv_2023|p1": 0.1816310778260231, + "shuf|OOD_arXiv_2023|p1": 0.22678083777427674, + "logit|OOD_arXiv_2023|p2": 0.19147473610937596, + "shuf|OOD_arXiv_2023|p2": 0.23733989745378495, + "logit|OOD_CommonPile|p0": 0.18915797509253024, + "shuf|OOD_CommonPile|p0": 0.23239998519420624, + "logit|OOD_CommonPile|p1": 0.18341674096882343, + "shuf|OOD_CommonPile|p1": 0.232904601842165, + "logit|OOD_CommonPile|p2": 0.18899137675762176, + "shuf|OOD_CommonPile|p2": 0.22944279089570047, + "logit|SHUFFLED_Pile-CC|p0": 0.18035524226725103, + "shuf|SHUFFLED_Pile-CC|p0": 0.22237672172486783, + "logit|SHUFFLED_Pile-CC|p1": 0.18587937839329244, + "shuf|SHUFFLED_Pile-CC|p1": 0.22617292143404483, + "logit|SHUFFLED_Pile-CC|p2": 0.1807046439498663, + "shuf|SHUFFLED_Pile-CC|p2": 0.2182137418538332 + } + }, + "control_C1_reproduces_stored_e52_matrix": { + "reference": "results/e52_factorial_410m.json", + "checked": false, + "applicable": false, + "fires": null, + "why": "the stored matrix was scored at the UNSTRIPPED readout. R1/C1 is the check that this same code reproduces it WITHOUT --rstrip; comparing the stripped arm to it would be comparing two conventions." + }, + "provenance": { + "schema_version": 1, + "experiment": "E57", + "utc": "2026-08-30T07:22:31+00:00", + "script": { + "path": "experiments/t52_factorial.py", + "sha256": "73ea8c493c2f36b26522e577752339a77fc84063610c786e9c199f6cfc390962", + "n_lines": 851 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "9b5e7c3d0df63ba700d0e85d1608e851e53fdc6f379ceb7d1e3414552b342ad8", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/t52_factorial.py", + "--device", + "cpu", + "--workers", + "6", + "--rstrip", + "--out", + "results/e52_factorial_410m_rstrip.json", + "--cells-out", + "results/e57_factorial_cells_410m_rstrip.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e52_factorial_410m.json", + "sha256": "81c0a06357e4ca91d2fa9dd9a748ff4696d02e9f78ad2e7edd65a60334c100fb", + "bytes": 41197, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "8bd3566ee0a991bd9839c7a0ee2e17db2920161f5eabb16dc5501258ac1e0aae" + } +} \ No newline at end of file diff --git a/results/e57_grid_variance_ci.json b/results/e57_grid_variance_ci.json new file mode 100644 index 0000000000000000000000000000000000000000..95f502bfafef4d94e92d49ebd248b5d7de940d1e --- /dev/null +++ b/results/e57_grid_variance_ci.json @@ -0,0 +1,215 @@ +{ + "experiment": "E57 \u2014 an interval on the headline variance decomposition", + "prereg": "specs/experiments/E57_grid_variance_ci.md, written before the run", + "status": "PRE-REGISTERED", + "inputs": [ + "results/e57_factorial_cells_410m.json", + "results/e52_factorial_410m.json" + ], + "n_draws_per_cell": 9, + "n_boot": 10000, + "measurement_side_C1": { + "reference": "results/e52_factorial_410m.json", + "checked": true, + "n_cells_compared": 128, + "max_abs_diff": 0.0, + "worst_cell": null, + "tolerance": 1e-12, + "fires": true + }, + "by_aggregation": { + "persist": { + "point": { + "SS_row": 0.00289945925135949, + "SS_col": 0.00022673347364850273, + "SS_resid": 5.234427980359093e-05, + "SS_total": 0.003178537004811584, + "fit_pct": 91.21993064640641, + "read_pct": 7.133265187892407, + "resid_pct": 1.6468041657011878 + }, + "bootstrap": { + "fit_pct": { + "point": 91.21993064640641, + "ci_lo": 87.60889894657397, + "ci_hi": 92.17106192216394 + }, + "read_pct": { + "point": 7.133265187892407, + "ci_lo": 6.310765498639046, + "ci_hi": 9.75205779444106 + }, + "resid_pct": { + "point": 1.6468041657011878, + "ci_lo": 1.5181725791970113, + "ci_hi": 2.7865513699088558 + }, + "fit_minus_read": { + "point": 84.086665458514, + "ci_lo": 77.85684115213292, + "ci_hi": 85.8602964235249, + "excludes_zero": true + }, + "unit": "fit seed block and prefix seed, resampled independently with replacement" + }, + "variance_components": { + "MS_within_cell": 2.366289779901387e-06, + "df_within": 512, + "MS_residual_of_cell_means": 1.0682506082365496e-06, + "df_residual": 49, + "expected_noise_in_MS_residual": 2.629210866557097e-07, + "noise_corrected_interaction_MS": 8.053295215808399e-07, + "residual_is_entirely_noise": false, + "fit_pct_after_noise_correction": 91.5911647006965, + "read_pct_after_noise_correction": 7.162295148091483, + "interaction_pct_after_noise_correction": 1.2465401512120193, + "note": "E[MS_resid] = theta_interaction + sigma^2/9. A theta <= 0 means the term the paper labels 'fit x read interaction' is not distinguishable from cell noise." + }, + "controls": { + "C1_draws_average_to_stored_matrix": { + "max_abs_diff_vs_stored_matrix": 0.0, + "tolerance": 1e-12, + "fires": true + }, + "C2_bootstrap_non_degenerate": { + "bootstrap_width_fit_pct": 4.56216297558997, + "fires": true + } + } + }, + "min": { + "point": { + "SS_row": 0.009152380847464421, + "SS_col": 0.0028264419878745047, + "SS_resid": 0.0004874723371463591, + "SS_total": 0.012466295172485285, + "fit_pct": 73.41700738536099, + "read_pct": 22.67267017800786, + "resid_pct": 3.910322436631158 + }, + "bootstrap": { + "fit_pct": { + "point": 73.41700738536099, + "ci_lo": 68.87079349603536, + "ci_hi": 75.69475005442473 + }, + "read_pct": { + "point": 22.67267017800786, + "ci_lo": 20.29090047208274, + "ci_hi": 26.822293731330955 + }, + "resid_pct": { + "point": 3.910322436631158, + "ci_lo": 3.690542901884536, + "ci_hi": 5.032002252783932 + }, + "fit_minus_read": { + "point": 50.74433720735313, + "ci_lo": 41.80212103651523, + "ci_hi": 55.32020204446164, + "excludes_zero": true + }, + "unit": "fit seed block and prefix seed, resampled independently with replacement" + }, + "variance_components": { + "MS_within_cell": 7.884861497544852e-06, + "df_within": 512, + "MS_residual_of_cell_means": 9.948415043803247e-06, + "df_residual": 49, + "expected_noise_in_MS_residual": 8.76095721949428e-07, + "noise_corrected_interaction_MS": 9.072319321853819e-06, + "residual_is_entirely_noise": false, + "fit_pct_after_noise_correction": 73.67069836218938, + "read_pct_after_noise_correction": 22.75101512898871, + "interaction_pct_after_noise_correction": 3.5782865088219125, + "note": "E[MS_resid] = theta_interaction + sigma^2/9. A theta <= 0 means the term the paper labels 'fit x read interaction' is not distinguishable from cell noise." + }, + "controls": { + "C1_draws_average_to_stored_matrix": { + "max_abs_diff_vs_stored_matrix": 0.0, + "tolerance": 1e-12, + "fires": true + }, + "C2_bootstrap_non_degenerate": { + "bootstrap_width_fit_pct": 6.823956558389369, + "fires": true + } + } + } + }, + "VERDICT": "ACCEPT \u2014 the fit axis exceeds the read axis by 84.1 points [77.9,85.9] under persist and 50.7 [41.8,55.3] under min; both intervals exclude zero. The headline ordering now carries an interval.", + "provenance": { + "schema_version": 1, + "experiment": "E57", + "utc": "2026-08-17T01:38:23+00:00", + "script": { + "path": "experiments/t57_grid_variance_ci.py", + "sha256": "55dd09ddc17256e9a6812fa2a1c9c512bd502eb8749738598af3753b786f3a6d", + "n_lines": 206 + }, + "git": { + "commit": "f8ef435b29365012c0492e023fb6b4dfece5f0f3", + "commit_short": "f8ef435b2936", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t53_ladder_summary.py", + "experiments/t55_matrix_robustness.py", + "experiments/t56_predictor_registry.py", + "experiments/t57_grid_variance_ci.py", + "experiments/t58_algebra_audit.py", + "experiments/t59_read_dose.py", + "experiments/t60_fitter_determinism.py", + "paper/paper.tex", + "results/e35_containment.json", + "results/e52_factorial_410m_rerun.json", + "results/e53_ladder_summary.json", + "results/e55_matrix_robustness.json", + "results/e56_predictor_registry.json", + "results/e57_factorial_cells_410m.json", + "results/e58_algebra_audit.json", + "results/e60_fitter_determinism.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/experiments/E57_grid_variance_ci.md", + "specs/experiments/E59_read_dose_response.md", + "tests/test_anchor_fidelity.py", + "xperiments/t52_factorial.py" + ], + "dirty_diff_sha256": "f81d33770c1942c6780156e67c676d525ee52e09d4ef1a49a09d7ead800f7ac8", + "describe": "f8ef435-dirty" + }, + "argv": [ + "experiments/t57_grid_variance_ci.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e57_factorial_cells_410m.json", + "sha256": "4c3b428670ba12195b4144d580ee0303426331207f60c34f6185a7cabecf8375", + "bytes": 82248, + "exists": true + }, + { + "path": "results/e52_factorial_410m.json", + "sha256": "6e81f3cae37d0c1906f8912af4bf8e75db0a0f259b2869f209701b74b6682590", + "bytes": 41190, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "752baf2b872fd2516a893c0f04c805da1fe167b216f28774f3323b55a094aa04" + } +} \ No newline at end of file diff --git a/results/e57_grid_variance_ci_rstrip.json b/results/e57_grid_variance_ci_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..eed5a68d5873794de38c0cd794a80467ff224a48 --- /dev/null +++ b/results/e57_grid_variance_ci_rstrip.json @@ -0,0 +1,269 @@ +{ + "experiment": "E57 \u2014 an interval on the headline variance decomposition", + "prereg": "docs/experiments/preregs/E57_grid_variance_ci.md, written before the run", + "status": "PRE-REGISTERED", + "inputs": [ + "results/e57_factorial_cells_410m_rstrip.json", + "results/e52_factorial_410m_rstrip.json" + ], + "readout_convention": "STRIPPED (anchor rule)", + "n_draws_per_cell": 9, + "n_boot": 10000, + "measurement_side_C1": { + "reference": "results/e52_factorial_410m.json", + "checked": false, + "applicable": false, + "fires": null, + "why": "the stored matrix was scored at the UNSTRIPPED readout. R1/C1 is the check that this same code reproduces it WITHOUT --rstrip; comparing the stripped arm to it would be comparing two conventions." + }, + "by_aggregation": { + "persist": { + "point": { + "SS_row": 0.002451851883005098, + "SS_col": 0.0023277982009026798, + "SS_resid": 5.661823871813297e-05, + "SS_total": 0.004836268322625911, + "fit_pct": 50.697184677169346, + "read_pct": 48.13211438274321, + "resid_pct": 1.1707009400874475 + }, + "bootstrap": { + "fit_pct": { + "point": 50.697184677169346, + "ci_lo": 45.18158148522331, + "ci_hi": 54.188775604491234 + }, + "read_pct": { + "point": 48.13211438274321, + "ci_lo": 44.402403827268486, + "ci_hi": 52.95370961200712 + }, + "resid_pct": { + "point": 1.1707009400874475, + "ci_lo": 1.057828420386843, + "ci_hi": 2.8243760415935606 + }, + "fit_minus_read": { + "point": 2.5650702944261354, + "ci_lo": -7.602188958362582, + "ci_hi": 9.795918277187646, + "excludes_zero": false + }, + "unit": "fit seed block and prefix seed, resampled independently with replacement" + }, + "variance_components": { + "MS_within_cell": 5.181613307979517e-06, + "df_within": 512, + "MS_residual_of_cell_means": 1.1554742595537341e-06, + "df_residual": 49, + "expected_noise_in_MS_residual": 5.757348119977241e-07, + "noise_corrected_interaction_MS": 5.7973944755601e-07, + "residual_is_entirely_noise": false, + "fit_pct_after_noise_correction": 50.994647555856034, + "read_pct_after_noise_correction": 48.41452685579747, + "interaction_pct_after_noise_correction": 0.5908255883464855, + "note": "E[MS_resid] = theta_interaction + sigma^2/9. A theta <= 0 means the term the paper labels 'fit x read interaction' is not distinguishable from cell noise." + }, + "controls": { + "C1_draws_average_to_stored_matrix": { + "max_abs_diff_vs_stored_matrix": 0.0, + "tolerance": 1e-12, + "fires": true + }, + "C2_bootstrap_non_degenerate": { + "bootstrap_width_fit_pct": 9.007194119267922, + "fires": true + } + } + }, + "min": { + "point": { + "SS_row": 0.007990259278199689, + "SS_col": 0.006680737315891797, + "SS_resid": 0.00030509150277111766, + "SS_total": 0.014976088096862603, + "fit_pct": 53.35344735234028, + "read_pct": 44.60936175509925, + "resid_pct": 2.0371908925604707 + }, + "bootstrap": { + "fit_pct": { + "point": 53.35344735234028, + "ci_lo": 49.05154716547634, + "ci_hi": 56.958779332004895 + }, + "read_pct": { + "point": 44.60936175509925, + "ci_lo": 40.95862929938772, + "ci_hi": 48.29168997564349 + }, + "resid_pct": { + "point": 2.0371908925604707, + "ci_lo": 1.975027493910122, + "ci_hi": 3.6127345308372907 + }, + "fit_minus_read": { + "point": 8.744085597241032, + "ci_lo": 0.796194967922311, + "ci_hi": 15.872143452423032, + "excludes_zero": true + }, + "unit": "fit seed block and prefix seed, resampled independently with replacement" + }, + "variance_components": { + "MS_within_cell": 1.4154011173741129e-05, + "df_within": 512, + "MS_residual_of_cell_means": 6.226357199410564e-06, + "df_residual": 49, + "expected_noise_in_MS_residual": 1.5726679081934588e-06, + "noise_corrected_interaction_MS": 4.653689291217106e-06, + "residual_is_entirely_noise": false, + "fit_pct_after_noise_correction": 53.629401974461324, + "read_pct_after_noise_correction": 44.84009023052267, + "interaction_pct_after_noise_correction": 1.5305077950159927, + "note": "E[MS_resid] = theta_interaction + sigma^2/9. A theta <= 0 means the term the paper labels 'fit x read interaction' is not distinguishable from cell noise." + }, + "controls": { + "C1_draws_average_to_stored_matrix": { + "max_abs_diff_vs_stored_matrix": 0.0, + "tolerance": 1e-12, + "fires": true + }, + "C2_bootstrap_non_degenerate": { + "bootstrap_width_fit_pct": 7.907232166528551, + "fires": true + } + } + } + }, + "VERDICT": "REJECT \u2014 the interval on fit-read includes zero under persist. The paper cannot claim the fit axis dominates; only a point estimate.", + "provenance": { + "schema_version": 1, + "experiment": "E57", + "utc": "2026-08-31T23:20:32+00:00", + "script": { + "path": "experiments/t57_grid_variance_ci.py", + "sha256": "cb96764a387a5a3665d6b5d618364132c5b98815920cc584dad1735d7bd7001c", + "n_lines": 213 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/t57_grid_variance_ci.py", + "--cells", + "results/e57_factorial_cells_410m_rstrip.json", + "--e52", + "results/e52_factorial_410m_rstrip.json", + "--out", + "results/e57_grid_variance_ci_rstrip.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e57_factorial_cells_410m_rstrip.json", + "sha256": "927541002dea5762ced90c315be67a1ae1c7ffa9028c9ef30011320ca181acd3", + "bytes": 84806, + "exists": true + }, + { + "path": "results/e52_factorial_410m.json", + "sha256": "81c0a06357e4ca91d2fa9dd9a748ff4696d02e9f78ad2e7edd65a60334c100fb", + "bytes": 41197, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "684eca7b4743fc7b77750267b3e929ab2b7294ca74dff8e280f4930d3501c25f" + } +} \ No newline at end of file diff --git a/results/e58_algebra_audit.json b/results/e58_algebra_audit.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae17a44065f15dbbeea836a3a77515c8ecca47f --- /dev/null +++ b/results/e58_algebra_audit.json @@ -0,0 +1,390 @@ +{ + "experiment": "E58 \u2014 algebra and measurement checks the validity audit left open", + "why": "E45's post-norm transpose, E45's degenerate C2, E36's asserted diagonal leakage", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "A_e45_orientation": { + "layernorm_gamma_mean": 3.4597463607788086, + "layernorm_gamma_sd": 0.2160930186510086, + "G_relative_asymmetry_per_layer": { + "9": 0.012462692335247993, + "10": 0.012448985129594803, + "11": 0.01279827393591404, + "12": 0.013043113052845001, + "13": 0.013180325739085674, + "14": 0.01295335404574871, + "15": 0.013749546371400356, + "16": 0.013555113226175308, + "17": 0.014134599827229977, + "18": 0.01464578602463007, + "19": 0.01459971722215414, + "20": 0.015326417051255703, + "21": 0.015958653762936592 + }, + "G_relative_asymmetry_median": 0.013555113226175308, + "stored_post_norm_r": 0.49614222895095655, + "by_orientation": { + "none": { + "r": 0.6169298718606291, + "worst_loo_set": [ + "multihop", + 0.53696139646654 + ], + "worst_loo_corpus": [ + "Github", + 0.20245342469864708 + ], + "survives": false, + "crosses_the_0p5_threshold": true + }, + "coded_G_transpose": { + "r": 0.49614222895095655, + "worst_loo_set": [ + "typo", + 0.38987876875470867 + ], + "worst_loo_corpus": [ + "Github", + 0.1524035039262419 + ], + "survives": false, + "crosses_the_0p5_threshold": false + }, + "correct_G": { + "r": 0.4994154833567744, + "worst_loo_set": [ + "typo", + 0.39499077980783137 + ], + "worst_loo_corpus": [ + "Github", + 0.16817400762862061 + ], + "survives": false, + "crosses_the_0p5_threshold": false + }, + "symmetrised": { + "r": 0.494970172207323, + "worst_loo_set": [ + "typo", + 0.3895896933294183 + ], + "worst_loo_corpus": [ + "Github", + 0.15312346914458108 + ], + "survives": false, + "crosses_the_0p5_threshold": false + } + }, + "threshold": 0.5, + "VERDICT": "IMMATERIAL \u2014 the corrected orientation lands on the same side of the pre-registered 0.5 threshold as the coded one; E45's stored VERDICT string stands", + "note": "in EVERY orientation the leave-one-corpus-out value is ~0.2 (Github), so E45 remains a failed predictor and no paper claim reverses. This is about the results file saying the right thing." + }, + "B_e45_proper_C2": { + "coded_C2_r": 0.030950976174021295, + "coded_C2_defect": "one Delta shared across all five corpora, so x has zero within-set variance and the control cannot fail", + "proper_C2_r_five_independent_norm_matched_deltas": 0.34909509131902045, + "threshold": 0.3, + "fires": false, + "VERDICT": "DOES NOT FIRE \u2014 a norm-matched random operator predicts as well as the real disagreement geometry; E45's primary is confounded" + }, + "controls": { + "C1_zero_delta_by_orientation": { + "none": 0.0, + "coded_G_transpose": 0.0, + "correct_G": 0.0, + "symmetrised": 0.0 + }, + "C1_fires": true, + "C3_pre_norm_reproduces_stored_e45": { + "stored_pre_norm_r": 0.6169298718606291, + "recomputed": 0.6169298718606291, + "abs_diff": 0.0, + "fires": true + } + }, + "C_e36_diagonal_prefix_fit_overlap": { + "what": "fraction of E36's prefix documents for corpus X that are among the 400 documents its own N=400 seed-0 operator was fitted on. Applies to the five DIAGONAL rungs of Appendix B, which are also its five most-contained rungs.", + "by_corpus": { + "Github": { + "n_fitting_docs": 400, + "n_docs_in_file": 2400, + "per_prefix_seed": { + "p0": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 81, + "overlap_fraction": 0.14972273567467653 + }, + "p1": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 106, + "overlap_fraction": 0.19593345656192238 + }, + "p2": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 89, + "overlap_fraction": 0.1645101663585952 + } + }, + "mean_overlap_fraction": 0.1700554528650647 + }, + "Wikipedia_en": { + "n_fitting_docs": 400, + "n_docs_in_file": 2400, + "per_prefix_seed": { + "p0": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 81, + "overlap_fraction": 0.14972273567467653 + }, + "p1": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 106, + "overlap_fraction": 0.19593345656192238 + }, + "p2": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 89, + "overlap_fraction": 0.1645101663585952 + } + }, + "mean_overlap_fraction": 0.1700554528650647 + }, + "StackExchange": { + "n_fitting_docs": 400, + "n_docs_in_file": 2400, + "per_prefix_seed": { + "p0": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 81, + "overlap_fraction": 0.14972273567467653 + }, + "p1": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 106, + "overlap_fraction": 0.19593345656192238 + }, + "p2": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 89, + "overlap_fraction": 0.1645101663585952 + } + }, + "mean_overlap_fraction": 0.1700554528650647 + }, + "Pile-CC": { + "n_fitting_docs": 400, + "n_docs_in_file": 2400, + "per_prefix_seed": { + "p0": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 81, + "overlap_fraction": 0.14972273567467653 + }, + "p1": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 107, + "overlap_fraction": 0.1977818853974122 + }, + "p2": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 89, + "overlap_fraction": 0.1645101663585952 + } + }, + "mean_overlap_fraction": 0.17067159581022798 + }, + "USPTO_Backgrounds": { + "n_fitting_docs": 400, + "n_docs_in_file": 2400, + "per_prefix_seed": { + "p0": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 82, + "overlap_fraction": 0.15157116451016636 + }, + "p1": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 106, + "overlap_fraction": 0.19593345656192238 + }, + "p2": { + "n_prefix_docs": 541, + "n_from_fitting_pool": 90, + "overlap_fraction": 0.16635859519408502 + } + }, + "mean_overlap_fraction": 0.17128773875539124 + } + }, + "mean_over_corpora": 0.17042513863216266, + "repo_assertion": "~17%", + "e52_by_contrast": "t52_factorial.py load_pool_heldout() excludes every fitting document and aborts rather than cycle \u2014 measured 0 by construction", + "direction_of_bias": "inflates one cell per row, so it lands mostly in the residual; on Appendix B's containment argument it lifts the high-containment end, which flattens any decline and therefore favours the paper's null" + }, + "provenance": { + "schema_version": 1, + "experiment": "E58", + "utc": "2026-08-30T00:20:10+00:00", + "script": { + "path": "experiments/t58_algebra_audit.py", + "sha256": "e6a4fd847367ef109ca8e015d86ab1c179d84a024af8990e5a1c729335c2c5a4", + "n_lines": 345 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reviews/REVIEW_INSIGHTS_2026-08-25.md", + "docs/validity/CONSTRUCT_VALIDITY.md" + ], + "dirty_diff_sha256": "6b143c292229b3d2480d1b73243675f0db5e23b27f66e4215e0a9c1947a761a8", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/t58_algebra_audit.py", + "--device", + "cpu" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e45_disagreement_geometry.json", + "sha256": "dfd1a336e310f2f0aaf8b966f0b4ae09395a4833cbd14b4481e516c043be8739", + "bytes": 4406, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Github_410m_n400_s0.pt", + "sha256": "4c87ed9a5f778ef0280a17ff0df5e623e8732fb7b706936fcc7caf624979511a", + "bytes": 46144865, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Wikipedia_en_410m_n400_s0.pt", + "sha256": "5e2af80098d34c57e6cdbfcf680a9c9303e18daae91b1577d3e9222e7eb4952b", + "bytes": 46145033, + "exists": true + }, + { + "path": "results/lenses/e28/e28_StackExchange_410m_n400_s0.pt", + "sha256": "3c3bbbca13ef5891fb19ce2c552334ebf12263c905f25545bc36a831b4bd2a00", + "bytes": 46145061, + "exists": true + }, + { + "path": "results/lenses/e28/e28_Pile-CC_410m_n400_s0.pt", + "sha256": "4754d23b9e6eb16ac8159661cc393328374d8cc8d286bbb9c971ee95166fca62", + "bytes": 46144893, + "exists": true + }, + { + "path": "results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0.pt", + "sha256": "4bc59371a760c85c5570f68e00a6e41a65a1ff7168079fc9ab5a695e2d70987b", + "bytes": 46145173, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "d635e656a85cfa0a3a059b5ad5653478393d3955f08fa9c045548dbdf74f038c" + } +} \ No newline at end of file diff --git a/results/e59_read_dose_410m.json b/results/e59_read_dose_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..5866b5194a69cff1792b7c65c448b7e99bfee821 --- /dev/null +++ b/results/e59_read_dose_410m.json @@ -0,0 +1,1275 @@ +{ + "experiment": "E59 \u2014 the read-axis dose-response", + "prereg": "docs/experiments/preregs/E59_read_dose_response.md, written before this ran", + "status": "PRE-REGISTERED", + "question": "is the 91%/7% fit-vs-read asymmetry a property of the method, or of a 200x dose ratio between the two axes?", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "excluded_corpus": { + "OOD_CommonPile": "0 documents of >=768 tokens, 147 of >=384 \u2014 physically cannot supply a longer read context. Measured 2026-08-16." + }, + "doses": [ + 128, + 384, + 768 + ], + "fit_seeds": [ + 0, + 1, + 2 + ], + "prefix_seeds": [ + 0, + 1, + 2 + ], + "n_items": 541, + "n_pairs": 893, + "smoke": false, + "dose_ratio_fit_vs_read": { + "128": 200.0, + "384": 66.66666666666667, + "768": 33.333333333333336 + }, + "operators": "results/e48/lens_*_410m_n200_s*.pt \u2014 identical to E52's; only the read axis moves", + "prefix_pool_construction": "drawn ONCE per rung from held-out documents of >= 768 tokens, then TRUNCATED to each dose, so the same documents are read at every dose", + "pool_meta": { + "Wikipedia_en|p0": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 520 + }, + "Wikipedia_en|p1": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 536 + }, + "Wikipedia_en|p2": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 532 + }, + "USPTO_Backgrounds|p0": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 500 + }, + "USPTO_Backgrounds|p1": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 495 + }, + "USPTO_Backgrounds|p2": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 518 + }, + "Pile-CC|p0": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 446 + }, + "Pile-CC|p1": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 464 + }, + "Pile-CC|p2": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 470 + }, + "StackExchange|p0": { + "n_distinct_docs": 458, + "cycling_factor": 1.1812227074235808, + "n_fitting_docs_excluded_seen": 600 + }, + "StackExchange|p1": { + "n_distinct_docs": 458, + "cycling_factor": 1.1812227074235808, + "n_fitting_docs_excluded_seen": 600 + }, + "StackExchange|p2": { + "n_distinct_docs": 458, + "cycling_factor": 1.1812227074235808, + "n_fitting_docs_excluded_seen": 600 + }, + "Github|p0": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 327 + }, + "Github|p1": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 315 + }, + "Github|p2": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 328 + }, + "OOD_News_2024|p0": { + "n_distinct_docs": 409, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 661 + }, + "OOD_News_2024|p1": { + "n_distinct_docs": 420, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 684 + }, + "OOD_News_2024|p2": { + "n_distinct_docs": 405, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 674 + }, + "OOD_arXiv_2023|p0": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 172 + }, + "OOD_arXiv_2023|p1": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 182 + }, + "OOD_arXiv_2023|p2": { + "n_distinct_docs": 541, + "cycling_factor": 1.0, + "n_fitting_docs_excluded_seen": 195 + } + }, + "by_dose": { + "128": { + "persist": { + "fit_pct": 88.68478030369347, + "read_pct": 9.45683761637228, + "residual_pct": 1.8583820799342452, + "row_means": { + "Wikipedia_en": 0.038766393582854004, + "USPTO_Backgrounds": 0.047598725785723044, + "Pile-CC": 0.04712269108128985, + "StackExchange": 0.04512989729576345, + "Github": 0.02580937210078691, + "OOD_News_2024": 0.03705063803742329, + "OOD_arXiv_2023": 0.039499257347132596 + }, + "col_means": { + "Wikipedia_en": 0.0399578698707508, + "USPTO_Backgrounds": 0.04243221749728989, + "Pile-CC": 0.041925042831442424, + "StackExchange": 0.03524890663139226, + "Github": 0.03902402628020989, + "OOD_News_2024": 0.041930259223879565, + "OOD_arXiv_2023": 0.04045865289600832 + }, + "fit_span": 0.021789353684936133, + "read_span": 0.00718331086589763, + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.04008721566966011, + "Wikipedia_en|USPTO_Backgrounds": 0.040731303937112294, + "Wikipedia_en|Pile-CC": 0.041783383248063424, + "Wikipedia_en|StackExchange": 0.03158689119025237, + "Wikipedia_en|Github": 0.0371421959846177, + "Wikipedia_en|OOD_News_2024": 0.04218884070869535, + "Wikipedia_en|OOD_arXiv_2023": 0.0378449243415768, + "USPTO_Backgrounds|Wikipedia_en": 0.04718475465455817, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.049209072841848764, + "USPTO_Backgrounds|Pile-CC": 0.04927536356262863, + "USPTO_Backgrounds|StackExchange": 0.043420098496911426, + "USPTO_Backgrounds|Github": 0.04697342338557872, + "USPTO_Backgrounds|OOD_News_2024": 0.048572873437984124, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.04855549412055148, + "Pile-CC|Wikipedia_en": 0.047614535679005913, + "Pile-CC|USPTO_Backgrounds": 0.04958673395319945, + "Pile-CC|Pile-CC": 0.04938663974300855, + "Pile-CC|StackExchange": 0.0409995816492786, + "Pile-CC|Github": 0.045152171807260146, + "Pile-CC|OOD_News_2024": 0.05006507673921685, + "Pile-CC|OOD_arXiv_2023": 0.04705409799805946, + "StackExchange|Wikipedia_en": 0.04362940489211016, + "StackExchange|USPTO_Backgrounds": 0.04815728486412101, + "StackExchange|Pile-CC": 0.04591440298698015, + "StackExchange|StackExchange": 0.041331839377784896, + "StackExchange|Github": 0.04514965671632025, + "StackExchange|OOD_News_2024": 0.04632412789182531, + "StackExchange|OOD_arXiv_2023": 0.04540256434120238, + "Github|Wikipedia_en": 0.025614112155098054, + "Github|USPTO_Backgrounds": 0.027038889370548227, + "Github|Pile-CC": 0.02642265702597797, + "Github|StackExchange": 0.022940734980834855, + "Github|Github": 0.025874506620069345, + "Github|OOD_News_2024": 0.026371424386484755, + "Github|OOD_arXiv_2023": 0.026403280166495176, + "OOD_News_2024|Wikipedia_en": 0.03729078197437856, + "OOD_News_2024|USPTO_Backgrounds": 0.04051533589243061, + "OOD_News_2024|Pile-CC": 0.04000505298996965, + "OOD_News_2024|StackExchange": 0.030654219861349297, + "OOD_News_2024|Github": 0.03395102156791836, + "OOD_News_2024|OOD_News_2024": 0.03968709014459617, + "OOD_News_2024|OOD_arXiv_2023": 0.037250963831320406, + "OOD_arXiv_2023|Wikipedia_en": 0.0382842840704446, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.041786901621768874, + "OOD_arXiv_2023|Pile-CC": 0.04068780026346859, + "OOD_arXiv_2023|StackExchange": 0.0358089808633344, + "OOD_arXiv_2023|Github": 0.038925207879704736, + "OOD_arXiv_2023|OOD_News_2024": 0.04030238125835442, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.04069924547285256 + }, + "logit_arm_mean": 0.030071528476574235, + "logit_arm_by_read_context": { + "Wikipedia_en": 0.027391525839144986, + "USPTO_Backgrounds": 0.028615072128983836, + "Pile-CC": 0.035671688499860466, + "StackExchange": 0.024893753851453464, + "Github": 0.028762414352968335, + "OOD_News_2024": 0.03820927715084205, + "OOD_arXiv_2023": 0.026956967512766522 + }, + "shuf_arm_mean": 0.021846755560753602, + "per_draw": [ + { + "fit_pct": 90.7922535224827, + "read_pct": 7.430378700724462 + }, + { + "fit_pct": 82.2073693621799, + "read_pct": 15.570104215586392 + }, + { + "fit_pct": 85.11217734376787, + "read_pct": 12.014235908362624 + }, + { + "fit_pct": 88.47924031073902, + "read_pct": 8.986864380793483 + }, + { + "fit_pct": 75.2678801875321, + "read_pct": 21.893887872727266 + }, + { + "fit_pct": 80.74643203054481, + "read_pct": 15.238661312534562 + }, + { + "fit_pct": 90.0915227937688, + "read_pct": 7.478362539728582 + }, + { + "fit_pct": 81.16931703953225, + "read_pct": 16.540243077889652 + }, + { + "fit_pct": 84.29144826123185, + "read_pct": 12.623170222243886 + } + ], + "fit_pct_draw_range": [ + 75.2678801875321, + 90.7922535224827 + ], + "read_pct_draw_range": [ + 7.430378700724462, + 21.893887872727266 + ] + }, + "min": { + "fit_pct": 77.88156213345998, + "read_pct": 19.309979397701433, + "residual_pct": 2.8084584688385887, + "row_means": { + "Wikipedia_en": 0.13037704450507012, + "USPTO_Backgrounds": 0.15757397184118865, + "Pile-CC": 0.1444637641547218, + "StackExchange": 0.14656658471221962, + "Github": 0.11268684133177712, + "OOD_News_2024": 0.13503231443760413, + "OOD_arXiv_2023": 0.1415638420316908 + }, + "col_means": { + "Wikipedia_en": 0.1414344311765735, + "USPTO_Backgrounds": 0.14428397829806994, + "Pile-CC": 0.14360296630433628, + "StackExchange": 0.12689127717758455, + "Github": 0.12948421641060756, + "OOD_News_2024": 0.14015553895797994, + "OOD_arXiv_2023": 0.1424119546891205 + }, + "fit_span": 0.04488713050941154, + "read_span": 0.017392701120485382, + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.13745829313993455, + "Wikipedia_en|USPTO_Backgrounds": 0.13685160357919005, + "Wikipedia_en|Pile-CC": 0.13761192481550905, + "Wikipedia_en|StackExchange": 0.11445725527074602, + "Wikipedia_en|Github": 0.1190687220957544, + "Wikipedia_en|OOD_News_2024": 0.13454470208121672, + "Wikipedia_en|OOD_arXiv_2023": 0.13264681055314012, + "USPTO_Backgrounds|Wikipedia_en": 0.1597413457516167, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.16240865459872617, + "USPTO_Backgrounds|Pile-CC": 0.16546140789157815, + "USPTO_Backgrounds|StackExchange": 0.14842800808449585, + "USPTO_Backgrounds|Github": 0.14715511527740294, + "USPTO_Backgrounds|OOD_News_2024": 0.15988392618795236, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.1599393450965484, + "Pile-CC|Wikipedia_en": 0.14869491424825457, + "Pile-CC|USPTO_Backgrounds": 0.14836930558085443, + "Pile-CC|Pile-CC": 0.14895093834234607, + "Pile-CC|StackExchange": 0.1337302645875348, + "Pile-CC|Github": 0.1361582430700461, + "Pile-CC|OOD_News_2024": 0.14780191228621536, + "Pile-CC|OOD_arXiv_2023": 0.1475407709678014, + "StackExchange|Wikipedia_en": 0.1471342760241694, + "StackExchange|USPTO_Backgrounds": 0.1546758765147792, + "StackExchange|Pile-CC": 0.15029816973126597, + "StackExchange|StackExchange": 0.13553505165295468, + "StackExchange|Github": 0.13915350776579644, + "StackExchange|OOD_News_2024": 0.14456868833965725, + "StackExchange|OOD_arXiv_2023": 0.15460052295691437, + "Github|Wikipedia_en": 0.11133564267721441, + "Github|USPTO_Backgrounds": 0.11846596379246976, + "Github|Pile-CC": 0.1167515120572514, + "Github|StackExchange": 0.10683833977414502, + "Github|Github": 0.11041632095972696, + "Github|OOD_News_2024": 0.1113966152899795, + "Github|OOD_arXiv_2023": 0.11360349477165275, + "OOD_News_2024|Wikipedia_en": 0.14004707295033667, + "OOD_News_2024|USPTO_Backgrounds": 0.14014158729049894, + "OOD_News_2024|Pile-CC": 0.1398885983145899, + "OOD_News_2024|StackExchange": 0.11874940356032716, + "OOD_News_2024|Github": 0.1256197339958615, + "OOD_News_2024|OOD_News_2024": 0.13932945198482938, + "OOD_News_2024|OOD_arXiv_2023": 0.14145035296678543, + "OOD_arXiv_2023|Wikipedia_en": 0.14562947344448832, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.14907485672997103, + "OOD_arXiv_2023|Pile-CC": 0.14625821297781336, + "OOD_arXiv_2023|StackExchange": 0.1305006173128883, + "OOD_arXiv_2023|Github": 0.12881787170966466, + "OOD_arXiv_2023|OOD_News_2024": 0.14356347653600904, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.14710238551100097 + }, + "logit_arm_mean": 0.12414802646353132, + "logit_arm_by_read_context": { + "Wikipedia_en": 0.12731547988951206, + "USPTO_Backgrounds": 0.12172445530692735, + "Pile-CC": 0.13018862046301366, + "StackExchange": 0.12258899013201396, + "Github": 0.1198522030065457, + "OOD_News_2024": 0.12813261710107327, + "OOD_arXiv_2023": 0.1192338193456332 + }, + "shuf_arm_mean": 0.13738427619848934, + "per_draw": [ + { + "fit_pct": 75.85457738948729, + "read_pct": 20.942905484740514 + }, + { + "fit_pct": 71.48368828053447, + "read_pct": 25.636088637101896 + }, + { + "fit_pct": 82.68936126798819, + "read_pct": 14.432396166613163 + }, + { + "fit_pct": 71.01341512227505, + "read_pct": 25.388112087777113 + }, + { + "fit_pct": 68.29719983919881, + "read_pct": 28.35628639231493 + }, + { + "fit_pct": 79.74364532744725, + "read_pct": 16.18109856352861 + }, + { + "fit_pct": 72.6255703643355, + "read_pct": 23.336692052219888 + }, + { + "fit_pct": 69.18986468299208, + "read_pct": 27.99601144529515 + }, + { + "fit_pct": 82.88407957539141, + "read_pct": 13.404811095556553 + } + ], + "fit_pct_draw_range": [ + 68.29719983919881, + 82.88407957539141 + ], + "read_pct_draw_range": [ + 13.404811095556553, + 28.35628639231493 + ] + } + }, + "384": { + "persist": { + "fit_pct": 75.63456836003762, + "read_pct": 22.816528933524566, + "residual_pct": 1.5489027064378116, + "row_means": { + "Wikipedia_en": 0.03799759197965383, + "USPTO_Backgrounds": 0.04694185388119271, + "Pile-CC": 0.04666424841612637, + "StackExchange": 0.04482117429826527, + "Github": 0.025447452692906297, + "OOD_News_2024": 0.03666624032434017, + "OOD_arXiv_2023": 0.03937325691286889 + }, + "col_means": { + "Wikipedia_en": 0.04156162330755107, + "USPTO_Backgrounds": 0.04450415482389785, + "Pile-CC": 0.042350994425618814, + "StackExchange": 0.032543399040069844, + "Github": 0.037407228231637016, + "OOD_News_2024": 0.04234790933680617, + "OOD_arXiv_2023": 0.037196509339772754 + }, + "fit_span": 0.02149440118828641, + "read_span": 0.011960755783828006, + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.04025864012849828, + "Wikipedia_en|USPTO_Backgrounds": 0.042519142579597734, + "Wikipedia_en|Pile-CC": 0.04200751210769845, + "Wikipedia_en|StackExchange": 0.029473496895904342, + "Wikipedia_en|Github": 0.03466411767682681, + "Wikipedia_en|OOD_News_2024": 0.04227044843137264, + "Wikipedia_en|OOD_arXiv_2023": 0.034789786037678516, + "USPTO_Backgrounds|Wikipedia_en": 0.04859956321823928, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.05076956789061013, + "USPTO_Backgrounds|Pile-CC": 0.050158720649778846, + "USPTO_Backgrounds|StackExchange": 0.03982415828957326, + "USPTO_Backgrounds|Github": 0.04593538187651171, + "USPTO_Backgrounds|OOD_News_2024": 0.04926195177767012, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.04404363346596559, + "Pile-CC|Wikipedia_en": 0.0487434890638623, + "Pile-CC|USPTO_Backgrounds": 0.052299690569957924, + "Pile-CC|Pile-CC": 0.049433195233965915, + "Pile-CC|StackExchange": 0.03752161374884761, + "Pile-CC|Github": 0.044174164903557134, + "Pile-CC|OOD_News_2024": 0.050277405170102915, + "Pile-CC|OOD_arXiv_2023": 0.044200180222590764, + "StackExchange|Wikipedia_en": 0.04594371466793948, + "StackExchange|USPTO_Backgrounds": 0.04960550417502721, + "StackExchange|Pile-CC": 0.046046345422251354, + "StackExchange|StackExchange": 0.039236117048292524, + "StackExchange|Github": 0.0440265918098804, + "StackExchange|OOD_News_2024": 0.0468190358641247, + "StackExchange|OOD_arXiv_2023": 0.04207091110034122, + "Github|Wikipedia_en": 0.027008157880562874, + "Github|USPTO_Backgrounds": 0.029334460297185512, + "Github|Pile-CC": 0.026484128098107046, + "Github|StackExchange": 0.02074170213503142, + "Github|Github": 0.023744346051373415, + "Github|OOD_News_2024": 0.026952486417980658, + "Github|OOD_arXiv_2023": 0.023866887970103157, + "OOD_News_2024|Wikipedia_en": 0.03918463856809669, + "OOD_News_2024|USPTO_Backgrounds": 0.04300294773549669, + "OOD_News_2024|Pile-CC": 0.04025177763154109, + "OOD_News_2024|StackExchange": 0.028380336987579035, + "OOD_News_2024|Github": 0.032016023313109244, + "OOD_News_2024|OOD_News_2024": 0.03980307214272519, + "OOD_News_2024|OOD_arXiv_2023": 0.03402488589183324, + "OOD_arXiv_2023|Wikipedia_en": 0.04119315962565856, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.04399777051940974, + "OOD_arXiv_2023|Pile-CC": 0.04207528183598899, + "OOD_arXiv_2023|StackExchange": 0.032626368175260725, + "OOD_arXiv_2023|Github": 0.03728997199020038, + "OOD_arXiv_2023|OOD_News_2024": 0.04105096555366698, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.03737928068989681 + }, + "logit_arm_mean": 0.029256576931636247, + "logit_arm_by_read_context": { + "Wikipedia_en": 0.02726959785601745, + "USPTO_Backgrounds": 0.031318317198505, + "Pile-CC": 0.035128501189562185, + "StackExchange": 0.024631802543687322, + "Github": 0.024749102389129498, + "OOD_News_2024": 0.037005650632393854, + "OOD_arXiv_2023": 0.024693066712158422 + }, + "shuf_arm_mean": 0.021348717387410857, + "per_draw": [ + { + "fit_pct": 66.67965516146387, + "read_pct": 31.401369980577368 + }, + { + "fit_pct": 78.54248907883829, + "read_pct": 19.116245440696414 + }, + { + "fit_pct": 82.50791793411943, + "read_pct": 14.635132739517688 + }, + { + "fit_pct": 59.11673219534861, + "read_pct": 38.840423117450634 + }, + { + "fit_pct": 75.70959518951881, + "read_pct": 21.968080546043357 + }, + { + "fit_pct": 78.16745155941258, + "read_pct": 18.153657548822387 + }, + { + "fit_pct": 63.58724536274107, + "read_pct": 34.26424233560957 + }, + { + "fit_pct": 78.95512806208424, + "read_pct": 18.735513621801648 + }, + { + "fit_pct": 82.55953797747671, + "read_pct": 13.912251394967521 + } + ], + "fit_pct_draw_range": [ + 59.11673219534861, + 82.55953797747671 + ], + "read_pct_draw_range": [ + 13.912251394967521, + 38.840423117450634 + ] + }, + "min": { + "fit_pct": 65.57953471918667, + "read_pct": 32.42265863722949, + "residual_pct": 1.9978066435838373, + "row_means": { + "Wikipedia_en": 0.12920543721743993, + "USPTO_Backgrounds": 0.15546386562701728, + "Pile-CC": 0.142793459460021, + "StackExchange": 0.14465200876196227, + "Github": 0.11106315517709368, + "OOD_News_2024": 0.13457579928494637, + "OOD_arXiv_2023": 0.13952944435120102 + }, + "col_means": { + "Wikipedia_en": 0.13945060858414288, + "USPTO_Backgrounds": 0.1470515641930794, + "Pile-CC": 0.14518587438182698, + "StackExchange": 0.11975481370316138, + "Github": 0.1271028390213374, + "OOD_News_2024": 0.14046075291754234, + "OOD_arXiv_2023": 0.13827671707859115 + }, + "fit_span": 0.044400710449923594, + "read_span": 0.02729675048991803, + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.13451480708188482, + "Wikipedia_en|USPTO_Backgrounds": 0.14049505276812446, + "Wikipedia_en|Pile-CC": 0.1385176325837771, + "Wikipedia_en|StackExchange": 0.11032706573605537, + "Wikipedia_en|Github": 0.11619209005600876, + "Wikipedia_en|OOD_News_2024": 0.13555838407741652, + "Wikipedia_en|OOD_arXiv_2023": 0.12883302821881243, + "USPTO_Backgrounds|Wikipedia_en": 0.15995610592265924, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.16200002274579473, + "USPTO_Backgrounds|Pile-CC": 0.16563954535457823, + "USPTO_Backgrounds|StackExchange": 0.13853617093215387, + "USPTO_Backgrounds|Github": 0.14881682350403733, + "USPTO_Backgrounds|OOD_News_2024": 0.15875758826732636, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.15454080266257125, + "Pile-CC|Wikipedia_en": 0.14579989719722006, + "Pile-CC|USPTO_Backgrounds": 0.15506606213748456, + "Pile-CC|Pile-CC": 0.1511478189378977, + "Pile-CC|StackExchange": 0.1238520700070593, + "Pile-CC|Github": 0.13305465252035195, + "Pile-CC|OOD_News_2024": 0.1470812367896239, + "Pile-CC|OOD_arXiv_2023": 0.14355247863050963, + "StackExchange|Wikipedia_en": 0.14514333394666512, + "StackExchange|USPTO_Backgrounds": 0.1560478561454349, + "StackExchange|Pile-CC": 0.15160742099914287, + "StackExchange|StackExchange": 0.12877141729825073, + "StackExchange|Github": 0.13644196275207732, + "StackExchange|OOD_News_2024": 0.14531836691829894, + "StackExchange|OOD_arXiv_2023": 0.14923370327386593, + "Github|Wikipedia_en": 0.10929596051573753, + "Github|USPTO_Backgrounds": 0.11938386716776424, + "Github|Pile-CC": 0.11792684280210071, + "Github|StackExchange": 0.10017448460890187, + "Github|Github": 0.10609646952814526, + "Github|OOD_News_2024": 0.11245915037062433, + "Github|OOD_arXiv_2023": 0.11210531124638187, + "OOD_News_2024|Wikipedia_en": 0.13834427835212815, + "OOD_News_2024|USPTO_Backgrounds": 0.14577262409859235, + "OOD_News_2024|Pile-CC": 0.14276208728551865, + "OOD_News_2024|StackExchange": 0.11330636822515064, + "OOD_News_2024|Github": 0.12285870640642113, + "OOD_News_2024|OOD_News_2024": 0.14103572045763335, + "OOD_News_2024|OOD_arXiv_2023": 0.13795081016918023, + "OOD_arXiv_2023|Wikipedia_en": 0.14309987707270516, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.15059546428836054, + "OOD_arXiv_2023|Pile-CC": 0.14869977270977364, + "OOD_arXiv_2023|StackExchange": 0.1233161191145579, + "OOD_arXiv_2023|Github": 0.12625916838232012, + "OOD_arXiv_2023|OOD_News_2024": 0.14301482354187303, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.14172088534881672 + }, + "logit_arm_mean": 0.12356656241629804, + "logit_arm_by_read_context": { + "Wikipedia_en": 0.12690831956764062, + "USPTO_Backgrounds": 0.12432758398354055, + "Pile-CC": 0.13045525314907233, + "StackExchange": 0.11729577171305815, + "Github": 0.11806795410811902, + "OOD_News_2024": 0.13108894154429435, + "OOD_arXiv_2023": 0.11682211284836133 + }, + "shuf_arm_mean": 0.1353385637558642, + "per_draw": [ + { + "fit_pct": 59.83203463681921, + "read_pct": 38.09722521721748 + }, + { + "fit_pct": 70.64349565280934, + "read_pct": 27.233642449051366 + }, + { + "fit_pct": 70.96516025666826, + "read_pct": 26.341693741956703 + }, + { + "fit_pct": 53.89154104531688, + "read_pct": 43.364068918164286 + }, + { + "fit_pct": 64.54517059258714, + "read_pct": 33.00082500907466 + }, + { + "fit_pct": 66.13061615835339, + "read_pct": 30.620896349706946 + }, + { + "fit_pct": 56.16745643330838, + "read_pct": 41.356347137385214 + }, + { + "fit_pct": 69.09085923136112, + "read_pct": 28.7698101740013 + }, + { + "fit_pct": 69.21803228081676, + "read_pct": 27.434170118540262 + } + ], + "fit_pct_draw_range": [ + 53.89154104531688, + 70.96516025666826 + ], + "read_pct_draw_range": [ + 26.341693741956703, + 43.364068918164286 + ] + } + }, + "768": { + "persist": { + "fit_pct": 82.86427950401284, + "read_pct": 15.70296342366248, + "residual_pct": 1.4327570723246812, + "row_means": { + "Wikipedia_en": 0.03748184013298698, + "USPTO_Backgrounds": 0.045851869697845174, + "Pile-CC": 0.046064474638403644, + "StackExchange": 0.04457260203318641, + "Github": 0.025559739633980727, + "OOD_News_2024": 0.036364406839914855, + "OOD_arXiv_2023": 0.03775972657437836 + }, + "col_means": { + "Wikipedia_en": 0.03928301404267254, + "USPTO_Backgrounds": 0.043609168822137966, + "Pile-CC": 0.04140896289055014, + "StackExchange": 0.03429500318117558, + "Github": 0.036026849873023965, + "OOD_News_2024": 0.04049782415718905, + "OOD_arXiv_2023": 0.03853383658394691 + }, + "fit_span": 0.020504735004422917, + "read_span": 0.009314165640962387, + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.03855007969929526, + "Wikipedia_en|USPTO_Backgrounds": 0.04197476356704202, + "Wikipedia_en|Pile-CC": 0.041326470507515804, + "Wikipedia_en|StackExchange": 0.03171069322464367, + "Wikipedia_en|Github": 0.03271053015357918, + "Wikipedia_en|OOD_News_2024": 0.040222854369009534, + "Wikipedia_en|OOD_arXiv_2023": 0.03587748940982338, + "USPTO_Backgrounds|Wikipedia_en": 0.04514276009156472, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.04994863613715602, + "USPTO_Backgrounds|Pile-CC": 0.04996790660855671, + "USPTO_Backgrounds|StackExchange": 0.040724655587433115, + "USPTO_Backgrounds|Github": 0.0431156854921331, + "USPTO_Backgrounds|OOD_News_2024": 0.04704198615832461, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.04502145780974792, + "Pile-CC|Wikipedia_en": 0.04616133691225615, + "Pile-CC|USPTO_Backgrounds": 0.05095177880591816, + "Pile-CC|Pile-CC": 0.047977329017077054, + "Pile-CC|StackExchange": 0.04090601362339738, + "Pile-CC|Github": 0.04219146918815871, + "Pile-CC|OOD_News_2024": 0.04835543760822879, + "Pile-CC|OOD_arXiv_2023": 0.04590795731378926, + "StackExchange|Wikipedia_en": 0.04347311726046933, + "StackExchange|USPTO_Backgrounds": 0.04957352479298909, + "StackExchange|Pile-CC": 0.045239990359793106, + "StackExchange|StackExchange": 0.040024418857259055, + "StackExchange|Github": 0.0430997302962674, + "StackExchange|OOD_News_2024": 0.04550796404687895, + "StackExchange|OOD_arXiv_2023": 0.04508946861864792, + "Github|Wikipedia_en": 0.026476274937805203, + "Github|USPTO_Backgrounds": 0.029239530111145644, + "Github|Pile-CC": 0.026605095812636946, + "Github|StackExchange": 0.021942350568456784, + "Github|Github": 0.023350246974991426, + "Github|OOD_News_2024": 0.02557414977086915, + "Github|OOD_arXiv_2023": 0.025730529261959924, + "OOD_News_2024|Wikipedia_en": 0.03727044302245809, + "OOD_News_2024|USPTO_Backgrounds": 0.04123268084497087, + "OOD_News_2024|Pile-CC": 0.038843108128963244, + "OOD_News_2024|StackExchange": 0.030999718376228377, + "OOD_News_2024|Github": 0.033006426686834955, + "OOD_News_2024|OOD_News_2024": 0.03876728730069266, + "OOD_News_2024|OOD_arXiv_2023": 0.03443118351925578, + "OOD_arXiv_2023|Wikipedia_en": 0.03790708637485902, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.04234326749574393, + "OOD_arXiv_2023|Pile-CC": 0.039902839799308115, + "OOD_arXiv_2023|StackExchange": 0.03375717203081068, + "OOD_arXiv_2023|Github": 0.034713860319202976, + "OOD_arXiv_2023|OOD_News_2024": 0.03801508984631962, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.03767877015440414 + }, + "logit_arm_mean": 0.028171038918108456, + "logit_arm_by_read_context": { + "Wikipedia_en": 0.027719283732585608, + "USPTO_Backgrounds": 0.02997406746726483, + "Pile-CC": 0.033705620227071145, + "StackExchange": 0.024724263853083055, + "Github": 0.02147600760993858, + "OOD_News_2024": 0.03502082105260342, + "OOD_arXiv_2023": 0.02457720848421256 + }, + "shuf_arm_mean": 0.020872791122556442, + "per_draw": [ + { + "fit_pct": 82.2388764510656, + "read_pct": 15.926934926450782 + }, + { + "fit_pct": 84.00751507631574, + "read_pct": 14.081469383246926 + }, + { + "fit_pct": 78.65002688603697, + "read_pct": 17.30756191238607 + }, + { + "fit_pct": 77.25197198455533, + "read_pct": 20.369343149802717 + }, + { + "fit_pct": 78.77322080725493, + "read_pct": 18.833033924742892 + }, + { + "fit_pct": 73.21445566898818, + "read_pct": 22.22756682754018 + }, + { + "fit_pct": 81.47943695078544, + "read_pct": 16.51605900481453 + }, + { + "fit_pct": 84.33946268988814, + "read_pct": 13.625404678923317 + }, + { + "fit_pct": 78.31145412683682, + "read_pct": 17.98267548569098 + } + ], + "fit_pct_draw_range": [ + 73.21445566898818, + 84.33946268988814 + ], + "read_pct_draw_range": [ + 13.625404678923317, + 22.22756682754018 + ] + }, + "min": { + "fit_pct": 68.28086817699625, + "read_pct": 29.394431516694866, + "residual_pct": 2.3247003063088796, + "row_means": { + "Wikipedia_en": 0.12665622011418382, + "USPTO_Backgrounds": 0.15184396701377062, + "Pile-CC": 0.14053657526771227, + "StackExchange": 0.14265726662226139, + "Github": 0.1088683233670299, + "OOD_News_2024": 0.13185123095200174, + "OOD_arXiv_2023": 0.13769647075484195 + }, + "col_means": { + "Wikipedia_en": 0.1349283139531811, + "USPTO_Backgrounds": 0.1432134219755729, + "Pile-CC": 0.14230418248131635, + "StackExchange": 0.12204343105355898, + "Github": 0.12159531436978825, + "OOD_News_2024": 0.13992733313805528, + "OOD_arXiv_2023": 0.13609805712032885 + }, + "fit_span": 0.042975643646740724, + "read_span": 0.021618107605784656, + "matrix": { + "Wikipedia_en|Wikipedia_en": 0.12981085069477558, + "Wikipedia_en|USPTO_Backgrounds": 0.13622477340201536, + "Wikipedia_en|Pile-CC": 0.1363568396617969, + "Wikipedia_en|StackExchange": 0.11037355785568555, + "Wikipedia_en|Github": 0.11124687848819627, + "Wikipedia_en|OOD_News_2024": 0.1345470730629232, + "Wikipedia_en|OOD_arXiv_2023": 0.12803356763389376, + "USPTO_Backgrounds|Wikipedia_en": 0.1546798170854648, + "USPTO_Backgrounds|USPTO_Backgrounds": 0.1581308063533571, + "USPTO_Backgrounds|Pile-CC": 0.16005559551219145, + "USPTO_Backgrounds|StackExchange": 0.14190943067272505, + "USPTO_Backgrounds|Github": 0.14008190093768969, + "USPTO_Backgrounds|OOD_News_2024": 0.1573105324887567, + "USPTO_Backgrounds|OOD_arXiv_2023": 0.1507396860462096, + "Pile-CC|Wikipedia_en": 0.14109714412026936, + "Pile-CC|USPTO_Backgrounds": 0.14907482208477127, + "Pile-CC|Pile-CC": 0.1490341159204642, + "Pile-CC|StackExchange": 0.12739595568014517, + "Pile-CC|Github": 0.1278026151160399, + "Pile-CC|OOD_News_2024": 0.14702963588966264, + "Pile-CC|OOD_arXiv_2023": 0.1423217380626334, + "StackExchange|Wikipedia_en": 0.14122954776717556, + "StackExchange|USPTO_Backgrounds": 0.15397988839281931, + "StackExchange|Pile-CC": 0.14890266905228297, + "StackExchange|StackExchange": 0.12994296993646356, + "StackExchange|Github": 0.13080228538148933, + "StackExchange|OOD_News_2024": 0.14606552579336696, + "StackExchange|OOD_arXiv_2023": 0.14767798003223206, + "Github|Wikipedia_en": 0.10518552031781939, + "Github|USPTO_Backgrounds": 0.11590438617600335, + "Github|Pile-CC": 0.11433741773168246, + "Github|StackExchange": 0.10287893902924326, + "Github|Github": 0.10260187768273883, + "Github|OOD_News_2024": 0.11049770038161012, + "Github|OOD_arXiv_2023": 0.1106724222501119, + "OOD_News_2024|Wikipedia_en": 0.13378461044695641, + "OOD_News_2024|USPTO_Backgrounds": 0.14130263796283138, + "OOD_News_2024|Pile-CC": 0.14122660292519462, + "OOD_News_2024|StackExchange": 0.11529074518216981, + "OOD_News_2024|Github": 0.1168397031724453, + "OOD_News_2024|OOD_News_2024": 0.1403403849237495, + "OOD_News_2024|OOD_arXiv_2023": 0.13417393205066522, + "OOD_arXiv_2023|Wikipedia_en": 0.13871070723980664, + "OOD_arXiv_2023|USPTO_Backgrounds": 0.14787663945721255, + "OOD_arXiv_2023|Pile-CC": 0.14621603656560184, + "OOD_arXiv_2023|StackExchange": 0.1265124190184805, + "OOD_arXiv_2023|Github": 0.1217919398099184, + "OOD_arXiv_2023|OOD_News_2024": 0.14370047942631775, + "OOD_arXiv_2023|OOD_arXiv_2023": 0.13906707376655605 + }, + "logit_arm_mean": 0.12161037313441435, + "logit_arm_by_read_context": { + "Wikipedia_en": 0.123526473219196, + "USPTO_Backgrounds": 0.12021252959966659, + "Pile-CC": 0.1303999507178863, + "StackExchange": 0.11568729877471924, + "Github": 0.11269759486118952, + "OOD_News_2024": 0.13086204007267951, + "OOD_arXiv_2023": 0.11788672469556331 + }, + "shuf_arm_mean": 0.13312444301942986, + "per_draw": [ + { + "fit_pct": 58.471407047795104, + "read_pct": 38.16253849640565 + }, + { + "fit_pct": 75.98872461275255, + "read_pct": 21.741190620382355 + }, + { + "fit_pct": 70.53429926459349, + "read_pct": 25.802948245105206 + }, + { + "fit_pct": 52.29494335895442, + "read_pct": 44.11952455509875 + }, + { + "fit_pct": 72.1099595297815, + "read_pct": 24.766509204956183 + }, + { + "fit_pct": 67.9237164248372, + "read_pct": 28.29418785863993 + }, + { + "fit_pct": 55.00972946505749, + "read_pct": 41.43569548387983 + }, + { + "fit_pct": 73.89591426242943, + "read_pct": 23.05406724058501 + }, + { + "fit_pct": 69.80703066630012, + "read_pct": 26.55158074906302 + } + ], + "fit_pct_draw_range": [ + 52.29494335895442, + 75.98872461275255 + ], + "read_pct_draw_range": [ + 21.741190620382355, + 44.11952455509875 + ] + } + } + }, + "controls": { + "C1_row_ordering_vs_e52": { + "spearman_row_ordering_vs_e52": 1.0, + "threshold": 0.8, + "fires": true, + "note": "cell VALUES cannot match E52 \u2014 the pool is the >=768-token subset \u2014 so the control is on the row ORDERING." + }, + "C2_logit_arm_moves_with_dose": { + "logit_arm_by_dose": { + "128": 0.030071528476574235, + "384": 0.029256576931636247, + "768": 0.028171038918108456 + }, + "range": 0.0019004895584657788, + "monotone": true, + "fires": true, + "note": "the unfitted arm has no operator, so its movement is pure context effect and bounds how much the read axis can move at all" + }, + "C3_prefix_fit_overlap": { + "n_documents": 0, + "fires": true + }, + "C4_cycling": { + "Wikipedia_en|p0": 1.0, + "Wikipedia_en|p1": 1.0, + "Wikipedia_en|p2": 1.0, + "USPTO_Backgrounds|p0": 1.0, + "USPTO_Backgrounds|p1": 1.0, + "USPTO_Backgrounds|p2": 1.0, + "Pile-CC|p0": 1.0, + "Pile-CC|p1": 1.0, + "Pile-CC|p2": 1.0, + "StackExchange|p0": 1.1812227074235808, + "StackExchange|p1": 1.1812227074235808, + "StackExchange|p2": 1.1812227074235808, + "Github|p0": 1.0, + "Github|p1": 1.0, + "Github|p2": 1.0, + "OOD_News_2024|p0": 1.0, + "OOD_News_2024|p1": 1.0, + "OOD_News_2024|p2": 1.0, + "OOD_arXiv_2023|p0": 1.0, + "OOD_arXiv_2023|p1": 1.0, + "OOD_arXiv_2023|p2": 1.0 + } + }, + "VERDICT": "PARTIAL \u2014 read share moves +6.2 points (9.5% -> 15.7%), monotone=False. Report the curve; the headline is dose-dependent with this slope.", + "provenance": { + "schema_version": 1, + "experiment": "E59", + "utc": "2026-08-30T02:57:46+00:00", + "script": { + "path": "experiments/t59_read_dose.py", + "sha256": "76473cf24c5b8b440ada67cc3229206f077355a4c024e4bda3201ed0089d2129", + "n_lines": 391 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/t1_module_audit.json" + ], + "dirty_diff_sha256": "6de268d981510d7cce49efe3da1c011b7dc2c81b40eae3e61c2f0fa57ef1b5ff", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/t59_read_dose.py", + "--device", + "cpu" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "7c1c6b762d47c819b5439a843284a7c79a55f5fcb1a0701cc50338479bd979c3" + } +} \ No newline at end of file diff --git a/results/e60_fitter_determinism.json b/results/e60_fitter_determinism.json new file mode 100644 index 0000000000000000000000000000000000000000..a16a2867f09d6ea2c74ed455387769a9726b9f6e --- /dev/null +++ b/results/e60_fitter_determinism.json @@ -0,0 +1,139 @@ +{ + "experiment": "E60 \u2014 the fitter equivalence, measured rather than asserted", + "why": "paper.tex asserted equality at 2e-05 and cited a test that did not contain the claim; 2e-05 was verify_against_anchor's atol default and nothing had run it.", + "model": "EleutherAI/pythia-70m-deduped", + "n_prompts": 8, + "target_layer": -2, + "skip_first": 16, + "max_seq_len": 128, + "by_thread_count": { + "1": { + "cross_implementation_abs": 0.0, + "cross_implementation_rel": 0.0, + "anchor_vs_anchor_at_base": 0.0, + "fast_vs_fast_at_base": 0.0, + "jacobian_absmax": 0.9221423864364624 + }, + "4": { + "cross_implementation_abs": 0.0, + "cross_implementation_rel": 0.0, + "anchor_vs_anchor_at_base": 0.0, + "fast_vs_fast_at_base": 0.0, + "jacobian_absmax": 0.9221423864364624 + }, + "8": { + "cross_implementation_abs": 0.0, + "cross_implementation_rel": 0.0, + "anchor_vs_anchor_at_base": 0.0, + "fast_vs_fast_at_base": 0.0, + "jacobian_absmax": 0.9221423864364624 + }, + "10": { + "cross_implementation_abs": 0.00010943412780761719, + "cross_implementation_rel": 0.00011867308750983206, + "anchor_vs_anchor_at_base": 0.00012826919555664062, + "fast_vs_fast_at_base": 0.00012826919555664062, + "jacobian_absmax": 0.922147810459137 + }, + "12": { + "cross_implementation_abs": 0.00010344386100769043, + "cross_implementation_rel": 0.00011217736161948043, + "anchor_vs_anchor_at_base": 0.00010056793689727783, + "fast_vs_fast_at_base": 0.000110626220703125, + "jacobian_absmax": 0.9221456050872803 + } + }, + "summary": { + "cross_implementation_max": 0.00010943412780761719, + "cross_implementation_exact_at_thread_counts": [ + "1", + "4", + "8" + ], + "anchor_self_disagreement_max_across_thread_counts": 0.00012826919555664062, + "cross_is_same_order_as_self": true, + "READING": "the two fitters agree EXACTLY at 1, 4, 8 threads, and differ by 1.1e-04 at higher counts -- the same order as the released fitter's disagreement with ITSELF across thread counts (1.3e-04). This is equality up to fp32 reduction order, not a reimplementation error. It also means FITS ARE NOT BIT-REPRODUCIBLE above 8 threads on this machine, which every stored lens SHA depends on." + }, + "controls": { + "C1_repeat_run_bit_identical": { + "anchor_repeat_diff": 0.0, + "fast_repeat_diff": 0.0, + "n_threads": 1, + "fires": true + }, + "C2_prompts_long_enough": { + "prompt_token_lengths": [ + 142, + 149, + 140, + 153, + 153, + 151, + 140, + 145 + ], + "skip_first": 16, + "all_above_skip_first_plus_1": true, + "fires": true + } + }, + "provenance": { + "schema_version": 1, + "experiment": "E60", + "utc": "2026-08-17T01:37:35+00:00", + "script": { + "path": "experiments/t60_fitter_determinism.py", + "sha256": "c397cfa36a13fe088bfa1b9106bb2aeac35284fbca411d650a4e92bf04fa4cab", + "n_lines": 173 + }, + "git": { + "commit": "f8ef435b29365012c0492e023fb6b4dfece5f0f3", + "commit_short": "f8ef435b2936", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/t53_ladder_summary.py", + "experiments/t55_matrix_robustness.py", + "experiments/t56_predictor_registry.py", + "experiments/t58_algebra_audit.py", + "experiments/t59_read_dose.py", + "experiments/t60_fitter_determinism.py", + "paper/paper.tex", + "results/e35_containment.json", + "results/e52_factorial_410m_rerun.json", + "results/e53_ladder_summary.json", + "results/e55_matrix_robustness.json", + "results/e56_predictor_registry.json", + "results/e57_factorial_cells_410m.json", + "results/e58_algebra_audit.json", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/experiments/E57_grid_variance_ci.md", + "specs/experiments/E59_read_dose_response.md", + "tests/test_anchor_fidelity.py", + "xperiments/t52_factorial.py" + ], + "dirty_diff_sha256": "4a72c81a812589b5e476dff564e05d42f643a7dd4b402e3ce4f3d7b37238d070", + "describe": "f8ef435-dirty" + }, + "argv": [ + "experiments/t60_fitter_determinism.py" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "c0ba0463ba224f7d4d39b3fb2b6a8524908a22cb304c3180f3e7e610a99106b9" + } +} \ No newline at end of file diff --git a/results/e61/tv_rand_Github_s0.json b/results/e61/tv_rand_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..c971931b686b0479e1a1828f08cad08e41f3c7aa --- /dev/null +++ b/results/e61/tv_rand_Github_s0.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Github", + "seed": 0, + "randomization": { + "seed": 61000, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2616271972656, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.016643550246953964, + "best1L": 0.016643550246953964, + "mean": 0.004267577165072518, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.009427121175186974, + "best1L": 0.006889049895107746, + "mean": 0.002900652670567589, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.05454545374959707, + "best1L": 0.05064935037600143, + "mean": 0.01418581387926159, + "persist": 0.006493506287889821 + }, + "poetry": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "typo": { + "min": 0.0029761905648878644, + "best1L": 0.0014880952824439322, + "mean": 0.0002289377319227372, + "persist": 0.0 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.01810710257120796, + "dispersion_mean": 0.008622323065442935, + "dispersion_slope": -0.0013899522953292542, + "stable_rank": 284.4104919433594, + "top8_spectral_share": 0.026922911405563354, + "condition_number": 5.4347825050354, + "unembed_alignment": 0.0619964599609375 + }, + "corpus": { + "mean_prompt_tokens": 2232.67, + "type_token_ratio": 0.030734501740069066, + "eval_vocab_overlap": 0.48778625954198473, + "model_cross_entropy": 13.58647276044966, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.01810710257120796, + "10": 0.01603658476336278, + "11": 0.014155309200581353, + "12": 0.01241941767544137, + "13": 0.010855843712034947, + "14": 0.009376712208396636, + "15": 0.008044349086911618, + "16": 0.006769412194158192, + "17": 0.005554271099685859, + "18": 0.004368770173106026, + "19": 0.003223864677479203, + "20": 0.002128987597864662, + "21": 0.0010495748905275607 + } + } + }, + "runtime_s": 431.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_Github_s1.json b/results/e61/tv_rand_Github_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..1954ffebab849613da9c6847320cc9bf38bddd0f --- /dev/null +++ b/results/e61/tv_rand_Github_s1.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Github", + "seed": 1, + "randomization": { + "seed": 61001, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2672119140625, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.002773925129856382, + "best1L": 0.001386962564928191, + "mean": 0.0003200682944485119, + "persist": 0.0 + }, + "multilingual": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "order-ops": { + "min": 0.010389610326715879, + "best1L": 0.010389610326715879, + "mean": 0.0018981019633689097, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.00728862958827189, + "best1L": 0.005830903670617512, + "mean": 0.0019062570562320097, + "persist": 0.0 + }, + "typo": { + "min": 0.0029761905648878644, + "best1L": 0.0029761905648878644, + "mean": 0.0011446887095059668, + "persist": 0.0014880952824439322 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00011944577064631241, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.020353319295948302, + "dispersion_mean": 0.009723318751226675, + "dispersion_slope": -0.0015868669554737545, + "stable_rank": 281.94757080078125, + "top8_spectral_share": 0.027638427913188934, + "condition_number": 5.596490859985352, + "unembed_alignment": 0.0625123679637909 + }, + "corpus": { + "mean_prompt_tokens": 2467.21, + "type_token_ratio": 0.031033029211133222, + "eval_vocab_overlap": 0.5282442748091603, + "model_cross_entropy": 13.83418978441943, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.020353319295948302, + "10": 0.01820333882067811, + "11": 0.01615037841790059, + "12": 0.014164283460931614, + "13": 0.012345597411371733, + "14": 0.0106633270513203, + "15": 0.009060131033217879, + "16": 0.007543569703476364, + "17": 0.00612555315322592, + "18": 0.004799966819435178, + "19": 0.003530901237365133, + "20": 0.0023191356728301075, + "21": 0.0011436416882455419 + } + } + }, + "runtime_s": 430.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_Github_s2.json b/results/e61/tv_rand_Github_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..3a47db8754a79da0cd2d57febea5570e7792a275 --- /dev/null +++ b/results/e61/tv_rand_Github_s2.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Github", + "seed": 2, + "randomization": { + "seed": 61002, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2533874511719, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "multilingual": { + "min": 0.0003625815734267235, + "best1L": 0.0003625815734267235, + "mean": 5.578178180647748e-05, + "persist": 0.0 + }, + "order-ops": { + "min": 0.07012986923967089, + "best1L": 0.05454545521310398, + "mean": 0.038761239332546084, + "persist": 0.04025973991623947 + }, + "poetry": { + "min": 0.00728862958827189, + "best1L": 0.004373177752963134, + "mean": 0.0012334604481501238, + "persist": 0.0 + }, + "typo": { + "min": 0.013392857142857142, + "best1L": 0.010416666977107525, + "mean": 0.004235348107093679, + "persist": 0.004464285847331796 + }, + "association": { + "min": 0.012422360479831696, + "best1L": 0.00776397529989481, + "mean": 0.006330625935723739, + "persist": 0.00776397529989481 + } + }, + "operator": { + "dispersion_L0": 0.020011057578519523, + "dispersion_mean": 0.009616780695716975, + "dispersion_slope": -0.0015571907996746392, + "stable_rank": 278.04644775390625, + "top8_spectral_share": 0.02773524448275566, + "condition_number": 5.671952247619629, + "unembed_alignment": 0.061117831617593765 + }, + "corpus": { + "mean_prompt_tokens": 2495.045, + "type_token_ratio": 0.02932011246290147, + "eval_vocab_overlap": 0.5290076335877862, + "model_cross_entropy": 12.201192804284998, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.020011057578519523, + "10": 0.017856119062642645, + "11": 0.015873106328487936, + "12": 0.014007370423370386, + "13": 0.012234554898308965, + "14": 0.010572736706165185, + "15": 0.00902733215462281, + "16": 0.0075539712287639085, + "17": 0.006152357057219628, + "18": 0.0048034911474155745, + "19": 0.0035197954244486985, + "20": 0.002291284358985153, + "21": 0.0011149726753702709 + } + } + }, + "runtime_s": 430.7, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_Pile-CC_s0.json b/results/e61/tv_rand_Pile-CC_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..dca217c8aef4c07cd121d075a50c0c24c9bd6916 --- /dev/null +++ b/results/e61/tv_rand_Pile-CC_s0.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Pile-CC", + "seed": 0, + "randomization": { + "seed": 61000, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2616271972656, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.015256587682025773, + "best1L": 0.015256587682025773, + "mean": 0.003200682786492897, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.00833937618881464, + "best1L": 0.005801305174827576, + "mean": 0.0023428347859797733, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.042857142963579724, + "best1L": 0.040259740714515956, + "mean": 0.010889110621064901, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "typo": { + "min": 0.004464285714285714, + "best1L": 0.0029761905648878644, + "mean": 0.00034340660619948594, + "persist": 0.0 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.013079269326338348, + "dispersion_mean": 0.006311264617405019, + "dispersion_slope": -0.0010029453656486731, + "stable_rank": 284.95269775390625, + "top8_spectral_share": 0.02693229541182518, + "condition_number": 5.432388782501221, + "unembed_alignment": 0.06169912964105606 + }, + "corpus": { + "mean_prompt_tokens": 1027.01, + "type_token_ratio": 0.09424445721073797, + "eval_vocab_overlap": 0.749618320610687, + "model_cross_entropy": 13.512615581890485, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.013079269326338348, + "10": 0.01163281379962777, + "11": 0.010314321733755183, + "12": 0.009077907344195982, + "13": 0.00795663609015981, + "14": 0.006888989765545794, + "15": 0.005924755113616857, + "16": 0.00500159816345326, + "17": 0.004115926770583883, + "18": 0.003245579222844007, + "19": 0.002411488806297226, + "20": 0.0016021686158403123, + "21": 0.0007949852740068299 + } + } + }, + "runtime_s": 430.5, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_Pile-CC_s1.json b/results/e61/tv_rand_Pile-CC_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..28736fd69f44cf5901c9d67f54ef306c10e30614 --- /dev/null +++ b/results/e61/tv_rand_Pile-CC_s1.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Pile-CC", + "seed": 1, + "randomization": { + "seed": 61001, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2672119140625, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.002773925129856382, + "best1L": 0.002773925129856382, + "mean": 0.0003200682861331318, + "persist": 0.0 + }, + "multilingual": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "order-ops": { + "min": 0.01168831145124776, + "best1L": 0.010389610326715879, + "mean": 0.002197802233110581, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.00874635563897235, + "best1L": 0.005830903670617512, + "mean": 0.002018389757722616, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.004464285847331796, + "best1L": 0.0029761905648878644, + "mean": 0.001144688659613686, + "persist": 0.0014880952824439322 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00011944577064631241, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.016215481327044222, + "dispersion_mean": 0.007823527312112432, + "dispersion_slope": -0.001264450845851342, + "stable_rank": 279.9710388183594, + "top8_spectral_share": 0.027748985216021538, + "condition_number": 5.6613078117370605, + "unembed_alignment": 0.06238432601094246 + }, + "corpus": { + "mean_prompt_tokens": 1244.555, + "type_token_ratio": 0.0835720398053923, + "eval_vocab_overlap": 0.7816793893129771, + "model_cross_entropy": 14.291111885964334, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.016215481327044222, + "10": 0.014549727708026493, + "11": 0.012937466081718677, + "12": 0.011396189339747448, + "13": 0.00995376724999822, + "14": 0.008609500806933474, + "15": 0.0073361836657061235, + "16": 0.006119270567627603, + "17": 0.0049798214808284376, + "18": 0.00390461426793039, + "19": 0.00287959767619553, + "20": 0.001889616141587152, + "21": 0.0009346187441178351 + } + } + }, + "runtime_s": 429.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_Pile-CC_s2.json b/results/e61/tv_rand_Pile-CC_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..6c3a6bd44eddf8dd1e7769ac1bc1f440f7fcfa7c --- /dev/null +++ b/results/e61/tv_rand_Pile-CC_s2.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "Pile-CC", + "seed": 2, + "randomization": { + "seed": 61002, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2533874511719, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "multilingual": { + "min": 0.000725163146853447, + "best1L": 0.000725163146853447, + "mean": 0.00011156356361295496, + "persist": 0.0 + }, + "order-ops": { + "min": 0.05974025877990893, + "best1L": 0.04935065044888428, + "mean": 0.03426573411395241, + "persist": 0.03506493488592761 + }, + "poetry": { + "min": 0.008746355505926269, + "best1L": 0.004373177752963134, + "mean": 0.0016819914536816733, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.013392857142857142, + "best1L": 0.011904762259551458, + "mean": 0.0045787547049777845, + "persist": 0.0029761905648878644 + }, + "association": { + "min": 0.012422360479831696, + "best1L": 0.009316770359873772, + "mean": 0.005733396907869194, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.01575130878074959, + "dispersion_mean": 0.007629881304049972, + "dispersion_slope": -0.0012247357508652719, + "stable_rank": 276.563720703125, + "top8_spectral_share": 0.02781270071864128, + "condition_number": 5.707178592681885, + "unembed_alignment": 0.06127714365720749 + }, + "corpus": { + "mean_prompt_tokens": 1350.62, + "type_token_ratio": 0.07824184448623595, + "eval_vocab_overlap": 0.7801526717557252, + "model_cross_entropy": 12.324972204259923, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.01575130878074959, + "10": 0.014098957724147746, + "11": 0.012547576131698879, + "12": 0.011090814529743184, + "13": 0.009715935866791403, + "14": 0.00840771575505589, + "15": 0.007195342711375439, + "16": 0.006027861043264268, + "17": 0.004917844963901832, + "18": 0.003853401306087277, + "19": 0.002830678828308683, + "20": 0.0018495158192966365, + "21": 0.0009015034922288114 + } + } + }, + "runtime_s": 432.5, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_USPTO_Backgrounds_s0.json b/results/e61/tv_rand_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..ca5681c504c3f5b2c8d26cc68492457d0a27a696 --- /dev/null +++ b/results/e61/tv_rand_USPTO_Backgrounds_s0.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "USPTO_Backgrounds", + "seed": 0, + "randomization": { + "seed": 61000, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2616271972656, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.018030512811882154, + "best1L": 0.018030512811882154, + "mean": 0.003734129880155836, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.00833937618881464, + "best1L": 0.005801305174827576, + "mean": 0.002342834769349013, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.05714285759521382, + "best1L": 0.049350649783653874, + "mean": 0.013486513726612819, + "persist": 0.006493506420935903 + }, + "poetry": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "typo": { + "min": 0.004464285714285714, + "best1L": 0.0029761905648878644, + "mean": 0.00034340660619948594, + "persist": 0.0 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.011990151127031879, + "dispersion_mean": 0.005761265600862756, + "dispersion_slope": -0.000920098898332595, + "stable_rank": 287.059814453125, + "top8_spectral_share": 0.026841018348932266, + "condition_number": 5.383846759796143, + "unembed_alignment": 0.06182160601019859 + }, + "corpus": { + "mean_prompt_tokens": 900.0, + "type_token_ratio": 0.0683, + "eval_vocab_overlap": 0.4083969465648855, + "model_cross_entropy": 13.49804958996472, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.011990151127031879, + "10": 0.010647017820002533, + "11": 0.009430899854145173, + "12": 0.008298288203860329, + "13": 0.007264686793594158, + "14": 0.00628182775865435, + "15": 0.0053959864724517845, + "16": 0.004551951976090891, + "17": 0.0037399316560316224, + "18": 0.002946595375603504, + "19": 0.0021850077974316802, + "20": 0.0014486187229229498, + "21": 0.0007154892533949762 + } + } + }, + "runtime_s": 430.2, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_USPTO_Backgrounds_s1.json b/results/e61/tv_rand_USPTO_Backgrounds_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..6af5d0f1c1ee236c2993ac638e74ca761406a7de --- /dev/null +++ b/results/e61/tv_rand_USPTO_Backgrounds_s1.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "USPTO_Backgrounds", + "seed": 1, + "randomization": { + "seed": 61001, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2672119140625, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.004160887694784573, + "best1L": 0.004160887694784573, + "mean": 0.0005334471435552197, + "persist": 0.0 + }, + "multilingual": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "order-ops": { + "min": 0.009090909069137914, + "best1L": 0.009090909069137914, + "mean": 0.0017982018679114325, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.007288629721317973, + "best1L": 0.005830903670617512, + "mean": 0.002018389757722616, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.004464285847331796, + "best1L": 0.0029761905648878644, + "mean": 0.0014880952491824115, + "persist": 0.0014880952824439322 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.014437738039171498, + "dispersion_mean": 0.0069598405177599505, + "dispersion_slope": -0.0011255661991958646, + "stable_rank": 279.68243408203125, + "top8_spectral_share": 0.0277714803814888, + "condition_number": 5.672665119171143, + "unembed_alignment": 0.06260673701763153 + }, + "corpus": { + "mean_prompt_tokens": 856.155, + "type_token_ratio": 0.07197294882351911, + "eval_vocab_overlap": 0.40458015267175573, + "model_cross_entropy": 14.2659513112661, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.014437738039171498, + "10": 0.012943968833245986, + "11": 0.011502950107930275, + "12": 0.01014204013111181, + "13": 0.008868055281275608, + "14": 0.0076654563807423476, + "15": 0.006517881655111688, + "16": 0.0054333247803166045, + "17": 0.004421017830841475, + "18": 0.003469084171445883, + "19": 0.0025619663384390634, + "20": 0.0016844254816143595, + "21": 0.0008300176996327401 + } + } + }, + "runtime_s": 430.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_rand_USPTO_Backgrounds_s2.json b/results/e61/tv_rand_USPTO_Backgrounds_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..0b39c512f859821691a3e3d8d0de88db34ccbc48 --- /dev/null +++ b/results/e61/tv_rand_USPTO_Backgrounds_s2.json @@ -0,0 +1,118 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "USPTO_Backgrounds", + "seed": 2, + "randomization": { + "seed": 61002, + "initialiser": "hf._init_weights (the model's own), initializer_range=0.02", + "n_blocks_reinitialised": 24, + "n_modules_uninitialised_flag_cleared": 288, + "block_param_fro_before": 1178.9830322265625, + "block_param_fro_after": 412.2533874511719, + "blocks_actually_changed": true, + "embedding_unchanged": true, + "unembedding_unchanged": true, + "final_norm_held_real": true, + "why_readout_is_held_fixed": "the read is unembed(norm(J h)); E38 shows W_U's concept-token geometry is what fails below 410M, so randomising it would answer a different question. This bounds 'no computation', not 'no model'." + }, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.0, + "best1L": 0.0, + "mean": 0.0, + "persist": 0.0 + }, + "multilingual": { + "min": 0.000725163146853447, + "best1L": 0.000725163146853447, + "mean": 0.00011156356361295496, + "persist": 0.0 + }, + "order-ops": { + "min": 0.054545454148735316, + "best1L": 0.048051949058260233, + "mean": 0.032967033330351114, + "persist": 0.03506493488592761 + }, + "poetry": { + "min": 0.007288629721317973, + "best1L": 0.007288629721317973, + "mean": 0.0022426552937499116, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.013392857541995389, + "best1L": 0.013392857541995389, + "mean": 0.004922161311177271, + "persist": 0.005952380996729646 + }, + "association": { + "min": 0.012422360346785613, + "best1L": 0.00776397529989481, + "mean": 0.0053750597788686195, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.013426118989142387, + "dispersion_mean": 0.006448293942088694, + "dispersion_slope": -0.0010442920813691443, + "stable_rank": 277.9263916015625, + "top8_spectral_share": 0.027701538056135178, + "condition_number": 5.651653289794922, + "unembed_alignment": 0.06108715385198593 + }, + "corpus": { + "mean_prompt_tokens": 724.32, + "type_token_ratio": 0.07828031809145129, + "eval_vocab_overlap": 0.4015267175572519, + "model_cross_entropy": 11.847573323292774, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.013426118989142387, + "10": 0.011976618485363938, + "11": 0.01063991960693444, + "12": 0.009383892084008194, + "13": 0.00820025301667415, + "14": 0.007087045948599108, + "15": 0.006056918509489898, + "16": 0.005067386806507418, + "17": 0.004126249311589419, + "18": 0.0032211866096750925, + "19": 0.0023620155917656355, + "20": 0.0015352511359947182, + "21": 0.0007449651514086375 + } + } + }, + "runtime_s": 429.0, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61/tv_real_USPTO_Backgrounds_s0.json b/results/e61/tv_real_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..2ff8430c7b9cdf2db1f46cb967548549e6da8f18 --- /dev/null +++ b/results/e61/tv_real_USPTO_Backgrounds_s0.json @@ -0,0 +1,106 @@ +{ + "model": "EleutherAI/pythia-410m-deduped", + "short": "410m", + "corpus": "USPTO_Backgrounds", + "seed": 0, + "randomization": null, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 24, + "ckpts": [ + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "200": { + "reads": { + "multihop": { + "min": 0.11650485305913857, + "best1L": 0.08183078920202595, + "mean": 0.030086418081607138, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829540678433, + "best1L": 0.11094996625823635, + "mean": 0.039883973609123914, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.12987013161182404, + "best1L": 0.101298702614648, + "mean": 0.0257742257256593, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20089285767504148, + "mean": 0.16346154042652675, + "persist": 0.17857142950275115 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.007763975166848728, + "mean": 0.0025083612085187008, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5678085150330651, + "dispersion_mean": 0.2671157123523049, + "dispersion_slope": -0.04671915013848609, + "stable_rank": 19.01607894897461, + "top8_spectral_share": 0.14552031457424164, + "condition_number": 1900.9591064453125, + "unembed_alignment": 0.1512080281972885 + }, + "corpus": { + "mean_prompt_tokens": 900.0, + "type_token_ratio": 0.0683, + "eval_vocab_overlap": 0.4083969465648855, + "model_cross_entropy": 2.3053268269375637, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "9": 0.5678085150330651, + "10": 0.5206723901105078, + "11": 0.4734474842240867, + "12": 0.3881995247717475, + "13": 0.34807402743528904, + "14": 0.3002357684057481, + "15": 0.2606290120402746, + "16": 0.19081561122698523, + "17": 0.15024058486034647, + "18": 0.11673163798141308, + "19": 0.08192126453996937, + "20": 0.04944734272060015, + "21": 0.024281097229930097 + } + } + }, + "runtime_s": 419.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/e61_randomized_null_410m.json b/results/e61_randomized_null_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..8991efc9489a5bf76a2d0e658614478e2e8f6e7b --- /dev/null +++ b/results/e61_randomized_null_410m.json @@ -0,0 +1,279 @@ +{ + "experiment": "E61 \u2014 the randomized-network null (dead salmon)", + "prereg": "docs/experiments/preregs/E61_randomized_network_null.md", + "status": "PRE-REGISTERED", + "reference": { + "real_model_logit_persist": 0.028439503419212996, + "real_model_logit_min": 0.1247396256774664, + "source": "results/e48_crossover_410m.json arms_admitted_mean.logit_I" + }, + "n_randomized_cells": 9, + "n_real_cells": 1, + "C1_competence_destroyed": { + "ce_by_cell": { + "Github|s0": 13.58647276044966, + "Github|s1": 13.83418978441943, + "Github|s2": 12.201192804284998, + "Pile-CC|s0": 13.512615581890485, + "Pile-CC|s1": 14.291111885964334, + "Pile-CC|s2": 12.324972204259923, + "USPTO_Backgrounds|s0": 13.49804958996472, + "USPTO_Backgrounds|s1": 14.2659513112661, + "USPTO_Backgrounds|s2": 11.847573323292774 + }, + "gate_nats": 8.0, + "min_ce": 11.847573323292774, + "fires": true, + "note": "a randomly initialised network is CONFIDENTLY WRONG, not uniform, so CE sits well above the uniform bound ln(50277)=10.83. The registered gate is >8.0 and is met as written." + }, + "C2_operator_well_formed": { + "n_bad": 0, + "bad": [], + "fires": true + }, + "C3_real_arm_reproduces_e48": { + "per_cell": { + "USPTO_Backgrounds|s0": { + "trainval_persist": 0.04964359144547156, + "e48_persist": 0.04964358932338655, + "N": 200, + "abs_diff": 2.1220850146552728e-09 + } + }, + "tolerance": 0.001, + "max_abs_diff": 2.1220850146552728e-09, + "fires": true + }, + "randomized_by_corpus": { + "USPTO_Backgrounds": { + "persist_per_seed": [ + 0.0019386753705995423, + 0.0008489044915352549, + 0.008495008360062327 + ], + "persist_mean": 0.003760862740732375, + "persist_sd": 0.004135940183438328, + "min_per_seed": [ + 0.017595406462039265, + 0.005000942466514451, + 0.015190420911780425 + ], + "min_mean": 0.012595589946778047, + "persist_vs_logit": -0.02467864067848062, + "persist_below_logit": true, + "min_vs_logit": -0.11214403573068835, + "min_below_logit": true + }, + "Github": { + "persist_per_seed": [ + 0.0023611004863466536, + 0.0005573593080043793, + 0.008944805152714253 + ], + "persist_mean": 0.003954421649021762, + "persist_sd": 0.004414897076383152, + "min_per_seed": [ + 0.016718463147325176, + 0.004685671121946403, + 0.01823478750884533 + ], + "min_mean": 0.013212973926038969, + "persist_vs_logit": -0.024485081770191233, + "persist_below_logit": true, + "min_vs_logit": -0.11152665175142742, + "min_below_logit": true + }, + "Pile-CC": { + "persist_per_seed": [ + 0.001678935092474733, + 0.0008489044915352549, + 0.00789977027369397 + ], + "persist_mean": 0.003475869952567986, + "persist_sd": 0.00385362274205298, + "min_per_seed": [ + 0.01418347850974117, + 0.005534575613481658, + 0.016520926915109157 + ], + "min_mean": 0.012079660346110662, + "persist_vs_logit": -0.02496363346664501, + "persist_below_logit": true, + "min_vs_logit": -0.11265996533135574, + "min_below_logit": true + } + }, + "n_corpora_below_the_free_baseline": "3/3", + "VERDICT": "FLOOR IS LOW \u2014 a J fitted on a weight-randomized model reads BELOW the unfitted logit lens on 3/3 corpora (closest approach -0.02449 against the 0.02844 free baseline). The readout reflects the model's computation, and the measurement has a floor where it should.", + "provenance": { + "schema_version": 1, + "experiment": "E61", + "utc": "2026-08-31T23:20:33+00:00", + "script": { + "path": "experiments/t61_adjudicate_null.py", + "sha256": "34e2aebaf4b95be65c94fc22bdcd788d5ebbb0031db48fbd2f783b88c501e6ca", + "n_lines": 159 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/t61_adjudicate_null.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e61/tv_rand_Github_s0.json", + "sha256": "29de5f70aba42dfc8aaa25956bee6d7c2319362b74597dcee98d2b218cc5aef6", + "bytes": 3080, + "exists": true + }, + { + "path": "results/e61/tv_rand_Github_s1.json", + "sha256": "d17ab3796ca4433a835700c245fa1d4d1a3c2ae98db32940aa571bf4efd33450", + "bytes": 3071, + "exists": true + }, + { + "path": "results/e61/tv_rand_Github_s2.json", + "sha256": "431ba0dbcf29d7500958373336b15ac6091a34c4d11fc16bc1d09f0e9c6e98f3", + "bytes": 3085, + "exists": true + }, + { + "path": "results/e61/tv_rand_Pile-CC_s0.json", + "sha256": "57c89ddcbe5c627993166c551831a30aa5175afe47839e4a391e9e53a257c0e4", + "bytes": 3087, + "exists": true + }, + { + "path": "results/e61/tv_rand_Pile-CC_s1.json", + "sha256": "7c8c11d76084c0bc2b8bc4875799e9823231c83056861562292bbc46b7bf4644", + "bytes": 3087, + "exists": true + }, + { + "path": "results/e61/tv_rand_Pile-CC_s2.json", + "sha256": "73f55c258710e59755131cf9951a417e857ece005406f9302baeaa45115322d2", + "bytes": 3098, + "exists": true + }, + { + "path": "results/e61/tv_rand_USPTO_Backgrounds_s0.json", + "sha256": "86a987841164ef54c21ba9e785f59f1fe0beba4f4cb2858b56fe505581c5f95c", + "bytes": 3083, + "exists": true + }, + { + "path": "results/e61/tv_rand_USPTO_Backgrounds_s1.json", + "sha256": "c8d0093b3353fe6938c4692c50827e267163a06c7f7f435d47b5459364ab9708", + "bytes": 3104, + "exists": true + }, + { + "path": "results/e61/tv_rand_USPTO_Backgrounds_s2.json", + "sha256": "5707062bd86530ec529048d6f278389ec9d0df290bef668291a409dec7213a67", + "bytes": 3113, + "exists": true + }, + { + "path": "results/e61/tv_real_USPTO_Backgrounds_s0.json", + "sha256": "2710222512a96a1d547913fc5705617ecbe54544d07ccde3b4f7b155240ff656", + "bytes": 2492, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "b43b004aebca91592db4cb23927478f5b1ca4abb810db0c033437e1fd98e6f66" + } +} \ No newline at end of file diff --git a/results/e62_band_adjudication.json b/results/e62_band_adjudication.json new file mode 100644 index 0000000000000000000000000000000000000000..44441c0136bb86d48a9047b46b2db39df472c273 --- /dev/null +++ b/results/e62_band_adjudication.json @@ -0,0 +1,343 @@ +{ + "experiment": "E62 \u2014 the 1B ladder on the declared band [6,13]", + "prereg": "specs/experiments/E62_ladder1b_corrected_band.md", + "status": "PRE-REGISTERED", + "n_cells_new": 15, + "n_cells_old": 15, + "C2_band": { + "expected": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_cells": 15, + "n_matching": 15, + "fires": true + }, + "C3_prompt_pool": { + "per_cell": { + "Github|s0": { + "new": 200, + "old": 200, + "same": true + }, + "Github|s1": { + "new": 200, + "old": 200, + "same": true + }, + "Github|s2": { + "new": 200, + "old": 200, + "same": true + }, + "Pile-CC|s0": { + "new": 200, + "old": 200, + "same": true + }, + "Pile-CC|s1": { + "new": 200, + "old": 200, + "same": true + }, + "Pile-CC|s2": { + "new": 200, + "old": 200, + "same": true + }, + "StackExchange|s0": { + "new": 200, + "old": 200, + "same": true + }, + "StackExchange|s1": { + "new": 200, + "old": 200, + "same": true + }, + "StackExchange|s2": { + "new": 200, + "old": 200, + "same": true + }, + "USPTO_Backgrounds|s0": { + "new": 200, + "old": 200, + "same": true + }, + "USPTO_Backgrounds|s1": { + "new": 200, + "old": 200, + "same": true + }, + "USPTO_Backgrounds|s2": { + "new": 200, + "old": 200, + "same": true + }, + "Wikipedia_en|s0": { + "new": 200, + "old": 200, + "same": true + }, + "Wikipedia_en|s1": { + "new": 200, + "old": 200, + "same": true + }, + "Wikipedia_en|s2": { + "new": 200, + "old": 200, + "same": true + } + }, + "fires": true + }, + "C1_shared_layer_dispersion": { + "layers": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_comparisons": 840, + "max_abs_diff": 3.6828356636209314e-05, + "worst_at": "Github|s0|N=10|L6", + "tolerance": 0.001, + "fires": true, + "why": "J is fitted per layer, so layers 6-13 cannot depend on whether layer 5 was also fitted. A failure means something other than the band changed." + }, + "asymptotes": { + "band_6_13": { + "Github": { + "mean": 0.025347685576638298, + "seed_sd": 0.0013861574903877498 + }, + "Wikipedia_en": { + "mean": 0.02517063217902822, + "seed_sd": 0.0002535570715574809 + }, + "StackExchange": { + "mean": 0.05391584097274712, + "seed_sd": 0.0009316870858664217 + }, + "Pile-CC": { + "mean": 0.04218367406699274, + "seed_sd": 0.000526333742266378 + }, + "USPTO_Backgrounds": { + "mean": 0.050449074909002294, + "seed_sd": 0.001534369357850337 + } + }, + "band_5_13_stored": { + "Github": { + "mean": 0.027256395461569938, + "seed_sd": 0.0013762403288060878 + }, + "Wikipedia_en": { + "mean": 0.02801095717586577, + "seed_sd": 0.00020285804501715256 + }, + "StackExchange": { + "mean": 0.05875215052316586, + "seed_sd": 0.0007357119491078887 + }, + "Pile-CC": { + "mean": 0.046013945933165296, + "seed_sd": 0.00039438237251602294 + }, + "USPTO_Backgrounds": { + "mean": 0.05470160290465824, + "seed_sd": 0.0017248115034397009 + } + }, + "delta": { + "Github": -0.0019087098849316399, + "Wikipedia_en": -0.0028403249968375482, + "StackExchange": -0.004836309550418742, + "Pile-CC": -0.0038302718661725535, + "USPTO_Backgrounds": -0.004252527995655946 + }, + "order_6_13": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "order_5_13": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "ordering_unchanged": false + }, + "PRIMARY": { + "band_6_13": { + "frac_interaction": 9.191520642250026, + "frac_corpus_main": 2.2424534939728105, + "frac_set_main": 88.50952415142129, + "interaction_over_corpus_main": 4.098867899358752, + "units": "percent, converted from t51's fractions" + }, + "band_5_13_stored": { + "frac_interaction": 9.158325214755475, + "frac_corpus_main": 2.2117202251311614 + }, + "delta_pp": 0.033195427494550955 + }, + "VERDICT": "CONFIRMS \u2014 interaction 9.192% on [6,13] vs 9.158% on [5,13] (+0.033 pp), still above the 2.242% corpus main effect. The band defect was immaterial; report the corrected number and retire the deviation.", + "provenance": { + "schema_version": 1, + "experiment": "E62", + "utc": "2026-08-17T17:30:37+00:00", + "script": { + "path": "experiments/t62_adjudicate_band.py", + "sha256": "f4a10aff036c8ee359bd4b581355178f75761cdfac2a2d65b1557bf7ad9cd748", + "n_lines": 186 + }, + "git": { + "commit": "529c025b7f5826b65cc019ad6ba8b7b12425d1b9", + "commit_short": "529c025b7f58", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "paper/paper.tex", + "repro/14_vast_pull_and_verify.sh", + "results/e59_read_dose_410m.json", + "results/e61/", + "results/e62_band_adjudication.json", + "results/e62_interaction_b613.json", + "results/ladder1b_b613/", + "specs/experiments/E59_read_dose_response.md", + "xperiments/t62_adjudicate_band.py" + ], + "dirty_diff_sha256": "a18f674eff0b6f9d465610ad2cfced1f2f4ea3fc13ba79642d0822d2c1a6461e", + "describe": "529c025-dirty" + }, + "argv": [ + "experiments/t62_adjudicate_band.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder1b_b613/tv_Github_s0.json", + "sha256": "d7cb38364bd55281dfd141256924444f0f6528381a2519fd27911b741b45577b", + "bytes": 14052, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Github_s1.json", + "sha256": "a88461c788cb37d62ad448803047a274943e31ccaccd3c02437e76039bb686ec", + "bytes": 14032, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Github_s2.json", + "sha256": "ce7908dd1bb119a24d6deae2abb60800f1bb4b97956a220d61dc15f64e67ec6d", + "bytes": 14074, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Pile-CC_s0.json", + "sha256": "8d30de01f9cc5fc7dba2994710198faf6a4ec959dc355686b020db43189599f8", + "bytes": 14023, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Pile-CC_s1.json", + "sha256": "92d720e2c333aac8c99c03a6918ba6854dbe55ce99e536f096e7268d983a127d", + "bytes": 14037, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Pile-CC_s2.json", + "sha256": "0cd4ab100c77f936a883e3556014d8ffa28bd5995ae961f0feeb59040bc2e117", + "bytes": 14092, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_StackExchange_s0.json", + "sha256": "088ea6d343decfcb2d3aa07fe7178f99e6f01681a41d6932d42142bf507ad0b7", + "bytes": 14197, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_StackExchange_s1.json", + "sha256": "f9ecf5e3b6908a01ad4e2f9b2b2021e0b7fbbad66ccd00a1c42d01ec0354506f", + "bytes": 14149, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_StackExchange_s2.json", + "sha256": "a6a603770a0fd601de7be319d528b05e20e4d4323882f31be79ffc70882588a1", + "bytes": 14179, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_USPTO_Backgrounds_s0.json", + "sha256": "55ace8cf28534827414685424b1661b0d6b673f375a656c15e3e094fd56995f6", + "bytes": 14171, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_USPTO_Backgrounds_s1.json", + "sha256": "b5cb9f7de464affae1b00cad82803a6f6beb3b74ab43a9c2e1e922e74dd27c5d", + "bytes": 14193, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_USPTO_Backgrounds_s2.json", + "sha256": "98026a284ade37f4d5d95873a907c7d1ada0ab825dc506c68e537ed8cdc2343a", + "bytes": 14185, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Wikipedia_en_s0.json", + "sha256": "5bf27fde005a80bd2ab33d5b369f5e90f3fcc1e0b0a54278ceeb5b37902313cb", + "bytes": 14187, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Wikipedia_en_s1.json", + "sha256": "4154b0dabcbcf3d58ab2e2a527a87cf5c9610ab32f5e7c96a84e0010b588fd27", + "bytes": 14261, + "exists": true + }, + { + "path": "results/ladder1b_b613/tv_Wikipedia_en_s2.json", + "sha256": "26c635061f0a43268a50b1fe374b709d9c628913471ce0d3ad9eb9bab6e77f65", + "bytes": 14257, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "5c112697381b3b5e2a3a0e2783df69286e84b06a963cd1707ebf04b027894eeb" + } +} \ No newline at end of file diff --git a/results/e62_interaction_b613.json b/results/e62_interaction_b613.json new file mode 100644 index 0000000000000000000000000000000000000000..7b52d58a12f39a7d2267989cb166d221bb88de50 --- /dev/null +++ b/results/e62_interaction_b613.json @@ -0,0 +1,723 @@ +{ + "experiment": "E51 \u2014 the corpus x concept-set interaction decomposition, STORED", + "why": "the programme's headline interaction numbers (6.8% / 1.9% / 4614x at 410M, 9.2% / 2.2% at 1B) appear in six documents including the paper abstract and in NO results JSON; no committed script computed them. This stores the decomposition and reports whether it reproduces them.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json" + ], + "aggregation": "persist", + "n_min": 75, + "decomposition": "balanced two-factor (eval set, corpus) with seed as the replication unit, after averaging over the NESTED N axis (SPINE A1.1)", + "scales": { + "410m": { + "admitted5": { + "grand_mean": 0.041563127900002834, + "SS_total": 0.22925593623758608, + "frac_set_main": 0.9127829399393488, + "frac_corpus_main": 0.01906450114831279, + "frac_interaction": 0.06780626428099203, + "frac_residual_seed": 0.00034629463134648467, + "SS_set": 0.2092609074774917, + "SS_corpus": 0.004370650059658984, + "SS_interaction": 0.015544988600512018, + "SS_residual_seed": 7.939009992338807e-05, + "interaction_over_noise_floor": 195.8051270311163, + "candidate_noise_floors": { + "seed_x_set": 2768.1445461585845, + "seed_x_corpus": 2547.878926111827, + "seed_main": 10268.19304676178, + "three_way_residual": 234.96272012579493, + "full_two_factor_residual": 195.8051270311163, + "mean_squares_F": 611.8910219722385 + }, + "SS_seed_main": 1.5138971900625057e-06, + "SS_seed_x_set": 5.615670837017576e-06, + "SS_seed_x_corpus": 6.10114885805596e-06, + "SS_three_way_residual": 6.615938303825178e-05, + "interaction_over_corpus_main": 3.5566765557352586, + "per_corpus_main": { + "Github": -0.01347907410647357, + "Pile-CC": 0.004152620384147217, + "StackExchange": 0.004112475200761065, + "USPTO_Backgrounds": 0.00817213694672702, + "Wikipedia_en": -0.0029581584251617415 + }, + "per_set_main": { + "multihop": -0.026766659081874818, + "multilingual": -0.007186749365197628, + "order-ops": -0.03100104450900741, + "poetry": -0.038628393975836214, + "typo": 0.10358284693191608 + }, + "interaction_cells": { + "Github|multihop": 0.0023481492099414816, + "Github|multilingual": 0.007358160156824216, + "Github|order-ops": 0.02190276691766114, + "Github|poetry": 0.014917517935270082, + "Github|typo": -0.046526594219696946, + "Pile-CC|multihop": -0.0020486935037057966, + "Pile-CC|multilingual": -0.002015691274651746, + "Pile-CC|order-ops": -0.009327498430404486, + "Pile-CC|poetry": -0.005629628390659461, + "Pile-CC|typo": 0.01902151159942149, + "StackExchange|multihop": -0.005006295451394596, + "StackExchange|multilingual": 0.0005318301469256631, + "StackExchange|order-ops": -0.012138998990460316, + "StackExchange|poetry": -0.002743446891852852, + "StackExchange|typo": 0.0193569111867821, + "USPTO_Backgrounds|multihop": -0.003962081805792314, + "USPTO_Backgrounds|multilingual": -0.0031076970647595306, + "USPTO_Backgrounds|order-ops": -0.010028111813667362, + "USPTO_Backgrounds|poetry": -0.007273591605950647, + "USPTO_Backgrounds|typo": 0.024371482290169853, + "Wikipedia_en|multihop": 0.008668921550951246, + "Wikipedia_en|multilingual": -0.0027666019643385886, + "Wikipedia_en|order-ops": 0.009591842316871033, + "Wikipedia_en|poetry": 0.0007291489531928777, + "Wikipedia_en|typo": -0.016223310856676575 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06780626428099203, + "null2_within_cell_mean": 0.023461171497061152, + "null2_max": 0.058449129594287226, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.06780626428099203, + "fires": null + }, + "C4_F_test": { + "F": 611.8910219722385, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03464607279448547, + "SS_total": 0.25078694938130414, + "frac_set_main": 0.9202689818852297, + "frac_corpus_main": 0.014505336726722001, + "frac_interaction": 0.06490812752157134, + "frac_residual_seed": 0.00031755386647712333, + "SS_set": 0.23079145057723538, + "SS_corpus": 0.003637749147443202, + "SS_interaction": 0.016278111291187544, + "SS_residual_seed": 7.963836543803573e-05, + "interaction_over_noise_floor": 204.40036911421873, + "candidate_noise_floors": { + "seed_x_set": 2664.38297317969, + "seed_x_corpus": 3357.1772819678595, + "seed_main": 15217.493238347859, + "three_way_residual": 240.763440450699, + "full_two_factor_residual": 204.40036911421873, + "mean_squares_F": 613.2011073426562 + }, + "SS_seed_main": 1.0696972908893393e-06, + "SS_seed_x_set": 6.109523839120302e-06, + "SS_seed_x_corpus": 4.848749387951859e-06, + "SS_three_way_residual": 6.761039492007427e-05, + "interaction_over_corpus_main": 4.474775645986651, + "per_corpus_main": { + "Github": -0.011242694633211085, + "Pile-CC": 0.003450384108972908, + "StackExchange": 0.003416929789484449, + "USPTO_Backgrounds": 0.006799981244456077, + "Wikipedia_en": -0.0024246005097023557 + }, + "per_set_main": { + "multihop": -0.019849603976357454, + "multilingual": -0.0002696942596802643, + "order-ops": -0.024083989403490047, + "poetry": -0.03171133887031885, + "typo": 0.11049990203743343, + "association": -0.03458527552758682 + }, + "interaction_cells": { + "Github|multihop": 0.0001117697366790002, + "Github|multilingual": 0.005121780683561734, + "Github|order-ops": 0.01966638744439865, + "Github|poetry": 0.0126811384620076, + "Github|typo": -0.04876297369295941, + "Github|association": 0.011181897366312435, + "Pile-CC|multihop": -0.0013464572285314877, + "Pile-CC|multilingual": -0.001313454999477437, + "Pile-CC|order-ops": -0.008625262155230178, + "Pile-CC|poetry": -0.004927392115485152, + "Pile-CC|typo": 0.019723747874595812, + "Pile-CC|association": -0.003511181375871558, + "StackExchange|multihop": -0.0043107500401179805, + "StackExchange|multilingual": 0.0012273755582022788, + "StackExchange|order-ops": -0.011443453579183704, + "StackExchange|poetry": -0.0020479014805762363, + "StackExchange|typo": 0.02005245659805873, + "StackExchange|association": -0.0034777270563830995, + "USPTO_Backgrounds|multihop": -0.0025899261035213714, + "USPTO_Backgrounds|multilingual": -0.001735541362488588, + "USPTO_Backgrounds|order-ops": -0.00865595611139642, + "USPTO_Backgrounds|poetry": -0.005901435903679704, + "USPTO_Backgrounds|typo": 0.025743637992440817, + "USPTO_Backgrounds|association": -0.006860778511354727, + "Wikipedia_en|multihop": 0.00813536363549186, + "Wikipedia_en|multilingual": -0.0033001598797979745, + "Wikipedia_en|order-ops": 0.009058284401411647, + "Wikipedia_en|poetry": 0.00019559103773349185, + "Wikipedia_en|typo": -0.016756868772135947, + "Wikipedia_en|association": 0.002667789577296943 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06490812752157134, + "null2_within_cell_mean": 0.022186420523225118, + "null2_max": 0.062092652683997974, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.06490812752157134, + "fires": null + }, + "C4_F_test": { + "F": 613.2011073426562, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + }, + "1b": { + "admitted5": { + "grand_mean": 0.03941338154088173, + "SS_total": 0.49531601839346706, + "frac_set_main": 0.8850952415142128, + "frac_corpus_main": 0.022424534939728104, + "frac_interaction": 0.09191520642250027, + "frac_residual_seed": 0.0005650171235587881, + "SS_set": 0.438401850925824, + "SS_corpus": 0.011107231360671311, + "SS_interaction": 0.04552707407500647, + "SS_residual_seed": 0.00027986203196526856, + "interaction_over_noise_floor": 162.67685100155518, + "candidate_noise_floors": { + "seed_x_set": 894.3447067827716, + "seed_x_corpus": 1032.5409304340064, + "seed_main": 4231.71458790936, + "three_way_residual": 261.4909879350379, + "full_two_factor_residual": 162.67685100155518, + "mean_squares_F": 508.3651593798599 + }, + "SS_seed_main": 1.0758540806387112e-05, + "SS_seed_x_set": 5.090551073845019e-05, + "SS_seed_x_corpus": 4.409227056584588e-05, + "SS_three_way_residual": 0.00017410570985458492, + "interaction_over_corpus_main": 4.098867899358752, + "per_corpus_main": { + "Github": -0.014065695964243438, + "Pile-CC": 0.0027702925261110095, + "StackExchange": 0.014502459431865386, + "USPTO_Backgrounds": 0.011035693368120568, + "Wikipedia_en": -0.014242749361853511 + }, + "per_set_main": { + "multihop": -0.03712489330875022, + "multilingual": -0.0372137199843391, + "order-ops": -0.03941338154088173, + "poetry": -0.03914613178931176, + "typo": 0.15289812662328284 + }, + "interaction_cells": { + "Github|multihop": 0.014551132861968304, + "Github|multilingual": 0.015914862033068418, + "Github|order-ops": 0.014065695964243438, + "Github|poetry": 0.013798446212673468, + "Github|typo": -0.05833013707195365, + "Pile-CC|multihop": -0.0049432005444985105, + "Pile-CC|multilingual": -0.0033685521333522775, + "Pile-CC|order-ops": -0.0027702925261110095, + "Pile-CC|poetry": -0.0030375422776809796, + "Pile-CC|typo": 0.01411958748164275, + "StackExchange|multihop": -0.01401702253414052, + "StackExchange|multilingual": -0.0144057710233721, + "StackExchange|order-ops": -0.014502459431865386, + "StackExchange|poetry": -0.014769709183435356, + "StackExchange|typo": 0.05769496217281337, + "USPTO_Backgrounds|multihop": -0.00916329390546751, + "USPTO_Backgrounds|multilingual": -0.012298685859977496, + "USPTO_Backgrounds|order-ops": -0.011035693368120568, + "USPTO_Backgrounds|poetry": -0.011302943119690538, + "USPTO_Backgrounds|typo": 0.04380061625325606, + "Wikipedia_en|multihop": 0.013572384122138223, + "Wikipedia_en|multilingual": 0.014158146983633435, + "Wikipedia_en|order-ops": 0.014242749361853511, + "Wikipedia_en|poetry": 0.015311748368133385, + "Wikipedia_en|typo": -0.057285028835758575 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.09191520642250027, + "null2_within_cell_mean": 0.030055321115558573, + "null2_max": 0.08513803132065896, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.09191520642250027, + "fires": null + }, + "C4_F_test": { + "F": 508.3651593798599, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.033431096081917605, + "SS_total": 0.5114854419963268, + "frac_set_main": 0.8886007231066569, + "frac_corpus_main": 0.0186506603267192, + "frac_interaction": 0.09218555119741657, + "frac_residual_seed": 0.0005630653692070697, + "SS_set": 0.4545063336164641, + "SS_corpus": 0.009539541240735328, + "SS_interaction": 0.04715156739988563, + "SS_residual_seed": 0.000287999739241703, + "interaction_over_noise_floor": 163.72086837312656, + "candidate_noise_floors": { + "seed_x_set": 899.9096118925797, + "seed_x_corpus": 1539.9638455679672, + "seed_main": 5076.473725476223, + "three_way_residual": 240.94172859920158, + "full_two_factor_residual": 163.72086837312656, + "mean_squares_F": 491.1626051193797 + }, + "SS_seed_main": 9.288252032755739e-06, + "SS_seed_x_set": 5.239589262828544e-05, + "SS_seed_x_corpus": 3.061861973941035e-05, + "SS_three_way_residual": 0.00019569697484125162, + "interaction_over_corpus_main": 4.942749992897047, + "per_corpus_main": { + "Github": -0.012308024768052359, + "Pile-CC": 0.002498363161326518, + "StackExchange": 0.011994803150642719, + "USPTO_Backgrounds": 0.009537163391960632, + "Wikipedia_en": -0.011722304935877524 + }, + "per_set_main": { + "multihop": -0.03114260784978609, + "multilingual": -0.031231434525374975, + "order-ops": -0.033431096081917605, + "poetry": -0.033163846330347635, + "typo": 0.15888041208224696, + "association": -0.02991142729482067 + }, + "interaction_cells": { + "Github|multihop": 0.012793461665777224, + "Github|multilingual": 0.014157190836877345, + "Github|order-ops": 0.012308024768052359, + "Github|poetry": 0.012040775016482388, + "Github|typo": -0.06008780826814472, + "Github|association": 0.008788355980955426, + "Pile-CC|multihop": -0.004671271179714019, + "Pile-CC|multilingual": -0.003096622768567786, + "Pile-CC|order-ops": -0.002498363161326518, + "Pile-CC|poetry": -0.002765612912896488, + "Pile-CC|typo": 0.014391516846427248, + "Pile-CC|association": -0.0013596468239224363, + "StackExchange|multihop": -0.011509366252917853, + "StackExchange|multilingual": -0.011898114742149433, + "StackExchange|order-ops": -0.011994803150642719, + "StackExchange|poetry": -0.012262052902212689, + "StackExchange|typo": 0.060202618454036044, + "StackExchange|association": -0.012538281406113308, + "USPTO_Backgrounds|multihop": -0.007664763929307573, + "USPTO_Backgrounds|multilingual": -0.01080015588381756, + "USPTO_Backgrounds|order-ops": -0.009537163391960632, + "USPTO_Backgrounds|poetry": -0.009804413143530602, + "USPTO_Backgrounds|typo": 0.04529914622941601, + "USPTO_Backgrounds|association": -0.007492649880799618, + "Wikipedia_en|multihop": 0.011051939696162236, + "Wikipedia_en|multilingual": 0.011637702557657448, + "Wikipedia_en|order-ops": 0.011722304935877524, + "Wikipedia_en|poetry": 0.012791303942157398, + "Wikipedia_en|typo": -0.059805473261734556, + "Wikipedia_en|association": 0.012602222129879964 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999998, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.09218555119741657, + "null2_within_cell_mean": 0.030983081160733592, + "null2_max": 0.08552296380183376, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.09218555119741657, + "fires": null + }, + "C4_F_test": { + "F": 491.1626051193797, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + } + }, + "ladder1b_dir": "results/ladder1b_b613", + "reproduces_the_prose": { + "410m": { + "claimed": { + "interaction": 6.8, + "corpus_main": 1.9, + "set_main": 91.3, + "noise_floor": 4614 + }, + "recomputed": { + "interaction": 6.780626428099203, + "corpus_main": 1.9064501148312791, + "set_main": 91.27829399393488, + "noise_floor": 195.8051270311163 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + }, + "1b": { + "claimed": { + "interaction": 9.2, + "corpus_main": 2.2, + "set_main": 88.6 + }, + "recomputed": { + "interaction": 9.191520642250026, + "corpus_main": 2.2424534939728105, + "set_main": 88.50952415142129, + "noise_floor": 162.67685100155518 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + } + }, + "noise_floor_4614_audit": { + "claimed": 4614, + "claimed_where": "PLAN.tex:52 and :270, HANDOFF.md:63 \u2014 described as 'the seed x set noise floor'. In no results file.", + "candidates_computed": { + "seed_x_set": 2768.1445461585845, + "seed_x_corpus": 2547.878926111827, + "seed_main": 10268.19304676178, + "three_way_residual": 234.96272012579493, + "full_two_factor_residual": 195.8051270311163, + "mean_squares_F": 611.8910219722385 + }, + "any_candidate_within_2pct": {}, + "reproduces": false, + "reading": "The six VARIANCE FRACTIONS reproduce exactly at both scales, so the decomposition specified in this file is the one that was used. The 4614x figure does NOT follow from it under any of six natural denominators. Cite a named, stored ratio from candidate_noise_floors instead, or drop the multiplier \u2014 do not restate 4614x." + }, + "VERDICT": "REPRODUCES \u2014 the headline interaction numbers now have a results file and match the prose to within 0.1 percentage points. || BUT the '4614x noise floor' does NOT reproduce under any natural denominator (nearest candidates: seed x set 2768x, seed x corpus 2548x, seed main 10268x, three-way residual 235x). The multiplier is unsupported and must not be restated until the operator names the intended denominator.", + "provenance": { + "schema_version": 1, + "experiment": "E51", + "utc": "2026-08-17T17:29:31+00:00", + "script": { + "path": "experiments/t51_interaction_variance.py", + "sha256": "849f3e2ab42163fc1e5c553f4b1100cec316d6cff89a8b20f0ef26fa008b542a", + "n_lines": 323 + }, + "git": { + "commit": "529c025b7f5826b65cc019ad6ba8b7b12425d1b9", + "commit_short": "529c025b7f58", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "aper/paper.tex", + "repro/14_vast_pull_and_verify.sh", + "results/e59_read_dose_410m.json", + "results/e61/", + "results/ladder1b_b613/", + "specs/experiments/E59_read_dose_response.md" + ], + "dirty_diff_sha256": "0dd437fb109aec18b4a567b53ad46a58c29c930b6f3bd48c436b1a261228354b", + "describe": "529c025-dirty" + }, + "argv": [ + "experiments/t51_interaction_variance.py", + "--ladder1b-dir", + "results/ladder1b_b613", + "--out", + "results/e62_interaction_b613.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "a6bab0d8687c9a48bc1041bd536b874b0915c1243224f1a968cd0c1f1664c400" + } +} \ No newline at end of file diff --git a/results/e64_instrument_cost.json b/results/e64_instrument_cost.json new file mode 100644 index 0000000000000000000000000000000000000000..a30184f7d75ae7b59ea26e77f155c1c95731232a --- /dev/null +++ b/results/e64_instrument_cost.json @@ -0,0 +1,323 @@ +{ + "experiment": "E64 \u2014 acquisition cost per checkpoint: transport operator vs sparse dictionary", + "question": "for a checkpoint series, where the instrument is refitted at every point, does a J-lens cost less than an SAE/crosscoder?", + "revision": "v2 2026-08-17 \u2014 v1 double-charged activation capture per layer; retracted", + "NOT_COMPARABLE_AS_OBJECTS": "a J-lens gives one transport map per source layer over the whole band in a single pass; an SAE gives a feature dictionary for ONE layer. This compares acquisition cost only, and says nothing about what either instrument tells you.", + "jlens_side": "MEASURED \u2014 repo cost table, converted at measured ~100% fp32 peak", + "sae_side": "MODELLED, and BRACKETED by one measured anchor (arXiv:2509.05291: ~6 A100-80GB h for a 2-way crosscoder at 1B, 400M tokens, F=16384). arXiv:2509.17196 reports token budgets (800M) but no hardware or GPU-hours.", + "assumptions": { + "activation_flops_per_token": "2 * P_model (forward only, base model frozen)", + "dictionary_flops_per_token": "6 * d_model * n_features (fwd+bwd+update), PER LAYER", + "activation_capture": "charged ONCE for the whole suite, not per layer (v2 fix)", + "n_tokens_swept": [ + 100000000.0, + 1000000000.0 + ], + "expansion_swept": [ + 8, + 32 + ], + "l40s_fp32_tflops": 91.6, + "utilisation": 1.0 + }, + "by_model": { + "410m": { + "n_layers": 24, + "d_model": 1024, + "n_params": 405012480, + "jlens_gpu_h_N200_L40S": 0.17, + "jlens_flops": 5.60592e+16, + "jlens_covers_n_source_layers": 22, + "sae": { + "100Mtok_x8": { + "n_features": 8192, + "flops_activation_capture_once": 8.1002496e+16, + "flops_dict_opt_one_layer": 5033164800000000.0, + "flops_all_layer_suite": 1.917321216e+17, + "capture_share_of_suite": 0.4224774405250205, + "gpu_h_suite_L40S": 0.5814292867540028, + "ratio_suite_over_jlens": 3.4201722750235466, + "RETRACTED_v1_ratio_per_layer_double_charged": 33.76403048206182 + }, + "100Mtok_x32": { + "n_features": 32768, + "flops_activation_capture_once": 8.1002496e+16, + "flops_dict_opt_one_layer": 2.01326592e+16, + "flops_all_layer_suite": 5.239209984e+17, + "capture_share_of_suite": 0.15460822575803063, + "gpu_h_suite_L40S": 1.5887948762736535, + "ratio_suite_over_jlens": 9.345852213374434, + "RETRACTED_v1_ratio_per_layer_double_charged": 39.68971042041271 + }, + "1000Mtok_x8": { + "n_features": 8192, + "flops_activation_capture_once": 8.1002496e+17, + "flops_dict_opt_one_layer": 5.0331648e+16, + "flops_all_layer_suite": 1.917321216e+18, + "capture_share_of_suite": 0.4224774405250205, + "gpu_h_suite_L40S": 5.814292867540029, + "ratio_suite_over_jlens": 34.20172275023547, + "RETRACTED_v1_ratio_per_layer_double_charged": 337.6403048206182 + }, + "1000Mtok_x32": { + "n_features": 32768, + "flops_activation_capture_once": 8.1002496e+17, + "flops_dict_opt_one_layer": 2.01326592e+17, + "flops_all_layer_suite": 5.239209984e+18, + "capture_share_of_suite": 0.15460822575803063, + "gpu_h_suite_L40S": 15.887948762736535, + "ratio_suite_over_jlens": 93.45852213374432, + "RETRACTED_v1_ratio_per_layer_double_charged": 396.89710420412706 + } + } + }, + "1b": { + "n_layers": 16, + "d_model": 2048, + "n_params": 1011351552, + "jlens_gpu_h_N200_L40S": 0.45, + "jlens_flops": 1.48392e+17, + "jlens_covers_n_source_layers": 14, + "sae": { + "100Mtok_x8": { + "n_features": 16384, + "flops_activation_capture_once": 2.022703104e+17, + "flops_dict_opt_one_layer": 2.01326592e+16, + "flops_all_layer_suite": 4.841275392e+17, + "capture_share_of_suite": 0.41780376868096164, + "gpu_h_suite_L40S": 1.4681208733624453, + "ratio_suite_over_jlens": 3.262490829694323, + "RETRACTED_v1_ratio_per_layer_double_charged": 20.982543360828075 + }, + "100Mtok_x32": { + "n_features": 65536, + "flops_activation_capture_once": 2.022703104e+17, + "flops_dict_opt_one_layer": 8.05306368e+16, + "flops_all_layer_suite": 1.3296992256e+18, + "capture_share_of_suite": 0.1521173409037142, + "gpu_h_suite_L40S": 4.032324192139738, + "ratio_suite_over_jlens": 8.960720426977195, + "RETRACTED_v1_ratio_per_layer_double_charged": 26.680772958110946 + }, + "1000Mtok_x8": { + "n_features": 16384, + "flops_activation_capture_once": 2.022703104e+18, + "flops_dict_opt_one_layer": 2.01326592e+17, + "flops_all_layer_suite": 4.841275392e+18, + "capture_share_of_suite": 0.41780376868096164, + "gpu_h_suite_L40S": 14.681208733624453, + "ratio_suite_over_jlens": 32.62490829694323, + "RETRACTED_v1_ratio_per_layer_double_charged": 209.82543360828075 + }, + "1000Mtok_x32": { + "n_features": 65536, + "flops_activation_capture_once": 2.022703104e+18, + "flops_dict_opt_one_layer": 8.05306368e+17, + "flops_all_layer_suite": 1.3296992256e+19, + "capture_share_of_suite": 0.1521173409037142, + "gpu_h_suite_L40S": 40.32324192139738, + "ratio_suite_over_jlens": 89.60720426977196, + "RETRACTED_v1_ratio_per_layer_double_charged": 266.80772958110947 + } + } + }, + "2.8b": { + "n_layers": 32, + "d_model": 2560, + "n_params": 2774138880, + "jlens_gpu_h_N200_L40S": 1.42, + "jlens_flops": 4.6825919999999994e+17, + "jlens_covers_n_source_layers": 30, + "sae": { + "100Mtok_x8": { + "n_features": 20480, + "flops_activation_capture_once": 5.54827776e+17, + "flops_dict_opt_one_layer": 3.145728e+16, + "flops_all_layer_suite": 1.498546176e+18, + "capture_share_of_suite": 0.3702440304382052, + "gpu_h_suite_L40S": 4.544354002911208, + "ratio_suite_over_jlens": 3.200249297824795, + "RETRACTED_v1_ratio_per_layer_double_charged": 37.56157205240175 + }, + "100Mtok_x32": { + "n_features": 81920, + "flops_activation_capture_once": 5.54827776e+17, + "flops_dict_opt_one_layer": 1.2582912e+17, + "flops_all_layer_suite": 4.329701376e+18, + "capture_share_of_suite": 0.12814458269003723, + "gpu_h_suite_L40S": 13.129856186317323, + "ratio_suite_over_jlens": 9.246377595998116, + "RETRACTED_v1_ratio_per_layer_double_charged": 43.60770035057507 + }, + "1000Mtok_x8": { + "n_features": 20480, + "flops_activation_capture_once": 5.54827776e+18, + "flops_dict_opt_one_layer": 3.145728e+17, + "flops_all_layer_suite": 1.498546176e+19, + "capture_share_of_suite": 0.3702440304382052, + "gpu_h_suite_L40S": 45.44354002911208, + "ratio_suite_over_jlens": 32.00249297824795, + "RETRACTED_v1_ratio_per_layer_double_charged": 375.6157205240175 + }, + "1000Mtok_x32": { + "n_features": 81920, + "flops_activation_capture_once": 5.54827776e+18, + "flops_dict_opt_one_layer": 1.2582912e+18, + "flops_all_layer_suite": 4.329701376e+19, + "capture_share_of_suite": 0.12814458269003723, + "gpu_h_suite_L40S": 131.29856186317323, + "ratio_suite_over_jlens": 92.46377595998115, + "RETRACTED_v1_ratio_per_layer_double_charged": 436.0770035057507 + } + } + }, + "6.9b": { + "n_layers": 32, + "d_model": 4096, + "n_params": 6854541312, + "jlens_gpu_h_N200_L40S": null, + "jlens_flops": null, + "jlens_covers_n_source_layers": 30, + "sae": { + "100Mtok_x8": { + "n_features": 32768, + "flops_activation_capture_once": 1.3709082624e+18, + "flops_dict_opt_one_layer": 8.05306368e+16, + "flops_all_layer_suite": 3.7868273664e+18, + "capture_share_of_suite": 0.36202026914769897, + "gpu_h_suite_L40S": 11.4835861426492, + "ratio_suite_over_jlens": null, + "RETRACTED_v1_ratio_per_layer_double_charged": null + }, + "100Mtok_x32": { + "n_features": 131072, + "flops_activation_capture_once": 1.3709082624e+18, + "flops_dict_opt_one_layer": 3.221225472e+17, + "flops_all_layer_suite": 1.10345846784e+19, + "capture_share_of_suite": 0.12423741376361253, + "gpu_h_suite_L40S": 33.46247173216885, + "ratio_suite_over_jlens": null, + "RETRACTED_v1_ratio_per_layer_double_charged": null + }, + "1000Mtok_x8": { + "n_features": 32768, + "flops_activation_capture_once": 1.3709082624e+19, + "flops_dict_opt_one_layer": 8.05306368e+17, + "flops_all_layer_suite": 3.7868273664e+19, + "capture_share_of_suite": 0.36202026914769897, + "gpu_h_suite_L40S": 114.835861426492, + "ratio_suite_over_jlens": null, + "RETRACTED_v1_ratio_per_layer_double_charged": null + }, + "1000Mtok_x32": { + "n_features": 131072, + "flops_activation_capture_once": 1.3709082624e+19, + "flops_dict_opt_one_layer": 3.221225472e+18, + "flops_all_layer_suite": 1.10345846784e+20, + "capture_share_of_suite": 0.12423741376361253, + "gpu_h_suite_L40S": 334.62471732168854, + "ratio_suite_over_jlens": null, + "RETRACTED_v1_ratio_per_layer_double_charged": null + } + } + } + }, + "MEASURED_ANCHOR": { + "source": "arXiv:2509.05291 Crosscoding Through Time, training details", + "what": "2-way crosscoder, middle layer of a 1B model, 400M tokens, F=16384, fp32 encoder", + "measured_a100_80gb_hours": 6.0, + "three_way_measured": 12.0, + "this_scripts_flop_model_predicts_h_at_a100_fp32_peak": 25.3, + "this_scripts_flop_model_predicts_h_at_a100_bf16_peak": 1.6, + "reading": "the measurement sits BETWEEN the model's fp32 and bf16 predictions, which is what mixed precision plus sub-peak utilisation looks like. The FLOP model is bracketed by the one real number, not validated against it.", + "DO_NOT": "divide 6 h by 2 to get a per-checkpoint cost. A crosscoder is a joint object whose cost depends on how many sources it ties (source's own warning)." + }, + "HEADLINE_1b": { + "jlens_one_fit_all_14_source_layers_gpu_h": 0.45, + "sae_all_layer_suite_gpu_h_range": [ + 1.4681208733624453, + 40.32324192139738 + ], + "ratio_range_suite_over_jlens": [ + 3.262490829694323, + 89.60720426977196 + ], + "capture_share_of_suite_range": [ + 0.1521173409037142, + 0.41780376868096164 + ], + "reading": "at 1b, one J-lens fit at N=200 covering all 14 source layers costs 0.45 GPU-h. An all-layer SAE suite \u2014 activation capture charged ONCE plus 14 dictionary optimisations \u2014 costs 3.3x to 90x that. v1 said 21x-267x per layer; that figure double-charged activation capture and is retracted.", + "CAVEAT": "the dictionary side remains a FLOP model, and SAE training is typically NOT compute-bound at near-peak: it is data-loader or storage-throughput constrained (arXiv:2410.20526 reports ~8 KB/token of latent activations, ~2 TB and >=500 MB/s for a pre-saving pipeline). Our workload runs at ~100% of fp32 peak, so a wall-clock comparison would favour the transport operator MORE than these FLOP ratios do. Quote the range and the accounting, never a point estimate." + }, + "BREAK_EVEN": { + "note": "J-lens cost is linear in N. Holding the dictionary side at its cheapest modelled setting (100M tokens, x8), the J-lens would have to fit this many prompts to cost the same as the ALL-LAYER SAE suite:", + "by_model": { + "410m": 684.0344550047093, + "1b": 652.4981659388646, + "2.8b": 640.049859564959 + } + }, + "provenance": { + "schema_version": 1, + "experiment": "E64", + "utc": "2026-08-17T21:03:10+00:00", + "script": { + "path": "experiments/t64_instrument_cost.py", + "sha256": "a701f985595f6317d217436a99c1ae000d4265a350008f187153e53e64ea0d6b", + "n_lines": 180 + }, + "git": { + "commit": "529c025b7f5826b65cc019ad6ba8b7b12425d1b9", + "commit_short": "529c025b7f58", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "RTIFACTS.md", + "experiments/t62_adjudicate_band.py", + "experiments/t64_instrument_cost.py", + "paper/paper.tex", + "repro/14_vast_pull_and_verify.sh", + "results/e59_read_dose_410m.json", + "results/e61/", + "results/e61_randomized_null_410m.json", + "results/e62_band_adjudication.json", + "results/e62_interaction_b613.json", + "results/e64_instrument_cost.json", + "results/ladder1b_b613/", + "specs/EXTREVIEW_PROMPT_INSTRUMENT_COST.md", + "specs/EXTREVIEW_PROMPT_INTRAINING.md", + "specs/CONFIG_MATRIX.md", + "specs/RESULTS_TAXONOMY.md", + "specs/experiments/E53_ladder_summary.md", + "specs/experiments/E55_matrix_robustness.md", + "specs/experiments/E56_predictor_registry.md", + "specs/experiments/E58_algebra_audit.md", + "specs/experiments/E59_read_dose_response.md", + "specs/experiments/E60_fitter_determinism.md", + "tools/build_config_matrix.py" + ], + "dirty_diff_sha256": "25d6bc3093862277076526b1dc9cd065cf6c9fbdf5abf9e0180bed2ffd0abdbe", + "describe": "529c025-dirty" + }, + "argv": [ + "experiments/t64_instrument_cost.py" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "87c7e7636b38c328f35c9a649927304117a5d862dda02e9f98f197841a52ca61" + } +} \ No newline at end of file diff --git a/results/e65_ckpt_geometry_410m.json b/results/e65_ckpt_geometry_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..db412a64d816508e54334feaf4fb6c56ff2f9966 --- /dev/null +++ b/results/e65_ckpt_geometry_410m.json @@ -0,0 +1,324 @@ +{ + "experiment": "E65 Phase 0 \u2014 the readout-geometry gate across training checkpoints", + "prereg": "docs/experiments/preregs/E65_training_axis_floor.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "checkpoints_requested": [ + 0, + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1000, + 2000, + 4000, + 8000, + 16000, + 32000, + 64000, + 143000 + ], + "statistic": "mean pairwise cosine and effective rank over the concept-token rows of W_U \u2014 E39's exact statistic, band-free and Jacobian-free", + "degeneracy_threshold_mean_cos": 0.031, + "threshold_source": "the paper's own 410M value, 0.031; above it the readout is in the regime E38/E39 shows no transport can separate", + "by_checkpoint": { + "step0": { + "step": 0, + "mean_cos": -1.8010245184996165e-05, + "eff_rank": 889.2781372070312, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.20205688476562 + }, + "step1": { + "step": 1, + "mean_cos": -1.8010245184996165e-05, + "eff_rank": 889.2781372070312, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.20205688476562 + }, + "step2": { + "step": 2, + "mean_cos": -1.8010174244409427e-05, + "eff_rank": 889.2781372070312, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.20205688476562 + }, + "step4": { + "step": 4, + "mean_cos": -1.8012380678555928e-05, + "eff_rank": 889.2772827148438, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.20205688476562 + }, + "step8": { + "step": 8, + "mean_cos": -1.805523606890347e-05, + "eff_rank": 889.2772827148438, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.20130920410156 + }, + "step16": { + "step": 16, + "mean_cos": -1.8175755030824803e-05, + "eff_rank": 889.2777099609375, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.19882202148438 + }, + "step32": { + "step": 32, + "mean_cos": -1.7882464817375876e-05, + "eff_rank": 889.2781372070312, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.19415283203125 + }, + "step64": { + "step": 64, + "mean_cos": -9.743186637933832e-06, + "eff_rank": 889.278564453125, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.1864013671875 + }, + "step128": { + "step": 128, + "mean_cos": 0.00021771581668872386, + "eff_rank": 889.0576171875, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 140.35028076171875 + }, + "step256": { + "step": 256, + "mean_cos": 0.004493770655244589, + "eff_rank": 885.3593139648438, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 142.33206176757812 + }, + "step512": { + "step": 512, + "mean_cos": 0.024525534361600876, + "eff_rank": 868.6627197265625, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 152.3126983642578 + }, + "step1000": { + "step": 1000, + "mean_cos": 0.04338376596570015, + "eff_rank": 833.4951782226562, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 181.98280334472656 + }, + "step2000": { + "step": 2000, + "mean_cos": 0.03989851847290993, + "eff_rank": 801.8504028320312, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 223.60780334472656 + }, + "step4000": { + "step": 4000, + "mean_cos": 0.037825021892786026, + "eff_rank": 783.4459228515625, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 246.458740234375 + }, + "step8000": { + "step": 8000, + "mean_cos": 0.0383550301194191, + "eff_rank": 765.5985107421875, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 258.25592041015625 + }, + "step16000": { + "step": 16000, + "mean_cos": 0.03865959867835045, + "eff_rank": 752.85400390625, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 259.9566650390625 + }, + "step32000": { + "step": 32000, + "mean_cos": 0.03873710334300995, + "eff_rank": 743.9141235351562, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 249.04891967773438 + }, + "step64000": { + "step": 64000, + "mean_cos": 0.03305966034531593, + "eff_rank": 736.3240356445312, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 218.1020965576172 + }, + "step143000": { + "step": 143000, + "mean_cos": 0.030666090548038483, + "eff_rank": 725.887939453125, + "n_concept_tokens": 1310, + "token_set_identical": true, + "wu_fro": 177.2579345703125 + } + }, + "n_degenerate": 7, + "degenerate_checkpoints": [ + "step1000", + "step2000", + "step4000", + "step8000", + "step16000", + "step32000", + "step64000" + ], + "n_clean": 12, + "earliest_clean_checkpoint": 0, + "controls": { + "C1b_final_checkpoint_anchor": { + "reference_e39_mean_cos": 0.030666086822748184, + "final_checkpoint_mean_cos": 0.030666090548038483, + "abs_diff": 3.725290298461914e-09, + "tolerance": 0.001, + "fires": true + }, + "C2b_token_set_identity": { + "n_checkpoints_checked": 19, + "all_token_sets_identical": true, + "fires": true + } + }, + "VERDICT": "GEOMETRY FLOOR AT step0 \u2014 7 early checkpoint(s) exceed the 410M threshold and are geometrically degenerate. Phase 1 should concentrate around the transition; readouts before it are uninterpretable for the same reason 70M/160M readouts are.", + "provenance": { + "schema_version": 1, + "experiment": "E65-P0", + "utc": "2026-08-30T06:31:58+00:00", + "script": { + "path": "experiments/t65_ckpt_geometry.py", + "sha256": "a70307ef12ae198b95e58982102da3773abe9cbfb86a68bbb45b77327c8091fc", + "n_lines": 178 + }, + "git": { + "commit": "c9c49697e295c0fec4225552836fc9733b279f75", + "commit_short": "c9c49697e295", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CLAUDE.md", + "README.md", + "corpora/manifest_ood.json", + "docs/HANDOFF.md", + "docs/REPO_REVIEW_PROMPT.md", + "docs/context/AGGREGATION_POLICY.md", + "docs/context/CONTEXT.md", + "docs/context/IMPLEMENTATION_RULES.md", + "docs/context/STATE.md", + "docs/experiments/INDEX.md", + "docs/experiments/METRICS.md", + "docs/experiments/PROPOSED.md", + "docs/experiments/PROVENANCE.md", + "docs/experiments/descriptions/E64.md", + "docs/experiments/descriptions/EN1.md", + "docs/experiments/preregs/CV3_margins.md", + "docs/experiments/preregs/CV4_capability_ladder.md", + "docs/experiments/preregs/CV5_metric_sensitivity.md", + "docs/experiments/preregs/CV6_per_family_ladder.md", + "docs/experiments/preregs/CV7_1b_rung.md", + "docs/experiments/preregs/DN1_design_nulls.md", + "docs/experiments/preregs/E33_logit_baseline.md", + "docs/experiments/preregs/E36_qladder.md", + "docs/experiments/preregs/E48_crossover.md", + "docs/experiments/preregs/E48b_containment.md", + "docs/experiments/preregs/E48c_exposure_vs_read.md", + "docs/experiments/preregs/E51_interaction.md", + "docs/experiments/preregs/E52_factorial.md", + "docs/experiments/preregs/E53_ladder_summary.md", + "docs/experiments/preregs/E55_matrix_robustness.md", + "docs/experiments/preregs/E56_predictor_registry.md", + "docs/experiments/preregs/E57_grid_variance_ci.md", + "docs/experiments/preregs/E58_algebra_audit.md", + "docs/experiments/preregs/E59_read_dose_response.md", + "docs/experiments/preregs/E60_fitter_determinism.md", + "docs/experiments/preregs/E61_randomized_network_null.md", + "docs/experiments/preregs/E62_ladder1b_corrected_band.md", + "docs/experiments/preregs/E65_training_axis_floor.md", + "docs/experiments/preregs/E66b_determinism_floor.md", + "docs/experiments/preregs/OS1_operator_space.md", + "docs/experiments/preregs/P0_t52_parallel_port.md", + "docs/experiments/preregs/R1_grid_rstrip.md", + "docs/experiments/preregs/R2_e48_rstrip_arm.md", + "docs/experiments/preregs/R3_ladder410_cpu.md", + "docs/experiments/preregs/R4_corrections.md", + "docs/experiments/preregs/R4b_e36_rstrip.md", + "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "docs/experiments/preregs/R6_within_source_resampling.md", + "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "docs/experiments/preregs/superseded/PREREG_E48_CROSSOVER.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/RELEASE.md", + "docs/reproducibility/REVIEWER_REPRO.md", + "docs/reproducibility/T1_FINDING_E33_BAND.md", + "docs/reproducibility/T1_SWEEP_PREDICTIONS.md", + "docs/reproducibility/T1_SWEEP_SPEC.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md" + ], + "dirty_diff_sha256": "ec61dde1905616f407f51d5f6fe3ea37c9c6e3e1ade6998cd0644527d080c600", + "describe": "c9c4969-dirty" + }, + "argv": [ + "experiments/t65_ckpt_geometry.py", + "--model", + "410m" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "bbe32c177ad9c0f80d66730260342bf64530277082594abfe433d15d19577877" + } +} \ No newline at end of file diff --git a/results/e65_ckpt_readout_1b.json b/results/e65_ckpt_readout_1b.json new file mode 100644 index 0000000000000000000000000000000000000000..14eb940c367bd5a6a276ca9d523a7cc08e7d48cd --- /dev/null +++ b/results/e65_ckpt_readout_1b.json @@ -0,0 +1,309 @@ +{ + "experiment": "E65 Phase 1 \u2014 the transport lens across training checkpoints", + "prereg": "specs/experiments/E65_training_axis_floor.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-1b-deduped", + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fitting_corpus": "Pile-CC", + "fitting_corpus_held_fixed": true, + "why_fixed": "the fitting corpus accounts for more variance in the read than the read context does; varying it across checkpoints would swamp the training signal", + "N": 200, + "seed_block": 0, + "n_derangements": 5, + "n_items": 541, + "n_pairs": 893, + "checkpoints": [ + 0, + 512, + 1000, + 2000, + 4000, + 8000, + 16000, + 32000, + 64000, + 143000 + ], + "by_checkpoint": { + "step0": { + "step": 0, + "jlens_persist": 0.002905893139541149, + "jlens_min": 0.006018015393055976, + "logit_persist": 0.0023162039462476967, + "logit_min": 0.008868563920259476, + "shuf_persist_mean": 0.0014558142621535807, + "shuf_persist_max": 0.0019526675110682844, + "shuf_draws": [ + 0.0019526675110682844, + 0.0014577260240912437, + 0.0014637998538091779, + 0.0012386971211526543, + 0.0011661808006465436 + ], + "cross_entropy": 11.048807058248434, + "clears_derangement": true, + "clears_logit": true + }, + "step512": { + "step": 512, + "jlens_persist": 0.0350887824781239, + "jlens_min": 0.06825746754184366, + "logit_persist": 0.06204907316714525, + "logit_min": 0.08371221926063299, + "shuf_persist_mean": 0.03565271611791104, + "shuf_persist_max": 0.037686183024197815, + "shuf_draws": [ + 0.037686183024197815, + 0.031805974128656087, + 0.03675868941936642, + 0.03766853562556207, + 0.03434419839177281 + ], + "cross_entropy": 5.286101985622096, + "clears_derangement": false, + "clears_logit": false + }, + "step1000": { + "step": 1000, + "jlens_persist": 0.00690901386551559, + "jlens_min": 0.027200930751860142, + "logit_persist": 0.01994552961550653, + "logit_min": 0.05539083071053028, + "shuf_persist_mean": 0.006664239396341145, + "shuf_persist_max": 0.006840809760615229, + "shuf_draws": [ + 0.006691464688628912, + 0.0065525057958439, + 0.006691464688628912, + 0.006840809760615229, + 0.006544952047988772 + ], + "cross_entropy": 4.230594643601426, + "clears_derangement": true, + "clears_logit": false + }, + "step2000": { + "step": 2000, + "jlens_persist": 0.011243635835126043, + "jlens_min": 0.03787338305264711, + "logit_persist": 0.02253168746829033, + "logit_min": 0.05762441186234355, + "shuf_persist_mean": 0.010253740432672203, + "shuf_persist_max": 0.010724156210199, + "shuf_draws": [ + 0.009993316465988755, + 0.010677939909510315, + 0.00977900899015367, + 0.010724156210199, + 0.010094280587509274 + ], + "cross_entropy": 3.597159059198053, + "clears_derangement": true, + "clears_logit": false + }, + "step4000": { + "step": 4000, + "jlens_persist": 0.018920646235346793, + "jlens_min": 0.06491318978369236, + "logit_persist": 0.03305366253480315, + "logit_min": 0.08779109446331859, + "shuf_persist_mean": 0.01822319828905165, + "shuf_persist_max": 0.019028346193954347, + "shuf_draws": [ + 0.019028346193954347, + 0.0176917667966336, + 0.018650685716420413, + 0.01770013142377138, + 0.018045061314478517 + ], + "cross_entropy": 3.248779348424963, + "clears_derangement": false, + "clears_logit": false + }, + "step8000": { + "step": 8000, + "jlens_persist": 0.024163772049359978, + "jlens_min": 0.08086016308516264, + "logit_persist": 0.028753516916185617, + "logit_min": 0.08457275207620114, + "shuf_persist_mean": 0.021487117889337243, + "shuf_persist_max": 0.02322088787332177, + "shuf_draws": [ + 0.02322088787332177, + 0.0231486827833578, + 0.01987059225793928, + 0.020690934243611993, + 0.020504492288455366 + ], + "cross_entropy": 3.0671937272355363, + "clears_derangement": true, + "clears_logit": false + }, + "step16000": { + "step": 16000, + "jlens_persist": 0.023416419001296162, + "jlens_min": 0.10030183345079421, + "logit_persist": 0.03830388281494379, + "logit_min": 0.11618245046120138, + "shuf_persist_mean": 0.0219226405210793, + "shuf_persist_max": 0.024532060301862658, + "shuf_draws": [ + 0.022489179740659893, + 0.024532060301862658, + 0.02247074255719781, + 0.019581223488785326, + 0.020539996516890823 + ], + "cross_entropy": 2.927255948384603, + "clears_derangement": false, + "clears_logit": false + }, + "step32000": { + "step": 32000, + "jlens_persist": 0.026716785226017236, + "jlens_min": 0.10749359335750341, + "logit_persist": 0.048226413875818254, + "logit_min": 0.13892803201451898, + "shuf_persist_mean": 0.02585170186124742, + "shuf_persist_max": 0.027695527439936994, + "shuf_draws": [ + 0.025241362396627665, + 0.027344086579978465, + 0.027695527439936994, + 0.025989882927387953, + 0.022987649962306022 + ], + "cross_entropy": 2.8721056585913307, + "clears_derangement": false, + "clears_logit": false + }, + "step64000": { + "step": 64000, + "jlens_persist": 0.029589090833906085, + "jlens_min": 0.11168387634679675, + "logit_persist": 0.0424603633582592, + "logit_min": 0.13002874851226806, + "shuf_persist_mean": 0.02966548749944195, + "shuf_persist_max": 0.031519839889369905, + "shuf_draws": [ + 0.031089026504196225, + 0.031519839889369905, + 0.029291472921613604, + 0.029066371335648, + 0.027360726846382022 + ], + "cross_entropy": 2.7797629382159257, + "clears_derangement": false, + "clears_logit": false + }, + "step143000": { + "step": 143000, + "jlens_persist": 0.04486040568444878, + "jlens_min": 0.14467011950910091, + "logit_persist": 0.022273217141628266, + "logit_min": 0.13399605853483082, + "shuf_persist_mean": 0.03907380965072662, + "shuf_persist_max": 0.040255373995751145, + "shuf_draws": [ + 0.040255373995751145, + 0.03975287983193994, + 0.0383127877721563, + 0.03957578467670828, + 0.037472221977077426 + ], + "cross_entropy": 2.6810064616504015, + "clears_derangement": true, + "clears_logit": true + } + }, + "t_star_steps": 143000, + "n_checkpoints_clearing_both": 2, + "e61_untrained_anchor": { + "randomized_block_persist": [ + 0.00376, + 0.00395, + 0.00348 + ], + "free_baseline_persist": 0.02844, + "source": "results/e61_randomized_null_410m.json" + }, + "controls": { + "C1_final_checkpoint_reproduces_e48": { + "reference": "e48_crossover_410m.json arms_admitted_mean['J|Pile-CC|s0'].persist", + "reference_value": 0.04046863599547318, + "final_checkpoint": 0.04486040568444878, + "abs_diff": 0.0043917696889756025, + "tolerance": 0.002146, + "fires": false + } + }, + "VERDICT": "UNCLEAR \u2014 the final checkpoint does not reproduce the stored result; this pipeline is not the paper's pipeline and nothing here is comparable", + "provenance": { + "schema_version": 1, + "experiment": "E65-P1", + "utc": "2026-08-18T06:02:33+00:00", + "script": { + "path": "experiments/t65_ckpt_readout.py", + "sha256": "c495cdd3d59c9aff7a20e252836f81eba8859eaf236ecafe2057c8ed14065de6", + "n_lines": 273 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/t65_ckpt_readout.py", + "--model", + "1b", + "--device", + "cuda" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.0", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "76ef634e932a087bb5354c8303679d438122773e10bcf2aef7473d6e054524bf" + } +} \ No newline at end of file diff --git a/results/e65_ckpt_readout_410m.json b/results/e65_ckpt_readout_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..e7144b2092e455cb6c8b40659ccd5062ddff06d2 --- /dev/null +++ b/results/e65_ckpt_readout_410m.json @@ -0,0 +1,314 @@ +{ + "experiment": "E65 Phase 1 \u2014 the transport lens across training checkpoints", + "prereg": "specs/experiments/E65_training_axis_floor.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fitting_corpus": "Pile-CC", + "fitting_corpus_held_fixed": true, + "why_fixed": "the fitting corpus accounts for more variance in the read than the read context does; varying it across checkpoints would swamp the training signal", + "N": 200, + "seed_block": 0, + "n_derangements": 5, + "n_items": 541, + "n_pairs": 893, + "checkpoints": [ + 0, + 512, + 1000, + 2000, + 4000, + 8000, + 16000, + 32000, + 64000, + 143000 + ], + "by_checkpoint": { + "step0": { + "step": 0, + "jlens_persist": 0.002113605686463416, + "jlens_min": 0.005292997974902391, + "logit_persist": 0.0035629860358312726, + "logit_min": 0.015512948529794813, + "shuf_persist_mean": 0.0013737007719464601, + "shuf_persist_max": 0.0016186642460525036, + "shuf_draws": [ + 0.0009913628804497422, + 0.0016186642460525036, + 0.0014877843263093383, + 0.0012103917601052673, + 0.001560300646815449 + ], + "cross_entropy": 11.027966370453706, + "clears_derangement": true, + "clears_logit": false + }, + "step512": { + "step": 512, + "jlens_persist": 0.03165703529957682, + "jlens_min": 0.058973956108093264, + "logit_persist": 0.024265835445839913, + "logit_min": 0.05828223363496363, + "shuf_persist_mean": 0.030829780052881686, + "shuf_persist_max": 0.03234609203645959, + "shuf_draws": [ + 0.0313972961390391, + 0.03234609203645959, + 0.03034068315755576, + 0.02999804059509188, + 0.030066788336262106 + ], + "cross_entropy": 5.677024222709037, + "clears_derangement": false, + "clears_logit": true + }, + "step1000": { + "step": 1000, + "jlens_persist": 0.007102607516571879, + "jlens_min": 0.055905706342309715, + "logit_persist": 0.02565042022615671, + "logit_min": 0.08593039389234036, + "shuf_persist_mean": 0.006594868991523981, + "shuf_persist_max": 0.006869167624972761, + "shuf_draws": [ + 0.006869167624972761, + 0.0064178973902016875, + 0.00628148305695504, + 0.00664554457180202, + 0.006760252313688398 + ], + "cross_entropy": 4.582304937345487, + "clears_derangement": true, + "clears_logit": false + }, + "step2000": { + "step": 2000, + "jlens_persist": 0.006561075593344868, + "jlens_min": 0.031062386743724347, + "logit_persist": 0.01820141498465091, + "logit_min": 0.055191149935126305, + "shuf_persist_mean": 0.003685156567953527, + "shuf_persist_max": 0.004730883403681219, + "shuf_draws": [ + 0.003934010327793658, + 0.004730883403681219, + 0.003266256139613688, + 0.003155860886909068, + 0.003338772081770003 + ], + "cross_entropy": 3.872286848119787, + "clears_derangement": true, + "clears_logit": false + }, + "step4000": { + "step": 4000, + "jlens_persist": 0.014180448930710554, + "jlens_min": 0.06610923931002617, + "logit_persist": 0.031676913844421506, + "logit_min": 0.07378817815333605, + "shuf_persist_mean": 0.01126657116226852, + "shuf_persist_max": 0.012213326897472144, + "shuf_draws": [ + 0.012213326897472144, + 0.012091761734336615, + 0.010384379792958498, + 0.011161026638001203, + 0.010482360748574138 + ], + "cross_entropy": 3.424504159807085, + "clears_derangement": true, + "clears_logit": false + }, + "step8000": { + "step": 8000, + "jlens_persist": 0.021808931208215653, + "jlens_min": 0.08360657729208469, + "logit_persist": 0.0348735500825569, + "logit_min": 0.0917188448831439, + "shuf_persist_mean": 0.016931592179462315, + "shuf_persist_max": 0.018179135164245963, + "shuf_draws": [ + 0.017465165327303113, + 0.01650818248745054, + 0.015719903795979918, + 0.016785574122332038, + 0.018179135164245963 + ], + "cross_entropy": 3.1989625638669676, + "clears_derangement": true, + "clears_logit": false + }, + "step16000": { + "step": 16000, + "jlens_persist": 0.026401121448725463, + "jlens_min": 0.09131083618849516, + "logit_persist": 0.033806373621337114, + "logit_min": 0.11723277438431978, + "shuf_persist_mean": 0.020870173866860568, + "shuf_persist_max": 0.0240499661071226, + "shuf_draws": [ + 0.021696795127354562, + 0.017674984387122096, + 0.01983940142672509, + 0.021089722285978496, + 0.0240499661071226 + ], + "cross_entropy": 3.077749570210775, + "clears_derangement": true, + "clears_logit": false + }, + "step32000": { + "step": 32000, + "jlens_persist": 0.025740327849052846, + "jlens_min": 0.1046044148504734, + "logit_persist": 0.04999426994472742, + "logit_min": 0.13164614448323847, + "shuf_persist_mean": 0.021414660862646998, + "shuf_persist_max": 0.02433080130722374, + "shuf_draws": [ + 0.02240841283928603, + 0.020808048243634403, + 0.018922870024107398, + 0.02060317189898342, + 0.02433080130722374 + ], + "cross_entropy": 2.9871180594504416, + "clears_derangement": true, + "clears_logit": false + }, + "step64000": { + "step": 64000, + "jlens_persist": 0.03726785376202315, + "jlens_min": 0.13848500996828078, + "logit_persist": 0.057896239683032036, + "logit_min": 0.14969237800687551, + "shuf_persist_mean": 0.02610415790230036, + "shuf_persist_max": 0.028284365963190793, + "shuf_draws": [ + 0.027911654300987722, + 0.02632141560316086, + 0.024690193217247726, + 0.02331316042691469, + 0.028284365963190793 + ], + "cross_entropy": 2.9292626681628526, + "clears_derangement": true, + "clears_logit": false + }, + "step143000": { + "step": 143000, + "jlens_persist": 0.050229771388694645, + "jlens_min": 0.1498804520815611, + "logit_persist": 0.03907186973374337, + "logit_min": 0.12607942409813405, + "shuf_persist_mean": 0.029170699063688518, + "shuf_persist_max": 0.032720142789185044, + "shuf_draws": [ + 0.029944545403122903, + 0.028072836622595788, + 0.02780707674100995, + 0.027308893762528895, + 0.032720142789185044 + ], + "cross_entropy": 2.874821559802906, + "clears_derangement": true, + "clears_logit": true + } + }, + "t_star_steps": 143000, + "n_checkpoints_clearing_both": 1, + "e61_untrained_anchor": { + "randomized_block_persist": [ + 0.00376, + 0.00395, + 0.00348 + ], + "free_baseline_persist": 0.02844, + "source": "results/e61_randomized_null_410m.json" + }, + "controls": { + "C1_final_checkpoint_reproduces_e48": { + "reference": "e48_crossover_410m.json arms_admitted_mean['J|Pile-CC|s0'].persist", + "reference_value": 0.04546590892132372, + "final_checkpoint": 0.050229771388694645, + "abs_diff": 0.004763862467370927, + "tolerance": 0.00037, + "fires": false + } + }, + "VERDICT": "UNCLEAR \u2014 the final checkpoint does not reproduce the stored result; this pipeline is not the paper's pipeline and nothing here is comparable", + "provenance": { + "schema_version": 1, + "experiment": "E65-P1", + "utc": "2026-08-17T23:15:58+00:00", + "script": { + "path": "experiments/t65_ckpt_readout.py", + "sha256": "c495cdd3d59c9aff7a20e252836f81eba8859eaf236ecafe2057c8ed14065de6", + "n_lines": 273 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/t65_ckpt_readout.py", + "--model", + "410m", + "--device", + "cuda" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.0", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-186-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "6cce283f6c9344414686d5d62ac5b0286ba735f9043f70b74b990f7ee64e90c2" + } +} \ No newline at end of file diff --git a/results/e65_ckpt_readout_410m_trainval.json b/results/e65_ckpt_readout_410m_trainval.json new file mode 100644 index 0000000000000000000000000000000000000000..0636fa7df52343076937f7c64fe83d9d3c43ffeb --- /dev/null +++ b/results/e65_ckpt_readout_410m_trainval.json @@ -0,0 +1,339 @@ +{ + "experiment": "E65 Phase 1 \u2014 the transport lens across training checkpoints", + "prereg": "specs/experiments/E65_training_axis_floor.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "fitting_corpus": "Pile-CC", + "fitting_corpus_held_fixed": true, + "why_fixed": "the fitting corpus accounts for more variance in the read than the read context does; varying it across checkpoints would swamp the training signal", + "N": 200, + "seed_block": 0, + "fitter": "trainval", + "n_derangements": 5, + "n_items": 541, + "n_pairs": 893, + "checkpoints": [ + 0, + 512, + 1000, + 2000, + 4000, + 8000, + 16000, + 32000, + 64000, + 143000 + ], + "by_checkpoint": { + "step0": { + "step": 0, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.002113605686463416, + "jlens_min": 0.005292997974902391, + "logit_persist": 0.0035629860358312726, + "logit_min": 0.015512948529794813, + "shuf_persist_mean": 0.0013737007719464601, + "shuf_persist_max": 0.0016186642460525036, + "shuf_draws": [ + 0.0009913628804497422, + 0.0016186642460525036, + 0.0014877843263093383, + 0.0012103917601052673, + 0.001560300646815449 + ], + "cross_entropy": 11.027966370453706, + "clears_derangement": true, + "clears_logit": false + }, + "step512": { + "step": 512, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.03165703529957682, + "jlens_min": 0.058973956108093264, + "logit_persist": 0.024265835445839913, + "logit_min": 0.05828223363496363, + "shuf_persist_mean": 0.030829780052881686, + "shuf_persist_max": 0.03234609203645959, + "shuf_draws": [ + 0.0313972961390391, + 0.03234609203645959, + 0.03034068315755576, + 0.02999804059509188, + 0.030066788336262106 + ], + "cross_entropy": 5.677024222709037, + "clears_derangement": false, + "clears_logit": true + }, + "step1000": { + "step": 1000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.007102607516571879, + "jlens_min": 0.055905706342309715, + "logit_persist": 0.02565042022615671, + "logit_min": 0.08593039389234036, + "shuf_persist_mean": 0.006594868991523981, + "shuf_persist_max": 0.006869167624972761, + "shuf_draws": [ + 0.006869167624972761, + 0.0064178973902016875, + 0.00628148305695504, + 0.00664554457180202, + 0.006760252313688398 + ], + "cross_entropy": 4.582304937345487, + "clears_derangement": true, + "clears_logit": false + }, + "step2000": { + "step": 2000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.006561075593344868, + "jlens_min": 0.031062386743724347, + "logit_persist": 0.01820141498465091, + "logit_min": 0.055191149935126305, + "shuf_persist_mean": 0.003685156567953527, + "shuf_persist_max": 0.004730883403681219, + "shuf_draws": [ + 0.003934010327793658, + 0.004730883403681219, + 0.003266256139613688, + 0.003155860886909068, + 0.003338772081770003 + ], + "cross_entropy": 3.872286848119787, + "clears_derangement": true, + "clears_logit": false + }, + "step4000": { + "step": 4000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.014180448930710554, + "jlens_min": 0.06610923931002617, + "logit_persist": 0.031676913844421506, + "logit_min": 0.07378817815333605, + "shuf_persist_mean": 0.01126657116226852, + "shuf_persist_max": 0.012213326897472144, + "shuf_draws": [ + 0.012213326897472144, + 0.012091761734336615, + 0.010384379792958498, + 0.011161026638001203, + 0.010482360748574138 + ], + "cross_entropy": 3.424504159807085, + "clears_derangement": true, + "clears_logit": false + }, + "step8000": { + "step": 8000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.021808931208215653, + "jlens_min": 0.08360657729208469, + "logit_persist": 0.0348735500825569, + "logit_min": 0.0917188448831439, + "shuf_persist_mean": 0.016931592179462315, + "shuf_persist_max": 0.018179135164245963, + "shuf_draws": [ + 0.017465165327303113, + 0.01650818248745054, + 0.015719903795979918, + 0.016785574122332038, + 0.018179135164245963 + ], + "cross_entropy": 3.1989625638669676, + "clears_derangement": true, + "clears_logit": false + }, + "step16000": { + "step": 16000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.026401121448725463, + "jlens_min": 0.09131083618849516, + "logit_persist": 0.033806373621337114, + "logit_min": 0.11723277438431978, + "shuf_persist_mean": 0.020870173866860568, + "shuf_persist_max": 0.0240499661071226, + "shuf_draws": [ + 0.021696795127354562, + 0.017674984387122096, + 0.01983940142672509, + 0.021089722285978496, + 0.0240499661071226 + ], + "cross_entropy": 3.077749570210775, + "clears_derangement": true, + "clears_logit": false + }, + "step32000": { + "step": 32000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.025740327849052846, + "jlens_min": 0.1046044148504734, + "logit_persist": 0.04999426994472742, + "logit_min": 0.13164614448323847, + "shuf_persist_mean": 0.021414660862646998, + "shuf_persist_max": 0.02433080130722374, + "shuf_draws": [ + 0.02240841283928603, + 0.020808048243634403, + 0.018922870024107398, + 0.02060317189898342, + 0.02433080130722374 + ], + "cross_entropy": 2.9871180594504416, + "clears_derangement": true, + "clears_logit": false + }, + "step64000": { + "step": 64000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.03726785376202315, + "jlens_min": 0.13848500996828078, + "logit_persist": 0.057896239683032036, + "logit_min": 0.14969237800687551, + "shuf_persist_mean": 0.02610415790230036, + "shuf_persist_max": 0.028284365963190793, + "shuf_draws": [ + 0.027911654300987722, + 0.02632141560316086, + 0.024690193217247726, + 0.02331316042691469, + 0.028284365963190793 + ], + "cross_entropy": 2.9292626681628526, + "clears_derangement": true, + "clears_logit": false + }, + "step143000": { + "step": 143000, + "fitter": "trainval", + "n_fit_used": 200, + "jlens_persist": 0.050229771388694645, + "jlens_min": 0.1498804520815611, + "logit_persist": 0.03907186973374337, + "logit_min": 0.12607942409813405, + "shuf_persist_mean": 0.029170699063688518, + "shuf_persist_max": 0.032720142789185044, + "shuf_draws": [ + 0.029944545403122903, + 0.028072836622595788, + 0.02780707674100995, + 0.027308893762528895, + 0.032720142789185044 + ], + "cross_entropy": 2.874821559802906, + "clears_derangement": true, + "clears_logit": true + } + }, + "t_star_steps": 143000, + "n_checkpoints_clearing_both": 1, + "e61_untrained_anchor": { + "randomized_block_persist": [ + 0.00376, + 0.00395, + 0.00348 + ], + "free_baseline_persist": 0.02844, + "source": "results/e61_randomized_null_410m.json" + }, + "controls": { + "C1_final_checkpoint_reproduces_e48": { + "reference": "e48_crossover_410m.json arms_admitted_mean['J|Pile-CC|s0'].persist", + "reference_value": 0.04546590892132372, + "final_checkpoint": 0.050229771388694645, + "abs_diff": 0.004763862467370927, + "tolerance": 0.00037, + "fires": false + } + }, + "VERDICT": "UNCLEAR \u2014 the final checkpoint does not reproduce the stored result; this pipeline is not the paper's pipeline and nothing here is comparable", + "provenance": { + "schema_version": 1, + "experiment": "E65-P1", + "utc": "2026-08-19T21:08:05+00:00", + "script": { + "path": "experiments/t65_ckpt_readout.py", + "sha256": "2ed9f93df24945f539fb8efb9c53062aed1808279b2640d50e5bc404a5d30324", + "n_lines": 308 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/t65_ckpt_readout.py", + "--model", + "410m", + "--device", + "cuda", + "--fitter", + "trainval", + "--save-lens-dir", + "results/e65_lenses" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.1", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-181-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "c9267544e80c984f3c2b20c3a28fbc29fa7f36e49763505d47373875e984fc9f" + } +} \ No newline at end of file diff --git a/results/e65_lenses/lens_1b_step0_trainval.pt b/results/e65_lenses/lens_1b_step0_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..edc110cfee755987b3f7e37bcf28d121f06f06dd --- /dev/null +++ b/results/e65_lenses/lens_1b_step0_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e505952688ddc550147c70d950ae8b4c10dcc12ea662a4de2ee1127fd31002d +size 67112671 diff --git a/results/e65_lenses/lens_410m_step0_trainval.pt b/results/e65_lenses/lens_410m_step0_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa31894c4f7856deda221f4c88542d6794934085 --- /dev/null +++ b/results/e65_lenses/lens_410m_step0_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4edb6ae594222b1fc817da66c2e25a279527b26612388555cfb0b79cdcdc1af6 +size 27268159 diff --git a/results/e65_lenses/lens_410m_step1000_trainval.pt b/results/e65_lenses/lens_410m_step1000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5cf501ee1884088a2f85f609eae99fdaf1f76b5 --- /dev/null +++ b/results/e65_lenses/lens_410m_step1000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:383fff33a9959c1db9baa71de3ea49d8bc39317b8404b871291e559bbf1452ee +size 27268216 diff --git a/results/e65_lenses/lens_410m_step143000_trainval.pt b/results/e65_lenses/lens_410m_step143000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..cab90ff9da9a4481938cd72eea74a0058b5a5e88 --- /dev/null +++ b/results/e65_lenses/lens_410m_step143000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d17344b57e890b4751ec92734a0cd6018dcfaa07cd68b1d63eae909b2bd84a6 +size 27268254 diff --git a/results/e65_lenses/lens_410m_step16000_trainval.pt b/results/e65_lenses/lens_410m_step16000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..1761f5d74988fb36ab415f657dd261ded670bb33 --- /dev/null +++ b/results/e65_lenses/lens_410m_step16000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a4cfa82184f4ef87b41cd02bc55880d5db36517efcf61e28c4573bb458d95dd +size 27268235 diff --git a/results/e65_lenses/lens_410m_step2000_trainval.pt b/results/e65_lenses/lens_410m_step2000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..05423685f991b33d011c5b388976e6ec746f2a70 --- /dev/null +++ b/results/e65_lenses/lens_410m_step2000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62d18987e53f3918e110534dee32a6332100d7a7b0f03ec2fb270fb642532429 +size 27268216 diff --git a/results/e65_lenses/lens_410m_step32000_trainval.pt b/results/e65_lenses/lens_410m_step32000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bb44c45c25ceaad472eaba3c53498cf94cef441 --- /dev/null +++ b/results/e65_lenses/lens_410m_step32000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48aadfd8f47f8db47e0ab2e310d29690c80ece4f35e196210b8784d072267814 +size 27268235 diff --git a/results/e65_lenses/lens_410m_step4000_trainval.pt b/results/e65_lenses/lens_410m_step4000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bab1d5f9ab42785ce9619c4ff1d8a5f431a6075 --- /dev/null +++ b/results/e65_lenses/lens_410m_step4000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f8f9a624e7725608a4b955bc54b7f5f2f99a8266b5ea4e5a70cc7ff6c287dfe +size 27268216 diff --git a/results/e65_lenses/lens_410m_step512_trainval.pt b/results/e65_lenses/lens_410m_step512_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcd259057c98845ab16618fcca9e4f74ea019e66 --- /dev/null +++ b/results/e65_lenses/lens_410m_step512_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d755e6de506ba3159b09db7d3a135bf8100a86eb987c1747a27e10b098b901c +size 27268197 diff --git a/results/e65_lenses/lens_410m_step64000_trainval.pt b/results/e65_lenses/lens_410m_step64000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fd0d54d6b3cca89666aac78e0f69a2d50e6e6a7 --- /dev/null +++ b/results/e65_lenses/lens_410m_step64000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3d26e17a8017d6dbeb0fd0cdcce29d31a6a040bfb7b9a688f32e15a485fd2a +size 27268235 diff --git a/results/e65_lenses/lens_410m_step8000_trainval.pt b/results/e65_lenses/lens_410m_step8000_trainval.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d20f7954cc571a56a96ada957457f000546970d --- /dev/null +++ b/results/e65_lenses/lens_410m_step8000_trainval.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0af1ce895557c0abcf27d64356ef176980249e8a2da2dc6fcfe0098d91da7c5b +size 27268216 diff --git a/results/e65r_phase1_rstrip_rescore.json b/results/e65r_phase1_rstrip_rescore.json new file mode 100644 index 0000000000000000000000000000000000000000..995a0a55325ed68b76ea22db594b6aa278c38add --- /dev/null +++ b/results/e65r_phase1_rstrip_rescore.json @@ -0,0 +1,606 @@ +{ + "experiment": "E65R \u2014 E65 Phase 1 rescored at the corrected readout", + "prereg": "docs/experiments/preregs/E65R_phase1_rstrip_rescore.md", + "status": "PRE-REGISTERED", + "rescore_only_no_refit": true, + "preserves": [ + "results/e65_ckpt_readout_410m.json", + "results/e65_ckpt_readout_410m_trainval.json", + "results/e65_ckpt_readout_1b.json" + ], + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "checkpoints": [ + 0, + 512, + 1000, + 2000, + 4000, + 8000, + 16000, + 32000, + 64000, + 143000 + ], + "fitting_corpus": "Pile-CC", + "N": 200, + "seed_block": 0, + "n_derangements": 5, + "device": "cpu", + "dtype": "float32", + "operators_from": "results/e65_lenses/lens_410m_step_trainval.pt (already fitted)", + "n_items": 541, + "n_pairs": 893, + "n_items_read_token_differs": 157, + "arms": { + "U": "unstripped (legacy) \u2014 reproduces the stored run", + "S": "STRIPPED (anchor rule) \u2014 prompt.rstrip() before tokenisation" + }, + "by_checkpoint_unstripped": { + "step0": { + "step": 0, + "jlens_persist": 0.002113605639897287, + "jlens_min": 0.005292997974902391, + "logit_persist": 0.0035629860358312726, + "logit_min": 0.015512948529794813, + "shuf_persist_mean": 0.0013737007719464601, + "shuf_persist_max": 0.0016186642460525036, + "shuf_draws": [ + 0.0009913628804497422, + 0.0016186642460525036, + 0.0014877843263093383, + 0.0012103917601052673, + 0.001560300646815449 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step512": { + "step": 512, + "jlens_persist": 0.03165703529957682, + "jlens_min": 0.05897395648062229, + "logit_persist": 0.024265833955723792, + "logit_min": 0.058282233355566856, + "shuf_persist_mean": 0.030829780052881686, + "shuf_persist_max": 0.03234609203645959, + "shuf_draws": [ + 0.0313972961390391, + 0.03234609203645959, + 0.03034068315755576, + 0.02999804059509188, + 0.030066788336262106 + ], + "clears_derangement": false, + "clears_logit": true + }, + "step1000": { + "step": 1000, + "jlens_persist": 0.0071026077028363945, + "jlens_min": 0.055905709695070985, + "logit_persist": 0.02565042022615671, + "logit_min": 0.08593039407860488, + "shuf_persist_mean": 0.006594869084656238, + "shuf_persist_max": 0.006869167624972761, + "shuf_draws": [ + 0.006869167624972761, + 0.0064178974833339454, + 0.006281483150087297, + 0.006645544758066535, + 0.006760252406820655 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step2000": { + "step": 2000, + "jlens_persist": 0.006561075267381966, + "jlens_min": 0.031062385998666286, + "logit_persist": 0.01820141423959285, + "logit_min": 0.055191153287887575, + "shuf_persist_mean": 0.003685156530700624, + "shuf_persist_max": 0.004730883403681219, + "shuf_draws": [ + 0.003934010327793658, + 0.004730883403681219, + 0.003266255767084658, + 0.0031558607006445527, + 0.0033387724542990327 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step4000": { + "step": 4000, + "jlens_persist": 0.014180449023842811, + "jlens_min": 0.06610923744738102, + "logit_persist": 0.03167691458947956, + "logit_min": 0.07378817591816186, + "shuf_persist_mean": 0.01126657078973949, + "shuf_persist_max": 0.012213327083736658, + "shuf_draws": [ + 0.012213327083736658, + 0.0120917615480721, + 0.010384379606693982, + 0.011161025706678629, + 0.010482360003516078 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step8000": { + "step": 8000, + "jlens_persist": 0.021808929718099536, + "jlens_min": 0.08360658027231693, + "logit_persist": 0.03487355231773108, + "logit_min": 0.09171884711831808, + "shuf_persist_mean": 0.016931593352928757, + "shuf_persist_max": 0.018179136468097566, + "shuf_draws": [ + 0.01746516542043537, + 0.01650818397756666, + 0.015719904541037978, + 0.016785576357506216, + 0.018179136468097566 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step16000": { + "step": 16000, + "jlens_persist": 0.02640112107619643, + "jlens_min": 0.09138335566967726, + "logit_persist": 0.033806375111453235, + "logit_min": 0.11723276991397143, + "shuf_persist_mean": 0.020870174835436045, + "shuf_persist_max": 0.0240499698324129, + "shuf_draws": [ + 0.021696795127354562, + 0.017674985877238213, + 0.01983940105419606, + 0.021089722285978496, + 0.0240499698324129 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step32000": { + "step": 32000, + "jlens_persist": 0.02574032766278833, + "jlens_min": 0.10460441745817661, + "logit_persist": 0.0499942734837532, + "logit_min": 0.13164614448323847, + "shuf_persist_mean": 0.021414660098962486, + "shuf_persist_max": 0.02433080130722374, + "shuf_draws": [ + 0.022408414701931178, + 0.020808045263402165, + 0.018922870117239653, + 0.020603169105015695, + 0.02433080130722374 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step64000": { + "step": 64000, + "jlens_persist": 0.0372678569285199, + "jlens_min": 0.13848500289022922, + "logit_persist": 0.05789624042809009, + "logit_min": 0.14969238098710774, + "shuf_persist_mean": 0.026104157008230686, + "shuf_persist_max": 0.02828436316922307, + "shuf_draws": [ + 0.02791165467351675, + 0.026321413926780224, + 0.024690193030983208, + 0.023313160240650176, + 0.02828436316922307 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step143000": { + "step": 143000, + "jlens_persist": 0.05022977418266237, + "jlens_min": 0.1498804535716772, + "logit_persist": 0.03907187047880143, + "logit_min": 0.1260794226080179, + "shuf_persist_mean": 0.02917069775983691, + "shuf_persist_max": 0.03272014316171408, + "shuf_draws": [ + 0.02994454503059387, + 0.02807283475995064, + 0.027807073947042227, + 0.027308891899883746, + 0.03272014316171408 + ], + "clears_derangement": true, + "clears_logit": true + } + }, + "by_checkpoint_stripped": { + "step0": { + "step": 0, + "jlens_persist": 0.001203932357020676, + "jlens_min": 0.0059417661279439924, + "logit_persist": 0.001796596124768257, + "logit_min": 0.007768653566017747, + "shuf_persist_mean": 0.0009305767435580492, + "shuf_persist_max": 0.0010907046031206845, + "shuf_draws": [ + 0.0007991594029590487, + 0.0007266431348398328, + 0.00101818828843534, + 0.00101818828843534, + 0.0010907046031206845 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step512": { + "step": 512, + "jlens_persist": 0.06675728072877973, + "jlens_min": 0.10008353600278497, + "logit_persist": 0.06558218972058967, + "logit_min": 0.09106245865114033, + "shuf_persist_mean": 0.0674365149694495, + "shuf_persist_max": 0.06831571569200605, + "shuf_draws": [ + 0.06831571569200605, + 0.06718658931786195, + 0.06751884480472654, + 0.06795543462503702, + 0.0662059904076159 + ], + "clears_derangement": false, + "clears_logit": true + }, + "step1000": { + "step": 1000, + "jlens_persist": 0.05345994462259114, + "jlens_min": 0.1280479329638183, + "logit_persist": 0.08449293887242675, + "logit_min": 0.11756009075324983, + "shuf_persist_mean": 0.05437850377522409, + "shuf_persist_max": 0.05764208657201379, + "shuf_draws": [ + 0.05764208657201379, + 0.05325940623879433, + 0.053382732137106356, + 0.0537467937450856, + 0.05386150018312037 + ], + "clears_derangement": false, + "clears_logit": false + }, + "step2000": { + "step": 2000, + "jlens_persist": 0.058130866405554114, + "jlens_min": 0.1246338315308094, + "logit_persist": 0.07751045541372151, + "logit_min": 0.13514827527105808, + "shuf_persist_mean": 0.05223490057978779, + "shuf_persist_max": 0.05474223552737385, + "shuf_draws": [ + 0.05314848779235035, + 0.05474223552737385, + 0.05016072227153927, + 0.05186850915197283, + 0.05125454815570265 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step4000": { + "step": 4000, + "jlens_persist": 0.08544831210747361, + "jlens_min": 0.16361099444329738, + "logit_persist": 0.0881994219031185, + "logit_min": 0.1538720102980733, + "shuf_persist_mean": 0.0767805318813771, + "shuf_persist_max": 0.07765907784923912, + "shuf_draws": [ + 0.07765907784923912, + 0.07759237755089998, + 0.07527534859254956, + 0.07642643805593252, + 0.07694941735826433 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step8000": { + "step": 8000, + "jlens_persist": 0.0949959086952731, + "jlens_min": 0.19125210866332054, + "logit_persist": 0.09802383973728865, + "logit_min": 0.17430603709071874, + "shuf_persist_mean": 0.08466399823315442, + "shuf_persist_max": 0.08661291457246989, + "shuf_draws": [ + 0.08661291457246989, + 0.08290851067285984, + 0.08364835174288601, + 0.0835603547981009, + 0.08658985937945544 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step16000": { + "step": 16000, + "jlens_persist": 0.08245790926739574, + "jlens_min": 0.19866943769156933, + "logit_persist": 0.09053566723596304, + "logit_min": 0.1729158954694867, + "shuf_persist_mean": 0.07242685643490404, + "shuf_persist_max": 0.07756438970100135, + "shuf_draws": [ + 0.07756438970100135, + 0.070901078148745, + 0.06815687406342477, + 0.07007170866709203, + 0.07544023159425706 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step32000": { + "step": 32000, + "jlens_persist": 0.09131660431157798, + "jlens_min": 0.22034837752580644, + "logit_persist": 0.1052584283053875, + "logit_min": 0.18938960144296288, + "shuf_persist_mean": 0.08420903585385531, + "shuf_persist_max": 0.08738383415620773, + "shuf_draws": [ + 0.08738383415620773, + 0.08611572065856307, + 0.07983260925393551, + 0.08225682561751455, + 0.08545618958305568 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step64000": { + "step": 64000, + "jlens_persist": 0.09709453869145364, + "jlens_min": 0.2321223948150873, + "logit_persist": 0.10741080678999423, + "logit_min": 0.20438538696616887, + "shuf_persist_mean": 0.08483874555677176, + "shuf_persist_max": 0.09157072007656097, + "shuf_draws": [ + 0.09157072007656097, + 0.08660385254770517, + 0.07790668588131666, + 0.08129324112087488, + 0.08681922815740109 + ], + "clears_derangement": true, + "clears_logit": false + }, + "step143000": { + "step": 143000, + "jlens_persist": 0.11742814308963716, + "jlens_min": 0.2631119828671217, + "logit_persist": 0.08492737400811165, + "logit_min": 0.19539144374430178, + "shuf_persist_mean": 0.08757544208317995, + "shuf_persist_max": 0.09173413701355457, + "shuf_draws": [ + 0.09081203192472458, + 0.0836572576314211, + 0.08607715740799904, + 0.08559662643820047, + 0.09173413701355457 + ], + "clears_derangement": true, + "clears_logit": true + } + }, + "reference_corrected": { + "source": "results/e48_crossover_410m_rstrip.json -> arms_admitted_mean['J|Pile-CC|s0'].persist", + "value": 0.11054239969234914, + "seed_values": [ + 0.11054239969234914, + 0.1118273551343009, + 0.11091751407366246 + ], + "seed_sd": 0.0006607611825883316, + "tolerance_2sd": 0.001321522365176663 + }, + "reference_as_stored_in_e65": { + "value": 0.04546590892132372, + "note": "the E65 run labels this e48_crossover_410m.json J|Pile-CC|s0 persist, but that field currently holds 0.04509159799199551; the stored value is the 3-seed mean" + }, + "PRIMARY": { + "C1_S_final": 0.11742814308963716, + "C1_S_reference": 0.11054239969234914, + "C1_S_abs_diff": 0.006885743397288022, + "C1_S_tolerance": 0.001321522365176663, + "C1_S_fires": false + }, + "phase1_rule_on_arm_S": null, + "t_star_steps_arm_S": 143000, + "n_checkpoints_clearing_both_arm_S": 1, + "controls": { + "C_U_reproduces_stored": { + "required": "abs_diff <= 1e-6 on 6 fields x 10 checkpoints vs e65_ckpt_readout_410m.json", + "max_abs_diff": 7.251948118210116e-05, + "n_compared": 60, + "worst_field": "step16000.jlens_min", + "fires": false + }, + "C_REF_differ": { + "required": "|stripped ref - unstripped ref| > C1 tolerance", + "stripped_ref": 0.11054239969234914, + "unstripped_ref_as_stored": 0.04546590892132372, + "abs_diff": 0.06507649077102542, + "tolerance": 0.001321522365176663, + "fires": true + }, + "C_DERANGE_distinct": { + "required": "5 distinct derangement draws per cell at step143000, both arms", + "U_distinct": 5, + "S_distinct": 5, + "fires": true + }, + "C_ITEMS_157": { + "required": "exactly 157 of 541 admitted items read a different token between arms", + "observed_differ": 157, + "observed_total": 541, + "fires": true + } + }, + "controls_fired": { + "C_U_reproduces_stored": false, + "C_REF_differ": true, + "C_DERANGE_distinct": true, + "C_ITEMS_157": true + }, + "branch": "UNINTERPRETABLE", + "VERDICT": "UNINTERPRETABLE \u2014 arm U does not reproduce the stored unstripped run (max abs_diff 7.252e-05 > 1e-6 on step16000.jlens_min). Neither arm is comparable to the stored file; both are discarded.", + "scope_limit": "410M only. Only 1 of 10 1B operators was persisted; the 1B arm of E65 Phase 1 remains at its stored UNCLEAR and is out of scope.", + "provenance": { + "schema_version": 1, + "experiment": "E65R \u2014 E65 Phase 1 rescored at the corrected readout", + "utc": "2026-09-01T20:44:28+00:00", + "script": { + "path": "experiments/e65r_phase1_rstrip_rescore.py", + "sha256": "e34193d4636a87309e9a40b830d49e7fe58a0f062a00ffad3b554679cc474d12", + "n_lines": 300 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "corpora/dataset_revisions.json", + "corpora/dataset_revisions.json.bak", + "corpora/manifest_ood.json", + "corpora/manifest_ood.json.bak", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E65R_phase1_rstrip_rescore.md", + "docs/experiments/preregs/E65T_threshold_defect_adjudication.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/EFFRANK1_effective_rank_adjudication.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py" + ], + "dirty_diff_sha256": "719949b81d9871a26c26430713cafa9ad3b949cc5eea0ad080ae38307d75f63d", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/e65r_phase1_rstrip_rescore.py" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "161f08444d84f1e7a4f5722ccd271f3bf2ecf7b10cc22904146ecb1fe2af111f" + } +} \ No newline at end of file diff --git a/results/e65t_threshold_adjudication.json b/results/e65t_threshold_adjudication.json new file mode 100644 index 0000000000000000000000000000000000000000..94dabaf5bc8ac7d3404e894329b2c930a5bdbe14 --- /dev/null +++ b/results/e65t_threshold_adjudication.json @@ -0,0 +1,484 @@ +{ + "experiment": "E65T \u2014 adjudication of E65 Phase 0's geometry-floor threshold", + "prereg": "docs/experiments/preregs/E65T_threshold_defect_adjudication.md", + "status": "PRE-REGISTERED", + "recomputes_not_remeasures": true, + "adjudicates": [ + "results/e65_ckpt_geometry_410m.json -> degenerate_checkpoints", + "docs/experiments/preregs/E65_training_axis_floor.md (PRE-REGISTRATION DEFECT, FLAGGED NOT FIXED)", + "paper/audit_paper_5pp.tex:918" + ], + "model": "EleutherAI/pythia-410m-deduped", + "checkpoints": [ + 0, + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1000, + 2000, + 4000, + 8000, + 16000, + 32000, + 64000, + 143000 + ], + "dtype": "float32", + "device": "cpu", + "stored_threshold_reproduced_only": 0.031, + "e38_degenerate_regime": { + "definition": "mean_cos >= 0.92 AND er_energy90 == 1", + "source": "results/e38_jgeometry.json -> E39 (70m/160m exhibit it; 410m/1b do not)", + "no_new_threshold_invented": true + }, + "by_checkpoint": { + "step0": { + "mean_cos": -1.8010245184996165e-05, + "er_energy90": 584, + "er_entropy": 889.2781372070312, + "sigma1_energy_share": 0.0034103586804121733, + "n": 1310, + "step": 0, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -1.8010245184996165e-05, + "stored_eff_rank_entropy": 889.2781372070312 + }, + "step1": { + "mean_cos": -1.8010245184996165e-05, + "er_energy90": 584, + "er_entropy": 889.2781372070312, + "sigma1_energy_share": 0.0034103586804121733, + "n": 1310, + "step": 1, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -1.8010245184996165e-05, + "stored_eff_rank_entropy": 889.2781372070312 + }, + "step2": { + "mean_cos": -1.8010174244409427e-05, + "er_energy90": 584, + "er_entropy": 889.2781372070312, + "sigma1_energy_share": 0.0034103605430573225, + "n": 1310, + "step": 2, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -1.8010174244409427e-05, + "stored_eff_rank_entropy": 889.2781372070312 + }, + "step4": { + "mean_cos": -1.8012380678555928e-05, + "er_energy90": 584, + "er_entropy": 889.2772827148438, + "sigma1_energy_share": 0.0034103619400411844, + "n": 1310, + "step": 4, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -1.8012380678555928e-05, + "stored_eff_rank_entropy": 889.2772827148438 + }, + "step8": { + "mean_cos": -1.805523606890347e-05, + "er_energy90": 584, + "er_entropy": 889.2772827148438, + "sigma1_energy_share": 0.003410352859646082, + "n": 1310, + "step": 8, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -1.805523606890347e-05, + "stored_eff_rank_entropy": 889.2772827148438 + }, + "step16": { + "mean_cos": -1.8175755030824803e-05, + "er_energy90": 584, + "er_entropy": 889.2777099609375, + "sigma1_energy_share": 0.0034101619385182858, + "n": 1310, + "step": 16, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -1.8175755030824803e-05, + "stored_eff_rank_entropy": 889.2777099609375 + }, + "step32": { + "mean_cos": -1.7882464817375876e-05, + "er_energy90": 584, + "er_entropy": 889.2781372070312, + "sigma1_energy_share": 0.003409935859963298, + "n": 1310, + "step": 32, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -1.7882464817375876e-05, + "stored_eff_rank_entropy": 889.2781372070312 + }, + "step64": { + "mean_cos": -9.743186637933832e-06, + "er_energy90": 584, + "er_entropy": 889.278564453125, + "sigma1_energy_share": 0.003409917000681162, + "n": 1310, + "step": 64, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": -9.743186637933832e-06, + "stored_eff_rank_entropy": 889.278564453125 + }, + "step128": { + "mean_cos": 0.00021771581668872386, + "er_energy90": 584, + "er_entropy": 889.0576171875, + "sigma1_energy_share": 0.004194085020571947, + "n": 1310, + "step": 128, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.00021771581668872386, + "stored_eff_rank_entropy": 889.0576171875 + }, + "step256": { + "mean_cos": 0.004493770655244589, + "er_energy90": 580, + "er_entropy": 885.3593139648438, + "sigma1_energy_share": 0.016765441745519638, + "n": 1310, + "step": 256, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.004493770655244589, + "stored_eff_rank_entropy": 885.3593139648438 + }, + "step512": { + "mean_cos": 0.024525534361600876, + "er_energy90": 561, + "er_entropy": 868.6627197265625, + "sigma1_energy_share": 0.04712479189038277, + "n": 1310, + "step": 512, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.024525534361600876, + "stored_eff_rank_entropy": 868.6627197265625 + }, + "step1000": { + "mean_cos": 0.04338376596570015, + "er_energy90": 517, + "er_entropy": 833.4951782226562, + "sigma1_energy_share": 0.06958720833063126, + "n": 1310, + "step": 1000, + "flagged_by_stored_rule": true, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.04338376596570015, + "stored_eff_rank_entropy": 833.4951782226562 + }, + "step2000": { + "mean_cos": 0.03989851847290993, + "er_energy90": 472, + "er_entropy": 801.8504028320312, + "sigma1_energy_share": 0.06794363260269165, + "n": 1310, + "step": 2000, + "flagged_by_stored_rule": true, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.03989851847290993, + "stored_eff_rank_entropy": 801.8504028320312 + }, + "step4000": { + "mean_cos": 0.037825021892786026, + "er_energy90": 444, + "er_entropy": 783.4459228515625, + "sigma1_energy_share": 0.06805678457021713, + "n": 1310, + "step": 4000, + "flagged_by_stored_rule": true, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.037825021892786026, + "stored_eff_rank_entropy": 783.4459228515625 + }, + "step8000": { + "mean_cos": 0.0383550301194191, + "er_energy90": 419, + "er_entropy": 765.5985107421875, + "sigma1_energy_share": 0.07073557376861572, + "n": 1310, + "step": 8000, + "flagged_by_stored_rule": true, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.0383550301194191, + "stored_eff_rank_entropy": 765.5985107421875 + }, + "step16000": { + "mean_cos": 0.03865959867835045, + "er_energy90": 402, + "er_entropy": 752.85400390625, + "sigma1_energy_share": 0.07120245695114136, + "n": 1310, + "step": 16000, + "flagged_by_stored_rule": true, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.03865959867835045, + "stored_eff_rank_entropy": 752.85400390625 + }, + "step32000": { + "mean_cos": 0.03873710334300995, + "er_energy90": 390, + "er_entropy": 743.9141235351562, + "sigma1_energy_share": 0.06831848621368408, + "n": 1310, + "step": 32000, + "flagged_by_stored_rule": true, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.03873710334300995, + "stored_eff_rank_entropy": 743.9141235351562 + }, + "step64000": { + "mean_cos": 0.03305966034531593, + "er_energy90": 382, + "er_entropy": 736.3240356445312, + "sigma1_energy_share": 0.056765343993902206, + "n": 1310, + "step": 64000, + "flagged_by_stored_rule": true, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.03305966034531593, + "stored_eff_rank_entropy": 736.3240356445312 + }, + "step143000": { + "mean_cos": 0.030666090548038483, + "er_energy90": 371, + "er_entropy": 725.887939453125, + "sigma1_energy_share": 0.05466432496905327, + "n": 1310, + "step": 143000, + "flagged_by_stored_rule": false, + "in_e38_degenerate_regime": false, + "stored_mean_cos": 0.030666090548038483, + "stored_eff_rank_entropy": 725.887939453125 + } + }, + "final_checkpoint_regime_check": { + "70m": { + "mean_cos": 0.9585258364677429, + "er_energy90": 1, + "er_entropy": 199.05316162109375, + "sigma1_energy_share": 0.9584628343582153, + "n": 1310, + "in_e38_degenerate_regime": true + }, + "160m": { + "mean_cos": 0.921846866607666, + "er_energy90": 1, + "er_entropy": 368.5517578125, + "sigma1_energy_share": 0.9217102527618408, + "n": 1310, + "in_e38_degenerate_regime": true + }, + "410m": { + "mean_cos": 0.030666090548038483, + "er_energy90": 371, + "er_entropy": 725.887939453125, + "sigma1_energy_share": 0.05466432496905327, + "n": 1310, + "in_e38_degenerate_regime": false + }, + "1b": { + "mean_cos": 0.019916437566280365, + "er_energy90": 508, + "er_entropy": 969.3308715820312, + "sigma1_energy_share": 0.042908456176519394, + "n": 1310, + "in_e38_degenerate_regime": false + } + }, + "PRIMARY": { + "flagged_by_stored_rule": [ + "step1000", + "step2000", + "step4000", + "step8000", + "step16000", + "step32000", + "step64000" + ], + "n_flagged": 7, + "flagged_and_in_e38_regime": [], + "n_flagged_and_in_regime": 0 + }, + "controls": { + "C1_reproduces_e65_phase0": { + "required": "mean_cos abs_diff <= 1e-6 and er_entropy abs_diff <= 1e-3, all 19", + "max_mean_cos_abs_diff": 0.0, + "max_entropy_abs_diff": 0.0, + "n_checkpoints_compared": 19, + "fires": true + }, + "C2_regime_not_vacuous": { + "required": "E38 degenerate regime TRUE for 70m and 160m final W_U", + "observed": { + "70m": true, + "160m": true + }, + "fires": true + }, + "C3_regime_not_universal": { + "required": "E38 degenerate regime FALSE for 410m and 1b final W_U", + "observed": { + "410m": false, + "1b": false + }, + "fires": true + }, + "C4_flagged_set_reproduces": { + "required": "mean_cos > 0.031 selects exactly ['step1000', 'step2000', 'step4000', 'step8000', 'step16000', 'step32000', 'step64000']", + "observed": [ + "step1000", + "step2000", + "step4000", + "step8000", + "step16000", + "step32000", + "step64000" + ], + "stored": [ + "step1000", + "step2000", + "step4000", + "step8000", + "step16000", + "step32000", + "step64000" + ], + "fires": true + } + }, + "controls_fired": { + "C1_reproduces_e65_phase0": true, + "C2_regime_not_vacuous": true, + "C3_regime_not_universal": true, + "C4_flagged_set_reproduces": true + }, + "branch": "FINDING NOT REAL", + "VERDICT": "FINDING NOT REAL \u2014 0 of the 7 checkpoints flagged by E65's stored 0.031 rule is in the degenerate regime as E38/E39 exhibits it (mean_cos >= 0.92 AND 90%-energy rank == 1). At the flagged checkpoints mean_cos is 0.0331-0.0434 against the regime's 0.92, and the 90%-energy rank is 382-517 against the regime's 1. The E65 pre-registration's self-flag is CONFIRMED: the 0.031 threshold does not select the regime it was justified by. The 'geometry floor' is an artifact of the threshold. paper/audit_paper_5pp.tex:918 ('degenerate from step 1000 through step 64000') restates it as fact and must be corrected or disclosed. No new threshold is proposed.", + "provenance": { + "schema_version": 1, + "experiment": "E65T \u2014 adjudication of E65 Phase 0's geometry-floor threshold", + "utc": "2026-09-01T20:28:11+00:00", + "script": { + "path": "experiments/e65t_threshold_adjudication.py", + "sha256": "640cb909ee2ba3419b53eda976e632c45c0f474db162d799bad98f55b6d2349a", + "n_lines": 196 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "corpora/dataset_revisions.json", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E65T_threshold_defect_adjudication.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/EFFRANK1_effective_rank_adjudication.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh" + ], + "dirty_diff_sha256": "83b3848cce07cd3ed221dd6ef145216d8164aee882acecaa2c67ded074747252", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/e65t_threshold_adjudication.py" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "91aba4ee892fe7a2489b3eb3a6216d379af375c42fc105706b19c1b59d7315a6" + } +} \ No newline at end of file diff --git a/results/e66_fitter_equivalence_cuda.json b/results/e66_fitter_equivalence_cuda.json new file mode 100644 index 0000000000000000000000000000000000000000..c7741518306d945f464d846c3865acc470608cba --- /dev/null +++ b/results/e66_fitter_equivalence_cuda.json @@ -0,0 +1,131 @@ +{ + "experiment": "E66 \u2014 do the programme's two fitters agree on CUDA?", + "prereg": "in-file docstring, fixed before running", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "corpus": "Pile-CC", + "n_fit": 200, + "n_items": 541, + "n_pairs": 893, + "arms": { + "A_fast_fit": { + "read": { + "min": 0.14523330219089986, + "persist": 0.04509159463923425 + } + }, + "B_trainval_loop": { + "read": { + "min": 0.14523330219089986, + "persist": 0.04509159463923425 + }, + "n_used": 200, + "n_skipped": 0 + }, + "C_stored_lens": { + "read": { + "min": 0.14523330219089986, + "persist": 0.04509159463923425 + }, + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt" + } + }, + "operator_differences": { + "A_vs_B": { + "max_abs": 1.1920928955078125e-07, + "max_rel": 1.0619564508650803e-07 + }, + "B_vs_C": { + "max_abs": 0.00110660120844841, + "max_rel": 0.002534676444779942 + }, + "A_vs_C": { + "max_abs": 0.00110660120844841, + "max_rel": 0.002534676444779942 + } + }, + "read_differences": { + "A_minus_B": 0.0, + "B_minus_C": 0.0, + "A_minus_C": 0.0, + "pile_cc_seed_sd": 0.000184 + }, + "controls": { + "C1_trainval_reproduces_stored": { + "what": "arm B (trainval, the path that produced the stored lens) vs arm C (stored)", + "operator": { + "max_abs": 0.00110660120844841, + "max_rel": 0.002534676444779942 + }, + "read_diff": 0.0, + "rel_tolerance": 0.001, + "fires": false + } + }, + "e65_v1_context": "E65 v1 fitted with fast_fit on CUDA and read 0.050230 against the stored lens's 0.045092 \u2014 a +11% gap under a configuration matching on prompts, weights, band, device and fitter-inputs.", + "VERDICT": "UNCLEAR \u2014 arm B does not reproduce the stored lens (max_rel 2.535e-03 > 0.001). The stored lens is not what its own provenance says it is, which is a larger problem than either branch of this experiment.", + "provenance": { + "schema_version": 1, + "experiment": "E66", + "utc": "2026-08-19T19:56:11+00:00", + "script": { + "path": "experiments/t66_fitter_equivalence_cuda.py", + "sha256": "21e2ed72dd14ee17b41ca1559798a9429f34555720b860dec134a0629ab8cce5", + "n_lines": 233 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/t66_fitter_equivalence_cuda.py", + "--device", + "cuda" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + } + ], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.1", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-181-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "48e1594bcc1da51992150cd8fcc2b99c533367701eabc0c41b5a20511a3e2c90" + } +} \ No newline at end of file diff --git a/results/e66b_determinism_floor.json b/results/e66b_determinism_floor.json new file mode 100644 index 0000000000000000000000000000000000000000..dd022172534403384e9854efb072bacf0a645ccc --- /dev/null +++ b/results/e66b_determinism_floor.json @@ -0,0 +1,214 @@ +{ + "experiment": "E66b \u2014 the fitter's determinism floor at 410M on CUDA", + "prereg": "docs/experiments/preregs/E66b_determinism_floor.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "corpus": "Pile-CC", + "seed_block": 0, + "n_fit": 200, + "device": "cuda", + "smoke": false, + "dim_batch_D1_D2": 32, + "dim_batch_D3": 64, + "resolves": "results/e66_fitter_equivalence_cuda.json (VERDICT: UNCLEAR)", + "e66_recap": { + "A_vs_B_max_abs": 1.1920928955078125e-07, + "B_vs_C_max_abs": 0.00110660120844841, + "read_differences_all_zero": true, + "note": "E66's PRIMARY (do the two fitters agree on CUDA?) returned EQUIVALENT. The UNCLEAR came from control C1, not from the primary." + }, + "fp16_half_ulp_at_absmax": 0.0004883, + "residual_R_fixed_in_registration": 0.0006183, + "S_fp16_refit_vs_stored": 0.0010986328125, + "F_identical_invocation": 0.0, + "F_prime_changed_reduction_order": 1.0728836059570312e-06, + "floor_used": 1.0728836059570312e-06, + "gpu_model": "NVIDIA L40S", + "gpu_capability": [ + 8, + 9 + ], + "stored_run_2026_08_14": { + "S": 0.0010986328125, + "F": 0.0, + "F_prime": 1.0281801223754883e-06, + "floor_used": 1.0281801223754883e-06, + "dim_batch_D1_D2": 32, + "n_fit": 200, + "torch": "2.11.0+cu128", + "gpu_model": null, + "VERDICT": "CONFIRMED DEFECT" + }, + "old_vs_new": { + "S": { + "stored": 0.0010986328125, + "new": 0.0010986328125, + "comparable_across_gpus": false, + "note": "NOT a reproduction target. Kernel- and card-dependent, and that dependence IS E66b's finding. See E66_C_COMPLETION_SPEC.md section 3." + }, + "F": { + "stored": 0.0, + "new": 0.0, + "comparable_across_gpus": true, + "note": "byte-identical invocation; must be 0.0 on any card" + }, + "F_prime": { + "stored": 1.0281801223754883e-06, + "new": 1.0728836059570312e-06, + "comparable_across_gpus": false, + "note": "doubled dim_batch changes reduction order, which is device-specific" + } + }, + "D1_persisted_to": "results/e66_D1_refit_410m_pilecc_s0.pt", + "D1_sha256": "00eebd6ddffff67532bb954e8d51674f0cd8d5021c0919d7139a942c607fff21", + "R_derivation_caveat": "R = 6.183e-04 was fixed in the registration as S_observed minus the fp16 HALF-ULP. Two independently rounded values straddling a boundary differ by a FULL ULP (9.766e-04), so R is likely overstated. The verdict survives either model: half-ULP gives R/3 = 2.061e-04 and full-ULP gives R/3 = 4.333e-05, and the measured floor clears both. The derivation is corrected here; CONFIRMED DEFECT is not reopened.", + "controls": { + "C1_tf32_is_off": { + "required": "observed flags False, asserted not assumed", + "matmul_allow_tf32": false, + "cudnn_allow_tf32": false, + "fires": true + }, + "C3_fp16_arithmetic": { + "required": "max|J| in [1,2) so the fp16 half-ULP is 4.883e-04", + "abs_max": 1.208984375, + "half_ulp": 0.0004883, + "fires": true + }, + "C4_floor_is_measurable": { + "required": "if F == F' == 0 the card is fully deterministic, the WITHIN-THE-FLOOR branch is unavailable by construction, and adjudication falls to S alone. This is the guard on the declared bias.", + "F_identical_invocation": 0.0, + "F_prime_changed_reduction_order": 1.0728836059570312e-06, + "floor_is_zero": false, + "fires": true + }, + "C5_same_prompts": { + "required": "REGISTERED VERBATIM: 'pool SHA-256 identical to tv_INSTREAM_Pile-CC_s0.json's inputs WHERE RECORDED'. Nothing is recorded, so the registered clause is VACUOUS -- it cannot fail, which rule 10 forbids. It is therefore adjudicated by reconstruction against the historical fitting code, with a negative control that makes it able to fail.", + "registered_form_vacuous": true, + "registered_form_computable": false, + "why_not": "results/e48/tv_INSTREAM_Pile-CC_s0.json has no provenance block; the stored .pt records only n_prompts/source_layers/d_model/model/corpus/seed; the mirror copy is byte-identical to local. E48 was fitted 20:03-20:04 PDT 2026-08-14, commit 1a6f5df added provenance.py at 20:05.", + "D1_pool_sha256": "708336f679b4fb14cf6a7196ca60489aeff8a8b4cbd2237f15596ba9c2f0ec8f", + "trainval_reconstruction_sha256": "708336f679b4fb14cf6a7196ca60489aeff8a8b4cbd2237f15596ba9c2f0ec8f", + "historical_code": "pythia/trainval.py @ 1a6f5df^ :156-159 (slice, filter len>=128, [:n_max])", + "pools_identical": true, + "corpus_sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "corpus_sha256_in_manifest": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "corpus_matches_manifest": true, + "n_pool": 200, + "stored_n_used": 200, + "n_matches_stored": true, + "negative_control_block1_sha256": "a2a6e27d9cc0a46bc3876cdd5ed907e99e6c53d147b4c21469477ca50ea68ffb", + "negative_control_differs": true, + "negative_control_prompt_overlap": 0, + "smoke": false, + "does_not_prove": "that the 2026-08-14 process consumed these prompts -- that run recorded nothing. It proves the pool is determined by corpus bytes that still match the manifest and by code still obtainable from git history.", + "fires": true + }, + "C2_same_estimand": { + "required": "D1's reads equal the stored operator's reads: read_diff == 0.0 under BOTH min and persist. E66 measured exactly that for its own three arms.", + "read_D1": { + "min": 0.14523330219089986, + "persist": 0.04509159463923425 + }, + "read_stored": { + "min": 0.14523330219089986, + "persist": 0.04509159463923425 + }, + "read_differences": { + "min": 0.0, + "persist": 0.0 + }, + "scoring_path": "t66_fitter_equivalence_cuda.py:124-141, UNSTRIPPED (legacy) readout; the convention cancels in a difference and this is not a corrected-readout number", + "n_items": 541, + "n_scored_pairs": 893, + "e66_reference_read_differences": { + "A_minus_B": 0.0, + "B_minus_C": 0.0, + "A_minus_C": 0.0 + }, + "fires": true + } + }, + "controls_fired": { + "C1_tf32_is_off": true, + "C3_fp16_arithmetic": true, + "C4_floor_is_measurable": true, + "C5_same_prompts": true, + "C2_same_estimand": true + }, + "VERDICT": "CONFIRMED DEFECT \u2014 the floor is 1.073e-06, more than three times below the residual R = 6.183e-04. Reduction order does not explain the gap. The stored operators do not reproduce and must be refit before release.", + "provenance": { + "schema_version": 1, + "experiment": "E66b", + "utc": "2026-08-31T04:13:31+00:00", + "script": { + "path": "experiments/e66b_determinism_floor.py", + "sha256": "900cc3a6244cb8e3a45ca211b39a1b4a9c9f1620d03c5455f2a1fe79686856ac", + "n_lines": 413 + }, + "git": { + "commit": "", + "commit_short": "", + "git_repository_present": false, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/e66b_determinism_floor.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e66_fitter_equivalence_cuda.json", + "sha256": "d6e9bd3cd0c9e9426fc69e9d8d711f9a46e934214aaa52f67e877d947ec55cd2", + "bytes": 3208, + "exists": true + } + ], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.16.1", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 8, + "torch_num_interop_threads": 96, + "OMP_NUM_THREADS": null, + "os_cpu_count": 96 + } + }, + "payload_sha256": "2bfee8c4753feabca0553d6140b7848457e74e06a6bb34b4c457377ac0b27fb2" + } +} \ No newline at end of file diff --git a/results/e66c_post_provenance_repro.json b/results/e66c_post_provenance_repro.json new file mode 100644 index 0000000000000000000000000000000000000000..b03297d330ea3361ed60f341d7dae91142a38bf5 --- /dev/null +++ b/results/e66c_post_provenance_repro.json @@ -0,0 +1,121 @@ +{ + "experiment": "E66c \u2014 do operators fitted under recorded provenance reproduce?", + "prereg": "docs/experiments/preregs/E66c_post_provenance_repro.md", + "status": "PRE-REGISTERED", + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "source": "Pile-CC", + "block": 0, + "n_fit": 200, + "dim_batch": 32, + "device": "cuda", + "target": "results/r6/lens_R6_Pile-CC_b0_410m_n200.pt", + "recorded_env_torch": "2.11.0+cu128", + "observed_torch": "2.11.0+cu128", + "fp16_ulp_threshold": 0.0009766, + "S6_fp16_refit_vs_stored": 0.000244140625, + "S_refit_vs_other_block": 0.022186279296875, + "why_this_target": "E48's panel was fitted 20:03-20:04 on 2026-08-14; commit 1a6f5df at 20:05 added provenance.py. r6 was fitted 2026-08-20 with a full environment block and a deterministic file-order pool.", + "controls": { + "C1_tf32_off": { + "required": "both flags False", + "matmul": false, + "cudnn": false, + "fires": true + }, + "C2_same_pool": { + "required": "200 prompts, blocks disjoint, endpoints recorded", + "n": 200, + "first_sha16": "abd0a56569c3d574", + "last_sha16": "faf7b33398773e30", + "b0_x_b1_overlap": 0, + "fires": true + }, + "C3_torch_matches_record": { + "required": "torch == 2.11.0+cu128", + "recorded": "2.11.0+cu128", + "observed": "2.11.0+cu128", + "fires": true + }, + "C4_comparison_can_fail": { + "required": "a different block's operator must exceed one fp16 ULP (9.766e-04)", + "max_abs_refit_b0_vs_stored_b1": 0.022186279296875, + "fires": true + } + }, + "controls_fired": { + "C1_tf32_off": true, + "C2_same_pool": true, + "C3_torch_matches_record": true, + "C4_comparison_can_fail": true + }, + "VERDICT": "REPRODUCES TO STORAGE \u2014 S6 = 2.441e-04, within one fp16 step (9.766e-04). The refit agrees to the resolution the file is stored at.", + "provenance": { + "schema_version": 1, + "experiment": "E66c", + "utc": "2026-08-29T22:03:15+00:00", + "script": { + "path": "experiments/e66c_post_provenance_repro.py", + "sha256": "5c4b606002bd92deb98fa9436c7de140e8ce2c2cfba59eaa11fa514a8bfce059", + "n_lines": 153 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "experiments/e66c_post_provenance_repro.py", + "--dim-batch", + "32" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/r6/lens_R6_Pile-CC_b0_410m_n200.pt", + "sha256": "33dae3b0d750d08e1cc7236e477e442e503d99f669a0f3ad0cfa4ef490e272e8", + "bytes": 27268171, + "exists": true + }, + { + "path": "corpora/Pile-CC.jsonl", + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "bytes": 13512530, + "exists": true + } + ], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.16.1", + "numpy": "2.4.6", + "platform": "Linux-6.8.0-85-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "871b4aee8fc8a9aceb48ef35657af86419a5d2121bf810ecd68a25382c4dc318" + } +} \ No newline at end of file diff --git a/results/e66d_s_quantization_limited.json b/results/e66d_s_quantization_limited.json new file mode 100644 index 0000000000000000000000000000000000000000..92268455cfc81bb77d52c3c30ba8e2f338d6d48d --- /dev/null +++ b/results/e66d_s_quantization_limited.json @@ -0,0 +1,294 @@ +{ + "experiment": "E66d \u2014 is S quantization-limited, or did the hardware just happen to match?", + "prereg": "docs/experiments/preregs/E66d_s_quantization_limited.md", + "status": "PRE-REGISTERED", + "adjudicates": "the EXPLANATION of S's stability only \u2014 not S, F, F', the floor, E66b's rule, its CONFIRMED DEFECT verdict, or any of C1-C5", + "recomputes_not_remeasures": true, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "torch_num_threads": 10, + "n_entries": 13631488, + "inputs": { + "D1": "results/e66_D1_refit_410m_pilecc_s0.pt", + "stored": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt" + }, + "fp16_ulp": 0.0009765625, + "fp16_half_ulp": 0.00048828125, + "F_prime_measured": 1.0728836059570312e-06, + "S": 0.0010986328125, + "PRIMARY": { + "statistic": "S recomputed after perturbing D1 by exactly F' with random signs", + "by_seed": { + "0": { + "S_pert": 0.0010986328125, + "unchanged": true, + "delta": 0.0 + }, + "1": { + "S_pert": 0.0010986328125, + "unchanged": true, + "delta": 0.0 + }, + "2": { + "S_pert": 0.0010986328125, + "unchanged": true, + "delta": 0.0 + }, + "3": { + "S_pert": 0.0010986328125, + "unchanged": true, + "delta": 0.0 + }, + "4": { + "S_pert": 0.0010986328125, + "unchanged": true, + "delta": 0.0 + } + }, + "all_seeds_unchanged": true, + "seeds": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + "MECHANISM_reported_not_adjudicated": { + "definition": "fraction of the 13,631,488 band entries whose fp16 image changes under +/- eps; an entry can only move if it lies within eps of a rounding boundary", + "by_eps": { + "1.000000e-08": { + "fraction_changed": 0.010738079364483174, + "n_changed": 146376, + "n_total": 13631488, + "is_F_prime": false + }, + "1.000000e-07": { + "fraction_changed": 0.08500986832838792, + "n_changed": 1158811, + "n_total": 13631488, + "is_F_prime": false + }, + "1.000000e-06": { + "fraction_changed": 0.44756280458890474, + "n_changed": 6100947, + "n_total": 13631488, + "is_F_prime": false + }, + "1.072884e-06": { + "fraction_changed": 0.46352701920729417, + "n_changed": 6318563, + "n_total": 13631488, + "is_F_prime": true + }, + "1.000000e-05": { + "fraction_changed": 0.9837102890014648, + "n_changed": 13409435, + "n_total": 13631488, + "is_F_prime": false + }, + "1.000000e-04": { + "fraction_changed": 0.999581190255972, + "n_changed": 13625779, + "n_total": 13631488, + "is_F_prime": false + }, + "4.882812e-04": { + "fraction_changed": 1.0, + "n_changed": 13631488, + "n_total": 13631488, + "is_F_prime": false + }, + "9.765625e-04": { + "fraction_changed": 1.0, + "n_changed": 13631488, + "n_total": 13631488, + "is_F_prime": false + } + } + }, + "controls": { + "Q1_S_recomputes": { + "required": "S recomputes from the persisted D1 to exactly the value the L40S run reported. If it does not, the object on disk is not what produced S.", + "S_recomputed": 0.0010986328125, + "S_reported_by_the_run": 0.0010986328125, + "exact": true, + "fires": true + }, + "Q2_perturbation_can_move_S": { + "required": "at eps = one full fp16 ULP (9.765625e-04) the recomputed S MUST change for at least one seed. A perturbation that never moves the answer cannot distinguish 'invariant' from 'inert'. IF Q2 DOES NOT FIRE, NO VERDICT ISSUES.", + "eps": 0.0009765625, + "by_seed": { + "0": { + "S_pert": 0.0020599365234375, + "changed": true + }, + "1": { + "S_pert": 0.0020751953125, + "changed": true + }, + "2": { + "S_pert": 0.0020751953125, + "changed": true + }, + "3": { + "S_pert": 0.00202178955078125, + "changed": true + }, + "4": { + "S_pert": 0.0020599365234375, + "changed": true + } + }, + "n_seeds_changed": 5, + "fires": true + }, + "Q3_ulp_constant": { + "required": "max|stored| in [1,2) so one fp16 step is 9.765625e-04", + "abs_max": 1.208984375, + "ulp": 0.0009765625, + "half_ulp": 0.00048828125, + "fires": true + }, + "Q4_inputs_are_recorded": { + "required": "the inputs are the recorded artifacts, asserted before measuring", + "D1_sha256": "00eebd6ddffff67532bb954e8d51674f0cd8d5021c0919d7139a942c607fff21", + "D1_sha256_expected": "00eebd6ddffff67532bb954e8d51674f0cd8d5021c0919d7139a942c607fff21", + "stored_sha256_first24": "4f00cc3d7450c3646b78f002", + "stored_sha256_expected_first24": "4f00cc3d7450c3646b78f002", + "D1_source_layers": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "stored_source_layers": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "D1_dtype": "float32", + "stored_dtype": "float16", + "fires": true + }, + "Q5_F_prime_is_measured": { + "required": "F' is the measured value read from the results file, not a chosen one, and it must sit below the fp16 half-ULP", + "F_prime": 1.0728836059570312e-06, + "F_prime_display": "1.072884e-06", + "source": "results/e66b_determinism_floor.json:F_prime_changed_reduction_order", + "fp16_half_ulp": 0.00048828125, + "F_prime_below_half_ulp": true, + "ratio_half_ulp_over_F_prime": 455.1111111111111, + "fires": true + } + }, + "controls_fired": { + "Q1": true, + "Q2": true, + "Q3": true, + "Q4": true, + "Q5": true + }, + "OUTCOME": "QUANTIZATION_LIMITED", + "VERDICT": "QUANTIZATION-LIMITED \u2014 perturbing D1 by exactly the measured device difference F' = 1.072884e-06, with random signs on all 5 seeds, leaves S bit-identical at 1.0986328125e-03. One fp16 step is 9.765625e-04, 455x the half-step above F', so fp16 rounding absorbs the kernel difference before S is computed. S's reproduction across the two runs is explained by rounding, not by the cards matching, and the completion spec's section 3 should be restated as bounded rather than absolute: S is comparable across devices for as long as the fitter's run-to-run difference stays below the fp16 rounding margin. THIS DOES NOT EXCLUDE that the 2026-08-14 box was also an L40S \u2014 that run recorded no GPU model and the question is unrecoverable. Sufficiency is established; exclusivity is not.", + "scope": "Establishes whether fp16 quantization SUFFICES to explain S's stability. It cannot exclude hardware coincidence: the 2026-08-14 run recorded no GPU model, which is E66b's own finding, and that question is unrecoverable.", + "provenance": { + "schema_version": 1, + "experiment": "E66d", + "utc": "2026-08-31T05:25:29+00:00", + "script": { + "path": "experiments/e66d_s_quantization_limited.py", + "sha256": "100431e1448223631cde76daed5d733ca09ed912f26bb2ec237afe3c3aa4c969", + "n_lines": 253 + }, + "git": { + "commit": "38b08b0de5897664f58271a5d12c879fa39a74d2", + "commit_short": "38b08b0de589", + "git_repository_present": true, + "branch": "pythia", + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": "38b08b0" + }, + "argv": [ + "experiments/e66d_s_quantization_limited.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/lenses/misc/e66_D1_refit_410m_pilecc_s0.pt", + "sha256": "00eebd6ddffff67532bb954e8d51674f0cd8d5021c0919d7139a942c607fff21", + "bytes": 54531448, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e66b_determinism_floor.json", + "sha256": "878e8441e3c21dbf1367a2a792217933f8c9c29421729cdad71a143c4d7b7a0a", + "bytes": 8452, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "b955114265f0fdca6cc09c8437aca496280c51c78fcaaf1b7a15648df0df65c9" + } +} \ No newline at end of file diff --git a/results/edata1_item_count_reconciliation.json b/results/edata1_item_count_reconciliation.json new file mode 100644 index 0000000000000000000000000000000000000000..dbf9a5626dfff512702e0f9e1e536b137a403315 --- /dev/null +++ b/results/edata1_item_count_reconciliation.json @@ -0,0 +1,51 @@ +{ + "experiment": "E-data1 \u2014 reconciling paper-reported vs released eval item counts", + "status": "EXPLORATORY \u2014 data audit, no model involved", + "source_paper": "RIGOROUS_ANTHROPIC.md sec A.6 (operator transcription of 2607.15495)", + "source_data": "anthropics/jacobian-lens @581d398 data/evaluations/*.json", + "per_set": { + "multihop": { + "released": 93, + "paper_reported": 50, + "ratio": 1.86, + "verdict": "~2x released" + }, + "multilingual": { + "released": 107, + "paper_reported": 54, + "ratio": 1.981, + "verdict": "~2x released" + }, + "order-ops": { + "released": 55, + "paper_reported": 55, + "ratio": 1.0, + "verdict": "EXACT MATCH" + }, + "poetry": { + "released": 98, + "paper_reported": 52, + "ratio": 1.885, + "verdict": "~2x released" + }, + "typo": { + "released": 96, + "paper_reported": 96, + "ratio": 1.0, + "verdict": "EXACT MATCH" + }, + "association": { + "released": 102, + "paper_reported": 50, + "ratio": 2.04, + "verdict": "~2x released" + } + }, + "finding": "Two sets (order-ops 55, typo 96) match the paper EXACTLY. The other four are 1.86-2.04x the paper's reported counts (mean 1.94). Most consistent explanation: the released sets were EXPANDED, roughly doubled, after the paper was written, for four of six sets; order-ops and typo were left at their published size.", + "rejected_explanations": [ + "tokenizer filtering: would not produce a near-uniform 2x, and would not leave two sets exact", + "subsampling for the figures: possible but would not so cleanly halve four sets and leave two intact", + "training/fitting divergence: impossible -- our fastfit is asserted bit-identical to jlens.fit" + ], + "decision": "USE THE RELEASED JSON. It is what the reference implementation consumes and what any replicator obtains. Consequence, disclosed: our absolute AUCs are NOT comparable to the paper's reported values. The J-vs-logit contrast is internal to our run and unaffected." +} \ No newline at end of file diff --git a/results/effrank1_adjudication.json b/results/effrank1_adjudication.json new file mode 100644 index 0000000000000000000000000000000000000000..ba601a47bda20d6cda95ca2725ba832661ba0a70 --- /dev/null +++ b/results/effrank1_adjudication.json @@ -0,0 +1,284 @@ +{ + "experiment": "EFFRANK1 \u2014 adjudication of eff_rank 371 (E38) vs 725.888 (E65)", + "prereg": "docs/experiments/preregs/EFFRANK1_effective_rank_adjudication.md", + "status": "PRE-REGISTERED", + "recomputes_not_remeasures": true, + "adjudicates": [ + "results/e38_jgeometry.json -> E39.*.eff_rank_I", + "results/e65_ckpt_geometry_410m.json -> by_checkpoint.step143000.eff_rank" + ], + "definitions": { + "er_energy90": "#{i : cumsum(sigma^2)_i / sum(sigma^2) < 0.90} + 1 [experiments/t38_jgeometry.py:167-169] INTEGER", + "er_entropy": "exp(-sum p log p), p = sigma / sum(sigma) [experiments/t65_ckpt_geometry.py:73-75] CONTINUOUS (Roy-Vetterli)", + "er_energy99": "as er_energy90 at 0.99 \u2014 reference only", + "er_stable": "||W||_F^2 / sigma_1^2 \u2014 reference only" + }, + "concept_token_set": "E39's 1310, via concept_token_ids over all six released eval sets", + "dtype": "float32", + "device": "cpu", + "by_model": { + "70m": { + "n": 1310, + "mean_cos": 0.9585258364677429, + "er_energy90": 1, + "er_energy99": 147, + "er_entropy": 199.05316162109375, + "er_stable": 1.0433372259140015, + "sigma1_energy_share": 0.9584628343582153, + "sigma_sum_sq": 12678.55078125, + "sigma_descending": true, + "sigma_top5": [ + 110.23574829101562, + 5.638315677642822, + 4.385774612426758, + 3.457092523574829, + 3.274916887283325 + ], + "model": "EleutherAI/pythia-70m-deduped", + "e38_stored_eff_rank_I": 1, + "e38_stored_mean_cos_I": 0.9585257768630981, + "er_energy90_reproduces_e38": true, + "mean_cos_abs_diff_vs_e38": 5.960464477539063e-08 + }, + "160m": { + "n": 1310, + "mean_cos": 0.921846866607666, + "er_energy90": 1, + "er_energy99": 293, + "er_entropy": 368.5517578125, + "er_stable": 1.0849395990371704, + "sigma1_energy_share": 0.9217102527618408, + "sigma_sum_sq": 17749.810546875, + "sigma_descending": true, + "sigma_top5": [ + 127.90692901611328, + 10.640669822692871, + 5.620233058929443, + 4.752574920654297, + 4.5224609375 + ], + "model": "EleutherAI/pythia-160m-deduped", + "e38_stored_eff_rank_I": 1, + "e38_stored_mean_cos_I": 0.921846866607666, + "er_energy90_reproduces_e38": true, + "mean_cos_abs_diff_vs_e38": 0.0 + }, + "410m": { + "n": 1310, + "mean_cos": 0.030666090548038483, + "er_energy90": 371, + "er_energy99": 719, + "er_entropy": 725.887939453125, + "er_stable": 18.293466567993164, + "sigma1_energy_share": 0.05466432496905327, + "sigma_sum_sq": 779.7222900390625, + "sigma_descending": true, + "sigma_top5": [ + 6.528628826141357, + 4.278520584106445, + 3.146270751953125, + 2.9240190982818604, + 2.8824281692504883 + ], + "model": "EleutherAI/pythia-410m-deduped", + "e38_stored_eff_rank_I": 371, + "e38_stored_mean_cos_I": 0.030666086822748184, + "er_energy90_reproduces_e38": true, + "mean_cos_abs_diff_vs_e38": 3.725290298461914e-09 + }, + "1b": { + "n": 1310, + "mean_cos": 0.019916437566280365, + "er_energy90": 508, + "er_energy99": 984, + "er_entropy": 969.3308715820312, + "er_stable": 23.305429458618164, + "sigma1_energy_share": 0.042908456176519394, + "sigma_sum_sq": 1206.3779296875, + "sigma_descending": true, + "sigma_top5": [ + 7.19470739364624, + 3.854170083999634, + 3.3111112117767334, + 3.2431817054748535, + 3.1600584983825684 + ], + "model": "EleutherAI/pythia-1b-deduped", + "e38_stored_eff_rank_I": 508, + "e38_stored_mean_cos_I": 0.019916435703635216, + "er_energy90_reproduces_e38": true, + "mean_cos_abs_diff_vs_e38": 1.862645149230957e-09 + } + }, + "e65_stored_410m_final": { + "mean_cos": 0.030666090548038483, + "eff_rank": 725.887939453125 + }, + "PRIMARY": { + "e38_energy90_reproduces_all_models": true, + "e65_entropy_reproduces_410m": true, + "e65_entropy_abs_diff": 0.0 + }, + "controls": { + "C1_same_object": { + "required": "mean_cos abs_diff <= 1e-6 vs E38 at all four models; n == 1310 at all four", + "max_mean_cos_abs_diff": 5.960464477539063e-08, + "n_concept_tokens": { + "70m": 1310, + "160m": 1310, + "410m": 1310, + "1b": 1310 + }, + "fires": true + }, + "C2_definitions_not_equal": { + "required": "er_entropy - er_energy90 > 1 at 410M", + "observed": 354.887939453125, + "fires": true + }, + "C3_rank_one_is_definitional": { + "required": "er_entropy(70M) > 1 where E38 stores eff_rank_I = 1", + "observed_er_entropy_70m": 199.05316162109375, + "observed_er_energy90_70m": 1, + "fires": true + }, + "C4_spectrum_integrity": { + "required": "sum(sigma^2) > 0 and sigma descending at every model", + "fires": true + } + }, + "controls_fired": { + "C1_same_object": true, + "C2_definitions_not_equal": true, + "C3_rank_one_is_definitional": true, + "C4_spectrum_integrity": true + }, + "branch": "NAME COLLISION", + "VERDICT": "NAME COLLISION \u2014 both stored numbers are correct and they are DIFFERENT STATISTICS of the same matrix. E38's `eff_rank_I` is the 90%-energy participation count (#{i: cumsum(sigma^2)/sum < 0.90} + 1), reproduced EXACTLY at all four models (70m/160m/410m/1b = 1/1/371/508). E65's `eff_rank` is the Roy-Vetterli entropy rank exp(-sum p log p), p = sigma/sum(sigma), reproduced at 410M to abs_diff 0.000e+00. Neither file is defective; the field NAME is. The paper's sentence 'The effective ranks are 1, 1, 371 and 508' is TRUE of the 90%-energy definition and must name it, because under the entropy definition the same four matrices give 199.1/368.6/725.9/969.3 \u2014 and in particular the 'rank-one' reading at 70M/160M is a property of the 90% cutoff, not of the matrix.", + "paper_claim_under_adjudication": { + "location": "paper/audit_paper_5pp.tex:693", + "text": "The effective ranks are 1, 1, 371 and 508.", + "energy90_values": [ + 1, + 1, + 371, + 508 + ], + "entropy_values": [ + 199.05316162109375, + 368.5517578125, + 725.887939453125, + 969.3308715820312 + ] + }, + "provenance": { + "schema_version": 1, + "experiment": "EFFRANK1 \u2014 adjudication of eff_rank 371 (E38) vs 725.888 (E65)", + "utc": "2026-09-01T20:26:07+00:00", + "script": { + "path": "experiments/effrank1_adjudication.py", + "sha256": "159c599971cb490a04dc61d650628498499dd02ba9ed5992d9c15b68011c3ffa", + "n_lines": 200 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "corpora/dataset_revisions.json", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/EFFRANK1_effective_rank_adjudication.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py" + ], + "dirty_diff_sha256": "83b3848cce07cd3ed221dd6ef145216d8164aee882acecaa2c67ded074747252", + "describe": "9987d82-dirty" + }, + "argv": [ + "experiments/effrank1_adjudication.py" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "cd592c0fc10cfcfa6b8065b5fe956de625a73484e29f4b83cf4fa68c4d5a774a" + } +} \ No newline at end of file diff --git a/results/en1_operator_error_at_n200.json b/results/en1_operator_error_at_n200.json new file mode 100644 index 0000000000000000000000000000000000000000..e76fbd5dd2710c61cf0cc373d0fd5a1b772ff26b --- /dev/null +++ b/results/en1_operator_error_at_n200.json @@ -0,0 +1,70 @@ +{ + "experiment": "E-n1 \u2014 operator estimation error at n=200, from measured dispersion", + "status": "EXPLORATORY \u2014 analytic, derived from stored dispersion; no model run", + "rule": "E||M_hat_n - M||_F^2 / ||M||_F^2 = disp/n => eps = sqrt(disp/n); n >= disp/eps^2", + "source": "external review B1 response (PROMPTS.md section 2, R-B1), executive answer item 2", + "n": 200, + "per_model": { + "1.4b": { + "n_layers": 24, + "disp_L0": 2.897373, + "eps_L0_at_200": 0.1204, + "eps_mid_at_200": 0.0503, + "eps_last_at_200": 0.0143, + "n_for_5pct_at_L0": 1159, + "n_for_10pct_at_L0": 290 + }, + "160m": { + "n_layers": 12, + "disp_L0": 0.641485, + "eps_L0_at_200": 0.0566, + "eps_mid_at_200": 0.0374, + "eps_last_at_200": 0.0179, + "n_for_5pct_at_L0": 257, + "n_for_10pct_at_L0": 64 + }, + "1b": { + "n_layers": 16, + "disp_L0": 1.73571, + "eps_L0_at_200": 0.0932, + "eps_mid_at_200": 0.0502, + "eps_last_at_200": 0.0137, + "n_for_5pct_at_L0": 694, + "n_for_10pct_at_L0": 174 + }, + "2.8b": { + "n_layers": 32, + "disp_L0": 4.418416, + "eps_L0_at_200": 0.1486, + "eps_mid_at_200": 0.0442, + "eps_last_at_200": 0.0079, + "n_for_5pct_at_L0": 1767, + "n_for_10pct_at_L0": 442 + }, + "410m": { + "n_layers": 24, + "disp_L0": 2.364437, + "eps_L0_at_200": 0.1087, + "eps_mid_at_200": 0.053, + "eps_last_at_200": 0.0106, + "n_for_5pct_at_L0": 946, + "n_for_10pct_at_L0": 236 + }, + "70m": { + "n_layers": 6, + "disp_L0": 0.229049, + "eps_L0_at_200": 0.0338, + "eps_mid_at_200": 0.0246, + "eps_last_at_200": 0.0169, + "n_for_5pct_at_L0": 92, + "n_for_10pct_at_L0": 23 + } + }, + "finding": "n=200 is NOT uniformly adequate. Layer-0 relative RMS Frobenius error runs 3.4% (70m) to 14.9% (2.8b) and grows with DEPTH, tracking dispersion. Late layers are fine (<1% at 2.8b). Reaching 5% error at 2.8b layer 0 would need n~1767 -- which retroactively justifies the anchor's n=1000 and shows our n=200 is marginal at the large end.", + "caveat": "This is a second-moment expectation for FROBENIUS error only. It is NOT a high-probability guarantee, NOT an operator-norm bound, and NOT valid for singular-subspace error or under heavy tails -- all flagged in the same source.", + "provenance": { + "payload_sha256_before_redaction": "df1cefcb1980fb4f08aff1b886a6ad86e2d032019aa4ec32f101f606b5759604", + "payload_sha256": "dfcf84ae7bcabd3079e37339fb380dafb912bda792f6d90a32a237a09c6f7963", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/fit_corpus_manifest_wikitext_n200.json b/results/fit_corpus_manifest_wikitext_n200.json new file mode 100644 index 0000000000000000000000000000000000000000..ad83ff2fe73ebe2b3f73fef9fc3c2b7e64388220 --- /dev/null +++ b/results/fit_corpus_manifest_wikitext_n200.json @@ -0,0 +1,221 @@ +{ + "purpose": "byte-level pin of the fitting corpus (RIGOR_SKILL axis 2)", + "corpus_arg": "wikitext", + "loader": "pythia/t2_fastfit.py:load_prompts", + "selection_rule": "first 200 streamed rows with len(text.strip()) >= 400 and not startswith('=')", + "hf_dataset": "Salesforce/wikitext:wikitext-103-raw-v1:train", + "hf_dataset_revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "datasets_version": "5.0.0", + "n_prompts": 200, + "sha256_concat": "1692366fba69257cc9a9aab4c055d4c0b40a0a003511e90363219cf527f10267", + "sha256_per_prompt_first12": [ + "0cf317d42146", + "b442045cfed4", + "1b82104b123b", + "ea1a652f366d", + "4d230f158de7", + "a16a7f374e95", + "d61dd16abe49", + "e81080034610", + "3ea158e7586e", + "6dc39aa29367", + "8d27c255bb89", + "913d9aaa81dc", + "9f66a1e0f0ba", + "3e1a4858f13d", + "0b7c2ad63a70", + "4b42e093b870", + "c00c1eaf603e", + "2c1d520e29a6", + "e91f8dab1ae9", + "7b6852cf95ba", + "db3716c1636b", + "f4b130feaf87", + "09ad2d35f0ba", + "cd8dcf875a2e", + "05d46db19bd8", + "a7d2675f1525", + "8c1829aaaf2d", + "118cdbf0d6af", + "34b01bf069ed", + "c4c009549a48", + "d52ff0ad702c", + "391f2753ecfe", + "1fcd914b15b8", + "43f7190f2a58", + "54e0b9b3254a", + "4128474319d2", + "cb25b5461084", + "1e7b9635596d", + "e87ad93bc5eb", + "98e4405d243e", + "e9d778779bf7", + "40c8c9e7d899", + "b603fd7b8906", + "e624c9a501ad", + "7ea94e882bfa", + "ab685ae2fd40", + "4f99315d375a", + "f416e2b71342", + "3197dcfd3b51", + "cdbf2dbf0c89", + "6e08d7ba3641", + "337b6d9018e9", + "d407985b994a", + "d7d442112afa", + "c47e891bc161", + "f9c79aad2b6c", + "9412d57bd0b2", + "c4b278d4f1a8", + "1ceb7f7d9cfd", + "bc554703cb81", + "7e8e556147d0", + "394b78f383af", + "bd1f731b0873", + "1cd906b65eb9", + "a8973a34a446", + "fba4b34f48a7", + "9a361e7af55f", + "af4076e1d1aa", + "01b8ae652288", + "b718e79ae12a", + "89235a56c1ec", + "115aaaec4403", + "9afa42a86f74", + "8b33a0900db7", + "37c5708bede8", + "59986807b7cd", + "d99d922b6376", + "a7b6dfa56377", + "d6385ec3c19b", + "f276cb523481", + "466701ab89cf", + "a4206cc88d83", + "957b4fa2c67e", + "6dc54c2b781d", + "1cef4d8e5361", + "79de4aa1b404", + "95bb6cdcc056", + "e73946eb2da5", + "a1509871363f", + "36b9a9b719cf", + "d08fc11fb61f", + "400c56096d25", + "9677544fd35e", + "c9c1acf68c6c", + "7aa27cd20db4", + "76ae3eb1786f", + "9d55d7add226", + "47b4378ca848", + "510ae8a5ba1e", + "362c58cb3008", + "eb4dc2caf60e", + "193bb61279d2", + "56970825b2ed", + "115869f8be2d", + "9e2851cf7440", + "fd426f61b9fb", + "531aa5ced61b", + "a66e730dec8b", + "c2ff58dc733c", + "25070d2494fa", + "97299dce6cb0", + "0487a02c6c2b", + "dcc2f82b4b1d", + "f80e642ced8a", + "93f36a27805c", + "60edacc93d08", + "cd43d4f8416d", + "9a3c7edc5506", + "8a7e5cb5285c", + "11da8d0d07a6", + "c90c248c5ac4", + "7ae4b6dbbd74", + "429568e361ee", + "6a0d84e7e660", + "1cfce7c9194c", + "fa4a93e168ee", + "76df28c245ab", + "3de3611adcc0", + "3d20268ee33b", + "4972975de56e", + "2ced3f3bec91", + "d9be4b17761e", + "29bf11834ef1", + "5cb97e1ed383", + "1c5017614788", + "706a5638cdd2", + "18a4c9a066b8", + "d032126bdffb", + "96261eeeb5e5", + "17c0f0e3c81c", + "49d4acfca2b7", + "0e00c18cc56d", + "1e11d1c2d991", + "f93ab4a33c53", + "255a45801094", + "1f0719211fa4", + "86e00bf1b109", + "55eff2c9a112", + "1fa82af508ea", + "c6820caf32db", + "333d16bd2e10", + "af87dbebee07", + "55e5fbbf04ed", + "3acaaceac968", + "a9b5d049514f", + "90d7a9e4105f", + "8c1f2fccab12", + "cc9279183bc2", + "6e1a1d076cd2", + "379136fae3ba", + "a535cb1e1608", + "7a7eabb28c6f", + "d5a3a051dfd9", + "03f28fb35b0a", + "e667d492dfcd", + "29605506fe8a", + "fa05d3b5660f", + "5afa9f566664", + "6e8c6e1c2f3a", + "1697b267d180", + "93262b1218e7", + "0fd1dc34ff9a", + "0412f00c66c9", + "6d047b6a870e", + "4d265c19ac8b", + "5994fa0bcaac", + "0fd7804b2c94", + "4efde498274f", + "a622255cd588", + "3f60c4243f14", + "6c958d02a2b6", + "5c226d59ea38", + "eb2ed8cb9573", + "2163922cce94", + "ce6c2b9dfeaa", + "1f6b1f1ff6bb", + "42fbd9c80a1e", + "3b8ce40debd2", + "8ed5923b806a", + "ab672fc0fda2", + "2af0c0111674", + "f482a9f59edb", + "d566ff6897e2", + "11d10923f345", + "4448885501e6", + "8d419bed572e", + "766a64b68e44", + "7ec67b3e8076", + "171c01b5e2d7", + "e825cf59a8d2" + ], + "chars_min_median_max": [ + 402, + 755, + 2113 + ], + "first_prompt_head": "Senj\u014d no Valkyria 3 : Unrecorded Chronicles ( Japanese : \u6226\u5834\u306e\u30f4\u30a1\u30eb\u30ad\u30e5\u30ea\u30a23 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ pl", + "last_prompt_head": "Certain features of divine images are more useful than others in determining a god 's identity . The head of a given divine image is particularly significant . In a hybrid image , the head represents ", + "runtime_s": 7.4 +} \ No newline at end of file diff --git a/results/found_shard0.tgz b/results/found_shard0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..b1ea26293a15c29a9fcb34624ed536826ca20d91 --- /dev/null +++ b/results/found_shard0.tgz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e804ce17a7ccec9c91c8839d1a447be723708c7925d49efc65711c0451b6e08 +size 12967246 diff --git a/results/ladder1b/tv_Github_s0.json b/results/ladder1b/tv_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..33dae0f1845dcf5e0a4dcab9487ce31db635845b --- /dev/null +++ b/results/ladder1b/tv_Github_s0.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Github", + "seed": 0, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17337031715682574, + "best1L": 0.15117891771452768, + "mean": 0.04207119744803224, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13850616344383784, + "best1L": 0.09535895447645869, + "mean": 0.024977842878018106, + "persist": 0.004350978881120682 + }, + "order-ops": { + "min": 0.02597402555069753, + "best1L": 0.014285714365541935, + "mean": 0.0036075035708823372, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408163645437787, + "best1L": 0.013119533391935485, + "mean": 0.003239391038992575, + "persist": 0.0 + }, + "typo": { + "min": 0.33928571322134565, + "best1L": 0.2351190522313118, + "mean": 0.15228174440562725, + "persist": 0.16220238299242087 + }, + "association": { + "min": 0.015527949801513128, + "best1L": 0.00776397529989481, + "mean": 0.0022429261090500014, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7697917099813595, + "dispersion_mean": 0.365344978520504, + "dispersion_slope": -0.09204706186487215, + "stable_rank": 39.81596755981445, + "top8_spectral_share": 0.06747092306613922, + "condition_number": 2708.509521484375, + "unembed_alignment": 0.05466344952583313 + }, + "corpus": { + "mean_prompt_tokens": 1392.0, + "type_token_ratio": 0.13376436781609197, + "eval_vocab_overlap": 0.1183206106870229, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7697917099813595, + "6": 0.638792522009224, + "7": 0.541402978456402, + "8": 0.44761461318612317, + "9": 0.3408533021705676, + "10": 0.26264247050565753, + "11": 0.15736812207319095, + "12": 0.09279380370037878, + "13": 0.036845284601633006 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17059639309133803, + "best1L": 0.151178916650159, + "mean": 0.0405301284044981, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13125453277357987, + "best1L": 0.09173313927437578, + "mean": 0.023164934079561914, + "persist": 0.0036258158673133168 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.014285714365541935, + "mean": 0.002453102497383952, + "persist": 0.0 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.010204081423580647, + "mean": 0.002915451968354838, + "persist": 0.0 + }, + "typo": { + "min": 0.29761905542441774, + "best1L": 0.23809523774044855, + "mean": 0.10466269643179008, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.012422359814601285, + "best1L": 0.00776397529989481, + "mean": 0.002415458971102323, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8197675653782605, + "dispersion_mean": 0.3892019662479752, + "dispersion_slope": -0.09800578627978401, + "stable_rank": 39.4859504699707, + "top8_spectral_share": 0.06885477900505066, + "condition_number": 1664.815185546875, + "unembed_alignment": 0.048217322677373886 + }, + "corpus": { + "mean_prompt_tokens": 1314.32, + "type_token_ratio": 0.09973218089962871, + "eval_vocab_overlap": 0.18854961832061068, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8197675653782605, + "6": 0.6796569956304669, + "7": 0.5771295834460367, + "8": 0.47746522684683596, + "9": 0.3640155788886007, + "10": 0.27775910932173514, + "11": 0.16857496767353078, + "12": 0.09928525549772702, + "13": 0.03916341354858342 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15395284231219972, + "mean": 0.04130066292626517, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13234227815909044, + "best1L": 0.09318346503589835, + "mean": 0.023849810806236098, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.019480518863669465, + "best1L": 0.014285714365541935, + "mean": 0.0023088023133043733, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.00874635563897235, + "mean": 0.0027534824164052096, + "persist": 0.0 + }, + "typo": { + "min": 0.3184523784688541, + "best1L": 0.23809523774044855, + "mean": 0.13376322628131934, + "persist": 0.1354166671101536 + }, + "association": { + "min": 0.012422359814601285, + "best1L": 0.00776397529989481, + "mean": 0.002070393313520721, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.825938455682667, + "dispersion_mean": 0.3926234371032556, + "dispersion_slope": -0.09901833463897901, + "stable_rank": 44.38139724731445, + "top8_spectral_share": 0.06547916680574417, + "condition_number": 1286.9332275390625, + "unembed_alignment": 0.05135410279035568 + }, + "corpus": { + "mean_prompt_tokens": 1159.04, + "type_token_ratio": 0.08555356156819437, + "eval_vocab_overlap": 0.26641221374045804, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.825938455682667, + "6": 0.6852853534672042, + "7": 0.5839163985991469, + "8": 0.48362634514198133, + "9": 0.3676195176218178, + "10": 0.28083807395117094, + "11": 0.16846980902102868, + "12": 0.09947666663654608, + "13": 0.03844031380773713 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16782246742929732, + "best1L": 0.15117891771452768, + "mean": 0.04176298342645168, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13560551112251623, + "best1L": 0.09463379079742092, + "mean": 0.02461526043979185, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.00874635563897235, + "mean": 0.0030774214870429467, + "persist": 0.0 + }, + "typo": { + "min": 0.32589285182101385, + "best1L": 0.23660713966403688, + "mean": 0.1407076721744878, + "persist": 0.14583333182547772 + }, + "association": { + "min": 0.013975154874580247, + "best1L": 0.00776397529989481, + "mean": 0.0024154589378408025, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.829551404376573, + "dispersion_mean": 0.3935487420827375, + "dispersion_slope": -0.09964724652228288, + "stable_rank": 45.77293014526367, + "top8_spectral_share": 0.06492404639720917, + "condition_number": 1642.5162353515625, + "unembed_alignment": 0.05251605808734894 + }, + "corpus": { + "mean_prompt_tokens": 1489.08, + "type_token_ratio": 0.06120109956035494, + "eval_vocab_overlap": 0.33129770992366414, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.829551404376573, + "6": 0.6870060978976706, + "7": 0.5870050828617581, + "8": 0.48689493398340533, + "9": 0.3674713896033034, + "10": 0.2808774120278359, + "11": 0.1668150030350979, + "12": 0.09848261836474972, + "13": 0.037834736594242885 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.171983360179833, + "best1L": 0.15256587948117936, + "mean": 0.04237941200179713, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13741841619568212, + "best1L": 0.09608411655894347, + "mean": 0.025018129298197373, + "persist": 0.004350978881120682 + }, + "order-ops": { + "min": 0.02337662343467985, + "best1L": 0.01688311674765178, + "mean": 0.0027417026992355075, + "persist": 0.0 + }, + "poetry": { + "min": 0.021865889163953916, + "best1L": 0.00874635563897235, + "mean": 0.0032393909891002943, + "persist": 0.0 + }, + "typo": { + "min": 0.32738095202616285, + "best1L": 0.23958333155938558, + "mean": 0.14236110874584743, + "persist": 0.14880952372082643 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0020703934133052826, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.834545754739069, + "dispersion_mean": 0.39604841832524706, + "dispersion_slope": -0.10023466117491917, + "stable_rank": 46.405799865722656, + "top8_spectral_share": 0.0655001550912857, + "condition_number": 2378.83349609375, + "unembed_alignment": 0.05375915765762329 + }, + "corpus": { + "mean_prompt_tokens": 1455.92, + "type_token_ratio": 0.055470080773668884, + "eval_vocab_overlap": 0.383206106870229, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.834545754739069, + "6": 0.6912912507037452, + "7": 0.591049886884231, + "8": 0.4891070530047572, + "9": 0.37026121265548023, + "10": 0.2826853997968901, + "11": 0.16810146256050448, + "12": 0.09927937590250281, + "13": 0.03811436868004338 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.16920943558216095, + "best1L": 0.15256587948117936, + "mean": 0.041917091235518456, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1363306718745402, + "best1L": 0.09535895367818219, + "mean": 0.024494399782270193, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.00874635563897235, + "mean": 0.002915451901831797, + "persist": 0.0 + }, + "typo": { + "min": 0.31547619083098005, + "best1L": 0.23511904584509985, + "mean": 0.13161375879177026, + "persist": 0.135416669371937 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.00776397529989481, + "mean": 0.0022429261090500014, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8362474421635532, + "dispersion_mean": 0.39527312432210243, + "dispersion_slope": -0.10030552792232653, + "stable_rank": 44.77740478515625, + "top8_spectral_share": 0.06598934531211853, + "condition_number": 2038.1468505859375, + "unembed_alignment": 0.05288536846637726 + }, + "corpus": { + "mean_prompt_tokens": 2064.4333333333334, + "type_token_ratio": 0.035418920446288735, + "eval_vocab_overlap": 0.433587786259542, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8362474421635532, + "6": 0.690390377949279, + "7": 0.5893715311756591, + "8": 0.48754770755251103, + "9": 0.36862075768365754, + "10": 0.28127553773686975, + "11": 0.1672722099737706, + "12": 0.09863017927260972, + "13": 0.038102375391012126 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1650485417672566, + "best1L": 0.14840499205248697, + "mean": 0.04130066266017301, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13451776467263699, + "best1L": 0.09427120935704027, + "mean": 0.024212391847478493, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.015584415623119898, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437594737326, + "best1L": 0.008746355505926269, + "mean": 0.0027534822999898878, + "persist": 0.0 + }, + "typo": { + "min": 0.30505952877657755, + "best1L": 0.2336309477686882, + "mean": 0.12119709088334016, + "persist": 0.11904761953545469 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0020703934133052826, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8485112169400966, + "dispersion_mean": 0.40083040089063127, + "dispersion_slope": -0.10159236809719534, + "stable_rank": 42.66108703613281, + "top8_spectral_share": 0.06666543334722519, + "condition_number": 2414.041259765625, + "unembed_alignment": 0.0514703169465065 + }, + "corpus": { + "mean_prompt_tokens": 2232.67, + "type_token_ratio": 0.030734501740069066, + "eval_vocab_overlap": 0.48778625954198473, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8485112169400966, + "6": 0.6994017295777756, + "7": 0.5968603133211917, + "8": 0.4932405236658103, + "9": 0.37379939409730256, + "10": 0.28578738871382836, + "11": 0.1705337147764044, + "12": 0.10054300498953923, + "13": 0.03879632193373284 + } + } + }, + "runtime_s": 1148.5, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Github_s1.json b/results/ladder1b/tv_Github_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..7d95fde5e13da1cbc956ca68a2c310f9105028fc --- /dev/null +++ b/results/ladder1b/tv_Github_s1.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Github", + "seed": 1, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.16782246849366597, + "best1L": 0.15533980833632605, + "mean": 0.039605485807572095, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12835387832352094, + "best1L": 0.08919506786125046, + "mean": 0.021754894671695574, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.029870130520846163, + "best1L": 0.015584415889212064, + "mean": 0.0036075035043592963, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.00874635563897235, + "mean": 0.003401360607572964, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3065476194024086, + "best1L": 0.23214286140033177, + "mean": 0.12185846508613654, + "persist": 0.1264880940850292 + }, + "association": { + "min": 0.02173913030752114, + "best1L": 0.006211180239915848, + "mean": 0.0032781229154872043, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.825677622701321, + "dispersion_mean": 0.4035518173423774, + "dispersion_slope": -0.09896070430535091, + "stable_rank": 45.56559371948242, + "top8_spectral_share": 0.06536078453063965, + "condition_number": 2283.919677734375, + "unembed_alignment": 0.05694640427827835 + }, + "corpus": { + "mean_prompt_tokens": 5978.6, + "type_token_ratio": 0.05735456461378918, + "eval_vocab_overlap": 0.20381679389312976, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.825677622701321, + "6": 0.6872710922207388, + "7": 0.5995495858694184, + "8": 0.5096057599985049, + "9": 0.3853272260851954, + "10": 0.300548624348268, + "11": 0.17740996520842406, + "12": 0.10608803247943373, + "13": 0.04048844717009244 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.1705963984131813, + "best1L": 0.15256587948117936, + "mean": 0.040684234617011886, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13778099790215492, + "best1L": 0.09390862791665963, + "mean": 0.02401095795046006, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.02597402555069753, + "best1L": 0.014285714365541935, + "mean": 0.003030303100656186, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985094198158, + "best1L": 0.007288629721317973, + "mean": 0.0029154518519395162, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3184523805975914, + "best1L": 0.23809523774044855, + "mean": 0.12533068949622767, + "persist": 0.12648809687899693 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.00776397529989481, + "mean": 0.0022429261090500014, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8287467528978196, + "dispersion_mean": 0.39365907357497737, + "dispersion_slope": -0.0993234166960348, + "stable_rank": 47.26062774658203, + "top8_spectral_share": 0.06629198789596558, + "condition_number": 1996.5281982421875, + "unembed_alignment": 0.05553808435797691 + }, + "corpus": { + "mean_prompt_tokens": 3429.04, + "type_token_ratio": 0.05325105568905583, + "eval_vocab_overlap": 0.2580152671755725, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8287467528978196, + "6": 0.681661214217127, + "7": 0.5865190101944758, + "8": 0.48890627314154944, + "9": 0.3705638880510327, + "10": 0.2834219290069629, + "11": 0.1674148342062547, + "12": 0.09853269324669577, + "13": 0.037165067212878405 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.1692094345177923, + "best1L": 0.15117891771452768, + "mean": 0.04207119798021657, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13741841832441942, + "best1L": 0.09354604647627898, + "mean": 0.024736121097313508, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.015584415490073817, + "mean": 0.002597402548417449, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437594737326, + "best1L": 0.00874635563897235, + "mean": 0.0032393909891002943, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30357142857142855, + "best1L": 0.23809523933700152, + "mean": 0.11375661433807441, + "persist": 0.11160714272409678 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0018978606177760021, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.9009888055780427, + "dispersion_mean": 0.4285164732721299, + "dispersion_slope": -0.10785655967761855, + "stable_rank": 44.32135009765625, + "top8_spectral_share": 0.06946738809347153, + "condition_number": 2011.893798828125, + "unembed_alignment": 0.053823117166757584 + }, + "corpus": { + "mean_prompt_tokens": 2819.96, + "type_token_ratio": 0.05382345848877289, + "eval_vocab_overlap": 0.34427480916030534, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.9009888055780427, + "6": 0.7448503028554948, + "7": 0.6389841524984788, + "8": 0.5296506534668731, + "9": 0.3991964985042581, + "10": 0.30603291257020365, + "11": 0.18455244953464248, + "12": 0.10997624071881547, + "13": 0.042416243722359685 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15256587735244206, + "mean": 0.04160887614956924, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13741841619568212, + "best1L": 0.09463379079742092, + "mean": 0.024494400514023646, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437062552998, + "best1L": 0.008746355505926269, + "mean": 0.0030774214870429467, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.24107143123235023, + "mean": 0.11822089898799147, + "persist": 0.12351190604801689 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0017253277889852012, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8896468148550352, + "dispersion_mean": 0.42328242690130247, + "dispersion_slope": -0.1064503900794756, + "stable_rank": 44.692657470703125, + "top8_spectral_share": 0.06912394613027573, + "condition_number": 1911.12646484375, + "unembed_alignment": 0.05373576283454895 + }, + "corpus": { + "mean_prompt_tokens": 2592.28, + "type_token_ratio": 0.05032892537328786, + "eval_vocab_overlap": 0.38091603053435114, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8896468148550352, + "6": 0.7356475088447318, + "7": 0.6303981059964792, + "8": 0.5217757466821866, + "9": 0.396156228046547, + "10": 0.3028537583397088, + "11": 0.18263175377248114, + "12": 0.10876662832193695, + "13": 0.041665297252615655 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.17198335698672704, + "best1L": 0.15395284018346242, + "mean": 0.04176298342645168, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13705583475530148, + "best1L": 0.09499637223780155, + "mean": 0.024574974019612585, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437594737326, + "best1L": 0.00874635563897235, + "mean": 0.0030774214870429467, + "persist": 0.0 + }, + "typo": { + "min": 0.30505952132599695, + "best1L": 0.24553571694663592, + "mean": 0.11574074333267552, + "persist": 0.1145833335550768 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0017253277889852012, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.9020600244666921, + "dispersion_mean": 0.42699671699165864, + "dispersion_slope": -0.1079612006023313, + "stable_rank": 43.7761116027832, + "top8_spectral_share": 0.06924284994602203, + "condition_number": 1832.1591796875, + "unembed_alignment": 0.0527254119515419 + }, + "corpus": { + "mean_prompt_tokens": 2471.41, + "type_token_ratio": 0.04500265030893296, + "eval_vocab_overlap": 0.42290076335877863, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.9020600244666921, + "6": 0.7439706601877969, + "7": 0.6359113522632066, + "8": 0.5263109675434047, + "9": 0.39780345412466694, + "10": 0.30449100733201717, + "11": 0.18263403196666392, + "12": 0.10830117706716773, + "13": 0.04148777797331266 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15256587735244206, + "mean": 0.04207119744803224, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13959390510405814, + "best1L": 0.09608411655894347, + "mean": 0.025259850347148522, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.01749271181012903, + "best1L": 0.008746355505926269, + "mean": 0.0027534824330359697, + "persist": 0.0 + }, + "typo": { + "min": 0.31250000106436865, + "best1L": 0.24107143176453455, + "mean": 0.12797619216144085, + "persist": 0.1309523820610983 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0017253277889852012, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8801329360078055, + "dispersion_mean": 0.41669253429388753, + "dispersion_slope": -0.10536700115661828, + "stable_rank": 45.22291564941406, + "top8_spectral_share": 0.06768527626991272, + "condition_number": 2298.35595703125, + "unembed_alignment": 0.05354642868041992 + }, + "corpus": { + "mean_prompt_tokens": 2525.48, + "type_token_ratio": 0.036734930917422956, + "eval_vocab_overlap": 0.49389312977099237, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8801329360078055, + "6": 0.7267599189941903, + "7": 0.6213164784973207, + "8": 0.5122947137612127, + "9": 0.3880963584816266, + "10": 0.2969945374560969, + "11": 0.1778959524617044, + "12": 0.10603499194707645, + "13": 0.04070692103795387 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15256587735244206, + "mean": 0.04207119744803224, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13886874222329684, + "best1L": 0.09572153511856284, + "mean": 0.025018129298197373, + "persist": 0.004350978881120682 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.008746355505926269, + "mean": 0.003077421437150666, + "persist": 0.0 + }, + "typo": { + "min": 0.31250000106436865, + "best1L": 0.24107142963579722, + "mean": 0.13326719483094557, + "persist": 0.13839285913854837 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0017253277889852012, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8815098107363234, + "dispersion_mean": 0.41710576644373903, + "dispersion_slope": -0.1055298457879471, + "stable_rank": 45.488460540771484, + "top8_spectral_share": 0.06723397970199585, + "condition_number": 2180.39501953125, + "unembed_alignment": 0.05372513830661774 + }, + "corpus": { + "mean_prompt_tokens": 2467.21, + "type_token_ratio": 0.031033029211133222, + "eval_vocab_overlap": 0.5282442748091603, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8815098107363234, + "6": 0.7277773218430262, + "7": 0.621885663647247, + "8": 0.5126854465902889, + "9": 0.38828278086489976, + "10": 0.2970092036084862, + "11": 0.1780036190250146, + "12": 0.10617141328964594, + "13": 0.04062663838871911 + } + } + }, + "runtime_s": 1171.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Github_s2.json b/results/ladder1b/tv_Github_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..9d5b856aef1ca07cb65935fee29a8e9dadb8f748 --- /dev/null +++ b/results/ladder1b/tv_Github_s2.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Github", + "seed": 2, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17059639309133803, + "best1L": 0.14563106532607759, + "mean": 0.04068423568138054, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1381435790764434, + "best1L": 0.09028281244848456, + "mean": 0.02344694228044578, + "persist": 0.0032632341608405113 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.014285714365541935, + "mean": 0.002597402548417449, + "persist": 0.0 + }, + "poetry": { + "min": 0.011661807740373271, + "best1L": 0.005830903670617512, + "mean": 0.002105604108822133, + "persist": 0.0 + }, + "typo": { + "min": 0.3110119051166943, + "best1L": 0.2470238134264946, + "mean": 0.12483465591711658, + "persist": 0.1250000007982765 + }, + "association": { + "min": 0.013975154874580247, + "best1L": 0.006211180106869766, + "mean": 0.0022429261090500014, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7206122645909392, + "dispersion_mean": 0.34635912168758914, + "dispersion_slope": -0.08607206486064055, + "stable_rank": 38.33663558959961, + "top8_spectral_share": 0.06750618666410446, + "condition_number": 2180.296630859375, + "unembed_alignment": 0.05314507335424423 + }, + "corpus": { + "mean_prompt_tokens": 4178.2, + "type_token_ratio": 0.07445790053132928, + "eval_vocab_overlap": 0.1801526717557252, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7206122645909392, + "6": 0.5963512718660686, + "7": 0.514949631393735, + "8": 0.4306331158648266, + "9": 0.3274143897769244, + "10": 0.25044438593059126, + "11": 0.15021200975037216, + "12": 0.0896171485148908, + "13": 0.03699787749995442 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.16920943238905498, + "best1L": 0.14979195381913865, + "mean": 0.04145476993705545, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13234227603035315, + "best1L": 0.09100797639361449, + "mean": 0.02284263912588358, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.014285714365541935, + "mean": 0.002453102497383952, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533391935485, + "best1L": 0.008746355505926269, + "mean": 0.0024295433457674725, + "persist": 0.0 + }, + "typo": { + "min": 0.3095238134264946, + "best1L": 0.24404762099896157, + "mean": 0.1137566122093371, + "persist": 0.10416666484837021 + }, + "association": { + "min": 0.009316769960735525, + "best1L": 0.006211180106869766, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8173472802286006, + "dispersion_mean": 0.38246649644534103, + "dispersion_slope": -0.09758807751642996, + "stable_rank": 37.6347770690918, + "top8_spectral_share": 0.07004207372665405, + "condition_number": 1570.223876953125, + "unembed_alignment": 0.0504717156291008 + }, + "corpus": { + "mean_prompt_tokens": 3050.28, + "type_token_ratio": 0.07094430675216701, + "eval_vocab_overlap": 0.30916030534351147, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8173472802286006, + "6": 0.6683476972712367, + "7": 0.5684701381549199, + "8": 0.4705184851239106, + "9": 0.3537668638665054, + "10": 0.27039658573677416, + "11": 0.1613289514340879, + "12": 0.09453803019702262, + "13": 0.03748443599501142 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17337032088211604, + "best1L": 0.15256587894899504, + "mean": 0.042995839778866084, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14213197518672263, + "best1L": 0.09608411709112781, + "mean": 0.02530013703341995, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.02467532442616565, + "best1L": 0.015584415490073817, + "mean": 0.002741702732497028, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533658027649, + "best1L": 0.007288629721317973, + "mean": 0.0024295433291367124, + "persist": 0.0 + }, + "typo": { + "min": 0.31696428677865435, + "best1L": 0.23660713913185255, + "mean": 0.12351190644715514, + "persist": 0.12351190312100309 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.00776397529989481, + "mean": 0.0020703933467822416, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8787068964669271, + "dispersion_mean": 0.4095688585851649, + "dispersion_slope": -0.10476182737359249, + "stable_rank": 39.76311111450195, + "top8_spectral_share": 0.0680774375796318, + "condition_number": 2528.64306640625, + "unembed_alignment": 0.05293760076165199 + }, + "corpus": { + "mean_prompt_tokens": 2748.88, + "type_token_ratio": 0.05657576904048194, + "eval_vocab_overlap": 0.36870229007633587, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8787068964669271, + "6": 0.7157048591969732, + "7": 0.6086075305092166, + "8": 0.5031150622837625, + "9": 0.37790149387946115, + "10": 0.28867416158034365, + "11": 0.17223439350363726, + "12": 0.10128162362699519, + "13": 0.03989370621916759 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16920943558216095, + "best1L": 0.15256587948117936, + "mean": 0.04222530419273036, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13995648654443876, + "best1L": 0.09499637223780155, + "mean": 0.024897268041968346, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.015584415623119898, + "mean": 0.0024531024641224314, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.008746355505926269, + "mean": 0.00242954319609063, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.23660713913185255, + "mean": 0.13310185154633863, + "persist": 0.1369047634569662 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.00776397529989481, + "mean": 0.0022429261755730423, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8828583087635737, + "dispersion_mean": 0.41533829125311683, + "dispersion_slope": -0.10548030323437586, + "stable_rank": 40.65681838989258, + "top8_spectral_share": 0.06702109426259995, + "condition_number": 3032.283203125, + "unembed_alignment": 0.05219731479883194 + }, + "corpus": { + "mean_prompt_tokens": 2601.7066666666665, + "type_token_ratio": 0.04838362510762166, + "eval_vocab_overlap": 0.4251908396946565, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8828583087635737, + "6": 0.7234056175790263, + "7": 0.617467393280582, + "8": 0.5127661454980146, + "9": 0.38541354721276844, + "10": 0.2959270339900536, + "11": 0.17624109568822988, + "12": 0.10373831664172364, + "13": 0.04022716262407917 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.1705963984131813, + "best1L": 0.15256587948117936, + "mean": 0.04253351927867958, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13886874435203417, + "best1L": 0.09535895421036653, + "mean": 0.024695834144949913, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.02077922105257, + "best1L": 0.014285714365541935, + "mean": 0.002597402548417449, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00728862958827189, + "mean": 0.002429543245982911, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.23660713913185255, + "mean": 0.12301587340022836, + "persist": 0.12351190312100309 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180106869766, + "mean": 0.0020703933467822416, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.9150831922531054, + "dispersion_mean": 0.4277248794746005, + "dispersion_slope": -0.1091370244586398, + "stable_rank": 39.653438568115234, + "top8_spectral_share": 0.06823047995567322, + "condition_number": 2565.923583984375, + "unembed_alignment": 0.05136628448963165 + }, + "corpus": { + "mean_prompt_tokens": 2183.76, + "type_token_ratio": 0.04825621863208411, + "eval_vocab_overlap": 0.44732824427480916, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.9150831922531054, + "6": 0.7475168473158279, + "7": 0.6355197895181823, + "8": 0.5245098710043492, + "9": 0.39487047769671696, + "10": 0.30292176842874463, + "11": 0.18132231117121672, + "12": 0.10647372882199377, + "13": 0.04130592906126796 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.16643550672701427, + "best1L": 0.14979195381913865, + "mean": 0.04191709043724196, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1374184185905116, + "best1L": 0.09318346503589835, + "mean": 0.02433325277109231, + "persist": 0.0032632341608405113 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.015584415623119898, + "mean": 0.0024531024641224314, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00728862958827189, + "mean": 0.002591512881086341, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3110119051166943, + "best1L": 0.24404762206333025, + "mean": 0.11789021029004029, + "persist": 0.11458333222461599 + }, + "association": { + "min": 0.009316769960735525, + "best1L": 0.006211180106869766, + "mean": 0.001552795010086681, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8957243297234698, + "dispersion_mean": 0.42003549622858216, + "dispersion_slope": -0.10731113040996351, + "stable_rank": 39.17411804199219, + "top8_spectral_share": 0.06958511471748352, + "condition_number": 2001.37548828125, + "unembed_alignment": 0.0505978986620903 + }, + "corpus": { + "mean_prompt_tokens": 2340.3533333333335, + "type_token_ratio": 0.036814953867364754, + "eval_vocab_overlap": 0.4946564885496183, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8957243297234698, + "6": 0.7351597630980645, + "7": 0.6265415460051238, + "8": 0.5183507216892502, + "9": 0.3874869189420406, + "10": 0.2967371231082473, + "11": 0.17659332456604285, + "12": 0.10368413065057337, + "13": 0.040041608274426675 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.16920943238905498, + "best1L": 0.15117891558579036, + "mean": 0.041917090703334124, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1374184185905116, + "best1L": 0.09427120935704027, + "mean": 0.024454113162521805, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.019480518996715546, + "best1L": 0.015584415623119898, + "mean": 0.0021645022290093558, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985493336405, + "best1L": 0.008746355505926269, + "mean": 0.0025915127979325397, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30803571215697695, + "best1L": 0.24255952718002455, + "mean": 0.12318121455609798, + "persist": 0.11904761873717819 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180106869766, + "mean": 0.0018978605845144816, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.8978252897709219, + "dispersion_mean": 0.42219372740477984, + "dispersion_slope": -0.10757791325620211, + "stable_rank": 40.95539474487305, + "top8_spectral_share": 0.06870953738689423, + "condition_number": 1940.07421875, + "unembed_alignment": 0.05159561708569527 + }, + "corpus": { + "mean_prompt_tokens": 2495.045, + "type_token_ratio": 0.02932011246290147, + "eval_vocab_overlap": 0.5290076335877862, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8978252897709219, + "6": 0.7383500872219555, + "7": 0.6302188887860449, + "8": 0.5201192510780657, + "9": 0.39062896280560855, + "10": 0.2992158263383373, + "11": 0.1777530106050933, + "12": 0.10501711366890504, + "13": 0.040615116368086034 + } + } + }, + "runtime_s": 1149.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Pile-CC_s0.json b/results/ladder1b/tv_Pile-CC_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..33bf61f355d986686e2c17f4a45a6f002c291ae7 --- /dev/null +++ b/results/ladder1b/tv_Pile-CC_s0.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Pile-CC", + "seed": 0, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18307905590959958, + "best1L": 0.16227461876613752, + "mean": 0.046077978265072615, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1446700505912304, + "best1L": 0.10442349554172584, + "mean": 0.02642816930477108, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.018181817872183665, + "mean": 0.0025974026482020107, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.007288629721317973, + "mean": 0.0022675737106640425, + "persist": 0.0 + }, + "typo": { + "min": 0.44345238166196005, + "best1L": 0.28571428677865435, + "mean": 0.22106481716036797, + "persist": 0.24404762099896157 + }, + "association": { + "min": 0.05124223631407533, + "best1L": 0.026397515886596272, + "mean": 0.01138716349045613, + "persist": 0.007763974900756564 + } + }, + "operator": { + "dispersion_L0": 0.6328822301576097, + "dispersion_mean": 0.2846188071328457, + "dispersion_slope": -0.07671334495497453, + "stable_rank": 43.10158157348633, + "top8_spectral_share": 0.06186465546488762, + "condition_number": 4874.767578125, + "unembed_alignment": 0.1052660346031189 + }, + "corpus": { + "mean_prompt_tokens": 1369.1, + "type_token_ratio": 0.24359068000876488, + "eval_vocab_overlap": 0.22061068702290076, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6328822301576097, + "6": 0.5199784944523959, + "7": 0.43270824888856085, + "8": 0.34012240601258337, + "9": 0.2527919647020777, + "10": 0.1865465082349515, + "11": 0.10844020214561817, + "12": 0.061621140454583466, + "13": 0.026478069147230252 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17475728318095207, + "best1L": 0.15117891771452768, + "mean": 0.04299583991191217, + "persist": 0.0 + }, + "multilingual": { + "min": 0.14068165155393736, + "best1L": 0.10007251399968352, + "mean": 0.025541857949325016, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.01688311648155962, + "best1L": 0.015584415357027735, + "mean": 0.002020202111452818, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.00874635563897235, + "mean": 0.0030774214870429467, + "persist": 0.0 + }, + "typo": { + "min": 0.40327381449086325, + "best1L": 0.25148809594767435, + "mean": 0.19196428624647005, + "persist": 0.20833333475249155 + }, + "association": { + "min": 0.04503105634025165, + "best1L": 0.029503105341323783, + "mean": 0.010351966567603605, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.7399904001926404, + "dispersion_mean": 0.33760130083028694, + "dispersion_slope": -0.08965364767332717, + "stable_rank": 42.70446014404297, + "top8_spectral_share": 0.0627836212515831, + "condition_number": 5210.1435546875, + "unembed_alignment": 0.097433902323246 + }, + "corpus": { + "mean_prompt_tokens": 1143.76, + "type_token_ratio": 0.2153598657060922, + "eval_vocab_overlap": 0.36412213740458016, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7399904001926404, + "6": 0.6094583968250934, + "7": 0.5119691264541039, + "8": 0.41090184111495, + "9": 0.3018877674103309, + "10": 0.22431968401252486, + "11": 0.13219633419183172, + "12": 0.07550695661124956, + "13": 0.03218120065985788 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17337031928556307, + "best1L": 0.15117891558579036, + "mean": 0.043458160146006515, + "persist": 0.0 + }, + "multilingual": { + "min": 0.140681654214859, + "best1L": 0.10007251399968352, + "mean": 0.025783578931753124, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.00173160174329366, + "persist": 0.0 + }, + "poetry": { + "min": 0.02332361468247005, + "best1L": 0.013119533524981566, + "mean": 0.004211208450474909, + "persist": 0.0 + }, + "typo": { + "min": 0.42708333262375425, + "best1L": 0.28422619508845465, + "mean": 0.21511243177311762, + "persist": 0.23958333155938558 + }, + "association": { + "min": 0.043478259949811866, + "best1L": 0.029503105341323783, + "mean": 0.009316770143673889, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.8223625422154351, + "dispersion_mean": 0.3620353137324591, + "dispersion_slope": -0.09863755451902344, + "stable_rank": 42.729347229003906, + "top8_spectral_share": 0.062420304864645004, + "condition_number": 38790.9140625, + "unembed_alignment": 0.09584115445613861 + }, + "corpus": { + "mean_prompt_tokens": 1013.8, + "type_token_ratio": 0.17648451371079107, + "eval_vocab_overlap": 0.4679389312977099, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8223625422154351, + "6": 0.665437363796339, + "7": 0.5429183868797084, + "8": 0.4311600708496003, + "9": 0.31210747054947696, + "10": 0.23155443700446396, + "11": 0.13923116055581425, + "12": 0.07969649136290402, + "13": 0.033849900378391064 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.170596394687891, + "best1L": 0.1484049909881183, + "mean": 0.04268762602337769, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1381435798747199, + "best1L": 0.09970993255930287, + "mean": 0.024977842013218572, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.015584415490073817, + "best1L": 0.015584415490073817, + "mean": 0.0018759018941117184, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.016034985227244242, + "mean": 0.004049238715586918, + "persist": 0.0 + }, + "typo": { + "min": 0.42261905116694315, + "best1L": 0.2797619008592197, + "mean": 0.2126322745212487, + "persist": 0.23809523986918585 + }, + "association": { + "min": 0.04192546475678682, + "best1L": 0.03260869599346604, + "mean": 0.00931677021019693, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.8031448211024771, + "dispersion_mean": 0.35637812155586923, + "dispersion_slope": -0.09636127365783294, + "stable_rank": 43.02328872680664, + "top8_spectral_share": 0.062223635613918304, + "condition_number": 14435.326171875, + "unembed_alignment": 0.0940600037574768 + }, + "corpus": { + "mean_prompt_tokens": 970.44, + "type_token_ratio": 0.1523844853880714, + "eval_vocab_overlap": 0.5259541984732824, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8031448211024771, + "6": 0.650446207662578, + "7": 0.5348110738856955, + "8": 0.42734002850997754, + "9": 0.30959632635435325, + "10": 0.23102222096972275, + "11": 0.13836802725547762, + "12": 0.0792521643218077, + "13": 0.033422223940733374 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.169209430792502, + "best1L": 0.14701802922146662, + "mean": 0.041917091235518456, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13669325411319733, + "best1L": 0.0989847696785416, + "mean": 0.024695834078426872, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285713833357607, + "best1L": 0.014285713833357607, + "mean": 0.0017316017765551805, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.016034985227244242, + "mean": 0.004049238715586918, + "persist": 0.0 + }, + "typo": { + "min": 0.42410714072840555, + "best1L": 0.28125, + "mean": 0.21362433742199624, + "persist": 0.23511904903820582 + }, + "association": { + "min": 0.046583851400230615, + "best1L": 0.03260869599346604, + "mean": 0.010179433955012687, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.7983356700055741, + "dispersion_mean": 0.3540998383581349, + "dispersion_slope": -0.09570479088419907, + "stable_rank": 43.140472412109375, + "top8_spectral_share": 0.06223316118121147, + "condition_number": 11483.5234375, + "unembed_alignment": 0.0937228798866272 + }, + "corpus": { + "mean_prompt_tokens": 886.71, + "type_token_ratio": 0.1412186622458301, + "eval_vocab_overlap": 0.5725190839694656, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7983356700055741, + "6": 0.6449450714443304, + "7": 0.5319294878749101, + "8": 0.4241933632931701, + "9": 0.30789320031328865, + "10": 0.23031594250863976, + "11": 0.1374256924932722, + "12": 0.07866501630896892, + "13": 0.0331951009810606 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.1692094329212393, + "best1L": 0.14563106532607759, + "mean": 0.041762983559497764, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13633067426936968, + "best1L": 0.0989847696785416, + "mean": 0.024776408515338386, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285713833357607, + "best1L": 0.014285713833357607, + "mean": 0.0015873016589986427, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.0174927112779447, + "mean": 0.004211208284167307, + "persist": 0.0 + }, + "typo": { + "min": 0.41071428997176035, + "best1L": 0.2678571439215115, + "mean": 0.2058531753718853, + "persist": 0.22470238272632873 + }, + "association": { + "min": 0.041925465288971155, + "best1L": 0.027950310680483068, + "mean": 0.008799171690563006, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.8064079856631152, + "dispersion_mean": 0.3599273578205989, + "dispersion_slope": -0.09665546716261487, + "stable_rank": 43.216461181640625, + "top8_spectral_share": 0.062286972999572754, + "condition_number": 41985.66796875, + "unembed_alignment": 0.0924883559346199 + }, + "corpus": { + "mean_prompt_tokens": 989.9266666666666, + "type_token_ratio": 0.10757699223511506, + "eval_vocab_overlap": 0.6763358778625954, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8064079856631152, + "6": 0.6526236075216245, + "7": 0.540178488444417, + "8": 0.4326765950750937, + "9": 0.31516081619887, + "10": 0.23608279779728486, + "11": 0.14139161121717572, + "12": 0.08095498667850302, + "13": 0.033869331789306366 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1678224690258503, + "best1L": 0.1442441035594259, + "mean": 0.04130066319235733, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13488034637910978, + "best1L": 0.09825960679778031, + "mean": 0.024413826343204294, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.012987012575779642, + "mean": 0.0015873016257371222, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.016034985227244242, + "mean": 0.00388726921352957, + "persist": 0.0 + }, + "typo": { + "min": 0.4047619083098003, + "best1L": 0.26488095415490015, + "mean": 0.2028769887983799, + "persist": 0.22172619295971735 + }, + "association": { + "min": 0.043478259949811866, + "best1L": 0.027950310680483068, + "mean": 0.009144237580975252, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.8165880510089281, + "dispersion_mean": 0.3629005519964024, + "dispersion_slope": -0.09781398746299994, + "stable_rank": 42.92951202392578, + "top8_spectral_share": 0.0625072717666626, + "condition_number": 58820.2890625, + "unembed_alignment": 0.09153644740581512 + }, + "corpus": { + "mean_prompt_tokens": 1027.01, + "type_token_ratio": 0.09424445721073797, + "eval_vocab_overlap": 0.749618320610687, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8165880510089281, + "6": 0.6585655615564759, + "7": 0.5445614655495898, + "8": 0.43615376149421115, + "9": 0.3168433866048269, + "10": 0.23693404980505792, + "11": 0.14171558827477684, + "12": 0.08085130753109761, + "13": 0.03389179614265781 + } + } + }, + "runtime_s": 1181.6, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Pile-CC_s1.json b/results/ladder1b/tv_Pile-CC_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..1423c68ae85073a7621c4db26b725cc573438cbd --- /dev/null +++ b/results/ladder1b/tv_Pile-CC_s1.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Pile-CC", + "seed": 1, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.1692094329212393, + "best1L": 0.1428571417927742, + "mean": 0.04299583884754351, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1392313257924148, + "best1L": 0.10043509544006415, + "mean": 0.025058415651853596, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.012987012575779642, + "mean": 0.00173160174329366, + "persist": 0.0 + }, + "poetry": { + "min": 0.01749271181012903, + "best1L": 0.010204081423580647, + "mean": 0.0025915127813017796, + "persist": 0.0 + }, + "typo": { + "min": 0.40624999148505075, + "best1L": 0.25595238379069735, + "mean": 0.2013888880610466, + "persist": 0.21726190485060215 + }, + "association": { + "min": 0.03726708064121859, + "best1L": 0.023291925766638348, + "mean": 0.006556245731189847, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.7342402562935304, + "dispersion_mean": 0.3335935762308396, + "dispersion_slope": -0.08863790400055685, + "stable_rank": 45.635704040527344, + "top8_spectral_share": 0.062016379088163376, + "condition_number": 2315.15869140625, + "unembed_alignment": 0.09036596864461899 + }, + "corpus": { + "mean_prompt_tokens": 2342.7, + "type_token_ratio": 0.1990011525163273, + "eval_vocab_overlap": 0.31679389312977096, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7342402562935304, + "6": 0.6002000313702758, + "7": 0.5065654174949676, + "8": 0.40605205174540976, + "9": 0.29507686368807934, + "10": 0.22263508998597384, + "11": 0.12973520475311906, + "12": 0.07498481649013058, + "13": 0.03285245425606977 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.170596394687891, + "best1L": 0.14563106532607759, + "mean": 0.04207119691584792, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13705583502139365, + "best1L": 0.09970993362367153, + "mean": 0.024655547658247606, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0015873016257371222, + "persist": 0.0 + }, + "poetry": { + "min": 0.023323615081608295, + "best1L": 0.018950437727783407, + "mean": 0.003887269296683371, + "persist": 0.0 + }, + "typo": { + "min": 0.4270833411387035, + "best1L": 0.2976190501025745, + "mean": 0.2342923278255122, + "persist": 0.26785714232495855 + }, + "association": { + "min": 0.04192546648638589, + "best1L": 0.029503105341323783, + "mean": 0.010179433772074324, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.8305339445513606, + "dispersion_mean": 0.3514039126985057, + "dispersion_slope": -0.09892907801803881, + "stable_rank": 44.3226432800293, + "top8_spectral_share": 0.062339648604393005, + "condition_number": 12413.6611328125, + "unembed_alignment": 0.0964149534702301 + }, + "corpus": { + "mean_prompt_tokens": 1220.56, + "type_token_ratio": 0.1920102248148391, + "eval_vocab_overlap": 0.3763358778625954, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8305339445513606, + "6": 0.653308188072103, + "7": 0.5315515222518188, + "8": 0.4069611413938811, + "9": 0.29112836001980247, + "10": 0.2172969325383237, + "11": 0.12744515071706944, + "12": 0.07344728570428963, + "13": 0.030962689037902592 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.16643550938793591, + "best1L": 0.1442441035594259, + "mean": 0.041146556580705304, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1385061607829162, + "best1L": 0.09898477074291025, + "mean": 0.02497784174712641, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0015873016257371222, + "persist": 0.0 + }, + "poetry": { + "min": 0.023323615746838704, + "best1L": 0.01895043732864516, + "mean": 0.0040492387821099585, + "persist": 0.0 + }, + "typo": { + "min": 0.4151785671710968, + "best1L": 0.2782738081046513, + "mean": 0.21445105863468988, + "persist": 0.23958333847778185 + }, + "association": { + "min": 0.046583850868046284, + "best1L": 0.029503105341323783, + "mean": 0.010351967049895652, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.8292186475515843, + "dispersion_mean": 0.35591535948441755, + "dispersion_slope": -0.09837839464223169, + "stable_rank": 43.76750183105469, + "top8_spectral_share": 0.06252644211053848, + "condition_number": 3886.000244140625, + "unembed_alignment": 0.09410043805837631 + }, + "corpus": { + "mean_prompt_tokens": 1063.98, + "type_token_ratio": 0.16949566721178969, + "eval_vocab_overlap": 0.49312977099236643, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8292186475515843, + "6": 0.6538805481986053, + "7": 0.5342034119528634, + "8": 0.41362698688514243, + "9": 0.30165248228309605, + "10": 0.2265116583357021, + "11": 0.13384898724357905, + "12": 0.07754531740048032, + "13": 0.03275019550870484 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16782247115458762, + "best1L": 0.1470180270927293, + "mean": 0.04145477060228586, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1381435836000102, + "best1L": 0.09862218823816095, + "mean": 0.024977842611925944, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0014430015081805842, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.0174927112779447, + "mean": 0.003725299777995263, + "persist": 0.0 + }, + "typo": { + "min": 0.41220238379069735, + "best1L": 0.27678571322134565, + "mean": 0.21395502611994743, + "persist": 0.23809523986918585 + }, + "association": { + "min": 0.04503105567502124, + "best1L": 0.031055900268256664, + "mean": 0.010351967049895652, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.8270314994629014, + "dispersion_mean": 0.35690895104394854, + "dispersion_slope": -0.09844779547251666, + "stable_rank": 43.38831329345703, + "top8_spectral_share": 0.06213821843266487, + "condition_number": 7379.37841796875, + "unembed_alignment": 0.09457827359437943 + }, + "corpus": { + "mean_prompt_tokens": 1195.1466666666668, + "type_token_ratio": 0.13649649694319246, + "eval_vocab_overlap": 0.5786259541984733, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8270314994629014, + "6": 0.6551423703129373, + "7": 0.5360551785661773, + "8": 0.41907985689114063, + "9": 0.3038550266317066, + "10": 0.22656851531499653, + "11": 0.13457681433285645, + "12": 0.07733174104907996, + "13": 0.03253955683374093 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.170596394687891, + "best1L": 0.1470180270927293, + "mean": 0.041762984091682095, + "persist": 0.0 + }, + "multilingual": { + "min": 0.14068165315049036, + "best1L": 0.09970993255930287, + "mean": 0.0254209973583264, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0015873016257371222, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.014577259309589863, + "mean": 0.003401360524419163, + "persist": 0.0 + }, + "typo": { + "min": 0.40773810020514895, + "best1L": 0.2678571439215115, + "mean": 0.20849867910146713, + "persist": 0.23065476225955145 + }, + "association": { + "min": 0.04658385033586195, + "best1L": 0.031055900268256664, + "mean": 0.010179433955012687, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.8223632688682858, + "dispersion_mean": 0.3567932899990031, + "dispersion_slope": -0.09806440879923463, + "stable_rank": 43.61874008178711, + "top8_spectral_share": 0.06214417889714241, + "condition_number": 8195.6142578125, + "unembed_alignment": 0.09564964473247528 + }, + "corpus": { + "mean_prompt_tokens": 1240.57, + "type_token_ratio": 0.11995292486518294, + "eval_vocab_overlap": 0.6412213740458015, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8223632688682858, + "6": 0.6535943946180139, + "7": 0.5358288086592564, + "8": 0.4215483407058179, + "9": 0.3055298991799822, + "10": 0.22754764082785303, + "11": 0.13481489020250706, + "12": 0.07724919955267534, + "13": 0.03266316737663624 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.1692094329212393, + "best1L": 0.14563106532607759, + "mean": 0.04160887628261532, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13705583715013095, + "best1L": 0.09862218823816095, + "mean": 0.024574973487428257, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0014430015081805842, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.016034985227244242, + "mean": 0.003563330076368792, + "persist": 0.0 + }, + "typo": { + "min": 0.40476190618106295, + "best1L": 0.2678571417927742, + "mean": 0.2020502627960273, + "persist": 0.22172619162925652 + }, + "association": { + "min": 0.04503105567502124, + "best1L": 0.027950310680483068, + "mean": 0.008799171690563006, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.8160012433933007, + "dispersion_mean": 0.35635511895815, + "dispersion_slope": -0.09736572548509799, + "stable_rank": 43.11106491088867, + "top8_spectral_share": 0.06246326491236687, + "condition_number": 8538.087890625, + "unembed_alignment": 0.09542164206504822 + }, + "corpus": { + "mean_prompt_tokens": 1317.5, + "type_token_ratio": 0.09565085388994307, + "eval_vocab_overlap": 0.7450381679389313, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8160012433933007, + "6": 0.649833676989224, + "7": 0.5348707387103377, + "8": 0.4236820220172154, + "9": 0.3074482393987945, + "10": 0.22883274451932878, + "11": 0.13603711480498407, + "12": 0.07788216241706795, + "13": 0.032608128373096214 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1692094329212393, + "best1L": 0.14563106532607759, + "mean": 0.041454769537917206, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13778100003089225, + "best1L": 0.09862218823816095, + "mean": 0.02477640818272318, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0014430015081805842, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.014577259309589863, + "mean": 0.003401360657465245, + "persist": 0.0 + }, + "typo": { + "min": 0.409226194024086, + "best1L": 0.26488095202616285, + "mean": 0.20535714019622123, + "persist": 0.22321428677865437 + }, + "association": { + "min": 0.04347825955067362, + "best1L": 0.029503105341323783, + "mean": 0.00914423739803689, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.8418152639280847, + "dispersion_mean": 0.36878047524388413, + "dispersion_slope": -0.10074603611413514, + "stable_rank": 43.21599197387695, + "top8_spectral_share": 0.0623183511197567, + "condition_number": 13309.560546875, + "unembed_alignment": 0.09507735818624496 + }, + "corpus": { + "mean_prompt_tokens": 1244.555, + "type_token_ratio": 0.0835720398053923, + "eval_vocab_overlap": 0.7816793893129771, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8418152639280847, + "6": 0.6736839062803632, + "7": 0.5546519641808079, + "8": 0.43929517359299447, + "9": 0.31852616006089807, + "10": 0.23709072504062811, + "11": 0.14027689948569674, + "12": 0.08023155285402479, + "13": 0.03345263177145872 + } + } + }, + "runtime_s": 1157.6, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Pile-CC_s2.json b/results/ladder1b/tv_Pile-CC_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..efa414a251d8e231f44005eea513b838655502ae --- /dev/null +++ b/results/ladder1b/tv_Pile-CC_s2.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Pile-CC", + "seed": 2, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.1719833596476487, + "best1L": 0.14701802815709794, + "mean": 0.04299583937972784, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13306744024157524, + "best1L": 0.09825960786214896, + "mean": 0.023728950215237483, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.00173160174329366, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00728862958827189, + "mean": 0.002429543245982911, + "persist": 0.0 + }, + "typo": { + "min": 0.38392856291362215, + "best1L": 0.24255952132599695, + "mean": 0.18485449893133982, + "persist": 0.20386904877211368 + }, + "association": { + "min": 0.03881987503596714, + "best1L": 0.029503105341323783, + "mean": 0.009144237364775367, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.6881825780832156, + "dispersion_mean": 0.3132262912367076, + "dispersion_slope": -0.0823711981881358, + "stable_rank": 46.46332931518555, + "top8_spectral_share": 0.060942985117435455, + "condition_number": 5801.087890625, + "unembed_alignment": 0.10181235522031784 + }, + "corpus": { + "mean_prompt_tokens": 468.8, + "type_token_ratio": 0.35942832764505117, + "eval_vocab_overlap": 0.15419847328244274, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6881825780832156, + "6": 0.5605166944479159, + "7": 0.470882875390044, + "8": 0.37910319354419875, + "9": 0.27685410470584154, + "10": 0.21055610993139298, + "11": 0.1283014360551244, + "12": 0.0728400492194333, + "13": 0.03179957975320171 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.16504854655691556, + "best1L": 0.14563106532607759, + "mean": 0.04037602072847741, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13306743891111442, + "best1L": 0.09753444365092687, + "mean": 0.023809523920395544, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0015873016257371222, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.00874635563897235, + "mean": 0.0024295432293521507, + "persist": 0.0 + }, + "typo": { + "min": 0.38244046909468515, + "best1L": 0.24107142750705993, + "mean": 0.18452380916901998, + "persist": 0.20386905090085097 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.026397516019642353, + "mean": 0.007936507596501283, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.8219787567082062, + "dispersion_mean": 0.36653984085588104, + "dispersion_slope": -0.09897648857476503, + "stable_rank": 43.01191711425781, + "top8_spectral_share": 0.06312838941812515, + "condition_number": 7258.0703125, + "unembed_alignment": 0.09489983320236206 + }, + "corpus": { + "mean_prompt_tokens": 1271.52, + "type_token_ratio": 0.20186862967157418, + "eval_vocab_overlap": 0.3893129770992366, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8219787567082062, + "6": 0.6710778742358862, + "7": 0.5564610335001586, + "8": 0.4317233667534194, + "9": 0.31784440453321916, + "10": 0.24131471993098128, + "11": 0.14330507416681545, + "12": 0.0813334509532652, + "13": 0.03381988692097759 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.16643550938793591, + "best1L": 0.14563106532607759, + "mean": 0.04037602099456957, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.13705583715013095, + "best1L": 0.09898476914635726, + "mean": 0.024937555526516268, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.00173160174329366, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816311325346, + "best1L": 0.013119533258889402, + "mean": 0.003401360541049923, + "persist": 0.0 + }, + "typo": { + "min": 0.4136904797383717, + "best1L": 0.26636904371636255, + "mean": 0.2055224829486438, + "persist": 0.23214285501411983 + }, + "association": { + "min": 0.038819875568151474, + "best1L": 0.029503105341323783, + "mean": 0.00914423753108297, + "persist": 0.007763974900756564 + } + }, + "operator": { + "dispersion_L0": 0.8155554632149757, + "dispersion_mean": 0.3619896784855022, + "dispersion_slope": -0.09839537098878172, + "stable_rank": 42.32051467895508, + "top8_spectral_share": 0.06277505308389664, + "condition_number": 5332.70361328125, + "unembed_alignment": 0.09477655589580536 + }, + "corpus": { + "mean_prompt_tokens": 1517.98, + "type_token_ratio": 0.13213612827573487, + "eval_vocab_overlap": 0.5229007633587787, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8155554632149757, + "6": 0.664118096832472, + "7": 0.5492855803642919, + "8": 0.4308797767567753, + "9": 0.31286030422598116, + "10": 0.2348022940690632, + "11": 0.138660520499449, + "12": 0.07879879517823275, + "13": 0.032946275228278844 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16782247115458762, + "best1L": 0.14563106532607759, + "mean": 0.040530126940991194, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13451776493872916, + "best1L": 0.09934735111892223, + "mean": 0.024091531456049, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285713833357607, + "best1L": 0.014285713833357607, + "mean": 0.0015873016589986427, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985493336405, + "best1L": 0.011661807341235024, + "mean": 0.0027534824330359697, + "persist": 0.0 + }, + "typo": { + "min": 0.4122023901769093, + "best1L": 0.26636904584509985, + "mean": 0.20734127185174397, + "persist": 0.22767857249294007 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.029503105341323783, + "mean": 0.00897170465239989, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.8068876829404937, + "dispersion_mean": 0.35612983417946376, + "dispersion_slope": -0.09692651959461872, + "stable_rank": 42.71046447753906, + "top8_spectral_share": 0.06253579258918762, + "condition_number": 4857.85009765625, + "unembed_alignment": 0.09291347861289978 + }, + "corpus": { + "mean_prompt_tokens": 1321.3866666666668, + "type_token_ratio": 0.122376493380691, + "eval_vocab_overlap": 0.5862595419847328, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8068876829404937, + "6": 0.6518732448014125, + "7": 0.5378526546001188, + "8": 0.42355402119650243, + "9": 0.30755656059560055, + "10": 0.23077415820331446, + "11": 0.1365103134018096, + "12": 0.07759565162479198, + "13": 0.032564220251129934 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.1664355072591986, + "best1L": 0.14563106532607759, + "mean": 0.04083834229303258, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.136693253581013, + "best1L": 0.0989847696785416, + "mean": 0.02473612129688263, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0015873016257371222, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725917654378, + "best1L": 0.01020408155662673, + "mean": 0.0025915127979325397, + "persist": 0.0 + }, + "typo": { + "min": 0.412202388048172, + "best1L": 0.2752976172736713, + "mean": 0.21031745789306505, + "persist": 0.23363095096179418 + }, + "association": { + "min": 0.04037267022899219, + "best1L": 0.029503105341323783, + "mean": 0.009144237630867533, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.793490362082176, + "dispersion_mean": 0.3513880354359665, + "dispersion_slope": -0.09532946081391795, + "stable_rank": 41.95417022705078, + "top8_spectral_share": 0.06297126412391663, + "condition_number": 5590.0224609375, + "unembed_alignment": 0.09449228644371033 + }, + "corpus": { + "mean_prompt_tokens": 1573.14, + "type_token_ratio": 0.10483491615495125, + "eval_vocab_overlap": 0.6938931297709924, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.793490362082176, + "6": 0.6427298353730239, + "7": 0.5302249150780143, + "8": 0.41824047738345455, + "9": 0.30420604334384516, + "10": 0.22827301720973983, + "11": 0.13571285976873312, + "12": 0.07708435833387024, + "13": 0.03253045035084127 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.1664355072591986, + "best1L": 0.14563106532607759, + "mean": 0.04068423581442663, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1359680907002517, + "best1L": 0.09825960679778031, + "mean": 0.02441382674234254, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0015873016257371222, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725917654378, + "best1L": 0.01020408155662673, + "mean": 0.0024295432958751917, + "persist": 0.0 + }, + "typo": { + "min": 0.4047619083098003, + "best1L": 0.26636904584509985, + "mean": 0.20287698454090528, + "persist": 0.22619047441652843 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.02950310560741595, + "mean": 0.008454106099504446, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.8158029107773455, + "dispersion_mean": 0.3617857851173141, + "dispersion_slope": -0.09795082235211189, + "stable_rank": 42.27999496459961, + "top8_spectral_share": 0.06306160986423492, + "condition_number": 7142.572265625, + "unembed_alignment": 0.09425834566354752 + }, + "corpus": { + "mean_prompt_tokens": 1431.0066666666667, + "type_token_ratio": 0.08970375167131762, + "eval_vocab_overlap": 0.7534351145038168, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.8158029107773455, + "6": 0.6598874946051783, + "7": 0.5452439874817082, + "8": 0.43261379011647383, + "9": 0.31376639171816567, + "10": 0.23535599671787613, + "11": 0.14012115407203665, + "12": 0.07979311625202642, + "13": 0.03348722431501632 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1664355072591986, + "best1L": 0.14563106532607759, + "mean": 0.04068423581442663, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1356055097920554, + "best1L": 0.09753444391701903, + "mean": 0.024373539789978946, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0014430015081805842, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985094198158, + "best1L": 0.013119533524981566, + "mean": 0.0027534823498821686, + "persist": 0.0 + }, + "typo": { + "min": 0.39880952664784025, + "best1L": 0.25892857142857145, + "mean": 0.1977513230272702, + "persist": 0.21577381129775727 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.027950310680483068, + "mean": 0.008281573470282768, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.826050786925598, + "dispersion_mean": 0.3689462247342912, + "dispersion_slope": -0.09899607145506167, + "stable_rank": 42.71167755126953, + "top8_spectral_share": 0.06279527395963669, + "condition_number": 7943.2724609375, + "unembed_alignment": 0.0924762636423111 + }, + "corpus": { + "mean_prompt_tokens": 1350.62, + "type_token_ratio": 0.07824184448623595, + "eval_vocab_overlap": 0.7801526717557252, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.826050786925598, + "6": 0.6685556627837327, + "7": 0.5543981781259117, + "8": 0.4422653861323561, + "9": 0.3229044924033892, + "10": 0.2435665026553704, + "11": 0.14527241526912732, + "12": 0.08295413531209249, + "13": 0.03454846300104175 + } + } + }, + "runtime_s": 1157.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_StackExchange_s0.json b/results/ladder1b/tv_StackExchange_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..8ffece0df5e86cc761aad3ca958c0760d9f58c28 --- /dev/null +++ b/results/ladder1b/tv_StackExchange_s0.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "StackExchange", + "seed": 0, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.1872399454670293, + "best1L": 0.15533980620758875, + "mean": 0.04854368990553277, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.150108773793493, + "best1L": 0.10478607644992215, + "mean": 0.027636774349957705, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.020779219988201345, + "mean": 0.0028860028500535657, + "persist": 0.0 + }, + "poetry": { + "min": 0.011661807740373271, + "best1L": 0.004373177752963134, + "mean": 0.0016196955693885684, + "persist": 0.0 + }, + "typo": { + "min": 0.4851190469094685, + "best1L": 0.3363095245191029, + "mean": 0.25148809594767435, + "persist": 0.27678571322134565 + }, + "association": { + "min": 0.03416148985603026, + "best1L": 0.01552795059978962, + "mean": 0.006211180023715964, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.5293641863905197, + "dispersion_mean": 0.2589897964730546, + "dispersion_slope": -0.06320400790527296, + "stable_rank": 44.194114685058594, + "top8_spectral_share": 0.06425436586141586, + "condition_number": 10336.8466796875, + "unembed_alignment": 0.08400242775678635 + }, + "corpus": { + "mean_prompt_tokens": 495.4, + "type_token_ratio": 0.28441663302381914, + "eval_vocab_overlap": 0.1099236641221374, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5293641863905197, + "6": 0.44817893453208807, + "7": 0.38380295339962545, + "8": 0.32169063457464303, + "9": 0.24173396652278364, + "10": 0.186855340203402, + "11": 0.11785116768585893, + "12": 0.06923199915354362, + "13": 0.03219898579502729 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.191400835024459, + "best1L": 0.1595006931040968, + "mean": 0.049776544395302026, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1530094231878008, + "best1L": 0.10696156535829816, + "mean": 0.02864394516551069, + "persist": 0.00290065272045987 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.020779219988201345, + "mean": 0.0031746032182127237, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.004373177886009216, + "mean": 0.0022675737106640425, + "persist": 0.0 + }, + "typo": { + "min": 0.49107142857142855, + "best1L": 0.3482142835855484, + "mean": 0.2632275117295129, + "persist": 0.29017857355730875 + }, + "association": { + "min": 0.029503105341323783, + "best1L": 0.01708074565976858, + "mean": 0.006038647194925163, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.558351725332855, + "dispersion_mean": 0.26817374410567285, + "dispersion_slope": -0.06718661472103284, + "stable_rank": 42.14235305786133, + "top8_spectral_share": 0.06516263633966446, + "condition_number": 21430.96484375, + "unembed_alignment": 0.0859011635184288 + }, + "corpus": { + "mean_prompt_tokens": 623.0, + "type_token_ratio": 0.20057784911717497, + "eval_vocab_overlap": 0.19236641221374046, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.558351725332855, + "6": 0.4711320245930222, + "7": 0.4003440908497536, + "8": 0.3322484778010576, + "9": 0.24772704786494626, + "10": 0.1881114565305894, + "11": 0.11689802384816378, + "12": 0.06836815340050517, + "13": 0.030382696730162303 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.19278779679111072, + "best1L": 0.16227462249142782, + "mean": 0.04931422309683902, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14757070104990685, + "best1L": 0.10551123933068343, + "mean": 0.027193619137895957, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003030303034133145, + "persist": 0.0 + }, + "poetry": { + "min": 0.0174927112779447, + "best1L": 0.005830903670617512, + "mean": 0.0024295432293521507, + "persist": 0.0 + }, + "typo": { + "min": 0.49851190618106295, + "best1L": 0.35416666737624575, + "mean": 0.26041666099003385, + "persist": 0.2827380948833057 + }, + "association": { + "min": 0.024844720294432982, + "best1L": 0.01552795059978962, + "mean": 0.005348516278900206, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.5247464461535675, + "dispersion_mean": 0.24889934155941823, + "dispersion_slope": -0.06311001505184503, + "stable_rank": 42.59612274169922, + "top8_spectral_share": 0.0649397000670433, + "condition_number": 6014.20361328125, + "unembed_alignment": 0.08685384690761566 + }, + "corpus": { + "mean_prompt_tokens": 770.32, + "type_token_ratio": 0.136281026067089, + "eval_vocab_overlap": 0.28091603053435116, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5247464461535675, + "6": 0.44113243924064377, + "7": 0.3716523302172266, + "8": 0.3053392641791929, + "9": 0.22756755871039935, + "10": 0.1724811899092498, + "11": 0.10711233279979404, + "12": 0.06252978296845257, + "13": 0.0275327298562374 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.18585298476474627, + "best1L": 0.15950069576501846, + "mean": 0.048851902330560346, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1450326330959797, + "best1L": 0.10478607618382998, + "mean": 0.02699218410998583, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003030303034133145, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437594737326, + "best1L": 0.005830903670617512, + "mean": 0.0025915127979325397, + "persist": 0.0 + }, + "typo": { + "min": 0.49851190618106295, + "best1L": 0.3556547590664455, + "mean": 0.2607473613960402, + "persist": 0.28422618976661135 + }, + "association": { + "min": 0.026397515886596272, + "best1L": 0.01552795059978962, + "mean": 0.005521049074429486, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.5308573071577152, + "dispersion_mean": 0.25049145159065933, + "dispersion_slope": -0.06398586054821946, + "stable_rank": 42.7908935546875, + "top8_spectral_share": 0.06541438400745392, + "condition_number": 7056.43115234375, + "unembed_alignment": 0.08688600361347198 + }, + "corpus": { + "mean_prompt_tokens": 680.6933333333334, + "type_token_ratio": 0.12363864295228394, + "eval_vocab_overlap": 0.3236641221374046, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5308573071577152, + "6": 0.44606751360628794, + "7": 0.3748155592875834, + "8": 0.30687930307429057, + "9": 0.2283807779210928, + "10": 0.1719039664664664, + "11": 0.10660653103696895, + "12": 0.06177489339650482, + "13": 0.027137212369023868 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18723994759576662, + "best1L": 0.16088765859603882, + "mean": 0.049006009341350625, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14648296098623956, + "best1L": 0.10478607618382998, + "mean": 0.02699218430955495, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.020779219988201345, + "best1L": 0.01818181773913758, + "mean": 0.0028860028500535657, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816391152995, + "best1L": 0.007288629721317973, + "mean": 0.003077421437150666, + "persist": 0.0 + }, + "typo": { + "min": 0.5029761918953487, + "best1L": 0.3601190447807312, + "mean": 0.2647156087415559, + "persist": 0.2901785767504147 + }, + "association": { + "min": 0.024844720161386898, + "best1L": 0.01552795059978962, + "mean": 0.005521049074429486, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5226103868276132, + "dispersion_mean": 0.24721371980556223, + "dispersion_slope": -0.06286157362064847, + "stable_rank": 42.66787338256836, + "top8_spectral_share": 0.0654623955488205, + "condition_number": 7518.5732421875, + "unembed_alignment": 0.08660872280597687 + }, + "corpus": { + "mean_prompt_tokens": 660.27, + "type_token_ratio": 0.11012161691429263, + "eval_vocab_overlap": 0.36946564885496186, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5226103868276132, + "6": 0.4398428207291554, + "7": 0.3689756612855299, + "8": 0.3020088517289251, + "9": 0.22538930520980646, + "10": 0.17044887667163322, + "11": 0.10646496869624471, + "12": 0.06187347590973264, + "13": 0.02730913119141865 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.19001387325780733, + "best1L": 0.16227462142705917, + "mean": 0.04946833063981363, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1482958687203271, + "best1L": 0.10587382077106408, + "mean": 0.027797921228089502, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.0033189034022923026, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816391152995, + "best1L": 0.007288629721317973, + "mean": 0.0029154519350933178, + "persist": 0.0 + }, + "typo": { + "min": 0.5104166695049831, + "best1L": 0.3645833304950169, + "mean": 0.27033729744809015, + "persist": 0.2976190522313118 + }, + "association": { + "min": 0.029503106139600277, + "best1L": 0.01552795059978962, + "mean": 0.006211180040346724, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5273011974228511, + "dispersion_mean": 0.24854110273914973, + "dispersion_slope": -0.0633996049032431, + "stable_rank": 42.59850311279297, + "top8_spectral_share": 0.06558497995138168, + "condition_number": 8003.591796875, + "unembed_alignment": 0.08617223799228668 + }, + "corpus": { + "mean_prompt_tokens": 630.2933333333333, + "type_token_ratio": 0.0962091724488069, + "eval_vocab_overlap": 0.4259541984732824, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5273011974228511, + "6": 0.442724331221343, + "7": 0.37091937493396215, + "8": 0.3029208134667473, + "9": 0.2263847366886081, + "10": 0.17116171673477773, + "11": 0.10649864989187699, + "12": 0.061834381191737356, + "13": 0.027124723100443792 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.19140083396009036, + "best1L": 0.16088765540293284, + "mean": 0.049622438182788234, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.15047135310513632, + "best1L": 0.10696156722094331, + "mean": 0.028039642410086735, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.0036075035043592963, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816391152995, + "best1L": 0.007288629721317973, + "mean": 0.003239391072254096, + "persist": 0.0 + }, + "typo": { + "min": 0.5133928614003318, + "best1L": 0.36607142857142855, + "mean": 0.27066799466099056, + "persist": 0.2991071396640369 + }, + "association": { + "min": 0.029503106139600277, + "best1L": 0.01552795059978962, + "mean": 0.006211180040346724, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5307789378173001, + "dispersion_mean": 0.24976324981384326, + "dispersion_slope": -0.06380759083849893, + "stable_rank": 42.35916519165039, + "top8_spectral_share": 0.06567764282226562, + "condition_number": 6247.00244140625, + "unembed_alignment": 0.08632923662662506 + }, + "corpus": { + "mean_prompt_tokens": 634.71, + "type_token_ratio": 0.08512549038143404, + "eval_vocab_overlap": 0.483206106870229, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5307789378173001, + "6": 0.4454429951229642, + "7": 0.37269763366009584, + "8": 0.30408845671225077, + "9": 0.2273746966912979, + "10": 0.17116481362238337, + "11": 0.1069127616102176, + "12": 0.06215313883453859, + "13": 0.02725581425354131 + } + } + }, + "runtime_s": 1149.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_StackExchange_s1.json b/results/ladder1b/tv_StackExchange_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..a40877ece6ea14ba1f1e91ca5c87e805123b51b9 --- /dev/null +++ b/results/ladder1b/tv_StackExchange_s1.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "StackExchange", + "seed": 1, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18862690829804965, + "best1L": 0.15533980514322007, + "mean": 0.04792726186237165, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.14140681443469866, + "best1L": 0.10333575042230743, + "mean": 0.026186447191451277, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.024675324958349978, + "best1L": 0.022077921644917557, + "mean": 0.0033189033357692616, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816324629954, + "best1L": 0.007288629721317973, + "mean": 0.002915451885201037, + "persist": 0.0 + }, + "typo": { + "min": 0.45386905329568045, + "best1L": 0.3154761961528233, + "mean": 0.22734787953751429, + "persist": 0.251488098608596 + }, + "association": { + "min": 0.026397516019642353, + "best1L": 0.012422360479831696, + "mean": 0.005866114582334246, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.553644111867256, + "dispersion_mean": 0.2641014414876631, + "dispersion_slope": -0.0656468794769945, + "stable_rank": 43.49940490722656, + "top8_spectral_share": 0.0644960030913353, + "condition_number": 145971.234375, + "unembed_alignment": 0.08604831248521805 + }, + "corpus": { + "mean_prompt_tokens": 697.0, + "type_token_ratio": 0.22682926829268293, + "eval_vocab_overlap": 0.11450381679389313, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.553644111867256, + "6": 0.4650189129321003, + "7": 0.38842935703679204, + "8": 0.3195338783395871, + "9": 0.24113374086030653, + "10": 0.18621816897600124, + "11": 0.12009307872408137, + "12": 0.07055821497670989, + "13": 0.03228350967613316 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.191400835024459, + "best1L": 0.15811373080526078, + "mean": 0.04808136913925409, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1468455402978829, + "best1L": 0.10659898365182537, + "mean": 0.027072757881666933, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.020779220520385673, + "mean": 0.0031746031516896827, + "persist": 0.0 + }, + "poetry": { + "min": 0.02332361468247005, + "best1L": 0.01020408155662673, + "mean": 0.00404923881537148, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.48363094457558226, + "best1L": 0.33482142857142855, + "mean": 0.24355158848421915, + "persist": 0.26190476066299845 + }, + "association": { + "min": 0.031055900800440992, + "best1L": 0.013975155406764575, + "mean": 0.006383712919029806, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5021151679486471, + "dispersion_mean": 0.2352878533952541, + "dispersion_slope": -0.05969713848682818, + "stable_rank": 41.95772933959961, + "top8_spectral_share": 0.06596612930297852, + "condition_number": 3984.37646484375, + "unembed_alignment": 0.08585882186889648 + }, + "corpus": { + "mean_prompt_tokens": 544.64, + "type_token_ratio": 0.19888366627497062, + "eval_vocab_overlap": 0.1717557251908397, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5021151679486471, + "6": 0.4189363051679192, + "7": 0.3481057345642283, + "8": 0.28206195128380696, + "9": 0.2119908555054993, + "10": 0.16173928639068375, + "11": 0.10491951980797336, + "12": 0.06075106705982715, + "13": 0.026970792828701642 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.1567267690386091, + "mean": 0.04946833037372146, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586818814278, + "best1L": 0.10514865762421063, + "mean": 0.027354766548212086, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.02597402608288186, + "best1L": 0.022077921644917557, + "mean": 0.003751803621915834, + "persist": 0.0 + }, + "poetry": { + "min": 0.01749271181012903, + "best1L": 0.00728862958827189, + "mean": 0.002915451868570277, + "persist": 0.0 + }, + "typo": { + "min": 0.48958333900996615, + "best1L": 0.3422619083098003, + "mean": 0.2566137569291251, + "persist": 0.284226194024086 + }, + "association": { + "min": 0.03416149118649108, + "best1L": 0.017080745260630335, + "mean": 0.006728778576611408, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5061538655842559, + "dispersion_mean": 0.23542427962004034, + "dispersion_slope": -0.06056432484170035, + "stable_rank": 42.734336853027344, + "top8_spectral_share": 0.0658789798617363, + "condition_number": 6460.17626953125, + "unembed_alignment": 0.08648285269737244 + }, + "corpus": { + "mean_prompt_tokens": 598.94, + "type_token_ratio": 0.1430193341570107, + "eval_vocab_overlap": 0.2633587786259542, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5061538655842559, + "6": 0.422093289461093, + "7": 0.3505242197666709, + "8": 0.2829485385281056, + "9": 0.2117808420232144, + "10": 0.15972883012817407, + "11": 0.10125244202380801, + "12": 0.05854685215622529, + "13": 0.02578963690881578 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.1567267690386091, + "mean": 0.049160116352140903, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14648295566439629, + "best1L": 0.10369833212878023, + "mean": 0.026992184575647116, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.019480519395853792, + "mean": 0.0033189033357692616, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408163645437787, + "best1L": 0.008746355505926269, + "mean": 0.003239390972469534, + "persist": 0.0 + }, + "typo": { + "min": 0.49553572280066355, + "best1L": 0.3482142857142857, + "mean": 0.2604166641831398, + "persist": 0.28869048080274035 + }, + "association": { + "min": 0.031055900401302745, + "best1L": 0.01552795059978962, + "mean": 0.006038647294709725, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5143912220436367, + "dispersion_mean": 0.24092926744243626, + "dispersion_slope": -0.06169549217798403, + "stable_rank": 43.14067459106445, + "top8_spectral_share": 0.06575492769479752, + "condition_number": 5844.2783203125, + "unembed_alignment": 0.08474697172641754 + }, + "corpus": { + "mean_prompt_tokens": 613.4, + "type_token_ratio": 0.12300836865558092, + "eval_vocab_overlap": 0.3236641221374046, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5143912220436367, + "6": 0.43121526155716655, + "7": 0.35882152988678434, + "8": 0.291395125234437, + "9": 0.21767100175762064, + "10": 0.16448922871959343, + "11": 0.10359733913907074, + "12": 0.060295374397191154, + "13": 0.02648732424642553 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.15811373080526078, + "mean": 0.04946833143809012, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1468455402978829, + "best1L": 0.10369833212878023, + "mean": 0.02719361847266555, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077921644917557, + "best1L": 0.019480519395853792, + "mean": 0.003030302967610104, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.008746355505926269, + "mean": 0.0030774214205199052, + "persist": 0.0 + }, + "typo": { + "min": 0.49851190192358835, + "best1L": 0.34375, + "mean": 0.2610780437077795, + "persist": 0.28422619083098005 + }, + "association": { + "min": 0.031055900401302745, + "best1L": 0.017080745260630335, + "mean": 0.006211180123500526, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5231732333477735, + "dispersion_mean": 0.24516981680353847, + "dispersion_slope": -0.06277487544653568, + "stable_rank": 43.071529388427734, + "top8_spectral_share": 0.06566400080919266, + "condition_number": 3965.353759765625, + "unembed_alignment": 0.0851605236530304 + }, + "corpus": { + "mean_prompt_tokens": 618.16, + "type_token_ratio": 0.1078685130063414, + "eval_vocab_overlap": 0.3603053435114504, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5231732333477735, + "6": 0.43833938255076116, + "7": 0.3656311692982848, + "8": 0.2969506915865429, + "9": 0.22158072477529267, + "10": 0.16739211262422626, + "11": 0.10525119307532604, + "12": 0.06130229023508486, + "13": 0.02690755373855419 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.15672676797424043, + "mean": 0.04962243791669607, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14793328461902483, + "best1L": 0.1051486578903028, + "mean": 0.02755620071132268, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.020779220520385673, + "mean": 0.0033189033357692616, + "persist": 0.0 + }, + "poetry": { + "min": 0.02186588982918433, + "best1L": 0.008746355505926269, + "mean": 0.0034013604911576423, + "persist": 0.0 + }, + "typo": { + "min": 0.49702381236212595, + "best1L": 0.34523809594767435, + "mean": 0.2614087292126247, + "persist": 0.28869047867400305 + }, + "association": { + "min": 0.031055900401302745, + "best1L": 0.017080745260630335, + "mean": 0.006556245781082127, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5303310222625943, + "dispersion_mean": 0.2483069070630179, + "dispersion_slope": -0.06370120671961166, + "stable_rank": 42.55085754394531, + "top8_spectral_share": 0.06605423986911774, + "condition_number": 3570.084716796875, + "unembed_alignment": 0.08521223068237305 + }, + "corpus": { + "mean_prompt_tokens": 697.54, + "type_token_ratio": 0.08590188376293832, + "eval_vocab_overlap": 0.4381679389312977, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5303310222625943, + "6": 0.44429698680184965, + "7": 0.3707772529810171, + "8": 0.30087152510995435, + "9": 0.22451898599724188, + "10": 0.1689809443378371, + "11": 0.10634146268042216, + "12": 0.06167112593244455, + "13": 0.02697285746380006 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.19278779679111072, + "best1L": 0.15811372974089213, + "mean": 0.04977654519357851, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1493836103805474, + "best1L": 0.10587382077106408, + "mean": 0.027596487064978907, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.003463203586371882, + "persist": 0.0 + }, + "poetry": { + "min": 0.02186588982918433, + "best1L": 0.008746355505926269, + "mean": 0.0035633301262610723, + "persist": 0.0 + }, + "typo": { + "min": 0.4880952366760799, + "best1L": 0.34375000425747465, + "mean": 0.2551256673676627, + "persist": 0.27976190724543165 + }, + "association": { + "min": 0.026397516019642353, + "best1L": 0.017080745260630335, + "mean": 0.005866114499180445, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.54439240311929, + "dispersion_mean": 0.2541662495780044, + "dispersion_slope": -0.06532337142695427, + "stable_rank": 42.21647644042969, + "top8_spectral_share": 0.0662483423948288, + "condition_number": 5749.08740234375, + "unembed_alignment": 0.08535419404506683 + }, + "corpus": { + "mean_prompt_tokens": 687.88, + "type_token_ratio": 0.07611065883584346, + "eval_vocab_overlap": 0.4618320610687023, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.54439240311929, + "6": 0.45522936730499186, + "7": 0.3789925756727131, + "8": 0.3075030324258364, + "9": 0.22952867623329007, + "10": 0.17256377831464076, + "11": 0.10865171080590647, + "12": 0.06306239668179243, + "13": 0.027572305643578 + } + } + }, + "runtime_s": 1152.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_StackExchange_s2.json b/results/ladder1b/tv_StackExchange_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..ac1394c10c2c73bd0fa9dc921647dadd101fac3b --- /dev/null +++ b/results/ladder1b/tv_StackExchange_s2.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "StackExchange", + "seed": 2, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18862690829804965, + "best1L": 0.15811373080526078, + "mean": 0.04746494136218514, + "persist": 0.0 + }, + "multilingual": { + "min": 0.14612037794930594, + "best1L": 0.10369833212878023, + "mean": 0.02666988962196878, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.02467532442616565, + "best1L": 0.019480518863669465, + "mean": 0.0037518036884388755, + "persist": 0.0 + }, + "poetry": { + "min": 0.021865888764815673, + "best1L": 0.01020408155662673, + "mean": 0.0034013606242037247, + "persist": 0.0 + }, + "typo": { + "min": 0.4598214307001659, + "best1L": 0.3005952313542366, + "mean": 0.22933201172522136, + "persist": 0.25297619083098005 + }, + "association": { + "min": 0.026397515886596272, + "best1L": 0.01552795059978962, + "mean": 0.005693581687020404, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.41496396668929597, + "dispersion_mean": 0.19849261772006568, + "dispersion_slope": -0.050301580777167065, + "stable_rank": 43.32089614868164, + "top8_spectral_share": 0.06558651477098465, + "condition_number": 5049.5625, + "unembed_alignment": 0.08648306876420975 + }, + "corpus": { + "mean_prompt_tokens": 444.6, + "type_token_ratio": 0.3090418353576248, + "eval_vocab_overlap": 0.10381679389312977, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.41496396668929597, + "6": 0.35037604687058493, + "7": 0.29728648001361174, + "8": 0.24810696380919314, + "9": 0.18423864302807844, + "10": 0.13829022050876857, + "11": 0.08361014872710379, + "12": 0.048185792723464224, + "13": 0.021375297110490625 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.19001387325780733, + "best1L": 0.16088765912822314, + "mean": 0.049160116352140903, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.15047135310513632, + "best1L": 0.10587382077106408, + "mean": 0.027918782218226364, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003030303034133145, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.008746355505926269, + "mean": 0.0034013606242037247, + "persist": 0.0 + }, + "typo": { + "min": 0.48214285501411985, + "best1L": 0.32291666524750845, + "mean": 0.24801587526287353, + "persist": 0.2708333374134132 + }, + "association": { + "min": 0.027950310680483068, + "best1L": 0.018633540320609297, + "mean": 0.006383712852506765, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.49407321758302863, + "dispersion_mean": 0.2349782717027455, + "dispersion_slope": -0.059529530675847965, + "stable_rank": 41.7374153137207, + "top8_spectral_share": 0.06669412553310394, + "condition_number": 8468.740234375, + "unembed_alignment": 0.08799387514591217 + }, + "corpus": { + "mean_prompt_tokens": 757.64, + "type_token_ratio": 0.18837442584868802, + "eval_vocab_overlap": 0.2381679389312977, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.49407321758302863, + "6": 0.41579303943268114, + "7": 0.3511912809940697, + "8": 0.28942305145333624, + "9": 0.21648524023054766, + "10": 0.16250644364537, + "11": 0.10079157040267173, + "12": 0.058546229261974926, + "13": 0.025994372321029326 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.19001387325780733, + "best1L": 0.16088765859603882, + "mean": 0.049006009341350625, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14720812173826353, + "best1L": 0.1040609133030687, + "mean": 0.02719361853918859, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.0031746032182127237, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408162847161293, + "best1L": 0.007288629721317973, + "mean": 0.003077421437150666, + "persist": 0.0 + }, + "typo": { + "min": 0.5, + "best1L": 0.3482142814568111, + "mean": 0.26306217163801193, + "persist": 0.29315475800207685 + }, + "association": { + "min": 0.027950310680483068, + "best1L": 0.018633540320609297, + "mean": 0.006211180023715964, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5099477653169052, + "dispersion_mean": 0.24195354173997416, + "dispersion_slope": -0.0614249504068182, + "stable_rank": 42.006961822509766, + "top8_spectral_share": 0.06613200157880783, + "condition_number": 23867.01953125, + "unembed_alignment": 0.08662077784538269 + }, + "corpus": { + "mean_prompt_tokens": 778.34, + "type_token_ratio": 0.13384896060847445, + "eval_vocab_overlap": 0.29923664122137406, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5099477653169052, + "6": 0.4292417773961189, + "7": 0.36174063520803634, + "8": 0.29671737077740823, + "9": 0.2221397433476685, + "10": 0.16669680012539628, + "11": 0.10424566357927438, + "12": 0.06037859667941675, + "13": 0.026473523229542745 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.18862691042678698, + "best1L": 0.15811373399836676, + "mean": 0.04900600907525846, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586818814278, + "best1L": 0.10514865762421063, + "mean": 0.027435339987277985, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.003463203586371882, + "persist": 0.0 + }, + "poetry": { + "min": 0.021865888764815673, + "best1L": 0.008746355505926269, + "mean": 0.0034013606242037247, + "persist": 0.0 + }, + "typo": { + "min": 0.501488093818937, + "best1L": 0.35416666311877115, + "mean": 0.2652116354022707, + "persist": 0.29613095202616285 + }, + "association": { + "min": 0.02950310574046203, + "best1L": 0.018633540320609297, + "mean": 0.0065562456480360454, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5174046904027215, + "dispersion_mean": 0.24476144157259005, + "dispersion_slope": -0.06214254081321488, + "stable_rank": 41.79080581665039, + "top8_spectral_share": 0.06602855771780014, + "condition_number": 8215.93359375, + "unembed_alignment": 0.08665631711483002 + }, + "corpus": { + "mean_prompt_tokens": 707.52, + "type_token_ratio": 0.12209784411276949, + "eval_vocab_overlap": 0.34885496183206105, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5174046904027215, + "6": 0.43413760674953533, + "7": 0.36469783851521653, + "8": 0.2987442481943267, + "9": 0.225026796609229, + "10": 0.16896430304566668, + "11": 0.10576259027763871, + "12": 0.061330026745490415, + "13": 0.02678487361348588 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18723994653139794, + "best1L": 0.15950069576501846, + "mean": 0.04885190312883684, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586605940545, + "best1L": 0.10587382050497192, + "mean": 0.027314478797572, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.0031746032182127237, + "persist": 0.0 + }, + "poetry": { + "min": 0.023323615081608295, + "best1L": 0.008746355505926269, + "mean": 0.0035633301262610723, + "persist": 0.0 + }, + "typo": { + "min": 0.4970238081046513, + "best1L": 0.34672618550913675, + "mean": 0.26008597548518864, + "persist": 0.2842261918953487 + }, + "association": { + "min": 0.027950311079621315, + "best1L": 0.01708074565976858, + "mean": 0.006038647161663643, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5239171505652214, + "dispersion_mean": 0.24627211797059978, + "dispersion_slope": -0.06289226407510626, + "stable_rank": 41.70997619628906, + "top8_spectral_share": 0.066600501537323, + "condition_number": 31562.021484375, + "unembed_alignment": 0.0865035355091095 + }, + "corpus": { + "mean_prompt_tokens": 746.24, + "type_token_ratio": 0.10425600343053174, + "eval_vocab_overlap": 0.3938931297709924, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5239171505652214, + "6": 0.43858962478997915, + "7": 0.3669866524674581, + "8": 0.29947818518725833, + "9": 0.22508836390956613, + "10": 0.1686186897279995, + "11": 0.10546101736592431, + "12": 0.06142511310119052, + "13": 0.026884264620800655 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.18862691042678698, + "best1L": 0.15811372974089213, + "mean": 0.049160116884325235, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586605940545, + "best1L": 0.10478607618382998, + "mean": 0.027314478930618082, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.0037518036884388755, + "persist": 0.0 + }, + "poetry": { + "min": 0.02186588982918433, + "best1L": 0.008746355505926269, + "mean": 0.0032393909891002943, + "persist": 0.0 + }, + "typo": { + "min": 0.49851190618106295, + "best1L": 0.3511904754808971, + "mean": 0.26190476225955145, + "persist": 0.2872023784688541 + }, + "association": { + "min": 0.02950310574046203, + "best1L": 0.018633540320609297, + "mean": 0.006383712819245245, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.526289873053294, + "dispersion_mean": 0.24718976024265263, + "dispersion_slope": -0.06326373629860059, + "stable_rank": 41.58824157714844, + "top8_spectral_share": 0.0667017474770546, + "condition_number": 61136.7265625, + "unembed_alignment": 0.08705676347017288 + }, + "corpus": { + "mean_prompt_tokens": 681.56, + "type_token_ratio": 0.09154488721951601, + "eval_vocab_overlap": 0.45725190839694657, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.526289873053294, + "6": 0.44093750221765615, + "7": 0.36905076177872626, + "8": 0.30027368162857593, + "9": 0.22572095528395392, + "10": 0.16902780869093528, + "11": 0.10545715584044071, + "12": 0.06121950899684627, + "13": 0.026730594693445336 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.18862691042678698, + "best1L": 0.15811372974089213, + "mean": 0.049160116884325235, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1475707053073815, + "best1L": 0.10551123933068343, + "mean": 0.02751591362591301, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.020779220520385673, + "mean": 0.0036075037039284196, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816391152995, + "best1L": 0.007288629721317973, + "mean": 0.003239390939208014, + "persist": 0.0 + }, + "typo": { + "min": 0.49553572280066355, + "best1L": 0.35119047335215975, + "mean": 0.2615740756903376, + "persist": 0.2842261918953487 + }, + "association": { + "min": 0.027950311079621315, + "best1L": 0.01708074565976858, + "mean": 0.006211179990454444, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.5308710102213203, + "dispersion_mean": 0.24900700608138615, + "dispersion_slope": -0.06378223794824027, + "stable_rank": 42.0009765625, + "top8_spectral_share": 0.06639274209737778, + "condition_number": 15629.2978515625, + "unembed_alignment": 0.08661937713623047 + }, + "corpus": { + "mean_prompt_tokens": 677.565, + "type_token_ratio": 0.07989639370392508, + "eval_vocab_overlap": 0.49694656488549616, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5308710102213203, + "6": 0.44422318181172143, + "7": 0.37139152795220126, + "8": 0.30237616688389446, + "9": 0.22742714196068256, + "10": 0.1703584304684972, + "11": 0.10608072326455865, + "12": 0.06148083346168696, + "13": 0.026854038707912694 + } + } + }, + "runtime_s": 1155.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_USPTO_Backgrounds_s0.json b/results/ladder1b/tv_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..eecf312fe589ddca50b6fc68a6517933f9ce5d4f --- /dev/null +++ b/results/ladder1b/tv_USPTO_Backgrounds_s0.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "USPTO_Backgrounds", + "seed": 0, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17753120458551816, + "best1L": 0.15672676690987178, + "mean": 0.04654029916439738, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13959390776497976, + "best1L": 0.10369833186268806, + "mean": 0.025904439655797824, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.018181818005229746, + "mean": 0.002597402548417449, + "persist": 0.0 + }, + "poetry": { + "min": 0.024781341797539165, + "best1L": 0.007288629721317973, + "mean": 0.003239391072254096, + "persist": 0.0 + }, + "typo": { + "min": 0.43898809381893705, + "best1L": 0.30952381023338865, + "mean": 0.23809523561171123, + "persist": 0.2619047670492104 + }, + "association": { + "min": 0.0465838507350002, + "best1L": 0.023291925500546182, + "mean": 0.009144237364775367, + "persist": 0.00776397529989481 + } + }, + "operator": { + "dispersion_L0": 0.47301620501098457, + "dispersion_mean": 0.233290447320659, + "dispersion_slope": -0.055890735497441155, + "stable_rank": 43.614784240722656, + "top8_spectral_share": 0.06967366486787796, + "condition_number": 2458.877685546875, + "unembed_alignment": 0.08536766469478607 + }, + "corpus": { + "mean_prompt_tokens": 1170.0, + "type_token_ratio": 0.16965811965811967, + "eval_vocab_overlap": 0.13358778625954199, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.47301620501098457, + "6": 0.4031538150149418, + "7": 0.3407946572573033, + "8": 0.28155163562904456, + "9": 0.21973978295825855, + "10": 0.17506722299644772, + "11": 0.11149321417179898, + "12": 0.06602053734270819, + "13": 0.0287769555044437 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.16504854496036256, + "best1L": 0.15117891505360603, + "mean": 0.043458160279052596, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1305293682962656, + "best1L": 0.1011602585869176, + "mean": 0.024333252970661436, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.022077921511871473, + "best1L": 0.018181818138275827, + "mean": 0.0028860028833150864, + "persist": 0.0 + }, + "poetry": { + "min": 0.026239067316055298, + "best1L": 0.008746355505926269, + "mean": 0.003563330109630312, + "persist": 0.0 + }, + "typo": { + "min": 0.44642856078488485, + "best1L": 0.3095238134264946, + "mean": 0.23429232729332788, + "persist": 0.2633928544819355 + }, + "association": { + "min": 0.03881987490292106, + "best1L": 0.02329192523445402, + "mean": 0.007591442487734769, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.5316837358240765, + "dispersion_mean": 0.26454599561160824, + "dispersion_slope": -0.06289958907614852, + "stable_rank": 44.49399948120117, + "top8_spectral_share": 0.06703855842351913, + "condition_number": 3901.375244140625, + "unembed_alignment": 0.08301647007465363 + }, + "corpus": { + "mean_prompt_tokens": 1036.16, + "type_token_ratio": 0.14858709079678814, + "eval_vocab_overlap": 0.2015267175572519, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5316837358240765, + "6": 0.45486623276199917, + "7": 0.38744816131348137, + "8": 0.3208030103674958, + "9": 0.24990696427725426, + "10": 0.2013657735175373, + "11": 0.1261313870909886, + "12": 0.07540569909822906, + "13": 0.033302996253412216 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17198335751891136, + "best1L": 0.14979195328695433, + "mean": 0.044536909221538475, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13560550872768676, + "best1L": 0.10043509464178767, + "mean": 0.024816694602902447, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.018181818005229746, + "best1L": 0.018181818005229746, + "mean": 0.002453102497383952, + "persist": 0.0 + }, + "poetry": { + "min": 0.033527696904327185, + "best1L": 0.011661807607327188, + "mean": 0.004859086641642664, + "persist": 0.0 + }, + "typo": { + "min": 0.44047618763787405, + "best1L": 0.30952380916901995, + "mean": 0.23677249093140876, + "persist": 0.27380952185818125 + }, + "association": { + "min": 0.041925465422017236, + "best1L": 0.026397515354411944, + "mean": 0.008109040624861206, + "persist": 0.007763975166848728 + } + }, + "operator": { + "dispersion_L0": 0.5339632621441137, + "dispersion_mean": 0.2638786653382646, + "dispersion_slope": -0.06283312447025062, + "stable_rank": 44.667335510253906, + "top8_spectral_share": 0.06643287092447281, + "condition_number": 9959.0849609375, + "unembed_alignment": 0.08671547472476959 + }, + "corpus": { + "mean_prompt_tokens": 863.9, + "type_token_ratio": 0.12239842574372034, + "eval_vocab_overlap": 0.24580152671755726, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5339632621441137, + "6": 0.4512158742206991, + "7": 0.3853444595505469, + "8": 0.32092475410534216, + "9": 0.24878643987680607, + "10": 0.1999281914199757, + "11": 0.12630840977466973, + "12": 0.07516452254827774, + "13": 0.03327207440395073 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.17337031928556307, + "best1L": 0.14979195328695433, + "mean": 0.04453690895544631, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1341551834983485, + "best1L": 0.10043509464178767, + "mean": 0.02469583441104208, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.01688311674765178, + "best1L": 0.01688311674765178, + "mean": 0.0023088023465658936, + "persist": 0.0 + }, + "poetry": { + "min": 0.03352769663823502, + "best1L": 0.013119533658027649, + "mean": 0.0046971170398007545, + "persist": 0.0 + }, + "typo": { + "min": 0.4419642814568111, + "best1L": 0.31101191150290625, + "mean": 0.2367724866739341, + "persist": 0.2693452404013702 + }, + "association": { + "min": 0.04347826061504228, + "best1L": 0.026397515354411944, + "mean": 0.008281573453652007, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5358612692974791, + "dispersion_mean": 0.2639928222135562, + "dispersion_slope": -0.06313682648387381, + "stable_rank": 44.648681640625, + "top8_spectral_share": 0.06630293279886246, + "condition_number": 14331.642578125, + "unembed_alignment": 0.0868721678853035 + }, + "corpus": { + "mean_prompt_tokens": 896.2666666666667, + "type_token_ratio": 0.10278191014578994, + "eval_vocab_overlap": 0.2870229007633588, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5358612692974791, + "6": 0.45252642019990386, + "7": 0.3860938474698626, + "8": 0.320953175177927, + "9": 0.24834381590479385, + "10": 0.19914104466916754, + "11": 0.12537100292985362, + "12": 0.07450672874609464, + "13": 0.033138095526923296 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.17475728211658342, + "best1L": 0.14979195328695433, + "mean": 0.04453690895544631, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1337926031223365, + "best1L": 0.10007251320140702, + "mean": 0.02469583441104208, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.018181818005229746, + "mean": 0.00259740261494049, + "persist": 0.0 + }, + "poetry": { + "min": 0.03061224560120276, + "best1L": 0.011661807474281107, + "mean": 0.004535147504481886, + "persist": 0.0 + }, + "typo": { + "min": 0.44196428997176035, + "best1L": 0.30654762365988325, + "mean": 0.23561507995639527, + "persist": 0.26785714232495855 + }, + "association": { + "min": 0.04192546515592507, + "best1L": 0.026397515354411944, + "mean": 0.008109040658122726, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5752491739912158, + "dispersion_mean": 0.2831266159896042, + "dispersion_slope": -0.06777504320423966, + "stable_rank": 45.06602478027344, + "top8_spectral_share": 0.0656706914305687, + "condition_number": 17480.2890625, + "unembed_alignment": 0.08675366640090942 + }, + "corpus": { + "mean_prompt_tokens": 928.07, + "type_token_ratio": 0.09325805165558633, + "eval_vocab_overlap": 0.3267175572519084, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5752491739912158, + "6": 0.48526126705351047, + "7": 0.41399108496028786, + "8": 0.34387511047450353, + "9": 0.26663898212887893, + "10": 0.21387162754430697, + "11": 0.1344811536542178, + "12": 0.07978332598471484, + "13": 0.034987818114801776 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.17059639575225966, + "best1L": 0.14979195328695433, + "mean": 0.044382801678563864, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13451776706746646, + "best1L": 0.10188542253204755, + "mean": 0.024856981821358204, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02337662356772593, + "best1L": 0.018181818005229746, + "mean": 0.003030302967610104, + "persist": 0.0 + }, + "poetry": { + "min": 0.0320699703214424, + "best1L": 0.013119533658027649, + "mean": 0.005021056077176971, + "persist": 0.0 + }, + "typo": { + "min": 0.44047618550913675, + "best1L": 0.30654762365988325, + "mean": 0.23346560767718724, + "persist": 0.26785714445369585 + }, + "association": { + "min": 0.0419254656881094, + "best1L": 0.026397515354411944, + "mean": 0.008109040658122726, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.5694789145885557, + "dispersion_mean": 0.2836138441089364, + "dispersion_slope": -0.06708562641606843, + "stable_rank": 44.44453048706055, + "top8_spectral_share": 0.06625822931528091, + "condition_number": 24259.751953125, + "unembed_alignment": 0.08471114188432693 + }, + "corpus": { + "mean_prompt_tokens": 893.02, + "type_token_ratio": 0.07713153120870753, + "eval_vocab_overlap": 0.3709923664122137, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5694789145885557, + "6": 0.48269562995086085, + "7": 0.41442734013392846, + "8": 0.34628077587659517, + "9": 0.26932605657261527, + "10": 0.21680172368019615, + "11": 0.13676321915470266, + "12": 0.08133149069434423, + "13": 0.035419446328629416 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.17337032034993172, + "best1L": 0.14979195328695433, + "mean": 0.04469101596623659, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13524292835167476, + "best1L": 0.10297316791755813, + "mean": 0.025340424118829624, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02207792137882539, + "best1L": 0.018181818005229746, + "mean": 0.002886002916576607, + "persist": 0.0 + }, + "poetry": { + "min": 0.03352769663823502, + "best1L": 0.013119533658027649, + "mean": 0.005183025579234319, + "persist": 0.0 + }, + "typo": { + "min": 0.4434523731470108, + "best1L": 0.3065476215311459, + "mean": 0.2333002643925803, + "persist": 0.2708333299628326 + }, + "association": { + "min": 0.0419254656881094, + "best1L": 0.024844720693571225, + "mean": 0.008109040658122726, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.576901796612843, + "dispersion_mean": 0.2858387999713513, + "dispersion_slope": -0.06789127703875861, + "stable_rank": 44.58827590942383, + "top8_spectral_share": 0.06589213758707047, + "condition_number": 35547.08203125, + "unembed_alignment": 0.08554147183895111 + }, + "corpus": { + "mean_prompt_tokens": 900.0, + "type_token_ratio": 0.0683, + "eval_vocab_overlap": 0.4083969465648855, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.576901796612843, + "6": 0.4871402197760389, + "7": 0.417590999025814, + "8": 0.34837453517647987, + "9": 0.2706164737212963, + "10": 0.21749043815985503, + "11": 0.1373216984204342, + "12": 0.08147823371624384, + "13": 0.035634805133156186 + } + } + }, + "runtime_s": 1195.6, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_USPTO_Backgrounds_s1.json b/results/ladder1b/tv_USPTO_Backgrounds_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..cc7b11ed6c96bd723349314deb588e1681819cf2 --- /dev/null +++ b/results/ladder1b/tv_USPTO_Backgrounds_s1.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "USPTO_Backgrounds", + "seed": 1, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17198335698672704, + "best1L": 0.151178916650159, + "mean": 0.044999229588678906, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13089195266366005, + "best1L": 0.10188542226595539, + "mean": 0.024413826276681254, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02467532522444214, + "best1L": 0.019480519129761627, + "mean": 0.0037518038214849575, + "persist": 0.0 + }, + "poetry": { + "min": 0.027696793100663593, + "best1L": 0.011661807740373271, + "mean": 0.004859086625011904, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.42559523028986795, + "best1L": 0.29613095096179415, + "mean": 0.2215608454176358, + "persist": 0.2544642851821014 + }, + "association": { + "min": 0.03726708064121859, + "best1L": 0.020186335247542177, + "mean": 0.00793650806216257, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.44168151700113717, + "dispersion_mean": 0.22586972280382778, + "dispersion_slope": -0.052015396513628666, + "stable_rank": 42.524635314941406, + "top8_spectral_share": 0.07137150317430496, + "condition_number": 6607.765625, + "unembed_alignment": 0.0845615565776825 + }, + "corpus": { + "mean_prompt_tokens": 1022.5, + "type_token_ratio": 0.19511002444987774, + "eval_vocab_overlap": 0.116793893129771, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.44168151700113717, + "6": 0.3800538735587793, + "7": 0.3299478105641887, + "8": 0.27780435828378053, + "9": 0.21793536914336242, + "10": 0.17506730423218725, + "11": 0.11282908490112194, + "12": 0.06709434695867773, + "13": 0.03041384059121496 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17891816848090716, + "best1L": 0.15672677169953073, + "mean": 0.04638619228665318, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13415518403053284, + "best1L": 0.10079767634826046, + "mean": 0.024977842212787697, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.02337662356772593, + "best1L": 0.018181818005229746, + "mean": 0.003030302967610104, + "persist": 0.0 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.011661807873419352, + "mean": 0.005344995214337749, + "persist": 0.0 + }, + "typo": { + "min": 0.4166666695049831, + "best1L": 0.28571428677865435, + "mean": 0.21560846003038542, + "persist": 0.24702381182994163 + }, + "association": { + "min": 0.041925465288971155, + "best1L": 0.0232919253675001, + "mean": 0.007763975183479488, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5133888206602486, + "dispersion_mean": 0.26027391038071374, + "dispersion_slope": -0.06018508936106801, + "stable_rank": 43.89020919799805, + "top8_spectral_share": 0.06779921054840088, + "condition_number": 9629.0751953125, + "unembed_alignment": 0.08524453639984131 + }, + "corpus": { + "mean_prompt_tokens": 1106.48, + "type_token_ratio": 0.14239751283348998, + "eval_vocab_overlap": 0.20076335877862594, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5133888206602486, + "6": 0.4365655506415959, + "7": 0.3792557140310934, + "8": 0.31928025180679387, + "9": 0.25064850791411386, + "10": 0.20201966201053517, + "11": 0.12984624040431622, + "12": 0.07761567387939067, + "13": 0.033844772078335646 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17891816848090716, + "best1L": 0.15533980780414172, + "mean": 0.04592387072209801, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1341551856270858, + "best1L": 0.10116025752254895, + "mean": 0.02489726847436811, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.016883116880697862, + "mean": 0.002741702732497028, + "persist": 0.0 + }, + "poetry": { + "min": 0.03061224493597235, + "best1L": 0.014577259575682027, + "mean": 0.004535147421328085, + "persist": 0.0 + }, + "typo": { + "min": 0.44196428784302305, + "best1L": 0.3110119008592197, + "mean": 0.23627645948103496, + "persist": 0.27529761993459295 + }, + "association": { + "min": 0.0419254656881094, + "best1L": 0.026397515620504106, + "mean": 0.0077639751335872075, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5207471427494479, + "dispersion_mean": 0.2618176014333029, + "dispersion_slope": -0.06108753995857512, + "stable_rank": 45.04499053955078, + "top8_spectral_share": 0.0662807747721672, + "condition_number": 31453.04296875, + "unembed_alignment": 0.08622291684150696 + }, + "corpus": { + "mean_prompt_tokens": 925.24, + "type_token_ratio": 0.12476762785871774, + "eval_vocab_overlap": 0.2603053435114504, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5207471427494479, + "6": 0.4412415104270314, + "7": 0.3811018933400514, + "8": 0.3206490408731014, + "9": 0.2512035072030619, + "10": 0.20242633717024933, + "11": 0.12882081781758065, + "12": 0.0764270937646372, + "13": 0.033741069554565184 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.17891816794872284, + "best1L": 0.1525658816099167, + "mean": 0.045615656833563535, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13306743891111442, + "best1L": 0.10043509464178767, + "mean": 0.02461526050631489, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02207792137882539, + "best1L": 0.016883116880697862, + "mean": 0.002886002916576607, + "persist": 0.0 + }, + "poetry": { + "min": 0.030612245335110595, + "best1L": 0.014577259575682027, + "mean": 0.004535147371435804, + "persist": 0.0 + }, + "typo": { + "min": 0.4330357036420277, + "best1L": 0.3080357110926083, + "mean": 0.23115078892026628, + "persist": 0.2708333299628326 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.024844720560525144, + "mean": 0.007936507746178125, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5251582984616096, + "dispersion_mean": 0.2629624541860797, + "dispersion_slope": -0.06171218989481235, + "stable_rank": 45.77305603027344, + "top8_spectral_share": 0.0656074583530426, + "condition_number": 2536345.75, + "unembed_alignment": 0.08496980369091034 + }, + "corpus": { + "mean_prompt_tokens": 883.7866666666666, + "type_token_ratio": 0.11121839357914429, + "eval_vocab_overlap": 0.3, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5251582984616096, + "6": 0.44387342682590625, + "7": 0.383579681512265, + "8": 0.3227691978726338, + "9": 0.25157843433492766, + "10": 0.20247657721853748, + "11": 0.12802678363704267, + "12": 0.07587825420722547, + "13": 0.033321433604570135 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18169209307857923, + "best1L": 0.15811373240181378, + "mean": 0.04669440577604941, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1334300216819559, + "best1L": 0.1015228418899434, + "mean": 0.02481669480247157, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.016883116880697862, + "mean": 0.002741702732497028, + "persist": 0.0 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.013119533791073732, + "mean": 0.004535147504481886, + "persist": 0.0 + }, + "typo": { + "min": 0.42559523241860525, + "best1L": 0.29166667269808905, + "mean": 0.2227182500064373, + "persist": 0.252976192427533 + }, + "association": { + "min": 0.04347826061504228, + "best1L": 0.026397515620504106, + "mean": 0.007936507929116488, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5253554748572146, + "dispersion_mean": 0.2610648703647865, + "dispersion_slope": -0.06183582550216353, + "stable_rank": 45.34809112548828, + "top8_spectral_share": 0.06621571630239487, + "condition_number": 21039.4609375, + "unembed_alignment": 0.08490266650915146 + }, + "corpus": { + "mean_prompt_tokens": 898.01, + "type_token_ratio": 0.09695883119341656, + "eval_vocab_overlap": 0.32061068702290074, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5253554748572146, + "6": 0.44306833995743705, + "7": 0.3817519622532571, + "8": 0.31984016952414046, + "9": 0.24802335137387285, + "10": 0.19892495489431614, + "11": 0.12566612423395254, + "12": 0.0742495449157625, + "13": 0.03270391127312572 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.18030513131192752, + "best1L": 0.15533980248229845, + "mean": 0.04623208474367857, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13415518403053284, + "best1L": 0.10261058541280883, + "mean": 0.02477640808293862, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02207792137882539, + "best1L": 0.016883116880697862, + "mean": 0.002886002916576607, + "persist": 0.0 + }, + "poetry": { + "min": 0.02915451888527189, + "best1L": 0.011661807873419352, + "mean": 0.004373177852747696, + "persist": 0.0 + }, + "typo": { + "min": 0.4226190469094685, + "best1L": 0.2827380948833057, + "mean": 0.22089947281139238, + "persist": 0.25297618923442705 + }, + "association": { + "min": 0.0403726708942226, + "best1L": 0.024844720294432982, + "mean": 0.007591442454473248, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5551490843315533, + "dispersion_mean": 0.274354911523677, + "dispersion_slope": -0.06529981135386063, + "stable_rank": 45.37302780151367, + "top8_spectral_share": 0.06589771062135696, + "condition_number": 46329.96875, + "unembed_alignment": 0.08424016833305359 + }, + "corpus": { + "mean_prompt_tokens": 906.9333333333333, + "type_token_ratio": 0.08178476918553367, + "eval_vocab_overlap": 0.37709923664122136, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5551490843315533, + "6": 0.4664419216133654, + "7": 0.40072301907666974, + "8": 0.3352980620771035, + "9": 0.2604202091529816, + "10": 0.2088632234488524, + "11": 0.13100265368262387, + "12": 0.07737513096875989, + "13": 0.03392089936118349 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.18169209414294787, + "best1L": 0.15672676850642478, + "mean": 0.04638619202056101, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13451776493872916, + "best1L": 0.10261058754154614, + "mean": 0.025179276641990458, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.02337662356772593, + "best1L": 0.018181818005229746, + "mean": 0.0028860028500535657, + "persist": 0.0 + }, + "poetry": { + "min": 0.033527696904327185, + "best1L": 0.011661807873419352, + "mean": 0.005021056043915451, + "persist": 0.0 + }, + "typo": { + "min": 0.4300595202616283, + "best1L": 0.29315476438828875, + "mean": 0.22486772281782968, + "persist": 0.26041666471532415 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.024844720294432982, + "mean": 0.007591442221642605, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5673477321600153, + "dispersion_mean": 0.2764095930757691, + "dispersion_slope": -0.06682527909977082, + "stable_rank": 45.38960266113281, + "top8_spectral_share": 0.0659954696893692, + "condition_number": 25814.265625, + "unembed_alignment": 0.08533263206481934 + }, + "corpus": { + "mean_prompt_tokens": 856.155, + "type_token_ratio": 0.07197294882351911, + "eval_vocab_overlap": 0.40458015267175573, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5673477321600153, + "6": 0.4748887145407973, + "7": 0.40453049849320216, + "8": 0.33529677192762986, + "9": 0.25926403400541287, + "10": 0.2070799701752128, + "11": 0.1294749356399967, + "12": 0.07634646922358966, + "13": 0.033457211516065594 + } + } + }, + "runtime_s": 1155.1, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_USPTO_Backgrounds_s2.json b/results/ladder1b/tv_USPTO_Backgrounds_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..8d39fc8e0aa844c54e5c897203ccc03bf1698bea --- /dev/null +++ b/results/ladder1b/tv_USPTO_Backgrounds_s2.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "USPTO_Backgrounds", + "seed": 2, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18723994653139794, + "best1L": 0.1595006941684655, + "mean": 0.04777315431939704, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.13705583768231527, + "best1L": 0.10333574855966228, + "mean": 0.024937554861285856, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.016883116880697862, + "best1L": 0.016883116880697862, + "mean": 0.0021645021957478355, + "persist": 0.0 + }, + "poetry": { + "min": 0.032069970454488485, + "best1L": 0.011661807740373271, + "mean": 0.004697116823600871, + "persist": 0.0 + }, + "typo": { + "min": 0.41220237740448545, + "best1L": 0.26934523986918585, + "mean": 0.20552248720611846, + "persist": 0.23958333102720125 + }, + "association": { + "min": 0.045031054743698666, + "best1L": 0.031055900135210583, + "mean": 0.009661835551794087, + "persist": 0.007763975166848728 + } + }, + "operator": { + "dispersion_L0": 0.5016068626304079, + "dispersion_mean": 0.24477083955803336, + "dispersion_slope": -0.05855149884610627, + "stable_rank": 43.5169792175293, + "top8_spectral_share": 0.06848541647195816, + "condition_number": 5833.79736328125, + "unembed_alignment": 0.0841280072927475 + }, + "corpus": { + "mean_prompt_tokens": 677.6, + "type_token_ratio": 0.2532467532467532, + "eval_vocab_overlap": 0.11297709923664122, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5016068626304079, + "6": 0.4226043837922462, + "7": 0.35128828822817304, + "8": 0.29427006421523594, + "9": 0.22701109624608268, + "10": 0.18639210463738407, + "11": 0.1183347168520479, + "12": 0.07078438422079401, + "13": 0.030645655199928453 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17753120671425546, + "best1L": 0.15533980780414172, + "mean": 0.04623208540890898, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1337926025901522, + "best1L": 0.10297316791755813, + "mean": 0.025219563095431243, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.002597402548417449, + "persist": 0.0 + }, + "poetry": { + "min": 0.03790087492338249, + "best1L": 0.016034985360290323, + "mean": 0.005668934135298643, + "persist": 0.0 + }, + "typo": { + "min": 0.4538690469094685, + "best1L": 0.32589285927159445, + "mean": 0.24619708955287933, + "persist": 0.28720238112977575 + }, + "association": { + "min": 0.046583851001092365, + "best1L": 0.02484472082661731, + "mean": 0.008454106282442808, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.5519794325019316, + "dispersion_mean": 0.2735380469937377, + "dispersion_slope": -0.0651494002031872, + "stable_rank": 45.42517852783203, + "top8_spectral_share": 0.06646056473255157, + "condition_number": 7522.41650390625, + "unembed_alignment": 0.08531197905540466 + }, + "corpus": { + "mean_prompt_tokens": 786.6, + "type_token_ratio": 0.16770912789219425, + "eval_vocab_overlap": 0.18167938931297709, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5519794325019316, + "6": 0.4696582747972445, + "7": 0.39805552103328795, + "8": 0.33114586534744817, + "9": 0.2599244673736741, + "10": 0.21059480671282468, + "11": 0.1296910475919537, + "12": 0.0779634826155796, + "13": 0.03282952496969516 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.18169209307857923, + "best1L": 0.15672677169953073, + "mean": 0.04684851358511618, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13524292835167476, + "best1L": 0.10369833399142538, + "mean": 0.02542099789051073, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.019480519129761627, + "mean": 0.002886002916576607, + "persist": 0.0 + }, + "poetry": { + "min": 0.03352769663823502, + "best1L": 0.013119533791073732, + "mean": 0.004859086541858103, + "persist": 0.0 + }, + "typo": { + "min": 0.45238095947674345, + "best1L": 0.32589285714285715, + "mean": 0.24239417697702134, + "persist": 0.2738095239869186 + }, + "association": { + "min": 0.046583849537585466, + "best1L": 0.026397515487458025, + "mean": 0.00897170468566141, + "persist": 0.00776397529989481 + } + }, + "operator": { + "dispersion_L0": 0.5595537977626145, + "dispersion_mean": 0.27749916557311255, + "dispersion_slope": -0.06573764722529124, + "stable_rank": 45.60226821899414, + "top8_spectral_share": 0.0654069036245346, + "condition_number": 6350.17578125, + "unembed_alignment": 0.08626112341880798 + }, + "corpus": { + "mean_prompt_tokens": 760.44, + "type_token_ratio": 0.1487559833780443, + "eval_vocab_overlap": 0.2549618320610687, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5595537977626145, + "6": 0.47301897209535965, + "7": 0.402987340141887, + "8": 0.33741732576484323, + "9": 0.2639401821578482, + "10": 0.213165417292874, + "11": 0.13347769690968128, + "12": 0.07944256337617926, + "13": 0.03448919465672652 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.18169209414294787, + "best1L": 0.15533980780414172, + "mean": 0.046694406042141576, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1356055097920554, + "best1L": 0.10333575042230743, + "mean": 0.02562243231971349, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.002597402548417449, + "persist": 0.0 + }, + "poetry": { + "min": 0.04081632689173732, + "best1L": 0.01457725970872811, + "mean": 0.00647878206135439, + "persist": 0.0 + }, + "typo": { + "min": 0.4479166716337204, + "best1L": 0.31249999680689405, + "mean": 0.23908729957682745, + "persist": 0.27380952345473425 + }, + "association": { + "min": 0.040372670362038274, + "best1L": 0.024844720294432982, + "mean": 0.0075914423214271665, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.5692146358635477, + "dispersion_mean": 0.2754348381555124, + "dispersion_slope": -0.0666930264994114, + "stable_rank": 45.682098388671875, + "top8_spectral_share": 0.06563644856214523, + "condition_number": 8010.80419921875, + "unembed_alignment": 0.08619038760662079 + }, + "corpus": { + "mean_prompt_tokens": 762.0533333333333, + "type_token_ratio": 0.12457570773699128, + "eval_vocab_overlap": 0.30381679389312977, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5692146358635477, + "6": 0.4746688567450134, + "7": 0.40009761820096623, + "8": 0.3314414595189906, + "9": 0.2569052236592565, + "10": 0.20641013303266903, + "11": 0.12964732469272625, + "12": 0.07689772951841879, + "13": 0.03363056216802306 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18307905590959958, + "best1L": 0.15811373240181378, + "mean": 0.04731083355311837, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1345177644065448, + "best1L": 0.10333575042230743, + "mean": 0.025300136700804745, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.002597402548417449, + "persist": 0.0 + }, + "poetry": { + "min": 0.039358600574944703, + "best1L": 0.01457725970872811, + "mean": 0.00615484279114753, + "persist": 0.0 + }, + "typo": { + "min": 0.4464285692998341, + "best1L": 0.31101190724543165, + "mean": 0.23908730436648643, + "persist": 0.2738095239869186 + }, + "association": { + "min": 0.04347826061504228, + "best1L": 0.026397515354411944, + "mean": 0.007936507929116488, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.5684543163267366, + "dispersion_mean": 0.2749056176767891, + "dispersion_slope": -0.06660221685344818, + "stable_rank": 45.49574661254883, + "top8_spectral_share": 0.06572692096233368, + "condition_number": 9729.783203125, + "unembed_alignment": 0.08719620853662491 + }, + "corpus": { + "mean_prompt_tokens": 698.86, + "type_token_ratio": 0.11398563374638697, + "eval_vocab_overlap": 0.3198473282442748, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5684543163267366, + "6": 0.4739228804921615, + "7": 0.3998765627530775, + "8": 0.3305808779655158, + "9": 0.2558745831258378, + "10": 0.20534632157122992, + "11": 0.12957004598511399, + "12": 0.07679939797896394, + "13": 0.03372557289246512 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.18030513131192752, + "best1L": 0.15395284284438407, + "mean": 0.04669440630823374, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13524293048041208, + "best1L": 0.10224800397242818, + "mean": 0.025219563727400134, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.002453102430860911, + "persist": 0.0 + }, + "poetry": { + "min": 0.03790087439119816, + "best1L": 0.013119533791073732, + "mean": 0.005830903770402074, + "persist": 0.0 + }, + "typo": { + "min": 0.4449404754808971, + "best1L": 0.31696428677865435, + "mean": 0.24140211726937974, + "persist": 0.27976190937416895 + }, + "association": { + "min": 0.040372670362038274, + "best1L": 0.024844720294432982, + "mean": 0.007418909492636365, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.5659470142374787, + "dispersion_mean": 0.2755412445690089, + "dispersion_slope": -0.06629493867564953, + "stable_rank": 45.89728546142578, + "top8_spectral_share": 0.06549137830734253, + "condition_number": 40846.4296875, + "unembed_alignment": 0.08579893410205841 + }, + "corpus": { + "mean_prompt_tokens": 697.4266666666666, + "type_token_ratio": 0.09326667558835337, + "eval_vocab_overlap": 0.366412213740458, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5659470142374787, + "6": 0.47343349946163427, + "7": 0.40042083161888825, + "8": 0.3320854808451176, + "9": 0.2577207986160589, + "10": 0.20722350881909413, + "11": 0.13103689382413256, + "12": 0.07799061238332561, + "13": 0.03401256131535097 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.18030513131192752, + "best1L": 0.15533980567540442, + "mean": 0.04638619175446885, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13633067267281668, + "best1L": 0.10261058541280883, + "mean": 0.025541858548032387, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.016883116880697862, + "mean": 0.00259740261494049, + "persist": 0.0 + }, + "poetry": { + "min": 0.036443148872682025, + "best1L": 0.013119533791073732, + "mean": 0.005830903670617512, + "persist": 0.0 + }, + "typo": { + "min": 0.44345237740448545, + "best1L": 0.31249999680689405, + "mean": 0.2390873059630394, + "persist": 0.2782738134264946 + }, + "association": { + "min": 0.04347826061504228, + "best1L": 0.026397515354411944, + "mean": 0.007763975100325686, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.5525729882369634, + "dispersion_mean": 0.27046866353902865, + "dispersion_slope": -0.06478534001632766, + "stable_rank": 45.486572265625, + "top8_spectral_share": 0.06580262631177902, + "condition_number": 55674.0, + "unembed_alignment": 0.08507412672042847 + }, + "corpus": { + "mean_prompt_tokens": 724.32, + "type_token_ratio": 0.07828031809145129, + "eval_vocab_overlap": 0.4015267175572519, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5525729882369634, + "6": 0.46339071155756445, + "7": 0.3933128306183753, + "8": 0.32698416487933113, + "9": 0.25406895422354026, + "10": 0.20455797524659375, + "11": 0.12907270340963406, + "12": 0.0767804440259174, + "13": 0.033477199653338834 + } + } + }, + "runtime_s": 1157.9, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Wikipedia_en_s0.json b/results/ladder1b/tv_Wikipedia_en_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..1d686977f59ae5992557ed086680fd867105b42e --- /dev/null +++ b/results/ladder1b/tv_Wikipedia_en_s0.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Wikipedia_en", + "seed": 0, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.15533980514322007, + "best1L": 0.13453536746757372, + "mean": 0.03852673726422446, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11493835811104093, + "best1L": 0.08520667188401733, + "mean": 0.020062847728175775, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.016883117013743947, + "best1L": 0.009090909202183996, + "mean": 0.0021645021624863148, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.00874635563897235, + "mean": 0.0027534824164052096, + "persist": 0.0 + }, + "typo": { + "min": 0.2916666641831398, + "best1L": 0.2038690485060215, + "mean": 0.11789020922567163, + "persist": 0.1235119044514639 + }, + "association": { + "min": 0.04503105540892908, + "best1L": 0.029503105341323783, + "mean": 0.009489302839418607, + "persist": 0.00776397529989481 + } + }, + "operator": { + "dispersion_L0": 0.5963914382640559, + "dispersion_mean": 0.3013923058418326, + "dispersion_slope": -0.0708221595471423, + "stable_rank": 38.92494201660156, + "top8_spectral_share": 0.0752456858754158, + "condition_number": 4004.5419921875, + "unembed_alignment": 0.08402330428361893 + }, + "corpus": { + "mean_prompt_tokens": 473.1, + "type_token_ratio": 0.3753963221306278, + "eval_vocab_overlap": 0.1648854961832061, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5963914382640559, + "6": 0.5055329353436832, + "7": 0.4405574528482072, + "8": 0.38068847956679686, + "9": 0.29278447583262873, + "10": 0.23429631822967809, + "11": 0.14256395298236008, + "12": 0.08364865070878358, + "13": 0.036067048800299546 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.15533980514322007, + "best1L": 0.1359223292342254, + "mean": 0.03868084480719907, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12291515804827213, + "best1L": 0.0866569976455399, + "mean": 0.022238337168736116, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.009090909202183996, + "mean": 0.0018759019273732389, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533258889402, + "best1L": 0.00874635563897235, + "mean": 0.002267573677402522, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30208332943064825, + "best1L": 0.20982143123235023, + "mean": 0.1263227534613439, + "persist": 0.13541666750929185 + }, + "association": { + "min": 0.04503105527588299, + "best1L": 0.029503105341323783, + "mean": 0.009316770027258567, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6923814597263568, + "dispersion_mean": 0.3480456270110184, + "dispersion_slope": -0.08240649900868452, + "stable_rank": 40.969810485839844, + "top8_spectral_share": 0.07022933661937714, + "condition_number": 2878.159912109375, + "unembed_alignment": 0.08298691362142563 + }, + "corpus": { + "mean_prompt_tokens": 607.48, + "type_token_ratio": 0.275037861328768, + "eval_vocab_overlap": 0.3099236641221374, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6923814597263568, + "6": 0.5860869560989266, + "7": 0.5118485644603324, + "8": 0.44185098201507067, + "9": 0.33257587580784176, + "10": 0.2654919988919001, + "11": 0.16372772287081994, + "12": 0.09779089992833936, + "13": 0.0406561832995785 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.15256588054554804, + "best1L": 0.13314840463655336, + "mean": 0.037602096529943604, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12073966727725097, + "best1L": 0.0866569976455399, + "mean": 0.021553460774677142, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.011688311584293842, + "best1L": 0.01038961045976196, + "mean": 0.0012987012908394849, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985094198158, + "best1L": 0.00728862958827189, + "mean": 0.0025915128478248206, + "persist": 0.0 + }, + "typo": { + "min": 0.30654762365988325, + "best1L": 0.21875000053218432, + "mean": 0.12499999893563134, + "persist": 0.1309523812628218 + }, + "association": { + "min": 0.040372670362038274, + "best1L": 0.027950310680483068, + "mean": 0.008626638695464603, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.7280667201454297, + "dispersion_mean": 0.3679835330243416, + "dispersion_slope": -0.08606234354321345, + "stable_rank": 40.463050842285156, + "top8_spectral_share": 0.07048916816711426, + "condition_number": 2914.02978515625, + "unembed_alignment": 0.07909112423658371 + }, + "corpus": { + "mean_prompt_tokens": 584.98, + "type_token_ratio": 0.21840062908133612, + "eval_vocab_overlap": 0.39236641221374047, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7280667201454297, + "6": 0.6142234321445779, + "7": 0.5383576642188906, + "8": 0.46562901981038896, + "9": 0.3536270521435394, + "10": 0.28454409603228276, + "11": 0.177566503585931, + "12": 0.10548542708546443, + "13": 0.044351882052569576 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.15117891877889633, + "best1L": 0.13176144286990166, + "mean": 0.037602096263851435, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11928933938699109, + "best1L": 0.0866569976455399, + "mean": 0.021311739326587746, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.011688311584293842, + "best1L": 0.01038961045976196, + "mean": 0.0012987012908394849, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.00874635563897235, + "mean": 0.003239391038992575, + "persist": 0.0 + }, + "typo": { + "min": 0.3020833304950169, + "best1L": 0.21279762099896157, + "mean": 0.12516534248633043, + "persist": 0.13244047508175885 + }, + "association": { + "min": 0.03881987516901323, + "best1L": 0.027950310680483068, + "mean": 0.008109040541707404, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.7043139426837298, + "dispersion_mean": 0.3557650883155636, + "dispersion_slope": -0.08303742042014134, + "stable_rank": 40.184852600097656, + "top8_spectral_share": 0.07099461555480957, + "condition_number": 2694.249755859375, + "unembed_alignment": 0.07934969663619995 + }, + "corpus": { + "mean_prompt_tokens": 628.08, + "type_token_ratio": 0.18373455610750222, + "eval_vocab_overlap": 0.48244274809160304, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7043139426837298, + "6": 0.5933705840055852, + "7": 0.5194068570828839, + "8": 0.44813234795960444, + "9": 0.34233076810506946, + "10": 0.27572015740071626, + "11": 0.1727859836794169, + "12": 0.1025243807832487, + "13": 0.04330077313981776 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.14979195701224463, + "best1L": 0.13176144286990166, + "mean": 0.03698566848678248, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11965192082737174, + "best1L": 0.08593183476477861, + "mean": 0.021392313364361013, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012708825725, + "best1L": 0.01038961045976196, + "mean": 0.0014430014749190637, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711544036865, + "best1L": 0.00874635563897235, + "mean": 0.003077421536935227, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.306547611951828, + "best1L": 0.21279762099896157, + "mean": 0.12748015911451407, + "persist": 0.1309523791340845 + }, + "association": { + "min": 0.03571428584733179, + "best1L": 0.026397516019642353, + "mean": 0.0075914423214271665, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.7157679795663753, + "dispersion_mean": 0.36048862996250175, + "dispersion_slope": -0.08441234226312, + "stable_rank": 40.50418472290039, + "top8_spectral_share": 0.0709468424320221, + "condition_number": 2106.712890625, + "unembed_alignment": 0.0804259181022644 + }, + "corpus": { + "mean_prompt_tokens": 835.6, + "type_token_ratio": 0.14358544758257538, + "eval_vocab_overlap": 0.5870229007633587, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7157679795663753, + "6": 0.6030089879750314, + "7": 0.5264103667363026, + "8": 0.45235865279326193, + "9": 0.3467530746045403, + "10": 0.2784865737632766, + "11": 0.17451315467022321, + "12": 0.10337066864847873, + "13": 0.043728210905025794 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.15117891771452768, + "best1L": 0.1303744800388813, + "mean": 0.037293881443994384, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12146482962582793, + "best1L": 0.08738216132457767, + "mean": 0.021795182023197412, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.0014430014083960227, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.007288629721317973, + "mean": 0.0029154519849855986, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.313988097012043, + "best1L": 0.21577381289431027, + "mean": 0.1294642843838249, + "persist": 0.1383928551471659 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.024844720294432982, + "mean": 0.007246376880045448, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.7395412099555592, + "dispersion_mean": 0.3697689046494183, + "dispersion_slope": -0.08751767067370902, + "stable_rank": 40.65960693359375, + "top8_spectral_share": 0.07069732248783112, + "condition_number": 2240.473876953125, + "unembed_alignment": 0.08022348582744598 + }, + "corpus": { + "mean_prompt_tokens": 1311.06, + "type_token_ratio": 0.09414773796266634, + "eval_vocab_overlap": 0.7244274809160305, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7395412099555592, + "6": 0.6223472528218907, + "7": 0.542101417214688, + "8": 0.4632725220085934, + "9": 0.3538911353218227, + "10": 0.283238302795678, + "11": 0.1760119109766456, + "12": 0.10370597344517886, + "13": 0.04381041730470781 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.1303744800388813, + "mean": 0.037139774433204105, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1207396678094353, + "best1L": 0.08701957988419703, + "mean": 0.021513173955359628, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.0014430014083960227, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.007288629721317973, + "mean": 0.0030774214870429467, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3110119008592197, + "best1L": 0.2127976231276989, + "mean": 0.13029100612870284, + "persist": 0.13839285727590322 + }, + "association": { + "min": 0.03571428531514747, + "best1L": 0.026397516019642353, + "mean": 0.00759144258751933, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.7373408756864744, + "dispersion_mean": 0.36795329561418133, + "dispersion_slope": -0.08724735761627327, + "stable_rank": 40.719383239746094, + "top8_spectral_share": 0.07073232531547546, + "condition_number": 2351.32958984375, + "unembed_alignment": 0.08063460886478424 + }, + "corpus": { + "mean_prompt_tokens": 1188.27, + "type_token_ratio": 0.08580120679643516, + "eval_vocab_overlap": 0.7618320610687023, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7373408756864744, + "6": 0.619669979776402, + "7": 0.5390880940895577, + "8": 0.46114849013370096, + "9": 0.3521219553927011, + "10": 0.2812355845906866, + "11": 0.17465770864629665, + "12": 0.1029622273190047, + "13": 0.04335474489280764 + } + } + }, + "runtime_s": 1149.6, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Wikipedia_en_s1.json b/results/ladder1b/tv_Wikipedia_en_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..a2189ab4f4b332ad989e06b05731e3458a4539da --- /dev/null +++ b/results/ladder1b/tv_Wikipedia_en_s1.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Wikipedia_en", + "seed": 1, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.15811373346618243, + "best1L": 0.13592233029859407, + "mean": 0.03945137866373573, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12073966967208045, + "best1L": 0.08520667228315558, + "mean": 0.022399483814037273, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.01688311727983611, + "best1L": 0.010389610592808043, + "mean": 0.0023088022800428526, + "persist": 0.0 + }, + "poetry": { + "min": 0.01020408155662673, + "best1L": 0.005830903670617512, + "mean": 0.0025915127813017796, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.33630951919725965, + "best1L": 0.22023809541549003, + "mean": 0.14302248694002628, + "persist": 0.1517857120239309 + }, + "association": { + "min": 0.037267080907310755, + "best1L": 0.02950310574046203, + "mean": 0.008109040491815125, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.5540066271512285, + "dispersion_mean": 0.2916952930345572, + "dispersion_slope": -0.06519770882004927, + "stable_rank": 40.90338134765625, + "top8_spectral_share": 0.07104503363370895, + "condition_number": 21029.5390625, + "unembed_alignment": 0.08136150985956192 + }, + "corpus": { + "mean_prompt_tokens": 1014.4, + "type_token_ratio": 0.25295741324921134, + "eval_vocab_overlap": 0.22366412213740458, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.5540066271512285, + "6": 0.47673740388441893, + "7": 0.4237973934323823, + "8": 0.37016608564327214, + "9": 0.2911884511166597, + "10": 0.23256782140793003, + "11": 0.14901670692761843, + "12": 0.08957276268993487, + "13": 0.03820438505757003 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.1539528433765684, + "best1L": 0.13592233029859407, + "mean": 0.038218524040920396, + "persist": 0.0 + }, + "multilingual": { + "min": 0.12364031986466475, + "best1L": 0.08665699777858597, + "mean": 0.022762065520510077, + "persist": 0.002900652587413788 + }, + "order-ops": { + "min": 0.02077922078647784, + "best1L": 0.014285714099449771, + "mean": 0.002453102430860911, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00728862958827189, + "mean": 0.003077421453781426, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.32142857036420275, + "best1L": 0.22172619455627032, + "mean": 0.1350859778800181, + "persist": 0.13392857289207832 + }, + "association": { + "min": 0.03726708064121859, + "best1L": 0.024844720294432982, + "mean": 0.008109040525076645, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6418016177682431, + "dispersion_mean": 0.3249557542444729, + "dispersion_slope": -0.07565173091876787, + "stable_rank": 40.99851989746094, + "top8_spectral_share": 0.07088685035705566, + "condition_number": 3713.778076171875, + "unembed_alignment": 0.08066453784704208 + }, + "corpus": { + "mean_prompt_tokens": 875.16, + "type_token_ratio": 0.22322775263951736, + "eval_vocab_overlap": 0.3473282442748092, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6418016177682431, + "6": 0.5414971293010173, + "7": 0.47350613727662944, + "8": 0.4100750154934172, + "9": 0.31477515085303076, + "10": 0.2502290866983696, + "11": 0.15843726777982484, + "12": 0.09454386048028784, + "13": 0.03973652254943669 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.151178916650159, + "best1L": 0.133148405700922, + "mean": 0.037447988986968994, + "persist": 0.0 + }, + "multilingual": { + "min": 0.12327773895646844, + "best1L": 0.08846990524658135, + "mean": 0.022278623522392342, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.015584415223981653, + "best1L": 0.012987012974917889, + "mean": 0.0017316017765551805, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.008746355505926269, + "mean": 0.0037252996948414613, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.30654761620930265, + "best1L": 0.22023809222238405, + "mean": 0.1259920642312084, + "persist": 0.13244047880704915 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.02950310574046203, + "mean": 0.008109040491815125, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6945654150703345, + "dispersion_mean": 0.3469236012405893, + "dispersion_slope": -0.08208663682576718, + "stable_rank": 40.09164047241211, + "top8_spectral_share": 0.0720309242606163, + "condition_number": 5550.63525390625, + "unembed_alignment": 0.0806603878736496 + }, + "corpus": { + "mean_prompt_tokens": 1091.84, + "type_token_ratio": 0.16892584994138335, + "eval_vocab_overlap": 0.5099236641221374, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6945654150703345, + "6": 0.5830861621549028, + "7": 0.5076271229970067, + "8": 0.43499803146962523, + "9": 0.33312641046145053, + "10": 0.26498112427897214, + "11": 0.1651817571094978, + "12": 0.0977559743283681, + "13": 0.04099041329514543 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.15117891771452768, + "best1L": 0.13314840463655336, + "mean": 0.03760209573166711, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12291515751608781, + "best1L": 0.0881073238062007, + "mean": 0.02231891067432506, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.015584415223981653, + "best1L": 0.012987012974917889, + "mean": 0.0017316017765551805, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.005830903670617512, + "mean": 0.003077421453781426, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.30059523774044855, + "best1L": 0.21428571694663592, + "mean": 0.12367724600647177, + "persist": 0.1294642858473318 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.027950310680483068, + "mean": 0.007591442271534886, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.7214863353224716, + "dispersion_mean": 0.35873150694134803, + "dispersion_slope": -0.08518291793236162, + "stable_rank": 39.963191986083984, + "top8_spectral_share": 0.07206524163484573, + "condition_number": 35202.94140625, + "unembed_alignment": 0.08057411760091782 + }, + "corpus": { + "mean_prompt_tokens": 1075.0933333333332, + "type_token_ratio": 0.14535172140093264, + "eval_vocab_overlap": 0.5938931297709924, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7214863353224716, + "6": 0.6033974317361552, + "7": 0.5242252780160045, + "8": 0.4490824687486879, + "9": 0.3440454931011, + "10": 0.2741628761296027, + "11": 0.16993041426660152, + "12": 0.10034117993148893, + "13": 0.041912085220019775 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.037447988986968994, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1232777378920998, + "best1L": 0.08738216079239335, + "mean": 0.022600918343024596, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.011688311850386006, + "mean": 0.0015873015924756015, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.008746355505926269, + "mean": 0.0035633301262610723, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.30803571960755755, + "best1L": 0.21279761674148695, + "mean": 0.12632275080042227, + "persist": 0.1309523812628218 + }, + "association": { + "min": 0.038819875568151474, + "best1L": 0.027950310680483068, + "mean": 0.007763975100325686, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.7386872056225807, + "dispersion_mean": 0.3661708389947264, + "dispersion_slope": -0.08736439512508315, + "stable_rank": 40.098236083984375, + "top8_spectral_share": 0.0717889741063118, + "condition_number": 9511.6533203125, + "unembed_alignment": 0.08157786726951599 + }, + "corpus": { + "mean_prompt_tokens": 1075.36, + "type_token_ratio": 0.13107238506174676, + "eval_vocab_overlap": 0.6595419847328244, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7386872056225807, + "6": 0.617690976012976, + "7": 0.5363415976156877, + "8": 0.45888718845815135, + "9": 0.3495675373646967, + "10": 0.27801839763016434, + "11": 0.17209503380993352, + "12": 0.10167849628963249, + "13": 0.04257111814871492 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.03760209573166711, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12001450492867402, + "best1L": 0.08665699791163206, + "mean": 0.02199661548781608, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.0014430014083960227, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.00874635563897235, + "mean": 0.0032393909891002943, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.31249999893563135, + "best1L": 0.2172619067132473, + "mean": 0.13111772680921213, + "persist": 0.13839285355061293 + }, + "association": { + "min": 0.038819875568151474, + "best1L": 0.027950310680483068, + "mean": 0.008109040757907289, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.733852971903743, + "dispersion_mean": 0.362280311851097, + "dispersion_slope": -0.08686380495453488, + "stable_rank": 40.45170974731445, + "top8_spectral_share": 0.07112841308116913, + "condition_number": 16232.2197265625, + "unembed_alignment": 0.08185826987028122 + }, + "corpus": { + "mean_prompt_tokens": 988.3066666666666, + "type_token_ratio": 0.11278550517383268, + "eval_vocab_overlap": 0.7259541984732825, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.733852971903743, + "6": 0.6133321013787665, + "7": 0.5314787114370657, + "8": 0.4533698655588226, + "9": 0.34349734636771384, + "10": 0.27384875766280475, + "11": 0.16929017715246206, + "12": 0.09993342984957446, + "13": 0.04191944534892005 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.03775620274245739, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12110224924981594, + "best1L": 0.0870195793520127, + "mean": 0.0219966155210776, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.0014430014083960227, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.00728862958827189, + "mean": 0.0032393909891002943, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.31250000319310595, + "best1L": 0.21577380863683565, + "mean": 0.13227513352675097, + "persist": 0.13839285621153458 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.026397516019642353, + "mean": 0.008109040491815125, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.7282724694211249, + "dispersion_mean": 0.35987468015675994, + "dispersion_slope": -0.08621044242852666, + "stable_rank": 40.894935607910156, + "top8_spectral_share": 0.07056150585412979, + "condition_number": 7075.61474609375, + "unembed_alignment": 0.08267121762037277 + }, + "corpus": { + "mean_prompt_tokens": 1070.71, + "type_token_ratio": 0.09227054944849679, + "eval_vocab_overlap": 0.7770992366412214, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7282724694211249, + "6": 0.6089320027468312, + "7": 0.5276671949254197, + "8": 0.4504241108176888, + "9": 0.3422152827647672, + "10": 0.27223317869211255, + "11": 0.1682092056234001, + "12": 0.09930845473496898, + "13": 0.041610221684525786 + } + } + }, + "runtime_s": 1148.7, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b/tv_Wikipedia_en_s2.json b/results/ladder1b/tv_Wikipedia_en_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..aa3a7735e2eed283d3af820c8eec7b5b315d1147 --- /dev/null +++ b/results/ladder1b/tv_Wikipedia_en_s2.json @@ -0,0 +1,496 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Wikipedia_en", + "seed": 2, + "band": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.1484049925846713, + "best1L": 0.133148405700922, + "mean": 0.03529049176722765, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11711384914815426, + "best1L": 0.08846990604485784, + "mean": 0.02066715045033821, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.0014430014083960227, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533258889402, + "best1L": 0.00728862958827189, + "mean": 0.0024295432127213906, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.28273809381893705, + "best1L": 0.21279761887022428, + "mean": 0.10763888832713876, + "persist": 0.10714285461498159 + }, + "association": { + "min": 0.029503106006554196, + "best1L": 0.02018633538058826, + "mean": 0.006211180173392806, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6086310107277106, + "dispersion_mean": 0.30876674528991876, + "dispersion_slope": -0.07183924745250886, + "stable_rank": 38.676780700683594, + "top8_spectral_share": 0.07327888160943985, + "condition_number": 3438.409912109375, + "unembed_alignment": 0.0828540250658989 + }, + "corpus": { + "mean_prompt_tokens": 1159.7, + "type_token_ratio": 0.2673105113391394, + "eval_vocab_overlap": 0.25572519083969464, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6086310107277106, + "6": 0.514910180178838, + "7": 0.4517956807526833, + "8": 0.3886835945953888, + "9": 0.2988448268912059, + "10": 0.2384881421053628, + "11": 0.14865566703466748, + "12": 0.09019120507076459, + "13": 0.03870040025264749 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.1567267690386091, + "best1L": 0.133148405700922, + "mean": 0.03837263051952634, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12182741319494587, + "best1L": 0.0881073238062007, + "mean": 0.022117476710783585, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.011688311584293842, + "best1L": 0.01038961045976196, + "mean": 0.0012987012908394849, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985094198158, + "best1L": 0.00728862958827189, + "mean": 0.0027534823498821686, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.2916666663118771, + "best1L": 0.21130952505128725, + "mean": 0.11970899546785015, + "persist": 0.12351190578192472 + }, + "association": { + "min": 0.029503106006554196, + "best1L": 0.024844720294432982, + "mean": 0.007246376880045448, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.6896478132278909, + "dispersion_mean": 0.3423375922209171, + "dispersion_slope": -0.08160813686982298, + "stable_rank": 40.91444396972656, + "top8_spectral_share": 0.06988204270601273, + "condition_number": 5702.87646484375, + "unembed_alignment": 0.08519232273101807 + }, + "corpus": { + "mean_prompt_tokens": 1155.12, + "type_token_ratio": 0.2179513816746312, + "eval_vocab_overlap": 0.4198473282442748, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6896478132278909, + "6": 0.5768516194244234, + "7": 0.5012084873902647, + "8": 0.4305869800609475, + "9": 0.32752746833960117, + "10": 0.25996825244888566, + "11": 0.16002553141029477, + "12": 0.09524617217632453, + "13": 0.03997600550962089 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.151178916650159, + "best1L": 0.133148405700922, + "mean": 0.03698566795459816, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11856417703841414, + "best1L": 0.08665699791163206, + "mean": 0.021593747727040733, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.0014430014083960227, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.005830903670617512, + "mean": 0.0025915128644555807, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.2946428603359631, + "best1L": 0.21428571481789863, + "mean": 0.12003968256924834, + "persist": 0.12202381089861904 + }, + "association": { + "min": 0.03726708037512643, + "best1L": 0.026397514955273697, + "mean": 0.008454106149396725, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.6830182428177147, + "dispersion_mean": 0.3379078530843464, + "dispersion_slope": -0.08081742805886624, + "stable_rank": 40.72758483886719, + "top8_spectral_share": 0.07092487066984177, + "condition_number": 2040.6922607421875, + "unembed_alignment": 0.08474588394165039 + }, + "corpus": { + "mean_prompt_tokens": 1273.0, + "type_token_ratio": 0.16691280439905734, + "eval_vocab_overlap": 0.533587786259542, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.6830182428177147, + "6": 0.5709934250210824, + "7": 0.494850301682867, + "8": 0.42348066293369796, + "9": 0.32242706077724764, + "10": 0.25564617263784073, + "11": 0.1579277621201939, + "12": 0.09362106685055721, + "13": 0.03920598291791576 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.14840499311685562, + "best1L": 0.13314840463655336, + "mean": 0.037447989253061156, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12110224924981594, + "best1L": 0.08629441647125143, + "mean": 0.021835469075345566, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.011688311850386006, + "mean": 0.0015873015924756015, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.005830903670617512, + "mean": 0.0029154519350933178, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.2976190501025745, + "best1L": 0.2098214291036129, + "mean": 0.12285052905125278, + "persist": 0.12351190365318741 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.024844720294432982, + "mean": 0.008281573520175048, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.7336836808187426, + "dispersion_mean": 0.36063485363161124, + "dispersion_slope": -0.08685896323361196, + "stable_rank": 41.1347770690918, + "top8_spectral_share": 0.07052216678857803, + "condition_number": 2013.7548828125, + "unembed_alignment": 0.08284927904605865 + }, + "corpus": { + "mean_prompt_tokens": 1527.08, + "type_token_ratio": 0.1274938662894762, + "eval_vocab_overlap": 0.6427480916030535, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7336836808187426, + "6": 0.611568600219889, + "7": 0.5278810668395582, + "8": 0.4518673657490676, + "9": 0.34248956113320417, + "10": 0.2718878815249586, + "11": 0.16662393982832902, + "12": 0.09844987811852367, + "13": 0.04126170845222875 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.14979195488350733, + "best1L": 0.13453536640320504, + "mean": 0.03791030948715551, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12001450279993671, + "best1L": 0.08629441647125143, + "mean": 0.021593747760302255, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.0014430014083960227, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.005830903670617512, + "mean": 0.0029154519350933178, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.30505952345473425, + "best1L": 0.21726190245577268, + "mean": 0.12417328064995152, + "persist": 0.12797619149621045 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.024844720294432982, + "mean": 0.007936508128685611, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.7309540053169609, + "dispersion_mean": 0.3606258536994695, + "dispersion_slope": -0.0865115984848107, + "stable_rank": 40.880088806152344, + "top8_spectral_share": 0.07076822221279144, + "condition_number": 2947.865478515625, + "unembed_alignment": 0.08200081437826157 + }, + "corpus": { + "mean_prompt_tokens": 1360.73, + "type_token_ratio": 0.11701807118238004, + "eval_vocab_overlap": 0.6732824427480916, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7309540053169609, + "6": 0.6108814672693653, + "7": 0.5280968129013105, + "8": 0.45148545984013594, + "9": 0.34272598505690816, + "10": 0.2723851311152808, + "11": 0.1679853621793761, + "12": 0.09938609430753068, + "13": 0.04173236530835718 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.03852673726422446, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12037708211158003, + "best1L": 0.08665699791163206, + "mean": 0.021875755794878517, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.011688311850386006, + "mean": 0.0015873015924756015, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.005830903670617512, + "mean": 0.0030774214870429467, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.30803571535008295, + "best1L": 0.21726190458450997, + "mean": 0.1312830684972661, + "persist": 0.1383928551471659 + }, + "association": { + "min": 0.04347826008285795, + "best1L": 0.0310559000021645, + "mean": 0.00948930298909545, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.7207363329812485, + "dispersion_mean": 0.35658185445661356, + "dispersion_slope": -0.08534955052699636, + "stable_rank": 41.18848419189453, + "top8_spectral_share": 0.0704093873500824, + "condition_number": 3682.5908203125, + "unembed_alignment": 0.08186440169811249 + }, + "corpus": { + "mean_prompt_tokens": 1263.6733333333334, + "type_token_ratio": 0.09840095805350539, + "eval_vocab_overlap": 0.7251908396946565, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.7207363329812485, + "6": 0.603557326820162, + "7": 0.522443307928167, + "8": 0.44657728887204534, + "9": 0.33938507684851826, + "10": 0.2699756518526756, + "11": 0.16687381947259874, + "12": 0.09836664664022284, + "13": 0.041321238693884044 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.03791031001933983, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12146482856145926, + "best1L": 0.0870195793520127, + "mean": 0.021956329699605703, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.011688311850386006, + "mean": 0.0015873015924756015, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.005830903670617512, + "mean": 0.0030774214870429467, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3110119008592197, + "best1L": 0.21875000266092165, + "mean": 0.13045635260641575, + "persist": 0.13690476132822887 + }, + "association": { + "min": 0.03881987503596714, + "best1L": 0.027950310680483068, + "mean": 0.008454106415488891, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.718326841473723, + "dispersion_mean": 0.35520746892377375, + "dispersion_slope": -0.0850394586568326, + "stable_rank": 40.954689025878906, + "top8_spectral_share": 0.07062297314405441, + "condition_number": 2881.38232421875, + "unembed_alignment": 0.08246830105781555 + }, + "corpus": { + "mean_prompt_tokens": 1159.06, + "type_token_ratio": 0.08913688678756923, + "eval_vocab_overlap": 0.7717557251908397, + "model_cross_entropy": NaN + }, + "dispersion_per_layer": { + "5": 0.718326841473723, + "6": 0.6011052642991852, + "7": 0.5200586701266299, + "8": 0.4449809389596702, + "9": 0.33838320370018443, + "10": 0.26889958271290665, + "11": 0.16609240455717597, + "12": 0.09781173116978978, + "13": 0.04120858331469855 + } + } + }, + "runtime_s": 1150.0, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/lens_Github_1b_n200_s0.pt b/results/ladder1b_b613/lens_Github_1b_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4898116546b01a34e124e75a71cb0b9a64c421f0 --- /dev/null +++ b/results/ladder1b_b613/lens_Github_1b_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:459f997d6661ff07494e3ac0829864e02d71b15e0609875a705e032371a30158 +size 67112671 diff --git a/results/ladder1b_b613/lens_Github_1b_n200_s1.pt b/results/ladder1b_b613/lens_Github_1b_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bfe2842efd380997ac1ed8865726fbfa2625869 --- /dev/null +++ b/results/ladder1b_b613/lens_Github_1b_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:187bd291fc66a6f3e5cf9a35c4640d9d963e1fd47434124dc3e7b7832f1a7950 +size 67112671 diff --git a/results/ladder1b_b613/lens_Github_1b_n200_s2.pt b/results/ladder1b_b613/lens_Github_1b_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fca3cef3e0879a29f8d22efe50a141174468b3a --- /dev/null +++ b/results/ladder1b_b613/lens_Github_1b_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e37d288145a6438be3c571ad3fac769b7fd314c00675573dc8e4e9b3812a2754 +size 67112671 diff --git a/results/ladder1b_b613/lens_Pile-CC_1b_n200_s0.pt b/results/ladder1b_b613/lens_Pile-CC_1b_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a658edd004d9f0efc639994bfb45a625f1a2253 --- /dev/null +++ b/results/ladder1b_b613/lens_Pile-CC_1b_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bf4ce26087e107fa74da45922f6ecc486d6eab6fb29b7fbbf9b356a0ca4eabd +size 67112685 diff --git a/results/ladder1b_b613/lens_Pile-CC_1b_n200_s1.pt b/results/ladder1b_b613/lens_Pile-CC_1b_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..12b3bf71742c58e8801c54e9d53ec3b061686af1 --- /dev/null +++ b/results/ladder1b_b613/lens_Pile-CC_1b_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f41713739a1f278fd0d1c5b5c23db6ecfe1df0991945aa6b83ad716200e9a8f0 +size 67112685 diff --git a/results/ladder1b_b613/lens_Pile-CC_1b_n200_s2.pt b/results/ladder1b_b613/lens_Pile-CC_1b_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..163013e996cdf7d259fbcfc3a84a3154fca52f39 --- /dev/null +++ b/results/ladder1b_b613/lens_Pile-CC_1b_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe7a6a5b4579d5f47f0119d943efc700eb70220fb16acdec55c1f724dcde0ac2 +size 67112685 diff --git a/results/ladder1b_b613/lens_StackExchange_1b_n200_s0.pt b/results/ladder1b_b613/lens_StackExchange_1b_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..e79a9a3c1cfc773c5a3b6d42c3be8b5c13a27dae --- /dev/null +++ b/results/ladder1b_b613/lens_StackExchange_1b_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5048cd087976e256a7ff9f30d8066dd95d442229f8c2b0ff5f4d273eef512c5b +size 67112769 diff --git a/results/ladder1b_b613/lens_StackExchange_1b_n200_s1.pt b/results/ladder1b_b613/lens_StackExchange_1b_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..de1dca5c00001b0cab356ce876a09f06710b90c9 --- /dev/null +++ b/results/ladder1b_b613/lens_StackExchange_1b_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:905a7e26d7af75d7e9a64a49ba5319156b0e220236c0bf8ec5c15d8989db2330 +size 67112769 diff --git a/results/ladder1b_b613/lens_StackExchange_1b_n200_s2.pt b/results/ladder1b_b613/lens_StackExchange_1b_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ba32bb03cb9bdbc095bdb80f50661e80ac5d27c --- /dev/null +++ b/results/ladder1b_b613/lens_StackExchange_1b_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78e10c625c615a13dc475fef35688ce8a90b8ab368ec84e1c8c7f88d70d3a040 +size 67112769 diff --git a/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s0.pt b/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..490524a95b0e5a93edc7adc151e77c5a28c63d71 --- /dev/null +++ b/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4acda201f3fbc46f77e96d3f6f6840ab6cc53b8d80f694ec5080dab7321b054a +size 67112825 diff --git a/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s1.pt b/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a97b8bda2fa3c81a76ff04b394e37fcacc2f569 --- /dev/null +++ b/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4be9599eae66c8895479b2d0151fde7975929aedc420916b1b29d7415234eac1 +size 67112825 diff --git a/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s2.pt b/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..f21dbb639925a663233507b9ad5d839510d72d18 --- /dev/null +++ b/results/ladder1b_b613/lens_USPTO_Backgrounds_1b_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f730745e88e5dae78b5f73b98a00ab55c28ed32dae8b7800ba8823445e090ff +size 67112825 diff --git a/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s0.pt b/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba0a83a607e7a5c5ec387c03b35d5261452bb537 --- /dev/null +++ b/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f60c8ee8186e4d53e9a0195b5f0535506160d2d313b3bc174d3129b80d9426b2 +size 67112755 diff --git a/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s1.pt b/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..42aa7db65adc60fb3a282743bfca90ecc24b241e --- /dev/null +++ b/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25a07e77788297cc89a4603c0a1771d9cf2f8580a481c015bf77d039adcd0464 +size 67112755 diff --git a/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s2.pt b/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9c4090a25230fc1fb4ce7f5cc42de9f0d96c3d3 --- /dev/null +++ b/results/ladder1b_b613/lens_Wikipedia_en_1b_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89838efaec6ce3b2f991ed142b320fd28c8f608e5205a8c934e0ac44675dd0fa +size 67112755 diff --git a/results/ladder1b_b613/tv_Github_s0.json b/results/ladder1b_b613/tv_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e1d02fb6a90ddbf18e67bf71fc91e8aad4cf1cc0 --- /dev/null +++ b/results/ladder1b_b613/tv_Github_s0.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Github", + "seed": 0, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17337031715682574, + "best1L": 0.15117891771452768, + "mean": 0.047330097295343876, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1381435798747199, + "best1L": 0.09535895447645869, + "mean": 0.028054749593138695, + "persist": 0.004350978881120682 + }, + "order-ops": { + "min": 0.02597402555069753, + "best1L": 0.014285714365541935, + "mean": 0.0040584415297156996, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408163645437787, + "best1L": 0.013119533391935485, + "mean": 0.002915451835308756, + "persist": 0.0 + }, + "typo": { + "min": 0.32738095202616285, + "best1L": 0.23363095628363745, + "mean": 0.15215773694217205, + "persist": 0.1502976186041321 + }, + "association": { + "min": 0.015527949801513128, + "best1L": 0.00776397529989481, + "mean": 0.0025232918560504913, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6388293503658602, + "dispersion_mean": 0.314796143048675, + "dispersion_slope": -0.08816634219845453, + "stable_rank": 39.976444244384766, + "top8_spectral_share": 0.06589789688587189, + "condition_number": 440.04644775390625, + "unembed_alignment": 0.06521646678447723 + }, + "corpus": { + "mean_prompt_tokens": 1392.0, + "type_token_ratio": 0.13376436781609197, + "eval_vocab_overlap": 0.1183206106870229, + "model_cross_entropy": 0.8978622951486089, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6388293503658602, + "7": 0.54141964755538, + "8": 0.447619170630371, + "9": 0.3408496398893195, + "10": 0.2626423269713415, + "11": 0.15737004928328202, + "12": 0.0927935476149672, + "13": 0.036845412078878446 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17059639309133803, + "best1L": 0.151178916650159, + "mean": 0.04576976437653814, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13125453277357987, + "best1L": 0.09173313927437578, + "mean": 0.02606055167104517, + "persist": 0.0036258158673133168 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.014285714365541935, + "mean": 0.0027597402721377356, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533658027649, + "best1L": 0.008746355505926269, + "mean": 0.002186588976266129, + "persist": 0.0 + }, + "typo": { + "min": 0.2946428635290691, + "best1L": 0.23809523774044855, + "mean": 0.10788690697933946, + "persist": 0.08630952305559601 + }, + "association": { + "min": 0.012422359814601285, + "best1L": 0.00776397529989481, + "mean": 0.0023291924901838812, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6796741351790971, + "dispersion_mean": 0.3353858592794878, + "dispersion_slope": -0.09387820562016158, + "stable_rank": 37.86494064331055, + "top8_spectral_share": 0.06856556981801987, + "condition_number": 468.4283752441406, + "unembed_alignment": 0.05846681445837021 + }, + "corpus": { + "mean_prompt_tokens": 1314.32, + "type_token_ratio": 0.09973218089962871, + "eval_vocab_overlap": 0.18854961832061068, + "model_cross_entropy": 0.6615764385032225, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6796741351790971, + "7": 0.5771412363829672, + "8": 0.47746283938988965, + "9": 0.36401896645582454, + "10": 0.2777650625263755, + "11": 0.16857581419472784, + "12": 0.09928545257963342, + "13": 0.03916336752738701 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15395284231219972, + "mean": 0.046463246058140485, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13234227815909044, + "best1L": 0.09318346503589835, + "mean": 0.026831036699669703, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.019480518863669465, + "best1L": 0.014285714365541935, + "mean": 0.00259740263157125, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533524981566, + "best1L": 0.00728862958827189, + "mean": 0.0020043732365593314, + "persist": 0.0 + }, + "typo": { + "min": 0.3139880948833057, + "best1L": 0.23809523774044855, + "mean": 0.1361607156161751, + "persist": 0.1294642865125622 + }, + "association": { + "min": 0.012422359814601285, + "best1L": 0.00776397529989481, + "mean": 0.0023291924901838812, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6852845574584784, + "dispersion_mean": 0.3384578010266831, + "dispersion_slope": -0.09502743526246149, + "stable_rank": 42.076602935791016, + "top8_spectral_share": 0.06584933400154114, + "condition_number": 428.05181884765625, + "unembed_alignment": 0.06267957389354706 + }, + "corpus": { + "mean_prompt_tokens": 1159.04, + "type_token_ratio": 0.08555356156819437, + "eval_vocab_overlap": 0.26641221374045804, + "model_cross_entropy": 0.7504909205636395, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6852845574584784, + "7": 0.583913553085828, + "8": 0.4836174976063741, + "9": 0.3676166202045078, + "10": 0.2808413313180005, + "11": 0.16847105877352067, + "12": 0.09947740869524963, + "13": 0.03844038107150628 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16782246742929732, + "best1L": 0.15117891771452768, + "mean": 0.046983356454542706, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13560551112251623, + "best1L": 0.09463379079742092, + "mean": 0.027692168152758052, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.01603498589247465, + "best1L": 0.00874635563897235, + "mean": 0.002368804616188364, + "persist": 0.0 + }, + "typo": { + "min": 0.3154761844447681, + "best1L": 0.23660713966403688, + "mean": 0.14174107435558522, + "persist": 0.1339285738234009 + }, + "association": { + "min": 0.013975154874580247, + "best1L": 0.00776397529989481, + "mean": 0.0027173913217016627, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6870159160641568, + "dispersion_mean": 0.3390515049265032, + "dispersion_slope": -0.09563959273921026, + "stable_rank": 43.78330993652344, + "top8_spectral_share": 0.06526567041873932, + "condition_number": 427.92047119140625, + "unembed_alignment": 0.06442983448505402 + }, + "corpus": { + "mean_prompt_tokens": 1489.08, + "type_token_ratio": 0.06120109956035494, + "eval_vocab_overlap": 0.33129770992366414, + "model_cross_entropy": 0.7504909205636395, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6870159160641568, + "7": 0.5870103894941271, + "8": 0.4868948458571734, + "9": 0.3674733556324608, + "10": 0.28088172793904204, + "11": 0.16681746223347418, + "12": 0.09848352347527649, + "13": 0.03783481871631519 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.171983360179833, + "best1L": 0.15256587948117936, + "mean": 0.04767683733786855, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13741841619568212, + "best1L": 0.09608411655894347, + "mean": 0.02814539521932602, + "persist": 0.004350978881120682 + }, + "order-ops": { + "min": 0.02337662343467985, + "best1L": 0.01688311674765178, + "mean": 0.0030844155699014664, + "persist": 0.0 + }, + "poetry": { + "min": 0.01749271114489862, + "best1L": 0.00874635563897235, + "mean": 0.0025510204057874425, + "persist": 0.0 + }, + "typo": { + "min": 0.3184523827263287, + "best1L": 0.23958333155938558, + "mean": 0.14397321774491242, + "persist": 0.13690476252564363 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.002329192523445402, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6913022827334321, + "dispersion_mean": 0.3412404222624482, + "dispersion_slope": -0.0962117597804341, + "stable_rank": 44.729373931884766, + "top8_spectral_share": 0.06577473133802414, + "condition_number": 419.0328063964844, + "unembed_alignment": 0.0655546635389328 + }, + "corpus": { + "mean_prompt_tokens": 1455.92, + "type_token_ratio": 0.055470080773668884, + "eval_vocab_overlap": 0.383206106870229, + "model_cross_entropy": 0.7504909205636395, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6913022827334321, + "7": 0.5910566074948372, + "8": 0.4891092303537212, + "9": 0.37026594262160556, + "10": 0.28269059157384185, + "11": 0.16810417190959556, + "12": 0.09928006922798838, + "13": 0.03811448218456369 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.16920943558216095, + "best1L": 0.15256587948117936, + "mean": 0.04715672614318984, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1363306718745402, + "best1L": 0.09535895367818219, + "mean": 0.02755619984652315, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00874635563897235, + "mean": 0.0021865889596353683, + "persist": 0.0 + }, + "typo": { + "min": 0.30654761514493395, + "best1L": 0.23511904584509985, + "mean": 0.13392857116247928, + "persist": 0.1294642866456083 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.00776397529989481, + "mean": 0.0025232918560504913, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6903904476034368, + "dispersion_mean": 0.3401504088289169, + "dispersion_slope": -0.09604597067895067, + "stable_rank": 43.119388580322266, + "top8_spectral_share": 0.06606467813253403, + "condition_number": 437.7383728027344, + "unembed_alignment": 0.0641467422246933 + }, + "corpus": { + "mean_prompt_tokens": 2064.4333333333334, + "type_token_ratio": 0.035418920446288735, + "eval_vocab_overlap": 0.433587786259542, + "model_cross_entropy": 0.7504909205636395, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6903904476034368, + "7": 0.5893686090340412, + "8": 0.48754267192598605, + "9": 0.3686199375986501, + "10": 0.2812762023218593, + "11": 0.16727274882424184, + "12": 0.09863021365870371, + "13": 0.038102439664416 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1650485417672566, + "best1L": 0.14840499205248697, + "mean": 0.04646324525986399, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13488034824175493, + "best1L": 0.09463379079742092, + "mean": 0.027284263500145504, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.015584415623119898, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533658027649, + "best1L": 0.007288629721317973, + "mean": 0.0020043732199285713, + "persist": 0.0 + }, + "typo": { + "min": 0.29910714285714285, + "best1L": 0.2336309477686882, + "mean": 0.12369791817452226, + "persist": 0.1116071440545576 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.002329192523445402, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6993961658908293, + "dispersion_mean": 0.3448668923511198, + "dispersion_slope": -0.09716482712181411, + "stable_rank": 40.94101333618164, + "top8_spectral_share": 0.06665398925542831, + "condition_number": 493.7596435546875, + "unembed_alignment": 0.062295183539390564 + }, + "corpus": { + "mean_prompt_tokens": 2232.67, + "type_token_ratio": 0.030734501740069066, + "eval_vocab_overlap": 0.48778625954198473, + "model_cross_entropy": 0.7504909205636395, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6993961658908293, + "7": 0.5968527503596057, + "8": 0.493231137681684, + "9": 0.37379537121920925, + "10": 0.2857856339554798, + "11": 0.17053425129846733, + "12": 0.10054348380547705, + "13": 0.03879634459820611 + } + } + }, + "runtime_s": 1597.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Github_s1.json b/results/ladder1b_b613/tv_Github_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..92b491f6a9cd40a5122817770dd65bb8c22c3ad0 --- /dev/null +++ b/results/ladder1b_b613/tv_Github_s1.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Github", + "seed": 1, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.16782246849366597, + "best1L": 0.15533980833632605, + "mean": 0.04455617203244141, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1287164597639016, + "best1L": 0.08919506786125046, + "mean": 0.024519579352012703, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.029870130520846163, + "best1L": 0.015584415889212064, + "mean": 0.004058441596238741, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.00874635563897235, + "mean": 0.002915451885201037, + "persist": 0.0 + }, + "typo": { + "min": 0.3020833336881229, + "best1L": 0.23214286140033177, + "mean": 0.12574404823992932, + "persist": 0.11904761860413211 + }, + "association": { + "min": 0.020186335646680424, + "best1L": 0.006211180239915848, + "mean": 0.0034937888184296234, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6872384835805264, + "dispersion_mean": 0.3507700231819892, + "dispersion_slope": -0.09613835072891791, + "stable_rank": 45.61228561401367, + "top8_spectral_share": 0.06465078145265579, + "condition_number": 431.8564758300781, + "unembed_alignment": 0.07063920795917511 + }, + "corpus": { + "mean_prompt_tokens": 5978.6, + "type_token_ratio": 0.05735456461378918, + "eval_vocab_overlap": 0.20381679389312976, + "model_cross_entropy": 1.1056717985385174, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6872384835805264, + "7": 0.5995145778317642, + "8": 0.509569300825792, + "9": 0.38530630800254845, + "10": 0.3005452747883475, + "11": 0.1774093452714614, + "12": 0.10608827660886004, + "13": 0.04048861854661326 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.1705963984131813, + "best1L": 0.15256587948117936, + "mean": 0.04576976490872247, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13778099790215492, + "best1L": 0.09390862791665963, + "mean": 0.02701232715376786, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.02597402555069753, + "best1L": 0.014285714365541935, + "mean": 0.003409090834403677, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533924119813, + "best1L": 0.007288629721317973, + "mean": 0.002368804715972926, + "persist": 0.0 + }, + "typo": { + "min": 0.3125, + "best1L": 0.23809523774044855, + "mean": 0.12834821481789863, + "persist": 0.11607142817229033 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.00776397529989481, + "mean": 0.002329192523445402, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6816587056156873, + "dispersion_mean": 0.3392682985601193, + "dispersion_slope": -0.0952729686822858, + "stable_rank": 47.224796295166016, + "top8_spectral_share": 0.06577228009700775, + "condition_number": 457.9798278808594, + "unembed_alignment": 0.06880629062652588 + }, + "corpus": { + "mean_prompt_tokens": 3429.04, + "type_token_ratio": 0.05325105568905583, + "eval_vocab_overlap": 0.2580152671755725, + "model_cross_entropy": 1.0346972928176055, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6816587056156873, + "7": 0.5865118258739526, + "8": 0.4888946278387601, + "9": 0.3705521442296567, + "10": 0.28341829544541913, + "11": 0.1674127405193732, + "12": 0.0985330662983877, + "13": 0.03716498265971797 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.1692094345177923, + "best1L": 0.15117891771452768, + "mean": 0.0473300963640213, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1370558368840388, + "best1L": 0.09354604647627898, + "mean": 0.027782813778945377, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.015584415490073817, + "mean": 0.0029220779127042207, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00874635563897235, + "mean": 0.002368804616188364, + "persist": 0.0 + }, + "typo": { + "min": 0.2991071364709309, + "best1L": 0.23809523933700152, + "mean": 0.11755952372082643, + "persist": 0.10416666883975267 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0019409938249737024, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7448599540257981, + "dispersion_mean": 0.3694574891592114, + "dispersion_slope": -0.10346006533556185, + "stable_rank": 44.448875427246094, + "top8_spectral_share": 0.06854691356420517, + "condition_number": 457.5372009277344, + "unembed_alignment": 0.06517452746629715 + }, + "corpus": { + "mean_prompt_tokens": 2819.96, + "type_token_ratio": 0.05382345848877289, + "eval_vocab_overlap": 0.34427480916030534, + "model_cross_entropy": 0.9684071734480493, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7448599540257981, + "7": 0.6389885321302544, + "8": 0.5296513829705921, + "9": 0.3991929846618054, + "10": 0.30602695993060863, + "11": 0.18454792413312884, + "12": 0.10997598803113799, + "13": 0.04241618739036555 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15256587735244206, + "mean": 0.04680998570152691, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13741841619568212, + "best1L": 0.09463379079742092, + "mean": 0.02755619984652315, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725984177419, + "best1L": 0.007288629721317973, + "mean": 0.0023688046827114056, + "persist": 0.0 + }, + "typo": { + "min": 0.29910713753529955, + "best1L": 0.23958333741341317, + "mean": 0.12202380810465131, + "persist": 0.11160714272409678 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0019409938249737024, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.735625993368757, + "dispersion_mean": 0.36497478941755074, + "dispersion_slope": -0.10210072371468462, + "stable_rank": 44.120994567871094, + "top8_spectral_share": 0.06859783083200455, + "condition_number": 459.625732421875, + "unembed_alignment": 0.06567065417766571 + }, + "corpus": { + "mean_prompt_tokens": 2592.28, + "type_token_ratio": 0.05032892537328786, + "eval_vocab_overlap": 0.38091603053435114, + "model_cross_entropy": 0.9684071734480493, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.735625993368757, + "7": 0.6303759420875721, + "8": 0.5217566200904792, + "9": 0.396139187865765, + "10": 0.302842623179965, + "11": 0.18262655778979145, + "12": 0.10876605656650859, + "13": 0.041665334391567437 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.17198335698672704, + "best1L": 0.15395284018346242, + "mean": 0.046983356454542706, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13705583475530148, + "best1L": 0.09499637223780155, + "mean": 0.027646846004894803, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00874635563897235, + "mean": 0.002368804616188364, + "persist": 0.0 + }, + "typo": { + "min": 0.3005952441266605, + "best1L": 0.24553571694663592, + "mean": 0.12016368923442704, + "persist": 0.10714285807417971 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0019409938249737024, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7439507501650792, + "dispersion_mean": 0.36760273300373114, + "dispersion_slope": -0.1033274833221042, + "stable_rank": 43.035926818847656, + "top8_spectral_share": 0.06867276132106781, + "condition_number": 496.5562438964844, + "unembed_alignment": 0.06418026983737946 + }, + "corpus": { + "mean_prompt_tokens": 2471.41, + "type_token_ratio": 0.04500265030893296, + "eval_vocab_overlap": 0.42290076335877863, + "model_cross_entropy": 0.9684071734480493, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7439507501650792, + "7": 0.6358918320943924, + "8": 0.5262953003437375, + "9": 0.3977876390283649, + "10": 0.3044802375469753, + "11": 0.18262842133340465, + "12": 0.10829996677105617, + "13": 0.0414877167468386 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15256587735244206, + "mean": 0.0473300963640213, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13959390510405814, + "best1L": 0.09608411655894347, + "mean": 0.0284173312996115, + "persist": 0.004713560587593487 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.011661807607327188, + "best1L": 0.007288629721317973, + "mean": 0.0020043731534055303, + "persist": 0.0 + }, + "typo": { + "min": 0.30803571322134565, + "best1L": 0.24107143176453455, + "mean": 0.13225446588226727, + "persist": 0.1220238074394209 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0019409938249737024, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7267346410284999, + "dispersion_mean": 0.3587499984695661, + "dispersion_slope": -0.10086591615759215, + "stable_rank": 44.00836181640625, + "top8_spectral_share": 0.06744460761547089, + "condition_number": 465.6758117675781, + "unembed_alignment": 0.06548624485731125 + }, + "corpus": { + "mean_prompt_tokens": 2525.48, + "type_token_ratio": 0.036734930917422956, + "eval_vocab_overlap": 0.49389312977099237, + "model_cross_entropy": 0.9684071734480493, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7267346410284999, + "7": 0.6212920401123465, + "8": 0.5122766405301069, + "9": 0.3880802639598921, + "10": 0.2969843800827041, + "11": 0.17789109206157858, + "12": 0.10603412503177216, + "13": 0.04070680494962892 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.17059639415570668, + "best1L": 0.15256587735244206, + "mean": 0.0473300963640213, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13886874222329684, + "best1L": 0.09572153511856284, + "mean": 0.028100072273186276, + "persist": 0.004350978881120682 + }, + "order-ops": { + "min": 0.022077922310147966, + "best1L": 0.015584415623119898, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725917654378, + "best1L": 0.007288629721317973, + "mean": 0.002368804732603686, + "persist": 0.0 + }, + "typo": { + "min": 0.30803571960755755, + "best1L": 0.24107142963579722, + "mean": 0.13653273667607987, + "persist": 0.1250000014635069 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0019409938249737024, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7277568791458068, + "dispersion_mean": 0.3590451117885415, + "dispersion_slope": -0.1009960379060959, + "stable_rank": 44.122291564941406, + "top8_spectral_share": 0.067069873213768, + "condition_number": 459.49365234375, + "unembed_alignment": 0.06585581600666046 + }, + "corpus": { + "mean_prompt_tokens": 2467.21, + "type_token_ratio": 0.031033029211133222, + "eval_vocab_overlap": 0.5282442748091603, + "model_cross_entropy": 0.9684071734480493, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7277568791458068, + "7": 0.6218659773089733, + "8": 0.5126708453786814, + "9": 0.38826891554539694, + "10": 0.29700065476319387, + "11": 0.17799990672705654, + "12": 0.10617103744201335, + "13": 0.04062667799720992 + } + } + }, + "runtime_s": 1605.3, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Github_s2.json b/results/ladder1b_b613/tv_Github_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..fe76ed4e2f2faca8633072152e6c196ab617b830 --- /dev/null +++ b/results/ladder1b_b613/tv_Github_s2.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Github", + "seed": 2, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17059639309133803, + "best1L": 0.14563106532607759, + "mean": 0.0459431345973696, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1381435790764434, + "best1L": 0.09028281244848456, + "mean": 0.026423133044902767, + "persist": 0.0032632341608405113 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.014285714365541935, + "mean": 0.0029220779127042207, + "persist": 0.0 + }, + "poetry": { + "min": 0.010204081822718893, + "best1L": 0.004373177752963134, + "mean": 0.0016399417405149766, + "persist": 0.0 + }, + "typo": { + "min": 0.30654762259551455, + "best1L": 0.2470238134264946, + "mean": 0.12797619269362517, + "persist": 0.12053571614835944 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.002329192523445402, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.596355064159279, + "dispersion_mean": 0.2995753707282723, + "dispersion_slope": -0.08286115638581353, + "stable_rank": 36.569400787353516, + "top8_spectral_share": 0.06739673018455505, + "condition_number": 766.620361328125, + "unembed_alignment": 0.06318318843841553 + }, + "corpus": { + "mean_prompt_tokens": 4178.2, + "type_token_ratio": 0.07445790053132928, + "eval_vocab_overlap": 0.1801526717557252, + "model_cross_entropy": 0.9091272325666101, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.596355064159279, + "7": 0.5149481405031391, + "8": 0.4306285560555555, + "9": 0.32739767171132445, + "10": 0.2504380046684493, + "11": 0.15021973602752509, + "12": 0.08961770327853108, + "13": 0.036998089422374716 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.16920943238905498, + "best1L": 0.14979195381913865, + "mean": 0.046636615480695455, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13234227603035315, + "best1L": 0.09100797639361449, + "mean": 0.02569796996457236, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.014285714365541935, + "mean": 0.0027597402721377356, + "persist": 0.0 + }, + "poetry": { + "min": 0.00874635563897235, + "best1L": 0.005830903670617512, + "mean": 0.0016399417238842165, + "persist": 0.0 + }, + "typo": { + "min": 0.30505952558347155, + "best1L": 0.24404762099896157, + "mean": 0.11737351332392011, + "persist": 0.09523809501635176 + }, + "association": { + "min": 0.009316769960735525, + "best1L": 0.006211180106869766, + "mean": 0.0017468943593225309, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.6683383639810172, + "dispersion_mean": 0.3280982760102152, + "dispersion_slope": -0.09281444417862528, + "stable_rank": 38.35292434692383, + "top8_spectral_share": 0.06814451515674591, + "condition_number": 635.273193359375, + "unembed_alignment": 0.06061464920639992 + }, + "corpus": { + "mean_prompt_tokens": 3050.28, + "type_token_ratio": 0.07094430675216701, + "eval_vocab_overlap": 0.30916030534351147, + "model_cross_entropy": 0.8334878241532557, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6683383639810172, + "7": 0.5684508487321674, + "8": 0.470500976973334, + "9": 0.35375177784778133, + "10": 0.27039136386919316, + "11": 0.16133031684031146, + "12": 0.09453802198216343, + "13": 0.03748453785575347 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17337032088211604, + "best1L": 0.15256587894899504, + "mean": 0.048370320083839555, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14213197518672263, + "best1L": 0.09608411709112781, + "mean": 0.028462653846613, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.02467532442616565, + "best1L": 0.015584415490073817, + "mean": 0.0030844155532707062, + "persist": 0.0 + }, + "poetry": { + "min": 0.010204081822718893, + "best1L": 0.00728862958827189, + "mean": 0.0018221574968525342, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999893563135, + "best1L": 0.23660713913185255, + "mean": 0.12630208528467587, + "persist": 0.11160714299018894 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.00776397529989481, + "mean": 0.002329192589968443, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7157087482129582, + "dispersion_mean": 0.3509249776019893, + "dispersion_slope": -0.09939499842064208, + "stable_rank": 39.95828628540039, + "top8_spectral_share": 0.0663888081908226, + "condition_number": 615.8179931640625, + "unembed_alignment": 0.06425055861473083 + }, + "corpus": { + "mean_prompt_tokens": 2748.88, + "type_token_ratio": 0.05657576904048194, + "eval_vocab_overlap": 0.36870229007633587, + "model_cross_entropy": 0.8928479163336929, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7157087482129582, + "7": 0.6086051877478175, + "8": 0.5031117533206145, + "9": 0.3778944433128748, + "10": 0.288669770258118, + "11": 0.1722349196000841, + "12": 0.10128125271596118, + "13": 0.039893745647486066 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16920943558216095, + "best1L": 0.15256587948117936, + "mean": 0.04750346818140575, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13995648654443876, + "best1L": 0.09499637223780155, + "mean": 0.028009426913091114, + "persist": 0.00398839744074004 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.015584415623119898, + "mean": 0.0027597402887684958, + "persist": 0.0 + }, + "poetry": { + "min": 0.011661807873419352, + "best1L": 0.008746355505926269, + "mean": 0.0020043732199285713, + "persist": 0.0 + }, + "typo": { + "min": 0.30952380916901995, + "best1L": 0.23660713913185255, + "mean": 0.13541666817452228, + "persist": 0.12797619309276342 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.00776397529989481, + "mean": 0.0025232919225735323, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7234079149694705, + "dispersion_mean": 0.3568964175972611, + "dispersion_slope": -0.10059439461926994, + "stable_rank": 40.5902214050293, + "top8_spectral_share": 0.06577693670988083, + "condition_number": 578.9374389648438, + "unembed_alignment": 0.06375765055418015 + }, + "corpus": { + "mean_prompt_tokens": 2601.7066666666665, + "type_token_ratio": 0.04838362510762166, + "eval_vocab_overlap": 0.4251908396946565, + "model_cross_entropy": 0.8928479163336929, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7234079149694705, + "7": 0.6174614785663671, + "8": 0.512763331967289, + "9": 0.38540744060801213, + "10": 0.29592376615463856, + "11": 0.1762421764933983, + "12": 0.10373791682894488, + "13": 0.04022731518996792 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.1705963984131813, + "best1L": 0.15256587948117936, + "mean": 0.04785020835697651, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13886874435203417, + "best1L": 0.09535895421036653, + "mean": 0.027782813512853215, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.02077922105257, + "best1L": 0.014285714365541935, + "mean": 0.0029220779127042207, + "persist": 0.0 + }, + "poetry": { + "min": 0.010204081822718893, + "best1L": 0.00728862958827189, + "mean": 0.0018221574968525342, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.23660713913185255, + "mean": 0.12500000159655297, + "persist": 0.11309524000223194 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180106869766, + "mean": 0.002329192589968443, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7475018678290234, + "dispersion_mean": 0.36679807001679804, + "dispersion_slope": -0.10369054104530342, + "stable_rank": 39.709503173828125, + "top8_spectral_share": 0.0668741837143898, + "condition_number": 557.2465209960938, + "unembed_alignment": 0.06276930123567581 + }, + "corpus": { + "mean_prompt_tokens": 2183.76, + "type_token_ratio": 0.04825621863208411, + "eval_vocab_overlap": 0.44732824427480916, + "model_cross_entropy": 0.8928479163336929, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7475018678290234, + "7": 0.6355030613887322, + "8": 0.5245011360928008, + "9": 0.39486200606472616, + "10": 0.3029157115251789, + "11": 0.18132149879234288, + "12": 0.10647340435439871, + "13": 0.04130587408718073 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.16643550672701427, + "best1L": 0.14979195381913865, + "mean": 0.04715672800583499, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1374184185905116, + "best1L": 0.09318346503589835, + "mean": 0.02737490912633283, + "persist": 0.0032632341608405113 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.015584415623119898, + "mean": 0.0027597402887684958, + "persist": 0.0 + }, + "poetry": { + "min": 0.010204081822718893, + "best1L": 0.00728862958827189, + "mean": 0.0020043731534055303, + "persist": 0.0 + }, + "typo": { + "min": 0.30357143389327185, + "best1L": 0.24404762206333025, + "mean": 0.12146577212427344, + "persist": 0.1056547597316759 + }, + "association": { + "min": 0.009316769960735525, + "best1L": 0.006211180106869766, + "mean": 0.0017468944424763322, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7351559124685678, + "dispersion_mean": 0.36057182873543014, + "dispersion_slope": -0.10233431111960932, + "stable_rank": 39.49131774902344, + "top8_spectral_share": 0.06823872029781342, + "condition_number": 519.7442016601562, + "unembed_alignment": 0.061255790293216705 + }, + "corpus": { + "mean_prompt_tokens": 2340.3533333333335, + "type_token_ratio": 0.036814953867364754, + "eval_vocab_overlap": 0.4946564885496183, + "model_cross_entropy": 0.8928479163336929, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7351559124685678, + "7": 0.6265375087760071, + "8": 0.5183484493761723, + "9": 0.3874812416095547, + "10": 0.2967331323868518, + "11": 0.1765924473488762, + "12": 0.10368432650223097, + "13": 0.04004161141518046 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.16920943238905498, + "best1L": 0.15117891558579036, + "mean": 0.04715672667537417, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1374184185905116, + "best1L": 0.09427120935704027, + "mean": 0.027510877964752063, + "persist": 0.003625815734267235 + }, + "order-ops": { + "min": 0.019480518996715546, + "best1L": 0.015584415623119898, + "mean": 0.002435064874589443, + "persist": 0.0 + }, + "poetry": { + "min": 0.010204081822718893, + "best1L": 0.005830903803663594, + "mean": 0.0018221574968525342, + "persist": 0.0 + }, + "typo": { + "min": 0.3035714328289032, + "best1L": 0.24255952718002455, + "mean": 0.12630208422030723, + "persist": 0.11458333195852381 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180106869766, + "mean": 0.0021350931908403125, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.7383406102365179, + "dispersion_mean": 0.36273627355774285, + "dispersion_slope": -0.10272096771027923, + "stable_rank": 40.56241226196289, + "top8_spectral_share": 0.06776784360408783, + "condition_number": 497.6051940917969, + "unembed_alignment": 0.062325045466423035 + }, + "corpus": { + "mean_prompt_tokens": 2495.045, + "type_token_ratio": 0.02932011246290147, + "eval_vocab_overlap": 0.5290076335877862, + "model_cross_entropy": 0.8928479163336929, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.7383406102365179, + "7": 0.6302121047570054, + "8": 0.5201169145821996, + "9": 0.3906247763381726, + "10": 0.29921155046091047, + "11": 0.17775196402104024, + "12": 0.1050171455623701, + "13": 0.040615122503726495 + } + } + }, + "runtime_s": 1602.9, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Pile-CC_s0.json b/results/ladder1b_b613/tv_Pile-CC_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..b9407eaac27a447e30b86237525df36f11e4026e --- /dev/null +++ b/results/ladder1b_b613/tv_Pile-CC_s0.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Pile-CC", + "seed": 0, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18307905590959958, + "best1L": 0.16227461876613752, + "mean": 0.051837725431791375, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1446700505912304, + "best1L": 0.10442349554172584, + "mean": 0.029641043394804, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.020779220121247426, + "best1L": 0.018181817872183665, + "mean": 0.0029220779293349813, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.007288629721317973, + "mean": 0.002186588926373848, + "persist": 0.0 + }, + "typo": { + "min": 0.42113094670431955, + "best1L": 0.28571428677865435, + "mean": 0.21912202079381263, + "persist": 0.22470238006540708 + }, + "association": { + "min": 0.05124223631407533, + "best1L": 0.026397515886596272, + "mean": 0.012228260631673038, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.5199762578034506, + "dispersion_mean": 0.24108439708843774, + "dispersion_slope": -0.07227614235366903, + "stable_rank": 48.28171920776367, + "top8_spectral_share": 0.058348432183265686, + "condition_number": 1309.95263671875, + "unembed_alignment": 0.12467511743307114 + }, + "corpus": { + "mean_prompt_tokens": 1369.1, + "type_token_ratio": 0.24359068000876488, + "eval_vocab_overlap": 0.22061068702290076, + "model_cross_entropy": 2.7138799443975223, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.5199762578034506, + "7": 0.4327055546604454, + "8": 0.3401194509648124, + "9": 0.252789589295988, + "10": 0.18654675389343536, + "11": 0.10843862447443368, + "12": 0.0616208421063417, + "13": 0.026478103508594676 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17475728318095207, + "best1L": 0.14979195594787598, + "mean": 0.048196948399501185, + "persist": 0.0 + }, + "multilingual": { + "min": 0.141044232994318, + "best1L": 0.10007251399968352, + "mean": 0.028598621620663574, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.01688311648155962, + "best1L": 0.015584415357027735, + "mean": 0.0022727272007614374, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.00874635563897235, + "mean": 0.0025510203725259218, + "persist": 0.0 + }, + "typo": { + "min": 0.38690476971013205, + "best1L": 0.25148809594767435, + "mean": 0.19029017963579722, + "persist": 0.19196428491600923 + }, + "association": { + "min": 0.04503105634025165, + "best1L": 0.029503105341323783, + "mean": 0.010869565053976007, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.6094533337261746, + "dispersion_mean": 0.2872991473629389, + "dispersion_slope": -0.08496266196492905, + "stable_rank": 47.7796630859375, + "top8_spectral_share": 0.05920329689979553, + "condition_number": 3349.883056640625, + "unembed_alignment": 0.11985383927822113 + }, + "corpus": { + "mean_prompt_tokens": 1143.76, + "type_token_ratio": 0.2153598657060922, + "eval_vocab_overlap": 0.36412213740458016, + "model_cross_entropy": 2.6439602963559263, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6094533337261746, + "7": 0.5119629867540827, + "8": 0.41089606070322066, + "9": 0.3018830598567136, + "10": 0.22431672818778245, + "11": 0.13219412054148685, + "12": 0.0755057034923116, + "13": 0.0321811856417385 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17337031928556307, + "best1L": 0.15117891558579036, + "mean": 0.04889042928282704, + "persist": 0.0 + }, + "multilingual": { + "min": 0.140681654214859, + "best1L": 0.10007251399968352, + "mean": 0.02882523581917797, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0019480519196284668, + "persist": 0.0 + }, + "poetry": { + "min": 0.02186588943004608, + "best1L": 0.013119533524981566, + "mean": 0.0036443149271820274, + "persist": 0.0 + }, + "typo": { + "min": 0.40922618550913675, + "best1L": 0.28422619508845465, + "mean": 0.21298362846885407, + "persist": 0.21874999893563135 + }, + "association": { + "min": 0.043478259949811866, + "best1L": 0.029503105341323783, + "mean": 0.01009316777344793, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.6654134275021735, + "dispersion_mean": 0.3044877252379986, + "dispersion_slope": -0.09158706206147303, + "stable_rank": 48.071617126464844, + "top8_spectral_share": 0.05917550250887871, + "condition_number": 47517.23046875, + "unembed_alignment": 0.11863064020872116 + }, + "corpus": { + "mean_prompt_tokens": 1013.8, + "type_token_ratio": 0.17648451371079107, + "eval_vocab_overlap": 0.4679389312977099, + "model_cross_entropy": 2.6810064616504015, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6654134275021735, + "7": 0.5429049633992575, + "8": 0.4311520995012402, + "9": 0.31210464526537124, + "10": 0.2315519782904562, + "11": 0.13922925701728278, + "12": 0.07969561286144881, + "13": 0.03384981806675863 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.170596394687891, + "best1L": 0.1484049909881183, + "mean": 0.048023578710854053, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1381435798747199, + "best1L": 0.09970993255930287, + "mean": 0.027918782085180283, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.015584415490073817, + "best1L": 0.015584415490073817, + "mean": 0.0021103895768257125, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.016034985227244242, + "mean": 0.0036443148939205067, + "persist": 0.0 + }, + "typo": { + "min": 0.40922618550913675, + "best1L": 0.2797619008592197, + "mean": 0.21112351172736712, + "persist": 0.21279761700757913 + }, + "association": { + "min": 0.04192546475678682, + "best1L": 0.03260869599346604, + "mean": 0.01009316777344793, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.6504307099188026, + "dispersion_mean": 0.3005275913696478, + "dispersion_slope": -0.08978917173298283, + "stable_rank": 48.27665710449219, + "top8_spectral_share": 0.058988429605960846, + "condition_number": 38281.7734375, + "unembed_alignment": 0.11717825382947922 + }, + "corpus": { + "mean_prompt_tokens": 970.44, + "type_token_ratio": 0.1523844853880714, + "eval_vocab_overlap": 0.5259541984732824, + "model_cross_entropy": 2.6810064616504015, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6504307099188026, + "7": 0.5348021394237101, + "8": 0.4273340271787625, + "9": 0.309593956690879, + "10": 0.23102006742221493, + "11": 0.13836612937042614, + "12": 0.0792515414967132, + "13": 0.03342215945567402 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.169209430792502, + "best1L": 0.14701802922146662, + "mean": 0.04715672654232809, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13669325411319733, + "best1L": 0.0989847696785416, + "mean": 0.027601523058755056, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285713833357607, + "best1L": 0.014285713833357607, + "mean": 0.0019480519362592272, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.016034985227244242, + "mean": 0.003826530683519585, + "persist": 0.0 + }, + "typo": { + "min": 0.40773809594767435, + "best1L": 0.28125, + "mean": 0.21205357515386172, + "persist": 0.21428571428571427 + }, + "association": { + "min": 0.046583851400230615, + "best1L": 0.03260869599346604, + "mean": 0.011063664436473377, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.644935885856992, + "dispersion_mean": 0.2985672584478525, + "dispersion_slope": -0.08912331891880862, + "stable_rank": 48.221275329589844, + "top8_spectral_share": 0.05921172350645065, + "condition_number": 39345.046875, + "unembed_alignment": 0.11645163595676422 + }, + "corpus": { + "mean_prompt_tokens": 886.71, + "type_token_ratio": 0.1412186622458301, + "eval_vocab_overlap": 0.5725190839694656, + "model_cross_entropy": 2.6810064616504015, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.644935885856992, + "7": 0.531924391853041, + "8": 0.4241895968242675, + "9": 0.3078911229387189, + "10": 0.23031338258992265, + "11": 0.13742389562742055, + "12": 0.07866466411181834, + "13": 0.033195127780638475 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.1692094329212393, + "best1L": 0.14563106532607759, + "mean": 0.046983355789312294, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13633067426936968, + "best1L": 0.0989847696785416, + "mean": 0.027646845738802637, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285713833357607, + "best1L": 0.014285713833357607, + "mean": 0.0017857142291697009, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.0174927112779447, + "mean": 0.004008746406595621, + "persist": 0.0 + }, + "typo": { + "min": 0.39880952664784025, + "best1L": 0.2678571439215115, + "mean": 0.20479910927159445, + "persist": 0.20386904664337635 + }, + "association": { + "min": 0.041925465288971155, + "best1L": 0.027950310680483068, + "mean": 0.009316770409766053, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.6526188744010458, + "dispersion_mean": 0.3041160018774359, + "dispersion_slope": -0.09024147143025452, + "stable_rank": 48.0203742980957, + "top8_spectral_share": 0.05940352380275726, + "condition_number": 21410.978515625, + "unembed_alignment": 0.11371073126792908 + }, + "corpus": { + "mean_prompt_tokens": 989.9266666666666, + "type_token_ratio": 0.10757699223511506, + "eval_vocab_overlap": 0.6763358778625954, + "model_cross_entropy": 2.6810064616504015, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6526188744010458, + "7": 0.5401761464329754, + "8": 0.43267529278072336, + "9": 0.31516037951059467, + "10": 0.23608183780809877, + "11": 0.1413911456932582, + "12": 0.08095506647691464, + "13": 0.03386927191587665 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1678224690258503, + "best1L": 0.1442441035594259, + "mean": 0.046463245659002235, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13488034637910978, + "best1L": 0.09825960679778031, + "mean": 0.027284264032329832, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.012987012575779642, + "mean": 0.0017857142790619815, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.016034985227244242, + "mean": 0.0036443148939205067, + "persist": 0.0 + }, + "typo": { + "min": 0.3928571492433548, + "best1L": 0.26488095415490015, + "mean": 0.20219493976661138, + "persist": 0.200892853683659 + }, + "association": { + "min": 0.043478259949811866, + "best1L": 0.027950310680483068, + "mean": 0.009899068440842842, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.658564351856671, + "dispersion_mean": 0.306189388444973, + "dispersion_slope": -0.09112478959237405, + "stable_rank": 47.8715705871582, + "top8_spectral_share": 0.059491582214832306, + "condition_number": 37859.51953125, + "unembed_alignment": 0.11356326937675476 + }, + "corpus": { + "mean_prompt_tokens": 1027.01, + "type_token_ratio": 0.09424445721073797, + "eval_vocab_overlap": 0.749618320610687, + "model_cross_entropy": 2.6810064616504015, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.658564351856671, + "7": 0.5445611549097247, + "8": 0.4361539079893096, + "9": 0.3168435070623155, + "10": 0.23693367322972486, + "11": 0.14171533244345694, + "12": 0.08085139411201263, + "13": 0.03389178595656914 + } + } + }, + "runtime_s": 1606.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Pile-CC_s1.json b/results/ladder1b_b613/tv_Pile-CC_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..93a8c0348c3cc5985f1754bea492720974378171 --- /dev/null +++ b/results/ladder1b_b613/tv_Pile-CC_s1.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Pile-CC", + "seed": 1, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.1692094329212393, + "best1L": 0.1428571417927742, + "mean": 0.04837031848728657, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1392313257924148, + "best1L": 0.10043509544006415, + "mean": 0.028009427046137198, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.012987012575779642, + "mean": 0.0019480519196284668, + "persist": 0.0 + }, + "poetry": { + "min": 0.01749271181012903, + "best1L": 0.010204081423580647, + "mean": 0.002915451885201037, + "persist": 0.0 + }, + "typo": { + "min": 0.39583333900996615, + "best1L": 0.25595238379069735, + "mean": 0.20256696375352995, + "persist": 0.20238095242530107 + }, + "association": { + "min": 0.03726708064121859, + "best1L": 0.023291925766638348, + "mean": 0.007569875767720597, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6002228508326923, + "dispersion_mean": 0.2835205201520867, + "dispersion_slope": -0.08370231798092526, + "stable_rank": 47.57102584838867, + "top8_spectral_share": 0.06011524423956871, + "condition_number": 3903.46240234375, + "unembed_alignment": 0.107410728931427 + }, + "corpus": { + "mean_prompt_tokens": 2342.7, + "type_token_ratio": 0.1990011525163273, + "eval_vocab_overlap": 0.31679389312977096, + "model_cross_entropy": 2.750524324984164, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6002228508326923, + "7": 0.5065818234284126, + "8": 0.40606266172292554, + "9": 0.2950828342963733, + "10": 0.2226389409464612, + "11": 0.12973704889164814, + "12": 0.07498545663516511, + "13": 0.0328525444630152 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.170596394687891, + "best1L": 0.14563106532607759, + "mean": 0.047330096763159545, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13778100003089225, + "best1L": 0.09970993362367153, + "mean": 0.027601523391370262, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0017857142790619815, + "persist": 0.0 + }, + "poetry": { + "min": 0.023323615081608295, + "best1L": 0.018950437727783407, + "mean": 0.0043731779026399764, + "persist": 0.0 + }, + "typo": { + "min": 0.41815475693770815, + "best1L": 0.2976190501025745, + "mean": 0.2349330335855484, + "persist": 0.24255952398691857 + }, + "association": { + "min": 0.04192546648638589, + "best1L": 0.029503105341323783, + "mean": 0.01086956515376057, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.6532984107885347, + "dispersion_mean": 0.29151348119549336, + "dispersion_slope": -0.08999114546044523, + "stable_rank": 49.02669906616211, + "top8_spectral_share": 0.059489935636520386, + "condition_number": 3353.388916015625, + "unembed_alignment": 0.116998091340065 + }, + "corpus": { + "mean_prompt_tokens": 1220.56, + "type_token_ratio": 0.1920102248148391, + "eval_vocab_overlap": 0.3763358778625954, + "model_cross_entropy": 2.8984559975014075, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6532984107885347, + "7": 0.5315505724932801, + "8": 0.4069663096206688, + "9": 0.29113370620513007, + "10": 0.217301829437578, + "11": 0.12744708269031552, + "12": 0.07344724571586143, + "13": 0.03096269261257857 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.16643550938793591, + "best1L": 0.1442441035594259, + "mean": 0.04628987543817077, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1385061607829162, + "best1L": 0.09898477074291025, + "mean": 0.027964104898273945, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0017857142790619815, + "persist": 0.0 + }, + "poetry": { + "min": 0.023323615746838704, + "best1L": 0.01895043732864516, + "mean": 0.004373177852747696, + "persist": 0.0 + }, + "typo": { + "min": 0.40625, + "best1L": 0.2782738081046513, + "mean": 0.21447172494871275, + "persist": 0.21726190777761595 + }, + "association": { + "min": 0.046583850868046284, + "best1L": 0.029503105341323783, + "mean": 0.01125776390212455, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.653869707636517, + "dispersion_mean": 0.29675045499232244, + "dispersion_slope": -0.08982809769180843, + "stable_rank": 48.294227600097656, + "top8_spectral_share": 0.05989233776926994, + "condition_number": 4690.37744140625, + "unembed_alignment": 0.11249332129955292 + }, + "corpus": { + "mean_prompt_tokens": 1063.98, + "type_token_ratio": 0.16949566721178969, + "eval_vocab_overlap": 0.49312977099236643, + "model_cross_entropy": 2.793578135000693, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.653869707636517, + "7": 0.5341972769226022, + "8": 0.41362510481859716, + "9": 0.30165236954671276, + "10": 0.22651362939629963, + "11": 0.1338499998939604, + "12": 0.0775453375930917, + "13": 0.03275021413079834 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16782247115458762, + "best1L": 0.1470180270927293, + "mean": 0.0466366158798337, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1381435836000102, + "best1L": 0.09862218823816095, + "mean": 0.02791878188561116, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0016233765719724552, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.0174927112779447, + "mean": 0.004008746406595621, + "persist": 0.0 + }, + "typo": { + "min": 0.4017857185431889, + "best1L": 0.27678571322134565, + "mean": 0.2137276791036129, + "persist": 0.22023809275456838 + }, + "association": { + "min": 0.04503105567502124, + "best1L": 0.031055900268256664, + "mean": 0.01125776390212455, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.6551361822705168, + "dispersion_mean": 0.29814251927560315, + "dispersion_slope": -0.09026863965830037, + "stable_rank": 48.506988525390625, + "top8_spectral_share": 0.059255219995975494, + "condition_number": 20736.703125, + "unembed_alignment": 0.11424291133880615 + }, + "corpus": { + "mean_prompt_tokens": 1195.1466666666668, + "type_token_ratio": 0.13649649694319246, + "eval_vocab_overlap": 0.5786259541984733, + "model_cross_entropy": 2.793578135000693, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6551361822705168, + "7": 0.5360513516850665, + "8": 0.41907904367979626, + "9": 0.3038551668014537, + "10": 0.2265700229035767, + "11": 0.1345771869802648, + "12": 0.07733169108543124, + "13": 0.03253950879871931 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.170596394687891, + "best1L": 0.1470180270927293, + "mean": 0.046983356321496625, + "persist": 0.0 + }, + "multilingual": { + "min": 0.14068165315049036, + "best1L": 0.09970993255930287, + "mean": 0.028372008752610003, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0017857142790619815, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.014577259309589863, + "mean": 0.0034620992207367507, + "persist": 0.0 + }, + "typo": { + "min": 0.3958333411387035, + "best1L": 0.2678571439215115, + "mean": 0.2077752985060215, + "persist": 0.21279762153114593 + }, + "association": { + "min": 0.04658385033586195, + "best1L": 0.031055900268256664, + "mean": 0.01106366456951946, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6535908682736605, + "dispersion_mean": 0.2985962473164186, + "dispersion_slope": -0.09020901287426361, + "stable_rank": 48.9556884765625, + "top8_spectral_share": 0.05908800661563873, + "condition_number": 7714.64208984375, + "unembed_alignment": 0.11538204550743103 + }, + "corpus": { + "mean_prompt_tokens": 1240.57, + "type_token_ratio": 0.11995292486518294, + "eval_vocab_overlap": 0.6412213740458015, + "model_cross_entropy": 2.793578135000693, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6535908682736605, + "7": 0.535826011021627, + "8": 0.4215473491439414, + "9": 0.30552986639683916, + "10": 0.2275486765133235, + "11": 0.13481500544771816, + "12": 0.0772490697909358, + "13": 0.03266313194330311 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.170596394687891, + "best1L": 0.1470180270927293, + "mean": 0.04680998610066516, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13705583715013095, + "best1L": 0.09862218823816095, + "mean": 0.027465554419904947, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0016233765719724552, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816337934562, + "best1L": 0.016034985227244242, + "mean": 0.0036443149604435477, + "persist": 0.0 + }, + "typo": { + "min": 0.3928571471146175, + "best1L": 0.2678571417927742, + "mean": 0.20163690458450997, + "persist": 0.20684523747435637 + }, + "association": { + "min": 0.04503105567502124, + "best1L": 0.027950310680483068, + "mean": 0.009510869875417225, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.6498304260099098, + "dispersion_mean": 0.29889871503707105, + "dispersion_slope": -0.08984567732931965, + "stable_rank": 48.498817443847656, + "top8_spectral_share": 0.05934905260801315, + "condition_number": 44123.6640625, + "unembed_alignment": 0.11502821743488312 + }, + "corpus": { + "mean_prompt_tokens": 1317.5, + "type_token_ratio": 0.09565085388994307, + "eval_vocab_overlap": 0.7450381679389313, + "model_cross_entropy": 2.793578135000693, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6498304260099098, + "7": 0.5348688347086978, + "8": 0.4236809966446499, + "9": 0.30744839385147477, + "10": 0.22883371099369848, + "11": 0.13603742749705047, + "12": 0.07788193194351276, + "13": 0.032607998647573834 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1692094329212393, + "best1L": 0.14563106532607759, + "mean": 0.0466366158798337, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13778100003089225, + "best1L": 0.09862218823816095, + "mean": 0.0276921685518963, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0016233765719724552, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.014577259309589863, + "mean": 0.0034620991542137097, + "persist": 0.0 + }, + "typo": { + "min": 0.39880952877657755, + "best1L": 0.26488095202616285, + "mean": 0.20442708368812287, + "persist": 0.20833333342203073 + }, + "association": { + "min": 0.04347825955067362, + "best1L": 0.029503105341323783, + "mean": 0.01009316769029413, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.6736834334988969, + "dispersion_mean": 0.30965177084113304, + "dispersion_slope": -0.09324059188940123, + "stable_rank": 48.34281921386719, + "top8_spectral_share": 0.05927416309714317, + "condition_number": 8496.62890625, + "unembed_alignment": 0.11571580916643143 + }, + "corpus": { + "mean_prompt_tokens": 1244.555, + "type_token_ratio": 0.0835720398053923, + "eval_vocab_overlap": 0.7816793893129771, + "model_cross_entropy": 2.793578135000693, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6736834334988969, + "7": 0.554652342457936, + "8": 0.43929620399430863, + "9": 0.31852754555645946, + "10": 0.23709266238377658, + "11": 0.14027773486957631, + "12": 0.0802316945788191, + "13": 0.03345254938929115 + } + } + }, + "runtime_s": 1607.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Pile-CC_s2.json b/results/ladder1b_b613/tv_Pile-CC_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..07870c41a5ce51a147ee3754dd272c1169bb6a92 --- /dev/null +++ b/results/ladder1b_b613/tv_Pile-CC_s2.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Pile-CC", + "seed": 2, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.1719833596476487, + "best1L": 0.14701802815709794, + "mean": 0.04837031862033265, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13306744024157524, + "best1L": 0.09825960786214896, + "mean": 0.026604423499000922, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0019480519196284668, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533258889402, + "best1L": 0.00728862958827189, + "mean": 0.0020043731700362904, + "persist": 0.0 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.24255952132599695, + "mean": 0.1835937462747097, + "persist": 0.1845238089029278 + }, + "association": { + "min": 0.03881987503596714, + "best1L": 0.029503105341323783, + "mean": 0.010093167856601732, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.560505676359184, + "dispersion_mean": 0.2663528416568537, + "dispersion_slope": -0.07749762017336517, + "stable_rank": 51.582916259765625, + "top8_spectral_share": 0.05737672373652458, + "condition_number": 2452.95458984375, + "unembed_alignment": 0.11552812159061432 + }, + "corpus": { + "mean_prompt_tokens": 468.8, + "type_token_ratio": 0.35942832764505117, + "eval_vocab_overlap": 0.15419847328244274, + "model_cross_entropy": 2.75623498865076, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.560505676359184, + "7": 0.4708733604551085, + "8": 0.37909832041805525, + "9": 0.27685161859080193, + "10": 0.2105538978042515, + "11": 0.12829967082081706, + "12": 0.07284060492042674, + "13": 0.03179958388618403 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.16504854655691556, + "best1L": 0.14563106532607759, + "mean": 0.045423023003552644, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13306743891111442, + "best1L": 0.09753444365092687, + "mean": 0.026695068925619125, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0017857142790619815, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.00874635563897235, + "mean": 0.002186588976266129, + "persist": 0.0 + }, + "typo": { + "min": 0.372023805975914, + "best1L": 0.24107142750705993, + "mean": 0.18508184381893703, + "persist": 0.18601190538278647 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.026397516019642353, + "mean": 0.008734471829874175, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.6710901158373438, + "dispersion_mean": 0.30961603991181297, + "dispersion_slope": -0.09259956060579246, + "stable_rank": 47.33066940307617, + "top8_spectral_share": 0.05978909507393837, + "condition_number": 2706.9501953125, + "unembed_alignment": 0.11336489021778107 + }, + "corpus": { + "mean_prompt_tokens": 1271.52, + "type_token_ratio": 0.20186862967157418, + "eval_vocab_overlap": 0.3893129770992366, + "model_cross_entropy": 2.7212641645551803, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6710901158373438, + "7": 0.5564710431980302, + "8": 0.43172997979991806, + "9": 0.3178522659245006, + "10": 0.24132054935444106, + "11": 0.14330855199563103, + "12": 0.08133587817228036, + "13": 0.03381993501235849 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.16643550938793591, + "best1L": 0.14563106532607759, + "mean": 0.04542302380182913, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.13705583715013095, + "best1L": 0.09898476914635726, + "mean": 0.02791878188561116, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0019480519196284668, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816311325346, + "best1L": 0.013119533258889402, + "mean": 0.0032798834145069122, + "persist": 0.0 + }, + "typo": { + "min": 0.4077381044626236, + "best1L": 0.26636904371636255, + "mean": 0.20610118817005837, + "persist": 0.2083333312932934 + }, + "association": { + "min": 0.038819875568151474, + "best1L": 0.029503105341323783, + "mean": 0.01009316769029413, + "persist": 0.007763974900756564 + } + }, + "operator": { + "dispersion_L0": 0.6641309800552728, + "dispersion_mean": 0.3052991049734024, + "dispersion_slope": -0.09197019398838799, + "stable_rank": 47.57178497314453, + "top8_spectral_share": 0.05946052074432373, + "condition_number": 5783.79833984375, + "unembed_alignment": 0.11444948613643646 + }, + "corpus": { + "mean_prompt_tokens": 1517.98, + "type_token_ratio": 0.13213612827573487, + "eval_vocab_overlap": 0.5229007633587787, + "model_cross_entropy": 2.743252049695264, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6641309800552728, + "7": 0.549295180774378, + "8": 0.43088536095018903, + "9": 0.3128658371458863, + "10": 0.2348064219208232, + "11": 0.13866283915194808, + "12": 0.07880004433352841, + "13": 0.0329461754551932 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.16782247115458762, + "best1L": 0.14563106532607759, + "mean": 0.04559639349047627, + "persist": 0.0 + }, + "multilingual": { + "min": 0.13451776493872916, + "best1L": 0.09934735111892223, + "mean": 0.027012327752475227, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285713833357607, + "best1L": 0.014285713833357607, + "mean": 0.0017857142291697009, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985493336405, + "best1L": 0.011661807341235024, + "mean": 0.0027332361454942395, + "persist": 0.0 + }, + "typo": { + "min": 0.4032738187483379, + "best1L": 0.26636904584509985, + "mean": 0.20721725853426115, + "persist": 0.21130952451910293 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.029503105341323783, + "mean": 0.009899068407581322, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.6518823986935576, + "dispersion_mean": 0.2997885052893947, + "dispersion_slope": -0.09017211951483015, + "stable_rank": 47.94319152832031, + "top8_spectral_share": 0.05936044082045555, + "condition_number": 4036.33984375, + "unembed_alignment": 0.11311537772417068 + }, + "corpus": { + "mean_prompt_tokens": 1321.3866666666668, + "type_token_ratio": 0.122376493380691, + "eval_vocab_overlap": 0.5862595419847328, + "model_cross_entropy": 2.743252049695264, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6518823986935576, + "7": 0.5378587390028484, + "8": 0.42355718095529576, + "9": 0.30756006586410567, + "10": 0.2307771969785979, + "11": 0.13651192331206569, + "12": 0.07759633706109893, + "13": 0.03256420044758802 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.1664355072591986, + "best1L": 0.14563106532607759, + "mean": 0.045943135528692176, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.136693253581013, + "best1L": 0.0989847696785416, + "mean": 0.027646845605756556, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0017857142790619815, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725917654378, + "best1L": 0.01020408155662673, + "mean": 0.0025510204889412436, + "persist": 0.0 + }, + "typo": { + "min": 0.40327381236212595, + "best1L": 0.2752976172736713, + "mean": 0.20907738006540708, + "persist": 0.21726190405232565 + }, + "association": { + "min": 0.04037267022899219, + "best1L": 0.029503105341323783, + "mean": 0.00989906839095056, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.6427371404574432, + "dispersion_mean": 0.2961279448871239, + "dispersion_slope": -0.08881777541869385, + "stable_rank": 47.28374481201172, + "top8_spectral_share": 0.05954214558005333, + "condition_number": 6215.93994140625, + "unembed_alignment": 0.11505630612373352 + }, + "corpus": { + "mean_prompt_tokens": 1573.14, + "type_token_ratio": 0.10483491615495125, + "eval_vocab_overlap": 0.6938931297709924, + "model_cross_entropy": 2.743252049695264, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6427371404574432, + "7": 0.5302299239482667, + "8": 0.4182430775583023, + "9": 0.3042088289799738, + "10": 0.22827540155594744, + "11": 0.13571393900513543, + "12": 0.07708471114338747, + "13": 0.032530536448534854 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.1664355072591986, + "best1L": 0.14563106532607759, + "mean": 0.04576976477567639, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1359680907002517, + "best1L": 0.09825960679778031, + "mean": 0.027284264165375913, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.014285714232495852, + "best1L": 0.014285714232495852, + "mean": 0.0017857142790619815, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725917654378, + "best1L": 0.01020408155662673, + "mean": 0.0025510204889412436, + "persist": 0.0 + }, + "typo": { + "min": 0.39583333475249155, + "best1L": 0.26636904584509985, + "mean": 0.20182291577969277, + "persist": 0.2038690455790077 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.02950310560741595, + "mean": 0.009316770409766053, + "persist": 0.004658385046890804 + } + }, + "operator": { + "dispersion_L0": 0.6598889896360449, + "dispersion_mean": 0.30503433814833447, + "dispersion_slope": -0.0912851664075321, + "stable_rank": 47.65371322631836, + "top8_spectral_share": 0.05967158079147339, + "condition_number": 12490.8935546875, + "unembed_alignment": 0.11475563049316406 + }, + "corpus": { + "mean_prompt_tokens": 1431.0066666666667, + "type_token_ratio": 0.08970375167131762, + "eval_vocab_overlap": 0.7534351145038168, + "model_cross_entropy": 2.743252049695264, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6598889896360449, + "7": 0.5452448940433332, + "8": 0.43261359036313374, + "9": 0.3137669929076349, + "10": 0.23535756741387998, + "11": 0.14012211551548584, + "12": 0.07979330883858017, + "13": 0.03348724646858302 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.1664355072591986, + "best1L": 0.14563106532607759, + "mean": 0.04576976477567639, + "persist": 0.0 + }, + "multilingual": { + "min": 0.1356055097920554, + "best1L": 0.09753444391701903, + "mean": 0.027238940953144004, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.012987012575779642, + "mean": 0.0016233765719724552, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985094198158, + "best1L": 0.013119533524981566, + "mean": 0.002915451951724078, + "persist": 0.0 + }, + "typo": { + "min": 0.3898809552192688, + "best1L": 0.25892857142857145, + "mean": 0.19717261833803995, + "persist": 0.1979166671101536 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.027950310680483068, + "mean": 0.008928571212371545, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6685550057883725, + "dispersion_mean": 0.3118082712630255, + "dispersion_slope": -0.09244734778284193, + "stable_rank": 47.63458251953125, + "top8_spectral_share": 0.05977129563689232, + "condition_number": 797330.4375, + "unembed_alignment": 0.11348909139633179 + }, + "corpus": { + "mean_prompt_tokens": 1350.62, + "type_token_ratio": 0.07824184448623595, + "eval_vocab_overlap": 0.7801526717557252, + "model_cross_entropy": 2.743252049695264, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6685550057883725, + "7": 0.5543980283058783, + "8": 0.44226505387420895, + "9": 0.3229046200311318, + "10": 0.24356751643313873, + "11": 0.1452731513566531, + "12": 0.08295439038190576, + "13": 0.034548403932915354 + } + } + }, + "runtime_s": 1612.2, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_StackExchange_s0.json b/results/ladder1b_b613/tv_StackExchange_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..0d818d06ffd75fff816858d943dfab35ab9410fe --- /dev/null +++ b/results/ladder1b_b613/tv_StackExchange_s0.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "StackExchange", + "seed": 0, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.1872399454670293, + "best1L": 0.15533980620758875, + "mean": 0.054611651226878166, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.150108773793493, + "best1L": 0.10478607644992215, + "mean": 0.031000725060169185, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.020779219988201345, + "mean": 0.0032467532603602323, + "persist": 0.0 + }, + "poetry": { + "min": 0.011661807740373271, + "best1L": 0.004373177752963134, + "mean": 0.0016399416906226958, + "persist": 0.0 + }, + "typo": { + "min": 0.4627976268529892, + "best1L": 0.3363095245191029, + "mean": 0.25130208475249155, + "persist": 0.25148809541549 + }, + "association": { + "min": 0.026397516019642353, + "best1L": 0.01552795059978962, + "mean": 0.005822981391767306, + "persist": 0.0 + } + }, + "operator": { + "dispersion_L0": 0.44817357108964256, + "dispersion_mean": 0.22518938427887492, + "dispersion_slope": -0.06132181272151912, + "stable_rank": 46.52853775024414, + "top8_spectral_share": 0.06171391159296036, + "condition_number": 1566.9361572265625, + "unembed_alignment": 0.10059109330177307 + }, + "corpus": { + "mean_prompt_tokens": 495.4, + "type_token_ratio": 0.28441663302381914, + "eval_vocab_overlap": 0.1099236641221374, + "model_cross_entropy": 2.2949726826435812, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.44817357108964256, + "7": 0.38379643808850117, + "8": 0.32168549702395854, + "9": 0.24172824445652816, + "10": 0.18685172688600288, + "11": 0.11784909415902946, + "12": 0.06923143603182429, + "13": 0.03219906649551257 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.191400835024459, + "best1L": 0.1595006931040968, + "mean": 0.055998612461345534, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1530094231878008, + "best1L": 0.10696156535829816, + "mean": 0.032133792260927815, + "persist": 0.00290065272045987 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.020779219988201345, + "mean": 0.003571428541493203, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725917654378, + "best1L": 0.004373177886009216, + "mean": 0.0020043731866670506, + "persist": 0.0 + }, + "typo": { + "min": 0.4702380916901997, + "best1L": 0.3482142835855484, + "mean": 0.26078868976661135, + "persist": 0.26934523561171125 + }, + "association": { + "min": 0.024844720294432982, + "best1L": 0.01708074565976858, + "mean": 0.005822981391767306, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.4711240191312084, + "dispersion_mean": 0.23189790121851034, + "dispersion_slope": -0.06488895961607717, + "stable_rank": 45.635475158691406, + "top8_spectral_share": 0.06247479468584061, + "condition_number": 1632.2113037109375, + "unembed_alignment": 0.10561275482177734 + }, + "corpus": { + "mean_prompt_tokens": 623.0, + "type_token_ratio": 0.20057784911717497, + "eval_vocab_overlap": 0.19236641221374046, + "model_cross_entropy": 2.2521506699570666, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4711240191312084, + "7": 0.4003343517651443, + "8": 0.3322422141037132, + "9": 0.24772412917563585, + "10": 0.1881115814393853, + "11": 0.1168967771206689, + "12": 0.06836740770360426, + "13": 0.030382729308722732 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.19278779679111072, + "best1L": 0.16227462249142782, + "mean": 0.055478502064943314, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14720812173826353, + "best1L": 0.10551123933068343, + "mean": 0.030411530153027604, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003409090900926718, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985493336405, + "best1L": 0.005830903670617512, + "mean": 0.002186588976266129, + "persist": 0.0 + }, + "typo": { + "min": 0.47916666311877115, + "best1L": 0.35416666737624575, + "mean": 0.2598586327263287, + "persist": 0.26785714605024885 + }, + "association": { + "min": 0.02173913044056722, + "best1L": 0.01552795059978962, + "mean": 0.005434782660034086, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.4411226183308903, + "dispersion_mean": 0.21441420876955875, + "dispersion_slope": -0.06060053537946188, + "stable_rank": 45.9186897277832, + "top8_spectral_share": 0.062009911984205246, + "condition_number": 1540.2703857421875, + "unembed_alignment": 0.10638751834630966 + }, + "corpus": { + "mean_prompt_tokens": 770.32, + "type_token_ratio": 0.136281026067089, + "eval_vocab_overlap": 0.28091603053435116, + "model_cross_entropy": 2.1957776739790633, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4411226183308903, + "7": 0.37164294872589454, + "8": 0.3053322033015148, + "9": 0.22756325755820614, + "10": 0.17247990085564735, + "11": 0.10711058837599992, + "12": 0.06252938475410479, + "13": 0.027532768254212258 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.18585298476474627, + "best1L": 0.15950069576501846, + "mean": 0.054958391934633255, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1450326330959797, + "best1L": 0.10514865762421063, + "mean": 0.030275561846792698, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003409090900926718, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437594737326, + "best1L": 0.005830903670617512, + "mean": 0.0023688046993421657, + "persist": 0.0 + }, + "typo": { + "min": 0.4806547590664455, + "best1L": 0.3556547590664455, + "mean": 0.25948660714285715, + "persist": 0.26488095202616285 + }, + "association": { + "min": 0.02018633551363434, + "best1L": 0.01552795059978962, + "mean": 0.0052406833274289966, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.44606156594261004, + "dispersion_mean": 0.21544325607770082, + "dispersion_slope": -0.061368267261811646, + "stable_rank": 46.27339553833008, + "top8_spectral_share": 0.06239614635705948, + "condition_number": 1407.9442138671875, + "unembed_alignment": 0.10641957819461823 + }, + "corpus": { + "mean_prompt_tokens": 680.6933333333334, + "type_token_ratio": 0.12363864295228394, + "eval_vocab_overlap": 0.3236641221374046, + "model_cross_entropy": 2.1957776739790633, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.44606156594261004, + "7": 0.37481046852909, + "8": 0.306875355396466, + "9": 0.22837814629560163, + "10": 0.17190329284974545, + "11": 0.10660542904308828, + "12": 0.061774523598752795, + "13": 0.027137266966252314 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18723994759576662, + "best1L": 0.16088765859603882, + "mean": 0.05513176242155688, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1461203774171216, + "best1L": 0.10478607618382998, + "mean": 0.03018491668626666, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.020779219988201345, + "best1L": 0.01818181773913758, + "mean": 0.0032467532603602323, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816391152995, + "best1L": 0.007288629721317973, + "mean": 0.0029154519350933178, + "persist": 0.0 + }, + "typo": { + "min": 0.4836309552192688, + "best1L": 0.3601190447807312, + "mean": 0.26320684594767435, + "persist": 0.26934524199792315 + }, + "association": { + "min": 0.02018633551363434, + "best1L": 0.01552795059978962, + "mean": 0.005434782660034086, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.4398382150209552, + "dispersion_mean": 0.21278719855330072, + "dispersion_slope": -0.06029476127477273, + "stable_rank": 46.108360290527344, + "top8_spectral_share": 0.06248774752020836, + "condition_number": 1309.3568115234375, + "unembed_alignment": 0.10673891752958298 + }, + "corpus": { + "mean_prompt_tokens": 660.27, + "type_token_ratio": 0.11012161691429263, + "eval_vocab_overlap": 0.36946564885496186, + "model_cross_entropy": 2.1957776739790633, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4398382150209552, + "7": 0.36897183036411724, + "8": 0.30200562241905327, + "9": 0.22538726377230925, + "10": 0.17044818356222313, + "11": 0.10646406704045794, + "12": 0.06187319374939667, + "13": 0.027309212497893077 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.19001387325780733, + "best1L": 0.16227462142705917, + "mean": 0.055651873616235595, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14793328515120915, + "best1L": 0.10587382077106408, + "mean": 0.031046047673693726, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.0037337663151057704, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437594737326, + "best1L": 0.007288629721317973, + "mean": 0.0027332361454942395, + "persist": 0.0 + }, + "typo": { + "min": 0.4925595223903656, + "best1L": 0.3645833304950169, + "mean": 0.26804315192358835, + "persist": 0.2708333358168602 + }, + "association": { + "min": 0.023291925101407936, + "best1L": 0.01552795059978962, + "mean": 0.0060170807077416354, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.4427218821496896, + "dispersion_mean": 0.21369492775807047, + "dispersion_slope": -0.06070328742590434, + "stable_rank": 46.14337158203125, + "top8_spectral_share": 0.0626375675201416, + "condition_number": 1000.0311889648438, + "unembed_alignment": 0.10553319752216339 + }, + "corpus": { + "mean_prompt_tokens": 630.2933333333333, + "type_token_ratio": 0.0962091724488069, + "eval_vocab_overlap": 0.4259541984732824, + "model_cross_entropy": 2.1957776739790633, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4427218821496896, + "7": 0.37091686474913926, + "8": 0.3029189838248952, + "9": 0.2263833765835112, + "10": 0.1711606055812049, + "11": 0.10649851654380707, + "12": 0.061834410281543906, + "13": 0.027124782350772782 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.19140083396009036, + "best1L": 0.16088765540293284, + "mean": 0.055825243437928815, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.15047135523387364, + "best1L": 0.10696156722094331, + "mean": 0.03136330696621111, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.004058441596238741, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816391152995, + "best1L": 0.007288629721317973, + "mean": 0.0029154519350933178, + "persist": 0.0 + }, + "typo": { + "min": 0.4910714200564793, + "best1L": 0.36607142857142855, + "mean": 0.26804315405232565, + "persist": 0.27380952345473425 + }, + "association": { + "min": 0.023291925101407936, + "best1L": 0.01552795059978962, + "mean": 0.0060170807077416354, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.44544269208718545, + "dispersion_mean": 0.21463580882949884, + "dispersion_slope": -0.061044769520306984, + "stable_rank": 45.98324203491211, + "top8_spectral_share": 0.06264745444059372, + "condition_number": 1037.608154296875, + "unembed_alignment": 0.10553774982690811 + }, + "corpus": { + "mean_prompt_tokens": 634.71, + "type_token_ratio": 0.08512549038143404, + "eval_vocab_overlap": 0.483206106870229, + "model_cross_entropy": 2.1957776739790633, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.44544269208718545, + "7": 0.37269641104430146, + "8": 0.3040875816821365, + "9": 0.22737421623563805, + "10": 0.1711644155109658, + "11": 0.10691225451598219, + "12": 0.06215312728465849, + "13": 0.027255772275123036 + } + } + }, + "runtime_s": 1604.3, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_StackExchange_s1.json b/results/ladder1b_b613/tv_StackExchange_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..bd6a819449e19681cc25af69599057fde4920ce7 --- /dev/null +++ b/results/ladder1b_b613/tv_StackExchange_s1.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "StackExchange", + "seed": 1, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18862690829804965, + "best1L": 0.15533980514322007, + "mean": 0.05391816994441407, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.14140681443469866, + "best1L": 0.10333575042230743, + "mean": 0.029278463085315058, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.024675324958349978, + "best1L": 0.022077921644917557, + "mean": 0.0037337661820596884, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816324629954, + "best1L": 0.007288629721317973, + "mean": 0.0029154518519395162, + "persist": 0.0 + }, + "typo": { + "min": 0.43749999361378805, + "best1L": 0.3154761961528233, + "mean": 0.22749256023338862, + "persist": 0.23363095202616282 + }, + "association": { + "min": 0.023291925101407936, + "best1L": 0.012422360479831696, + "mean": 0.006017080774264676, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.4650132247909998, + "dispersion_mean": 0.227905492902315, + "dispersion_slope": -0.0627578658954328, + "stable_rank": 45.60431671142578, + "top8_spectral_share": 0.06259722262620926, + "condition_number": 733.0093994140625, + "unembed_alignment": 0.10309186577796936 + }, + "corpus": { + "mean_prompt_tokens": 697.0, + "type_token_ratio": 0.22682926829268293, + "eval_vocab_overlap": 0.11450381679389313, + "model_cross_entropy": 2.097923113848712, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4650132247909998, + "7": 0.38842346659985083, + "8": 0.31952948051131264, + "9": 0.24112899126149476, + "10": 0.18621532393831935, + "11": 0.12009159502902597, + "12": 0.07055826626134162, + "13": 0.032283594826175085 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.191400835024459, + "best1L": 0.15811373080526078, + "mean": 0.05409154003219945, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1468455402978829, + "best1L": 0.10659898365182537, + "mean": 0.0303208847264094, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.020779220520385673, + "mean": 0.003571428541493203, + "persist": 0.0 + }, + "poetry": { + "min": 0.02332361468247005, + "best1L": 0.01020408155662673, + "mean": 0.0038265306502580643, + "persist": 0.0 + }, + "typo": { + "min": 0.46130951600415365, + "best1L": 0.33482142857142855, + "mean": 0.24218749787126267, + "persist": 0.24107143336108752 + }, + "association": { + "min": 0.02484472082661731, + "best1L": 0.013975155406764575, + "mean": 0.006211179990454444, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.4189372210164216, + "dispersion_mean": 0.20193495374979567, + "dispersion_slope": -0.056693116497411516, + "stable_rank": 44.58869171142578, + "top8_spectral_share": 0.06321461498737335, + "condition_number": 1074.916259765625, + "unembed_alignment": 0.10266400873661041 + }, + "corpus": { + "mean_prompt_tokens": 544.64, + "type_token_ratio": 0.19888366627497062, + "eval_vocab_overlap": 0.1717557251908397, + "model_cross_entropy": 2.3700751117328265, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4189372210164216, + "7": 0.3481068613050758, + "8": 0.28206295072812265, + "9": 0.21199034691165555, + "10": 0.16174010356026858, + "11": 0.104919972958529, + "12": 0.06075148505505656, + "13": 0.026970688463235457 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.1567267690386091, + "mean": 0.05565187148749828, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586818814278, + "best1L": 0.10514865762421063, + "mean": 0.030638143553265502, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.02597402608288186, + "best1L": 0.022077921644917557, + "mean": 0.0042207792368052265, + "persist": 0.0 + }, + "poetry": { + "min": 0.01749271181012903, + "best1L": 0.00728862958827189, + "mean": 0.0025510204224182026, + "persist": 0.0 + }, + "typo": { + "min": 0.46726189979485105, + "best1L": 0.3422619083098003, + "mean": 0.25390624893563135, + "persist": 0.2589285708963871 + }, + "association": { + "min": 0.026397515487458025, + "best1L": 0.017080745260630335, + "mean": 0.006405279589151698, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.42209511709326947, + "dispersion_mean": 0.20158468016394926, + "dispersion_slope": -0.05751406427358335, + "stable_rank": 45.936729431152344, + "top8_spectral_share": 0.06311541050672531, + "condition_number": 784.7188720703125, + "unembed_alignment": 0.10305953025817871 + }, + "corpus": { + "mean_prompt_tokens": 598.94, + "type_token_ratio": 0.1430193341570107, + "eval_vocab_overlap": 0.2633587786259542, + "model_cross_entropy": 2.3230903341963485, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.42209511709326947, + "7": 0.3505266605677967, + "8": 0.2829513782800685, + "9": 0.2117830240666191, + "10": 0.15973101866297856, + "11": 0.1012533937392425, + "12": 0.058547129887173116, + "13": 0.025789719014445895 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.1567267690386091, + "mean": 0.05530513104583536, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14648295566439629, + "best1L": 0.10369833212878023, + "mean": 0.030230239166745117, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.019480519395853792, + "mean": 0.0037337661820596884, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408163645437787, + "best1L": 0.008746355505926269, + "mean": 0.002915451901831797, + "persist": 0.0 + }, + "typo": { + "min": 0.4717261812516621, + "best1L": 0.3482142857142857, + "mean": 0.25688244295971735, + "persist": 0.2619047611951828 + }, + "association": { + "min": 0.023291925101407936, + "best1L": 0.01552795059978962, + "mean": 0.005628881992639175, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.43122030456999744, + "dispersion_mean": 0.20674880225106626, + "dispersion_slope": -0.05883771996107284, + "stable_rank": 46.31576919555664, + "top8_spectral_share": 0.0630844384431839, + "condition_number": 778.70849609375, + "unembed_alignment": 0.10305175185203552 + }, + "corpus": { + "mean_prompt_tokens": 613.4, + "type_token_ratio": 0.12300836865558092, + "eval_vocab_overlap": 0.3236641221374046, + "model_cross_entropy": 2.3230903341963485, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.43122030456999744, + "7": 0.358826079465303, + "8": 0.29139884502597513, + "9": 0.21767335596253642, + "10": 0.16449108072752175, + "11": 0.10359785798308986, + "12": 0.06029548548434728, + "13": 0.026487408789758998 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.15811373080526078, + "mean": 0.05565187255186694, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1468455402978829, + "best1L": 0.10369833212878023, + "mean": 0.030456853099167347, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.022077921644917557, + "best1L": 0.019480519395853792, + "mean": 0.003409090900926718, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.008746355505926269, + "mean": 0.002733236162125, + "persist": 0.0 + }, + "typo": { + "min": 0.47172618976661135, + "best1L": 0.34375, + "mean": 0.25762649199792315, + "persist": 0.2604166663118771 + }, + "association": { + "min": 0.023291925101407936, + "best1L": 0.017080745260630335, + "mean": 0.005822981325244265, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.4383450472027046, + "dispersion_mean": 0.21042162767887881, + "dispersion_slope": -0.05989319869320194, + "stable_rank": 46.145782470703125, + "top8_spectral_share": 0.06316930800676346, + "condition_number": 682.4490356445312, + "unembed_alignment": 0.10305862128734589 + }, + "corpus": { + "mean_prompt_tokens": 618.16, + "type_token_ratio": 0.1078685130063414, + "eval_vocab_overlap": 0.3603053435114504, + "model_cross_entropy": 2.3230903341963485, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4383450472027046, + "7": 0.3656362789209364, + "8": 0.2969540665729556, + "9": 0.22158286377573075, + "10": 0.16739322039674573, + "11": 0.10525157138047461, + "12": 0.061302324259650376, + "13": 0.026907648921832575 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.19001387112907, + "best1L": 0.15672676797424043, + "mean": 0.0558252427726984, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586605940545, + "best1L": 0.1051486578903028, + "mean": 0.030910079101366655, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.020779220520385673, + "mean": 0.0037337661820596884, + "persist": 0.0 + }, + "poetry": { + "min": 0.02186588982918433, + "best1L": 0.008746355505926269, + "mean": 0.003097667674800115, + "persist": 0.0 + }, + "typo": { + "min": 0.47172618550913675, + "best1L": 0.34523809594767435, + "mean": 0.25855654797383715, + "persist": 0.2633928582072258 + }, + "association": { + "min": 0.02484472082661731, + "best1L": 0.017080745260630335, + "mean": 0.006405279589151698, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.44430054037335864, + "dispersion_mean": 0.21305509189311, + "dispersion_slope": -0.06078541135559573, + "stable_rank": 45.847476959228516, + "top8_spectral_share": 0.06334731727838516, + "condition_number": 647.826416015625, + "unembed_alignment": 0.10269877314567566 + }, + "corpus": { + "mean_prompt_tokens": 697.54, + "type_token_ratio": 0.08590188376293832, + "eval_vocab_overlap": 0.4381679389312977, + "model_cross_entropy": 2.3230903341963485, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.44430054037335864, + "7": 0.3707803464933988, + "8": 0.30087321705451653, + "9": 0.22452009065764383, + "10": 0.168981282289491, + "11": 0.1063412866107052, + "12": 0.06167112037515709, + "13": 0.02697285129060919 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.19278779679111072, + "best1L": 0.15811372974089213, + "mean": 0.055998612993529866, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1493836103805474, + "best1L": 0.10587382077106408, + "mean": 0.030864757219595567, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.0038961038226261735, + "persist": 0.0 + }, + "poetry": { + "min": 0.02186588982918433, + "best1L": 0.008746355505926269, + "mean": 0.003097667674800115, + "persist": 0.0 + }, + "typo": { + "min": 0.4672619125672749, + "best1L": 0.34375000425747465, + "mean": 0.25297619295971735, + "persist": 0.2559523827263287 + }, + "association": { + "min": 0.02484472082661731, + "best1L": 0.017080745260630335, + "mean": 0.006211179990454444, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.45523233640158345, + "dispersion_mean": 0.21788916084641474, + "dispersion_slope": -0.06222388043417205, + "stable_rank": 45.60831069946289, + "top8_spectral_share": 0.0634746327996254, + "condition_number": 685.8636474609375, + "unembed_alignment": 0.10301263630390167 + }, + "corpus": { + "mean_prompt_tokens": 687.88, + "type_token_ratio": 0.07611065883584346, + "eval_vocab_overlap": 0.4618320610687023, + "model_cross_entropy": 2.3230903341963485, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.45523233640158345, + "7": 0.37899508460156195, + "8": 0.3075046116491916, + "9": 0.22952992963262947, + "10": 0.1725644405368773, + "11": 0.10865203463263828, + "12": 0.06306245186199867, + "13": 0.027572397454837177 + } + } + }, + "runtime_s": 1603.2, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_StackExchange_s2.json b/results/ladder1b_b613/tv_StackExchange_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..ab16f7eba428ff818645cadc32b21d2355199008 --- /dev/null +++ b/results/ladder1b_b613/tv_StackExchange_s2.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "StackExchange", + "seed": 2, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18862690829804965, + "best1L": 0.15811373080526078, + "mean": 0.053398058616689274, + "persist": 0.0 + }, + "multilingual": { + "min": 0.14539521293980734, + "best1L": 0.10369833212878023, + "mean": 0.029822334447609528, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02467532442616565, + "best1L": 0.019480518863669465, + "mean": 0.004220779103759144, + "persist": 0.0 + }, + "poetry": { + "min": 0.021865888764815673, + "best1L": 0.01020408155662673, + "mean": 0.003462099120952189, + "persist": 0.0 + }, + "typo": { + "min": 0.44345237953322275, + "best1L": 0.3005952313542366, + "mean": 0.23009672920618737, + "persist": 0.24107143070016587 + }, + "association": { + "min": 0.02484472095966339, + "best1L": 0.01552795059978962, + "mean": 0.006017080641218594, + "persist": 0.001552795059978962 + } + }, + "operator": { + "dispersion_L0": 0.35037223011490604, + "dispersion_mean": 0.17143239398830412, + "dispersion_slope": -0.048665171488073235, + "stable_rank": 47.66275405883789, + "top8_spectral_share": 0.06239287927746773, + "condition_number": 636.2601928710938, + "unembed_alignment": 0.10538221895694733 + }, + "corpus": { + "mean_prompt_tokens": 444.6, + "type_token_ratio": 0.3090418353576248, + "eval_vocab_overlap": 0.10381679389312977, + "model_cross_entropy": 2.1864613988378028, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.35037223011490604, + "7": 0.29728085474301924, + "8": 0.24810326881923883, + "9": 0.18423836689744308, + "10": 0.13829147943348177, + "11": 0.08361148706948797, + "12": 0.04818627078374565, + "13": 0.02137519404511039 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.19001387325780733, + "best1L": 0.16088765912822314, + "mean": 0.05530513051365103, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.15047135310513632, + "best1L": 0.10587382077106408, + "mean": 0.0312726608743625, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003409090900926718, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.008746355505926269, + "mean": 0.003097667608277074, + "persist": 0.0 + }, + "typo": { + "min": 0.4613095245191029, + "best1L": 0.32291666524750845, + "mean": 0.24739583049501693, + "persist": 0.2544642851821014 + }, + "association": { + "min": 0.027950310680483068, + "best1L": 0.018633540320609297, + "mean": 0.006793478320884917, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.4157884374915875, + "dispersion_mean": 0.20259002997210054, + "dispersion_slope": -0.057281300081676714, + "stable_rank": 45.540897369384766, + "top8_spectral_share": 0.06312388926744461, + "condition_number": 936.2377319335938, + "unembed_alignment": 0.10655812174081802 + }, + "corpus": { + "mean_prompt_tokens": 757.64, + "type_token_ratio": 0.18837442584868802, + "eval_vocab_overlap": 0.2381679389312977, + "model_cross_entropy": 2.2932498691318273, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4157884374915875, + "7": 0.3511874363041807, + "8": 0.2894212107310079, + "9": 0.2164841644703078, + "10": 0.16250631631245294, + "11": 0.10079188543668735, + "12": 0.05854633103603652, + "13": 0.025994457994543706 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.19001387325780733, + "best1L": 0.16088765859603882, + "mean": 0.055131761357188225, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14720812173826353, + "best1L": 0.1040609133030687, + "mean": 0.03045685283307518, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003571428541493203, + "persist": 0.0 + }, + "poetry": { + "min": 0.020408162847161293, + "best1L": 0.007288629721317973, + "mean": 0.0027332361454942395, + "persist": 0.0 + }, + "typo": { + "min": 0.4776785671710968, + "best1L": 0.3482142814568111, + "mean": 0.2604166620544025, + "persist": 0.26636904584509985 + }, + "association": { + "min": 0.026397516019642353, + "best1L": 0.018633540320609297, + "mean": 0.006405279389582574, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.42924787911000656, + "dispersion_mean": 0.20845691693249607, + "dispersion_slope": -0.05903713455514291, + "stable_rank": 45.7269172668457, + "top8_spectral_share": 0.06278593093156815, + "condition_number": 894.0880737304688, + "unembed_alignment": 0.10539476573467255 + }, + "corpus": { + "mean_prompt_tokens": 778.34, + "type_token_ratio": 0.13384896060847445, + "eval_vocab_overlap": 0.29923664122137406, + "model_cross_entropy": 2.308186775929219, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.42924787911000656, + "7": 0.36174429935939134, + "8": 0.2967217708351206, + "9": 0.22214325888820127, + "10": 0.1667001689451436, + "11": 0.10424573002812207, + "12": 0.06037851887941542, + "13": 0.026473709414567795 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.18862691042678698, + "best1L": 0.15811373399836676, + "mean": 0.05513176109109606, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586818814278, + "best1L": 0.10514865762421063, + "mean": 0.030774111593408243, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.0038961038226261735, + "persist": 0.0 + }, + "poetry": { + "min": 0.021865888764815673, + "best1L": 0.008746355505926269, + "mean": 0.002915451885201037, + "persist": 0.0 + }, + "typo": { + "min": 0.4761904754808971, + "best1L": 0.35416666311877115, + "mean": 0.2632068448833057, + "persist": 0.2678571439215115 + }, + "association": { + "min": 0.026397516019642353, + "best1L": 0.018633540320609297, + "mean": 0.006599378722187664, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.43413891949662065, + "dispersion_mean": 0.21068171078040784, + "dispersion_slope": -0.059563425016684185, + "stable_rank": 45.515987396240234, + "top8_spectral_share": 0.06262922286987305, + "condition_number": 905.05029296875, + "unembed_alignment": 0.10589657723903656 + }, + "corpus": { + "mean_prompt_tokens": 707.52, + "type_token_ratio": 0.12209784411276949, + "eval_vocab_overlap": 0.34885496183206105, + "model_cross_entropy": 2.308186775929219, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.43413891949662065, + "7": 0.364697853590322, + "8": 0.29874527456366534, + "9": 0.22502827177490542, + "10": 0.16896573909946347, + "11": 0.10576272406212892, + "12": 0.06133006919328301, + "13": 0.02678483446287415 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18723994653139794, + "best1L": 0.15950069576501846, + "mean": 0.054958391402448924, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586605940545, + "best1L": 0.10587382050497192, + "mean": 0.030592821139310087, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.022077922177101885, + "best1L": 0.019480518863669465, + "mean": 0.003571428541493203, + "persist": 0.0 + }, + "poetry": { + "min": 0.023323615081608295, + "best1L": 0.008746355505926269, + "mean": 0.003097667674800115, + "persist": 0.0 + }, + "typo": { + "min": 0.47321429210049765, + "best1L": 0.34672618550913675, + "mean": 0.2579985133239201, + "persist": 0.26190476438828875 + }, + "association": { + "min": 0.024844720294432982, + "best1L": 0.01708074565976858, + "mean": 0.006017080724372396, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.4385892235032095, + "dispersion_mean": 0.21156656624276468, + "dispersion_slope": -0.060098321983935464, + "stable_rank": 45.396366119384766, + "top8_spectral_share": 0.06318563967943192, + "condition_number": 917.6392822265625, + "unembed_alignment": 0.10630789399147034 + }, + "corpus": { + "mean_prompt_tokens": 746.24, + "type_token_ratio": 0.10425600343053174, + "eval_vocab_overlap": 0.3938931297709924, + "model_cross_entropy": 2.308186775929219, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4385892235032095, + "7": 0.366985579378595, + "8": 0.2994784372427782, + "9": 0.22508916727469727, + "10": 0.1686197989547984, + "11": 0.10546098243799733, + "12": 0.06142512027559388, + "13": 0.026884220874447813 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.18862691042678698, + "best1L": 0.15811372974089213, + "mean": 0.05530513237629618, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.14829586605940545, + "best1L": 0.10478607618382998, + "mean": 0.030592821671494415, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.023376623301633766, + "best1L": 0.019480518863669465, + "mean": 0.004220779103759144, + "persist": 0.0 + }, + "poetry": { + "min": 0.02186588982918433, + "best1L": 0.008746355505926269, + "mean": 0.002915451885201037, + "persist": 0.0 + }, + "typo": { + "min": 0.47916666737624575, + "best1L": 0.3511904754808971, + "mean": 0.25985863166196005, + "persist": 0.2633928582072258 + }, + "association": { + "min": 0.026397516019642353, + "best1L": 0.018633540320609297, + "mean": 0.006405279389582574, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.44093673116226334, + "dispersion_mean": 0.2123017686939808, + "dispersion_slope": -0.06047300575697167, + "stable_rank": 45.36140441894531, + "top8_spectral_share": 0.0633113756775856, + "condition_number": 855.451904296875, + "unembed_alignment": 0.10617785155773163 + }, + "corpus": { + "mean_prompt_tokens": 681.56, + "type_token_ratio": 0.09154488721951601, + "eval_vocab_overlap": 0.45725190839694657, + "model_cross_entropy": 2.308186775929219, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.44093673116226334, + "7": 0.36904914467241706, + "8": 0.30027294710785346, + "9": 0.22572054598751914, + "10": 0.16902795063683557, + "11": 0.10545680147500394, + "12": 0.061219404704068336, + "13": 0.02673062380588547 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.18862691042678698, + "best1L": 0.15811372974089213, + "mean": 0.055305131311927526, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1475707053073815, + "best1L": 0.10551123933068343, + "mean": 0.030819434073886702, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.023376623833818094, + "best1L": 0.020779220520385673, + "mean": 0.004058441463192659, + "persist": 0.0 + }, + "poetry": { + "min": 0.02040816391152995, + "best1L": 0.007288629721317973, + "mean": 0.0027332361454942395, + "persist": 0.0 + }, + "typo": { + "min": 0.47619047122342245, + "best1L": 0.35119047335215975, + "mean": 0.2593005948833057, + "persist": 0.26041666897279875 + }, + "association": { + "min": 0.024844720294432982, + "best1L": 0.01708074565976858, + "mean": 0.0062111800569774845, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.44422067982670704, + "dispersion_mean": 0.21377314615415052, + "dispersion_slope": -0.06091735558840611, + "stable_rank": 45.893985748291016, + "top8_spectral_share": 0.06314670294523239, + "condition_number": 780.1212158203125, + "unembed_alignment": 0.10471590608358383 + }, + "corpus": { + "mean_prompt_tokens": 677.565, + "type_token_ratio": 0.07989639370392508, + "eval_vocab_overlap": 0.49694656488549616, + "model_cross_entropy": 2.308186775929219, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.44422067982670704, + "7": 0.37138916628234797, + "8": 0.30237469231931074, + "9": 0.2274264576686649, + "10": 0.17035846324039464, + "11": 0.10608086744068339, + "12": 0.06148085367446993, + "13": 0.02685398878062542 + } + } + }, + "runtime_s": 1601.6, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_USPTO_Backgrounds_s0.json b/results/ladder1b_b613/tv_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..62bea66cd9b40a100aa8071d84e3eedb8f6ee684 --- /dev/null +++ b/results/ladder1b_b613/tv_USPTO_Backgrounds_s0.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "USPTO_Backgrounds", + "seed": 0, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17753120458551816, + "best1L": 0.15672676690987178, + "mean": 0.05235783636037793, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.13959390776497976, + "best1L": 0.10369833186268806, + "mean": 0.02891588117927313, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.018181818005229746, + "mean": 0.0029220778794427004, + "persist": 0.0 + }, + "poetry": { + "min": 0.023323615480746542, + "best1L": 0.007288629721317973, + "mean": 0.00346209917084447, + "persist": 0.0 + }, + "typo": { + "min": 0.42113094670431955, + "best1L": 0.30952381023338865, + "mean": 0.2366071417927742, + "persist": 0.25297619136316435 + }, + "association": { + "min": 0.04347826141331877, + "best1L": 0.023291925500546182, + "mean": 0.009316769993997045, + "persist": 0.00776397529989481 + } + }, + "operator": { + "dispersion_L0": 0.40315082302349786, + "dispersion_mean": 0.2033244833807695, + "dispersion_slope": -0.05415885248193656, + "stable_rank": 41.67527389526367, + "top8_spectral_share": 0.06769061088562012, + "condition_number": 1945.2562255859375, + "unembed_alignment": 0.09913688898086548 + }, + "corpus": { + "mean_prompt_tokens": 1170.0, + "type_token_ratio": 0.16965811965811967, + "eval_vocab_overlap": 0.13358778625954199, + "model_cross_entropy": 2.151170115427928, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.40315082302349786, + "7": 0.3407958731133363, + "8": 0.28155186178820857, + "9": 0.21973897809082935, + "10": 0.17506823265939242, + "11": 0.11149324652915021, + "12": 0.06601992671753944, + "13": 0.02877692512420156 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.16504854496036256, + "best1L": 0.15117891505360603, + "mean": 0.04906380110021148, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13089195186538355, + "best1L": 0.1011602585869176, + "mean": 0.02719361847266555, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.022077921511871473, + "best1L": 0.018181818138275827, + "mean": 0.003246753310252513, + "persist": 0.0 + }, + "poetry": { + "min": 0.026239067316055298, + "best1L": 0.008746355505926269, + "mean": 0.0040087464398571426, + "persist": 0.0 + }, + "typo": { + "min": 0.42261903839451925, + "best1L": 0.3095238134264946, + "mean": 0.2321428544819355, + "persist": 0.2470238097012043 + }, + "association": { + "min": 0.03881987490292106, + "best1L": 0.02329192523445402, + "mean": 0.008152173815428146, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.45486343208026697, + "dispersion_mean": 0.23115340449901398, + "dispersion_slope": -0.06123430300986029, + "stable_rank": 43.186607360839844, + "top8_spectral_share": 0.06467950344085693, + "condition_number": 2429.585693359375, + "unembed_alignment": 0.09520602226257324 + }, + "corpus": { + "mean_prompt_tokens": 1036.16, + "type_token_ratio": 0.14858709079678814, + "eval_vocab_overlap": 0.2015267175572519, + "model_cross_entropy": 2.182336464271889, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.45486343208026697, + "7": 0.38744721024602846, + "8": 0.3208019212532327, + "9": 0.24990753668098856, + "10": 0.2013662073393487, + "11": 0.1261324948619621, + "12": 0.07540540161639647, + "13": 0.03330303191388791 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17198335751891136, + "best1L": 0.14979195328695433, + "mean": 0.050104022425200255, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13560550872768676, + "best1L": 0.10043509464178767, + "mean": 0.02764684533966439, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.018181818005229746, + "best1L": 0.018181818005229746, + "mean": 0.002759740238876215, + "persist": 0.0 + }, + "poetry": { + "min": 0.033527696904327185, + "best1L": 0.011661807607327188, + "mean": 0.0054664723741422805, + "persist": 0.0 + }, + "typo": { + "min": 0.41369047335215975, + "best1L": 0.30952380916901995, + "mean": 0.23214285980377877, + "persist": 0.250000001596553 + }, + "association": { + "min": 0.041925465422017236, + "best1L": 0.026397515354411944, + "mean": 0.008734471896397216, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.4512115375620367, + "dispersion_mean": 0.23011654363348225, + "dispersion_slope": -0.06082333551920514, + "stable_rank": 43.143367767333984, + "top8_spectral_share": 0.06378254294395447, + "condition_number": 1146.066162109375, + "unembed_alignment": 0.0985567718744278 + }, + "corpus": { + "mean_prompt_tokens": 863.9, + "type_token_ratio": 0.12239842574372034, + "eval_vocab_overlap": 0.24580152671755726, + "model_cross_entropy": 2.1572919192614854, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4512115375620367, + "7": 0.38534115225186527, + "8": 0.32092226245436534, + "9": 0.2487853491264309, + "10": 0.19992725928323787, + "11": 0.12630842993457803, + "12": 0.07516429060225026, + "13": 0.033272067853093466 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.17337031928556307, + "best1L": 0.14979195328695433, + "mean": 0.050104022957384586, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1341551834983485, + "best1L": 0.10043509464178767, + "mean": 0.02746555461947407, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.01688311674765178, + "best1L": 0.01688311674765178, + "mean": 0.0025974025816789697, + "persist": 0.0 + }, + "poetry": { + "min": 0.03352769663823502, + "best1L": 0.013119533658027649, + "mean": 0.005284256617804723, + "persist": 0.0 + }, + "typo": { + "min": 0.4196428528853825, + "best1L": 0.31101191150290625, + "mean": 0.23251487580793245, + "persist": 0.2485119067132473 + }, + "association": { + "min": 0.04347826061504228, + "best1L": 0.026397515354411944, + "mean": 0.008928571229002305, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.45252171695085597, + "dispersion_mean": 0.23000707884769608, + "dispersion_slope": -0.06106597880743251, + "stable_rank": 43.504512786865234, + "top8_spectral_share": 0.06342680007219315, + "condition_number": 1177.92626953125, + "unembed_alignment": 0.09887100756168365 + }, + "corpus": { + "mean_prompt_tokens": 896.2666666666667, + "type_token_ratio": 0.10278191014578994, + "eval_vocab_overlap": 0.2870229007633588, + "model_cross_entropy": 2.1572919192614854, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.45252171695085597, + "7": 0.3860898173385124, + "8": 0.32094967036854344, + "9": 0.24834146594064055, + "10": 0.199139046641558, + "11": 0.1253702113393555, + "12": 0.07450661675192552, + "13": 0.033138085450177386 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.17475728211658342, + "best1L": 0.14979195328695433, + "mean": 0.050104022957384586, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1337926031223365, + "best1L": 0.10007251320140702, + "mean": 0.027465555417750562, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.018181818005229746, + "mean": 0.0029220779459657414, + "persist": 0.0 + }, + "poetry": { + "min": 0.03061224560120276, + "best1L": 0.011661807474281107, + "mean": 0.005102040927990207, + "persist": 0.0 + }, + "typo": { + "min": 0.41815475480897085, + "best1L": 0.30654762365988325, + "mean": 0.23084077345473425, + "persist": 0.24702381289431027 + }, + "association": { + "min": 0.040372670495084355, + "best1L": 0.026397515354411944, + "mean": 0.008540372530530606, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.485257955343297, + "dispersion_mean": 0.24661011446416353, + "dispersion_slope": -0.06552215240861133, + "stable_rank": 43.746421813964844, + "top8_spectral_share": 0.06272465735673904, + "condition_number": 1205.3673095703125, + "unembed_alignment": 0.09912055730819702 + }, + "corpus": { + "mean_prompt_tokens": 928.07, + "type_token_ratio": 0.09325805165558633, + "eval_vocab_overlap": 0.3267175572519084, + "model_cross_entropy": 2.1572919192614854, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.485257955343297, + "7": 0.41398848519292925, + "8": 0.34387351540458017, + "9": 0.26663780906024637, + "10": 0.21387090910950518, + "11": 0.13448099530263324, + "12": 0.07978347639993111, + "13": 0.0349877699001859 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.17059639575225966, + "best1L": 0.14979195328695433, + "mean": 0.04993065167218447, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13451776706746646, + "best1L": 0.10188542253204755, + "mean": 0.027646845871848718, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02337662356772593, + "best1L": 0.018181818005229746, + "mean": 0.003409090934188238, + "persist": 0.0 + }, + "poetry": { + "min": 0.0320699703214424, + "best1L": 0.013119533658027649, + "mean": 0.005648688230264399, + "persist": 0.0 + }, + "typo": { + "min": 0.4151785650423595, + "best1L": 0.30654762365988325, + "mean": 0.22898065245577268, + "persist": 0.24702381289431027 + }, + "association": { + "min": 0.04037267102726868, + "best1L": 0.026397515354411944, + "mean": 0.008540372530530606, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.48269494489053194, + "dispersion_mean": 0.2478806731825312, + "dispersion_slope": -0.06520810922047912, + "stable_rank": 42.5976448059082, + "top8_spectral_share": 0.06383633613586426, + "condition_number": 1530.1171875, + "unembed_alignment": 0.09728412330150604 + }, + "corpus": { + "mean_prompt_tokens": 893.02, + "type_token_ratio": 0.07713153120870753, + "eval_vocab_overlap": 0.3709923664122137, + "model_cross_entropy": 2.1572919192614854, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.48269494489053194, + "7": 0.4144268283028257, + "8": 0.3462805956478996, + "9": 0.2693264737522899, + "10": 0.2168022619515449, + "11": 0.13676328341294483, + "12": 0.0813316263911378, + "13": 0.03541937111107507 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.17337032034993172, + "best1L": 0.14979195328695433, + "mean": 0.050277392113847394, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13524292835167476, + "best1L": 0.10297316791755813, + "mean": 0.02823604011375989, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02207792137882539, + "best1L": 0.018181818005229746, + "mean": 0.003246753293621753, + "persist": 0.0 + }, + "poetry": { + "min": 0.03352769663823502, + "best1L": 0.013119533658027649, + "mean": 0.005830903953340437, + "persist": 0.0 + }, + "typo": { + "min": 0.41815475268023355, + "best1L": 0.3065476215311459, + "mean": 0.22879464605024882, + "persist": 0.24553571801100457 + }, + "association": { + "min": 0.04037267102726868, + "best1L": 0.024844720693571225, + "mean": 0.008540372530530606, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.487138486065882, + "dispersion_mean": 0.24945497953188453, + "dispersion_slope": -0.06580193986061443, + "stable_rank": 43.00107192993164, + "top8_spectral_share": 0.06327935308218002, + "condition_number": 1503.6197509765625, + "unembed_alignment": 0.09831929951906204 + }, + "corpus": { + "mean_prompt_tokens": 900.0, + "type_token_ratio": 0.0683, + "eval_vocab_overlap": 0.4083969465648855, + "model_cross_entropy": 2.1572919192614854, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.487138486065882, + "7": 0.41758931918123415, + "8": 0.3483729017351545, + "9": 0.2706155867509618, + "10": 0.21748928042972973, + "11": 0.13732130207135443, + "12": 0.08147828237847679, + "13": 0.03563467764228289 + } + } + }, + "runtime_s": 1610.5, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_USPTO_Backgrounds_s1.json b/results/ladder1b_b613/tv_USPTO_Backgrounds_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..1f97b183d8b47b6b276bae2e679980093e046c6d --- /dev/null +++ b/results/ladder1b_b613/tv_USPTO_Backgrounds_s1.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "USPTO_Backgrounds", + "seed": 1, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.17198335698672704, + "best1L": 0.151178916650159, + "mean": 0.0506241341520633, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1305293712232794, + "best1L": 0.10188542226595539, + "mean": 0.027148296324802295, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02467532522444214, + "best1L": 0.019480519129761627, + "mean": 0.004220779137020665, + "persist": 0.0 + }, + "poetry": { + "min": 0.027696793100663593, + "best1L": 0.011661807740373271, + "mean": 0.005284256534650922, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.4017857164144516, + "best1L": 0.29613095096179415, + "mean": 0.21800594936524118, + "persist": 0.23511904957039015 + }, + "association": { + "min": 0.03726708064121859, + "best1L": 0.020186335247542177, + "mean": 0.008734472029443299, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.380053788848394, + "dispersion_mean": 0.19889362132677027, + "dispersion_slope": -0.05118496662168532, + "stable_rank": 40.344058990478516, + "top8_spectral_share": 0.06880591809749603, + "condition_number": 3461.15283203125, + "unembed_alignment": 0.09775885939598083 + }, + "corpus": { + "mean_prompt_tokens": 1022.5, + "type_token_ratio": 0.19511002444987774, + "eval_vocab_overlap": 0.116793893129771, + "model_cross_entropy": 2.1789153915267807, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.380053788848394, + "7": 0.32994707680154667, + "8": 0.27780548145141226, + "9": 0.21793553186243628, + "10": 0.17506967859033148, + "11": 0.11282869545596903, + "12": 0.06709505112257416, + "13": 0.030413666481498368 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17891816848090716, + "best1L": 0.15672677169953073, + "mean": 0.05218446560736213, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13415518403053284, + "best1L": 0.10079767634826046, + "mean": 0.02778281391199146, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02337662356772593, + "best1L": 0.018181818005229746, + "mean": 0.003409090934188238, + "persist": 0.0 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.011661807873419352, + "mean": 0.005830903753771314, + "persist": 0.0 + }, + "typo": { + "min": 0.38988095096179415, + "best1L": 0.28571428677865435, + "mean": 0.2120535708963871, + "persist": 0.2247023843228817 + }, + "association": { + "min": 0.040372670628130436, + "best1L": 0.0232919253675001, + "mean": 0.008152173815428146, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.4365652062756269, + "dispersion_mean": 0.22863522195297722, + "dispersion_slope": -0.05885927103264099, + "stable_rank": 41.80659866333008, + "top8_spectral_share": 0.06532493233680725, + "condition_number": 1350.9720458984375, + "unembed_alignment": 0.09787148237228394 + }, + "corpus": { + "mean_prompt_tokens": 1106.48, + "type_token_ratio": 0.14239751283348998, + "eval_vocab_overlap": 0.20076335877862594, + "model_cross_entropy": 2.2068332816459035, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4365652062756269, + "7": 0.3792562815924193, + "8": 0.31928186581661633, + "9": 0.2506497703493988, + "10": 0.20202135874585944, + "11": 0.12984644486056424, + "12": 0.07761608813099588, + "13": 0.03384475985233684 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.17891816848090716, + "best1L": 0.15533980780414172, + "mean": 0.051664355477052074, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1341551856270858, + "best1L": 0.10116025752254895, + "mean": 0.02769216848537326, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.016883116880697862, + "mean": 0.0030844156530552675, + "persist": 0.0 + }, + "poetry": { + "min": 0.03061224493597235, + "best1L": 0.014577259575682027, + "mean": 0.0051020407450518435, + "persist": 0.0 + }, + "typo": { + "min": 0.41517857355730875, + "best1L": 0.3110119008592197, + "mean": 0.23195684541549003, + "persist": 0.2470238097012043 + }, + "association": { + "min": 0.04347826034895012, + "best1L": 0.026397515620504106, + "mean": 0.008540372530530606, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.4412338830375938, + "dispersion_mean": 0.22944867636093103, + "dispersion_slope": -0.05952436512897374, + "stable_rank": 44.16301727294922, + "top8_spectral_share": 0.06326144188642502, + "condition_number": 910.6898803710938, + "unembed_alignment": 0.09883494675159454 + }, + "corpus": { + "mean_prompt_tokens": 925.24, + "type_token_ratio": 0.12476762785871774, + "eval_vocab_overlap": 0.2603053435114504, + "model_cross_entropy": 2.255969021771405, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4412338830375938, + "7": 0.38109582875379894, + "8": 0.32064513912774834, + "9": 0.2512010789640722, + "10": 0.20242556273758963, + "11": 0.12882052585826825, + "12": 0.07642636831267902, + "13": 0.03374102409569783 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.17891816794872284, + "best1L": 0.1525658816099167, + "mean": 0.05131761343883617, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13306743891111442, + "best1L": 0.10043509464178767, + "mean": 0.02742023187290345, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02207792137882539, + "best1L": 0.016883116880697862, + "mean": 0.003246753293621753, + "persist": 0.0 + }, + "poetry": { + "min": 0.030612245335110595, + "best1L": 0.014577259575682027, + "mean": 0.005102040927990207, + "persist": 0.0 + }, + "typo": { + "min": 0.4077380916901997, + "best1L": 0.3080357110926083, + "mean": 0.2273065485060215, + "persist": 0.24999999840344703 + }, + "association": { + "min": 0.03881987490292106, + "best1L": 0.024844720560525144, + "mean": 0.007958074482823057, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.4438703951030434, + "dispersion_mean": 0.2301870074706463, + "dispersion_slope": -0.06006737356294479, + "stable_rank": 44.382625579833984, + "top8_spectral_share": 0.0629059299826622, + "condition_number": 1053.400390625, + "unembed_alignment": 0.09776949882507324 + }, + "corpus": { + "mean_prompt_tokens": 883.7866666666666, + "type_token_ratio": 0.11121839357914429, + "eval_vocab_overlap": 0.3, + "model_cross_entropy": 2.255969021771405, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4438703951030434, + "7": 0.3835771694183672, + "8": 0.32276780059155263, + "9": 0.2515772032377574, + "10": 0.20247629341606274, + "11": 0.12802760160167492, + "12": 0.07587821722994113, + "13": 0.033321379166771324 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18169209307857923, + "best1L": 0.15811373240181378, + "mean": 0.05253120671425547, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1334300216819559, + "best1L": 0.1015228418899434, + "mean": 0.027646844940526143, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.016883116880697862, + "mean": 0.0030844156530552675, + "persist": 0.0 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.013119533791073732, + "mean": 0.005102040794944125, + "persist": 0.0 + }, + "typo": { + "min": 0.4032738081046513, + "best1L": 0.29166667269808905, + "mean": 0.21912202185818128, + "persist": 0.23214285767504148 + }, + "association": { + "min": 0.041925464889832904, + "best1L": 0.026397515620504106, + "mean": 0.008346273231187038, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.44306598605816067, + "dispersion_mean": 0.22802779727413378, + "dispersion_slope": -0.060019694010794135, + "stable_rank": 44.481204986572266, + "top8_spectral_share": 0.06309165060520172, + "condition_number": 1001.7794189453125, + "unembed_alignment": 0.09758026897907257 + }, + "corpus": { + "mean_prompt_tokens": 898.01, + "type_token_ratio": 0.09695883119341656, + "eval_vocab_overlap": 0.32061068702290074, + "model_cross_entropy": 2.255969021771405, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.44306598605816067, + "7": 0.38175023454509777, + "8": 0.31983922841728213, + "9": 0.2480221992168734, + "10": 0.19892480801633106, + "11": 0.12566668000397682, + "12": 0.07424943944053741, + "13": 0.032703802494810974 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.18030513131192752, + "best1L": 0.15533980248229845, + "mean": 0.05201109605176108, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13343002035149507, + "best1L": 0.10261058541280883, + "mean": 0.027510877499090775, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02207792137882539, + "best1L": 0.016883116880697862, + "mean": 0.003246753293621753, + "persist": 0.0 + }, + "poetry": { + "min": 0.02915451888527189, + "best1L": 0.011661807873419352, + "mean": 0.004919825021975807, + "persist": 0.0 + }, + "typo": { + "min": 0.40029761833803995, + "best1L": 0.2827380948833057, + "mean": 0.21781994242753303, + "persist": 0.23511904531291553 + }, + "association": { + "min": 0.03881987516901323, + "best1L": 0.024844720294432982, + "mean": 0.008152173848689668, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.4664431553768745, + "dispersion_mean": 0.2392562697792057, + "dispersion_slope": -0.06320051410399934, + "stable_rank": 44.48917007446289, + "top8_spectral_share": 0.0628930851817131, + "condition_number": 1087.0709228515625, + "unembed_alignment": 0.096527099609375 + }, + "corpus": { + "mean_prompt_tokens": 906.9333333333333, + "type_token_ratio": 0.08178476918553367, + "eval_vocab_overlap": 0.37709923664122136, + "model_cross_entropy": 2.255969021771405, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4664431553768745, + "7": 0.4007240441363299, + "8": 0.3352987813842738, + "9": 0.260420703779637, + "10": 0.2088640302031294, + "11": 0.13100348511486168, + "12": 0.07737501085060243, + "13": 0.033920947387936905 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.18169209414294787, + "best1L": 0.15672676850642478, + "mean": 0.052184466804776876, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13451776493872916, + "best1L": 0.10261058754154614, + "mean": 0.028009426979614154, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02337662356772593, + "best1L": 0.018181818005229746, + "mean": 0.003246753227098712, + "persist": 0.0 + }, + "poetry": { + "min": 0.033527696904327185, + "best1L": 0.011661807873419352, + "mean": 0.0056486881304798385, + "persist": 0.0 + }, + "typo": { + "min": 0.4062499978712627, + "best1L": 0.29315476438828875, + "mean": 0.22135416684406145, + "persist": 0.24107142697487557 + }, + "association": { + "min": 0.03881987596728972, + "best1L": 0.024844720294432982, + "mean": 0.008152173815428146, + "persist": 0.006211180239915848 + } + }, + "operator": { + "dispersion_L0": 0.4748792461552829, + "dispersion_mean": 0.24004025786098576, + "dispersion_slope": -0.06429152082800739, + "stable_rank": 44.43504333496094, + "top8_spectral_share": 0.06289353221654892, + "condition_number": 1011.5020751953125, + "unembed_alignment": 0.09829408675432205 + }, + "corpus": { + "mean_prompt_tokens": 856.155, + "type_token_ratio": 0.07197294882351911, + "eval_vocab_overlap": 0.40458015267175573, + "model_cross_entropy": 2.255969021771405, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4748792461552829, + "7": 0.4045249623674996, + "8": 0.3352946977710724, + "9": 0.25926356918831495, + "10": 0.20708047643269928, + "11": 0.12947549669790065, + "12": 0.07634640928941376, + "13": 0.033457204985702614 + } + } + }, + "runtime_s": 1609.8, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_USPTO_Backgrounds_s2.json b/results/ladder1b_b613/tv_USPTO_Backgrounds_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..ce110df611d7c02f030ebc3cb86bc623946b3a0a --- /dev/null +++ b/results/ladder1b_b613/tv_USPTO_Backgrounds_s2.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "USPTO_Backgrounds", + "seed": 2, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.18723994653139794, + "best1L": 0.1595006941684655, + "mean": 0.053571428172290325, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13705583768231527, + "best1L": 0.10333574855966228, + "mean": 0.02778281391199146, + "persist": 0.001450326293706894 + }, + "order-ops": { + "min": 0.016883116880697862, + "best1L": 0.016883116880697862, + "mean": 0.0024350648912202033, + "persist": 0.0 + }, + "poetry": { + "min": 0.032069970454488485, + "best1L": 0.011661807740373271, + "mean": 0.005102040844836405, + "persist": 0.0 + }, + "typo": { + "min": 0.38690475693770815, + "best1L": 0.26934523986918585, + "mean": 0.20219493870224273, + "persist": 0.2157738134264946 + }, + "association": { + "min": 0.045031054743698666, + "best1L": 0.031055900135210583, + "mean": 0.010093167623771089, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.422582343334678, + "dispersion_mean": 0.21266225937768315, + "dispersion_slope": -0.05612462982890251, + "stable_rank": 42.93210220336914, + "top8_spectral_share": 0.06593415886163712, + "condition_number": 994.7443237304688, + "unembed_alignment": 0.09280547499656677 + }, + "corpus": { + "mean_prompt_tokens": 677.6, + "type_token_ratio": 0.2532467532467532, + "eval_vocab_overlap": 0.11297709923664122, + "model_cross_entropy": 2.236127011410825, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.422582343334678, + "7": 0.35128392216436555, + "8": 0.2942663474694232, + "9": 0.22701049844739818, + "10": 0.18639013543951946, + "11": 0.11833449861063967, + "12": 0.07078464438354819, + "13": 0.030645685171892992 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.17753120671425546, + "best1L": 0.15533980780414172, + "mean": 0.052011095918715, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13415518403053284, + "best1L": 0.10297316791755813, + "mean": 0.028190717433712313, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.0029220778794427004, + "persist": 0.0 + }, + "poetry": { + "min": 0.03790087492338249, + "best1L": 0.016034985360290323, + "mean": 0.006377551072676267, + "persist": 0.0 + }, + "typo": { + "min": 0.42559523454734255, + "best1L": 0.32589285927159445, + "mean": 0.24237351013081415, + "persist": 0.2619047627917358 + }, + "association": { + "min": 0.04503105634025165, + "best1L": 0.02484472082661731, + "mean": 0.009122670561607395, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.469653628712383, + "dispersion_mean": 0.23873494946566803, + "dispersion_slope": -0.06323750037471447, + "stable_rank": 43.80411148071289, + "top8_spectral_share": 0.06446799635887146, + "condition_number": 1992.5205078125, + "unembed_alignment": 0.09582406282424927 + }, + "corpus": { + "mean_prompt_tokens": 786.6, + "type_token_ratio": 0.16770912789219425, + "eval_vocab_overlap": 0.18167938931297709, + "model_cross_entropy": 2.2282100973043355, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.469653628712383, + "7": 0.39805907933358786, + "8": 0.3311504880744945, + "9": 0.2599324500392719, + "10": 0.21059730546539743, + "11": 0.12969265005706876, + "12": 0.07796426674861552, + "13": 0.032829727294525456 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.18169209307857923, + "best1L": 0.15672677169953073, + "mean": 0.052704576935086934, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1348803469112941, + "best1L": 0.10369833399142538, + "mean": 0.02828136339251484, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.019480519129761627, + "mean": 0.003246753293621753, + "persist": 0.0 + }, + "poetry": { + "min": 0.03352769663823502, + "best1L": 0.013119533791073732, + "mean": 0.005284256551281682, + "persist": 0.0 + }, + "typo": { + "min": 0.42857142857142855, + "best1L": 0.32589285714285715, + "mean": 0.23921130916901998, + "persist": 0.25744047814181875 + }, + "association": { + "min": 0.04503105487674475, + "best1L": 0.026397515487458025, + "mean": 0.009510869376494415, + "persist": 0.00776397529989481 + } + }, + "operator": { + "dispersion_L0": 0.4730173267828236, + "dispersion_mean": 0.242244168297503, + "dispersion_slope": -0.06369095436918644, + "stable_rank": 44.90222930908203, + "top8_spectral_share": 0.06282716989517212, + "condition_number": 1466.7344970703125, + "unembed_alignment": 0.09712330996990204 + }, + "corpus": { + "mean_prompt_tokens": 760.44, + "type_token_ratio": 0.1487559833780443, + "eval_vocab_overlap": 0.2549618320610687, + "model_cross_entropy": 2.2471217851381042, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4730173267828236, + "7": 0.4029908012813693, + "8": 0.33742097567480067, + "9": 0.2639456325262663, + "10": 0.2131675726574113, + "11": 0.13347877607376568, + "12": 0.0794430220690991, + "13": 0.03448923931448806 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.18169209414294787, + "best1L": 0.15533980780414172, + "mean": 0.05253120618207114, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1356055097920554, + "best1L": 0.10333575042230743, + "mean": 0.02846265404618212, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.0029220778794427004, + "persist": 0.0 + }, + "poetry": { + "min": 0.04081632689173732, + "best1L": 0.01457725970872811, + "mean": 0.007106414147918778, + "persist": 0.0 + }, + "typo": { + "min": 0.4181547611951828, + "best1L": 0.31249999680689405, + "mean": 0.23549107568604605, + "persist": 0.25446428624647005 + }, + "association": { + "min": 0.038819875701197555, + "best1L": 0.024844720294432982, + "mean": 0.007958074383038496, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.474666468623353, + "dispersion_mean": 0.23871334231325667, + "dispersion_slope": -0.06379934870638895, + "stable_rank": 45.00823974609375, + "top8_spectral_share": 0.06275606155395508, + "condition_number": 1383.854736328125, + "unembed_alignment": 0.09773260354995728 + }, + "corpus": { + "mean_prompt_tokens": 762.0533333333333, + "type_token_ratio": 0.12457570773699128, + "eval_vocab_overlap": 0.30381679389312977, + "model_cross_entropy": 2.2471217851381042, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.474666468623353, + "7": 0.4000994355490106, + "8": 0.3314438418147997, + "9": 0.25690922624371126, + "10": 0.20641155139492479, + "11": 0.12964785639887874, + "12": 0.07689785575061284, + "13": 0.033630502730762415 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.18307905590959958, + "best1L": 0.15811373240181378, + "mean": 0.05322468812976565, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1345177644065448, + "best1L": 0.10333575042230743, + "mean": 0.028145394753664732, + "persist": 0.000725163146853447 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.0029220778794427004, + "persist": 0.0 + }, + "poetry": { + "min": 0.039358600574944703, + "best1L": 0.01457725970872811, + "mean": 0.006924198225273618, + "persist": 0.0 + }, + "typo": { + "min": 0.41964285501411985, + "best1L": 0.31101190724543165, + "mean": 0.23511904744165285, + "persist": 0.252976192427533 + }, + "association": { + "min": 0.04192546595420156, + "best1L": 0.026397515354411944, + "mean": 0.008346273231187038, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.47391941286198225, + "dispersion_mean": 0.2382122848982527, + "dispersion_slope": -0.06369418944134694, + "stable_rank": 44.79694747924805, + "top8_spectral_share": 0.06277165561914444, + "condition_number": 1345.786376953125, + "unembed_alignment": 0.09942447394132614 + }, + "corpus": { + "mean_prompt_tokens": 698.86, + "type_token_ratio": 0.11398563374638697, + "eval_vocab_overlap": 0.3198473282442748, + "model_cross_entropy": 2.2471217851381042, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.47391941286198225, + "7": 0.3998771855749534, + "8": 0.33058228294931724, + "9": 0.2558771172371771, + "10": 0.20534699164088607, + "11": 0.12957032863203063, + "12": 0.07679941432203716, + "13": 0.03372554596763776 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.18030513131192752, + "best1L": 0.15395284284438407, + "mean": 0.05253120658120939, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13524293048041208, + "best1L": 0.10224800397242818, + "mean": 0.028054749393569573, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.019480519129761627, + "best1L": 0.016883116880697862, + "mean": 0.002759740172353174, + "persist": 0.0 + }, + "poetry": { + "min": 0.03790087439119816, + "best1L": 0.013119533791073732, + "mean": 0.006559766729229263, + "persist": 0.0 + }, + "typo": { + "min": 0.4196428507566452, + "best1L": 0.31696428677865435, + "mean": 0.23790922654526575, + "persist": 0.2574404738843441 + }, + "association": { + "min": 0.038819875701197555, + "best1L": 0.024844720294432982, + "mean": 0.007763975050433406, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.4734285131005585, + "dispersion_mean": 0.23924018532264874, + "dispersion_slope": -0.0635917566374505, + "stable_rank": 44.346622467041016, + "top8_spectral_share": 0.06288588792085648, + "condition_number": 1486.6685791015625, + "unembed_alignment": 0.09801170974969864 + }, + "corpus": { + "mean_prompt_tokens": 697.4266666666666, + "type_token_ratio": 0.09326667558835337, + "eval_vocab_overlap": 0.366412213740458, + "model_cross_entropy": 2.2471217851381042, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.4734285131005585, + "7": 0.4004206156325796, + "8": 0.3320859798267626, + "9": 0.2577223955936699, + "10": 0.20722378104224895, + "11": 0.13103714402339633, + "12": 0.07799056962566742, + "13": 0.03401248373630683 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.18030513131192752, + "best1L": 0.15533980567540442, + "mean": 0.05218446667173079, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.13633067267281668, + "best1L": 0.10261058541280883, + "mean": 0.028417331100042378, + "persist": 0.0010877447202801704 + }, + "order-ops": { + "min": 0.02077922025429351, + "best1L": 0.016883116880697862, + "mean": 0.0029220779459657414, + "persist": 0.0 + }, + "poetry": { + "min": 0.036443148872682025, + "best1L": 0.013119533791073732, + "mean": 0.0065597666959677425, + "persist": 0.0 + }, + "typo": { + "min": 0.4151785650423595, + "best1L": 0.31249999680689405, + "mean": 0.23530505704028265, + "persist": 0.2544642819889954 + }, + "association": { + "min": 0.04192546595420156, + "best1L": 0.026397515354411944, + "mean": 0.008152173898581947, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.463386650375068, + "dispersion_mean": 0.23520527556285778, + "dispersion_slope": -0.06232468217797565, + "stable_rank": 43.90788650512695, + "top8_spectral_share": 0.06329284608364105, + "condition_number": 1604.33154296875, + "unembed_alignment": 0.09725585579872131 + }, + "corpus": { + "mean_prompt_tokens": 724.32, + "type_token_ratio": 0.07828031809145129, + "eval_vocab_overlap": 0.4015267175572519, + "model_cross_entropy": 2.2471217851381042, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.463386650375068, + "7": 0.39331254785324643, + "8": 0.32698436997515057, + "9": 0.2540700979528158, + "10": 0.20455803387099453, + "11": 0.12907291943485566, + "12": 0.07678034532032939, + "13": 0.03347723972040196 + } + } + }, + "runtime_s": 1614.0, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Wikipedia_en_s0.json b/results/ladder1b_b613/tv_Wikipedia_en_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..d461a0d7d223f2d58fa8aa5ab5d27951c31e179c --- /dev/null +++ b/results/ladder1b_b613/tv_Wikipedia_en_s0.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Wikipedia_en", + "seed": 0, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.15533980514322007, + "best1L": 0.13453536746757372, + "mean": 0.04334257982139077, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11493835811104093, + "best1L": 0.08520667188401733, + "mean": 0.022434735637424246, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012575779642, + "best1L": 0.009090909202183996, + "mean": 0.001948051986151508, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259309589863, + "best1L": 0.00728862958827189, + "mean": 0.002733236095601959, + "persist": 0.0 + }, + "typo": { + "min": 0.2827380948833057, + "best1L": 0.2038690485060215, + "mean": 0.12258184674595084, + "persist": 0.11755952438605684 + }, + "association": { + "min": 0.04503105540892908, + "best1L": 0.029503105341323783, + "mean": 0.010869565369960452, + "persist": 0.00776397529989481 + } + }, + "operator": { + "dispersion_L0": 0.5055491660110195, + "dispersion_mean": 0.26452520895151743, + "dispersion_slope": -0.06956980965034845, + "stable_rank": 37.6092414855957, + "top8_spectral_share": 0.07327023893594742, + "condition_number": 3882.177734375, + "unembed_alignment": 0.09873466938734055 + }, + "corpus": { + "mean_prompt_tokens": 473.1, + "type_token_ratio": 0.3753963221306278, + "eval_vocab_overlap": 0.1648854961832061, + "model_cross_entropy": 2.2009749816344666, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.5055491660110195, + "7": 0.44057096460831363, + "8": 0.380700620211202, + "9": 0.2927923665877, + "10": 0.23430286516178997, + "11": 0.1425681944590533, + "12": 0.08365035441981669, + "13": 0.03606714015324404 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.15533980514322007, + "best1L": 0.1359223292342254, + "mean": 0.04351595017526831, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12291515804827213, + "best1L": 0.0866569976455399, + "mean": 0.02479151589795947, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.01168831145124776, + "best1L": 0.009090909202183996, + "mean": 0.001948051986151508, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533258889402, + "best1L": 0.00874635563897235, + "mean": 0.002368804616188364, + "persist": 0.0 + }, + "typo": { + "min": 0.29464285394975115, + "best1L": 0.20982143123235023, + "mean": 0.1298363097012043, + "persist": 0.12351190578192472 + }, + "association": { + "min": 0.04503105527588299, + "best1L": 0.029503105341323783, + "mean": 0.01028726708942226, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.5861165501607526, + "dispersion_mean": 0.30501381717544507, + "dispersion_slope": -0.08083436859846575, + "stable_rank": 38.274864196777344, + "top8_spectral_share": 0.068900465965271, + "condition_number": 1750.0152587890625, + "unembed_alignment": 0.09926523268222809 + }, + "corpus": { + "mean_prompt_tokens": 607.48, + "type_token_ratio": 0.275037861328768, + "eval_vocab_overlap": 0.3099236641221374, + "model_cross_entropy": 2.133059348716392, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.5861165501607526, + "7": 0.5118682531769825, + "8": 0.44186439873620575, + "9": 0.33258414066029535, + "10": 0.26549739465138433, + "11": 0.163730818904064, + "12": 0.09779261377636853, + "13": 0.04065636733750718 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.15256588054554804, + "best1L": 0.13314840463655336, + "mean": 0.04230235729898725, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12073966727725097, + "best1L": 0.0866569976455399, + "mean": 0.024066352218921696, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.011688311584293842, + "best1L": 0.01038961045976196, + "mean": 0.0014610389480367303, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985094198158, + "best1L": 0.00728862958827189, + "mean": 0.002915451885201037, + "persist": 0.0 + }, + "typo": { + "min": 0.3020833336881229, + "best1L": 0.21875000053218432, + "mean": 0.12872024066746235, + "persist": 0.12053571388657604 + }, + "association": { + "min": 0.040372670362038274, + "best1L": 0.027950310680483068, + "mean": 0.009704968842145587, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6142485444322514, + "dispersion_mean": 0.32298257419499493, + "dispersion_slope": -0.08436933543617559, + "stable_rank": 37.002628326416016, + "top8_spectral_share": 0.06930404156446457, + "condition_number": 2699.783447265625, + "unembed_alignment": 0.09596860408782959 + }, + "corpus": { + "mean_prompt_tokens": 584.98, + "type_token_ratio": 0.21840062908133612, + "eval_vocab_overlap": 0.39236641221374047, + "model_cross_entropy": 2.2552570587879903, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6142485444322514, + "7": 0.5383761358975673, + "8": 0.4656415799959505, + "9": 0.3536355682211139, + "10": 0.2845504727854986, + "11": 0.1775696904899605, + "12": 0.10548656716744509, + "13": 0.04435203457017252 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.15117891877889633, + "best1L": 0.13176144286990166, + "mean": 0.042302358496401994, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11928933938699109, + "best1L": 0.0866569976455399, + "mean": 0.023749093724680797, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.011688311584293842, + "best1L": 0.01038961045976196, + "mean": 0.0014610389480367303, + "persist": 0.0 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.00874635563897235, + "mean": 0.0036443149271820274, + "persist": 0.0 + }, + "typo": { + "min": 0.2976190447807312, + "best1L": 0.21279762099896157, + "mean": 0.12909225933253765, + "persist": 0.12053571282220739 + }, + "association": { + "min": 0.03881987516901323, + "best1L": 0.027950310680483068, + "mean": 0.009122670511715114, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.5933936034749386, + "dispersion_mean": 0.31220583563466986, + "dispersion_slope": -0.08128367171545389, + "stable_rank": 37.57901382446289, + "top8_spectral_share": 0.0691654309630394, + "condition_number": 2778.657470703125, + "unembed_alignment": 0.09568147361278534 + }, + "corpus": { + "mean_prompt_tokens": 628.08, + "type_token_ratio": 0.18373455610750222, + "eval_vocab_overlap": 0.48244274809160304, + "model_cross_entropy": 2.2552570587879903, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.5933936034749386, + "7": 0.519425063039319, + "8": 0.4481455619814698, + "9": 0.3423401465869334, + "10": 0.2757264905261664, + "11": 0.17278920589446434, + "12": 0.10252572763682358, + "13": 0.04330088593724345 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.14979195701224463, + "best1L": 0.13176144286990166, + "mean": 0.04160887588347707, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11965192082737174, + "best1L": 0.08593183476477861, + "mean": 0.023839738818683794, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012708825725, + "best1L": 0.01038961045976196, + "mean": 0.0016233765886032156, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711544036865, + "best1L": 0.00874635563897235, + "mean": 0.00346209920410599, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3005952366760799, + "best1L": 0.21279762099896157, + "mean": 0.13039434727813518, + "persist": 0.11904761900327035 + }, + "association": { + "min": 0.03571428584733179, + "best1L": 0.026397516019642353, + "mean": 0.008540372513899845, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6030284078798687, + "dispersion_mean": 0.3160865431446337, + "dispersion_slope": -0.08252618824140588, + "stable_rank": 38.06486511230469, + "top8_spectral_share": 0.06895136833190918, + "condition_number": 2569.2001953125, + "unembed_alignment": 0.09756442159414291 + }, + "corpus": { + "mean_prompt_tokens": 835.6, + "type_token_ratio": 0.14358544758257538, + "eval_vocab_overlap": 0.5870229007633587, + "model_cross_entropy": 2.2552570587879903, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6030284078798687, + "7": 0.5264256275282095, + "8": 0.4523696018774117, + "9": 0.3467607675758948, + "10": 0.278491950644125, + "11": 0.17451585520574373, + "12": 0.10337185182595203, + "13": 0.04372828261986397 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.15117891771452768, + "best1L": 0.1303744800388813, + "mean": 0.04195561832083123, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12146482962582793, + "best1L": 0.08738216132457767, + "mean": 0.02429296648395913, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.007288629721317973, + "mean": 0.003279883464399193, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30803571322134565, + "best1L": 0.21577381289431027, + "mean": 0.13281249973390782, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.024844720294432982, + "mean": 0.008152173898581947, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6223560726020907, + "dispersion_mean": 0.32355109570849794, + "dispersion_slope": -0.08540805154702978, + "stable_rank": 38.004600524902344, + "top8_spectral_share": 0.06896200031042099, + "condition_number": 2404.32568359375, + "unembed_alignment": 0.0976831465959549 + }, + "corpus": { + "mean_prompt_tokens": 1311.06, + "type_token_ratio": 0.09414773796266634, + "eval_vocab_overlap": 0.7244274809160305, + "model_cross_entropy": 2.2552570587879903, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6223560726020907, + "7": 0.5421077778821591, + "8": 0.4632782010557158, + "9": 0.3538954765185729, + "10": 0.28324164697573356, + "11": 0.17601269520638652, + "12": 0.1037064266127582, + "13": 0.04381046881456662 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.1303744800388813, + "mean": 0.041782247966953685, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1207396678094353, + "best1L": 0.08701957988419703, + "mean": 0.024021029339304993, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.007288629721317973, + "mean": 0.003279883464399193, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30505952132599695, + "best1L": 0.2127976231276989, + "mean": 0.1333705365125622, + "persist": 0.12351190258881875 + }, + "association": { + "min": 0.03571428531514747, + "best1L": 0.026397516019642353, + "mean": 0.008540372513899845, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.6196782351725778, + "dispersion_mean": 0.32178287025467384, + "dispersion_slope": -0.08506293863983168, + "stable_rank": 38.293304443359375, + "top8_spectral_share": 0.06886643916368484, + "condition_number": 2229.14697265625, + "unembed_alignment": 0.09784462302923203 + }, + "corpus": { + "mean_prompt_tokens": 1188.27, + "type_token_ratio": 0.08580120679643516, + "eval_vocab_overlap": 0.7618320610687023, + "model_cross_entropy": 2.2552570587879903, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6196782351725778, + "7": 0.5390936397336722, + "8": 0.4611523824781842, + "9": 0.35212484445493986, + "10": 0.28123789862704746, + "11": 0.1746583375073148, + "12": 0.10296264428726687, + "13": 0.043354979776387276 + } + } + }, + "runtime_s": 1599.3, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Wikipedia_en_s1.json b/results/ladder1b_b613/tv_Wikipedia_en_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e7a8a50b3f384aa017a75e5c10b39f79b2d8259f --- /dev/null +++ b/results/ladder1b_b613/tv_Wikipedia_en_s1.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Wikipedia_en", + "seed": 1, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.15811373346618243, + "best1L": 0.13453536853194237, + "mean": 0.04420943172382457, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12073966967208045, + "best1L": 0.08520667228315558, + "mean": 0.025018129664074098, + "persist": 0.0025380710139870644 + }, + "order-ops": { + "min": 0.01688311727983611, + "best1L": 0.010389610592808043, + "mean": 0.0025974025816789697, + "persist": 0.0 + }, + "poetry": { + "min": 0.01020408155662673, + "best1L": 0.005830903670617512, + "mean": 0.002733236095601959, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3258928582072258, + "best1L": 0.22023809541549003, + "mean": 0.14583333342203073, + "persist": 0.14136904837297543 + }, + "association": { + "min": 0.037267080907310755, + "best1L": 0.02950310574046203, + "mean": 0.009122670561607395, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.47674265674226557, + "dispersion_mean": 0.258907584500404, + "dispersion_slope": -0.06503528175275816, + "stable_rank": 38.597877502441406, + "top8_spectral_share": 0.06908620148897171, + "condition_number": 2919.098388671875, + "unembed_alignment": 0.09765419363975525 + }, + "corpus": { + "mean_prompt_tokens": 1014.4, + "type_token_ratio": 0.25295741324921134, + "eval_vocab_overlap": 0.22366412213740458, + "model_cross_entropy": 2.2704979149070947, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.47674265674226557, + "7": 0.42379948191439815, + "8": 0.37016707836595597, + "9": 0.2911882100051881, + "10": 0.23256682543214774, + "11": 0.1490181481146308, + "12": 0.08957370656867093, + "13": 0.03820456885997495 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.1539528433765684, + "best1L": 0.13592233029859407, + "mean": 0.0429958389805896, + "persist": 0.0 + }, + "multilingual": { + "min": 0.12364031986466475, + "best1L": 0.08665699777858597, + "mean": 0.025380711403808424, + "persist": 0.002900652587413788 + }, + "order-ops": { + "min": 0.019480519661945955, + "best1L": 0.014285714099449771, + "mean": 0.00259740259830973, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.00728862958827189, + "mean": 0.0032798834311376724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31250000319310595, + "best1L": 0.22023809860859597, + "mean": 0.13746280010257447, + "persist": 0.12500000093132257 + }, + "association": { + "min": 0.03571428598037788, + "best1L": 0.024844720294432982, + "mean": 0.008540372530530606, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.541488005953075, + "dispersion_mean": 0.2853466299450252, + "dispersion_slope": -0.07412459582528741, + "stable_rank": 38.18363952636719, + "top8_spectral_share": 0.06939663738012314, + "condition_number": 1246.62646484375, + "unembed_alignment": 0.09588560461997986 + }, + "corpus": { + "mean_prompt_tokens": 875.16, + "type_token_ratio": 0.22322775263951736, + "eval_vocab_overlap": 0.3473282442748092, + "model_cross_entropy": 2.201320877762528, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.541488005953075, + "7": 0.47349803471198115, + "8": 0.4100700096505195, + "9": 0.3147708792856251, + "10": 0.2502266464883049, + "11": 0.15843799784514645, + "12": 0.09454491262905887, + "13": 0.03973655299649039 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.151178916650159, + "best1L": 0.133148405700922, + "mean": 0.04230235796421766, + "persist": 0.0 + }, + "multilingual": { + "min": 0.12327773895646844, + "best1L": 0.08846990524658135, + "mean": 0.02492748373853309, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.015584415223981653, + "best1L": 0.012987012974917889, + "mean": 0.0019480519029977067, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.008746355505926269, + "mean": 0.003826530633627304, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30059523241860525, + "best1L": 0.22023809222238405, + "mean": 0.13002232142857142, + "persist": 0.12053571601531335 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.02950310574046203, + "mean": 0.00912267096074564, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.5830995403540831, + "dispersion_mean": 0.30347273014285125, + "dispersion_slope": -0.08002094360641031, + "stable_rank": 37.91212463378906, + "top8_spectral_share": 0.07018131762742996, + "condition_number": 2100.91796875, + "unembed_alignment": 0.0968603566288948 + }, + "corpus": { + "mean_prompt_tokens": 1091.84, + "type_token_ratio": 0.16892584994138335, + "eval_vocab_overlap": 0.5099236641221374, + "model_cross_entropy": 2.2350154051909574, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.5830995403540831, + "7": 0.5076358995771296, + "8": 0.4350026249475804, + "9": 0.3331297829066282, + "10": 0.2649820356995505, + "11": 0.16518400461958105, + "12": 0.09775752282547848, + "13": 0.04099043021277809 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.15117891771452768, + "best1L": 0.13314840463655336, + "mean": 0.042302358496401994, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12291515751608781, + "best1L": 0.0881073238062007, + "mean": 0.024882161058485508, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.012987012974917889, + "mean": 0.0017857142624312214, + "persist": 0.0 + }, + "poetry": { + "min": 0.014577259575682027, + "best1L": 0.005830903670617512, + "mean": 0.0032798834311376724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.29613095628363745, + "best1L": 0.21428571694663592, + "mean": 0.12779017803924425, + "persist": 0.11755952518433332 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.027950310680483068, + "mean": 0.008540372430746044, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6034067834548965, + "dispersion_mean": 0.3133899928685439, + "dispersion_slope": -0.0828245063887631, + "stable_rank": 37.115291595458984, + "top8_spectral_share": 0.07043488323688507, + "condition_number": 2106.64208984375, + "unembed_alignment": 0.09702761471271515 + }, + "corpus": { + "mean_prompt_tokens": 1075.0933333333332, + "type_token_ratio": 0.14535172140093264, + "eval_vocab_overlap": 0.5938931297709924, + "model_cross_entropy": 2.2350154051909574, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6034067834548965, + "7": 0.524231391591567, + "8": 0.44908621818134925, + "9": 0.3440473631916343, + "10": 0.2741628266529632, + "11": 0.16993099899222813, + "12": 0.10034221329697347, + "13": 0.041912147586739744 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.04212898827557053, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.1232777378920998, + "best1L": 0.08738216079239335, + "mean": 0.02519941928663424, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.008746355505926269, + "mean": 0.003826530700150345, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30059523561171125, + "best1L": 0.21279761674148695, + "mean": 0.12983630704028265, + "persist": 0.11904761900327035 + }, + "association": { + "min": 0.038819875568151474, + "best1L": 0.027950310680483068, + "mean": 0.008734471896397216, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6176942060927859, + "dispersion_mean": 0.3196079588174255, + "dispersion_slope": -0.08489432124055589, + "stable_rank": 37.457008361816406, + "top8_spectral_share": 0.06999088078737259, + "condition_number": 1994.1351318359375, + "unembed_alignment": 0.09836126118898392 + }, + "corpus": { + "mean_prompt_tokens": 1075.36, + "type_token_ratio": 0.13107238506174676, + "eval_vocab_overlap": 0.6595419847328244, + "model_cross_entropy": 2.2350154051909574, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6176942060927859, + "7": 0.5363450046582763, + "8": 0.45889021121235424, + "9": 0.3495691134494323, + "10": 0.2780187064103931, + "11": 0.1720959028699652, + "12": 0.10167943356149081, + "13": 0.04257109228470583 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.04230235902858632, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12001450492867402, + "best1L": 0.08665699791163206, + "mean": 0.024564902264890925, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.00874635563897235, + "mean": 0.003462099120952189, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30654761514493395, + "best1L": 0.2172619067132473, + "mean": 0.13448660634458065, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.038819875568151474, + "best1L": 0.027950310680483068, + "mean": 0.009122670561607395, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6133445558848083, + "dispersion_mean": 0.3158385337964163, + "dispersion_slope": -0.08428146512518078, + "stable_rank": 38.13249969482422, + "top8_spectral_share": 0.06926519423723221, + "condition_number": 2264.159423828125, + "unembed_alignment": 0.09866895526647568 + }, + "corpus": { + "mean_prompt_tokens": 988.3066666666666, + "type_token_ratio": 0.11278550517383268, + "eval_vocab_overlap": 0.7259541984732825, + "model_cross_entropy": 2.2350154051909574, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6133445558848083, + "7": 0.5314868432021086, + "8": 0.45337690920636975, + "9": 0.343502257182123, + "10": 0.27385189115893116, + "11": 0.1692919039429988, + "12": 0.09993447502780595, + "13": 0.041919434766184434 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.042475729515509944, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12110224924981594, + "best1L": 0.0870195793520127, + "mean": 0.024564902298152447, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.00728862958827189, + "mean": 0.00346209918747523, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30505952558347155, + "best1L": 0.21577380863683565, + "mean": 0.13541666977107525, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.026397516019642353, + "mean": 0.008928571229002305, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.608940813482654, + "dispersion_mean": 0.3138283938384911, + "dispersion_slope": -0.0836877743304026, + "stable_rank": 38.76348876953125, + "top8_spectral_share": 0.06858880072832108, + "condition_number": 1996.6590576171875, + "unembed_alignment": 0.09973999857902527 + }, + "corpus": { + "mean_prompt_tokens": 1070.71, + "type_token_ratio": 0.09227054944849679, + "eval_vocab_overlap": 0.7770992366412214, + "model_cross_entropy": 2.2350154051909574, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.608940813482654, + "7": 0.5276728791756566, + "8": 0.4504292523487832, + "9": 0.34221897323860484, + "10": 0.2722355440136175, + "11": 0.16821038053543333, + "12": 0.09930903211208728, + "13": 0.04161027580109193 + } + } + }, + "runtime_s": 1598.4, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder1b_b613/tv_Wikipedia_en_s2.json b/results/ladder1b_b613/tv_Wikipedia_en_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..e78a74edac054508b7a6c347a78ed419b7fe7edd --- /dev/null +++ b/results/ladder1b_b613/tv_Wikipedia_en_s2.json @@ -0,0 +1,510 @@ +{ + "model": "EleutherAI/pythia-1b-deduped", + "short": "1b", + "corpus": "Wikipedia_en", + "seed": 2, + "randomization": null, + "band": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers": 16, + "ckpts": [ + 10, + 25, + 50, + 75, + 100, + 150, + 200 + ], + "device": "cuda", + "n_items": 541, + "by_N": { + "10": { + "reads": { + "multihop": { + "min": 0.149791954351323, + "best1L": 0.133148405700922, + "mean": 0.03987517354211637, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11711384914815426, + "best1L": 0.08846990604485784, + "mean": 0.02302393047804279, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.013119533258889402, + "best1L": 0.00728862958827189, + "mean": 0.002733236162125, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.2738095223903656, + "best1L": 0.21279761887022428, + "mean": 0.11160714205886636, + "persist": 0.10416666697710752 + }, + "association": { + "min": 0.029503106006554196, + "best1L": 0.02018633538058826, + "mean": 0.006793478304254157, + "persist": 0.003105590119957924 + } + }, + "operator": { + "dispersion_L0": 0.5148948648084336, + "dispersion_mean": 0.2712794389326583, + "dispersion_slope": -0.07049728555135176, + "stable_rank": 38.59467315673828, + "top8_spectral_share": 0.07043042778968811, + "condition_number": 1800.6309814453125, + "unembed_alignment": 0.09852178394794464 + }, + "corpus": { + "mean_prompt_tokens": 1159.7, + "type_token_ratio": 0.2673105113391394, + "eval_vocab_overlap": 0.25572519083969464, + "model_cross_entropy": 2.2789809905730927, + "model_cross_entropy_n_scored_tokens": 1110, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.5148948648084336, + "7": 0.4517854925290836, + "8": 0.38867928127066903, + "9": 0.2988452258922578, + "10": 0.2384857388375442, + "11": 0.14865442444397478, + "12": 0.09018989911471223, + "13": 0.0387005845645915 + } + }, + "25": { + "reads": { + "multihop": { + "min": 0.1567267690386091, + "best1L": 0.133148405700922, + "mean": 0.04316920906837497, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12182741319494587, + "best1L": 0.0881073238062007, + "mean": 0.024746192752250602, + "persist": 0.0018129078671336174 + }, + "order-ops": { + "min": 0.011688311584293842, + "best1L": 0.01038961045976196, + "mean": 0.0014610389480367303, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985094198158, + "best1L": 0.00728862958827189, + "mean": 0.002915451885201037, + "persist": 0.0 + }, + "typo": { + "min": 0.28869047867400305, + "best1L": 0.21130952505128725, + "mean": 0.12351190777761596, + "persist": 0.11011904757469893 + }, + "association": { + "min": 0.029503106006554196, + "best1L": 0.024844720294432982, + "mean": 0.008152173898581947, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.5768447734936551, + "dispersion_mean": 0.29892112035655466, + "dispersion_slope": -0.07937038774590832, + "stable_rank": 39.74794387817383, + "top8_spectral_share": 0.06760132312774658, + "condition_number": 1292.6646728515625, + "unembed_alignment": 0.10098008066415787 + }, + "corpus": { + "mean_prompt_tokens": 1155.12, + "type_token_ratio": 0.2179513816746312, + "eval_vocab_overlap": 0.4198473282442748, + "model_cross_entropy": 2.1596785261824323, + "model_cross_entropy_n_scored_tokens": 2775, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.5768447734936551, + "7": 0.5012041542533735, + "8": 0.43058301132947985, + "9": 0.32752588245948827, + "10": 0.25996543996203186, + "11": 0.16002468655255986, + "12": 0.09524503585928373, + "13": 0.039975978942564744 + } + }, + "50": { + "reads": { + "multihop": { + "min": 0.151178916650159, + "best1L": 0.133148405700922, + "mean": 0.04160887694784573, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.11856417703841414, + "best1L": 0.08665699791163206, + "mean": 0.02406635201935257, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.005830903670617512, + "mean": 0.0029154519350933178, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.28869047654526575, + "best1L": 0.21428571481789863, + "mean": 0.12388393070016589, + "persist": 0.11160714245800461 + }, + "association": { + "min": 0.03726708037512643, + "best1L": 0.026397514955273697, + "mean": 0.009316770426396812, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.570988372803899, + "dispersion_mean": 0.29476614177422517, + "dispersion_slope": -0.07847669366406439, + "stable_rank": 40.2789306640625, + "top8_spectral_share": 0.0682266503572464, + "condition_number": 946.0098266601562, + "unembed_alignment": 0.1008974015712738 + }, + "corpus": { + "mean_prompt_tokens": 1273.0, + "type_token_ratio": 0.16691280439905734, + "eval_vocab_overlap": 0.533587786259542, + "model_cross_entropy": 2.215524011904055, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.570988372803899, + "7": 0.4948457809755886, + "8": 0.4234780559429146, + "9": 0.3224230526628319, + "10": 0.2556420372881048, + "11": 0.15792543965787262, + "12": 0.09362032654222417, + "13": 0.039206068320365696 + } + }, + "75": { + "reads": { + "multihop": { + "min": 0.14840499311685562, + "best1L": 0.13314840463655336, + "mean": 0.04212898761034012, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12110224924981594, + "best1L": 0.08629441647125143, + "mean": 0.024338288864653026, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.011688311850386006, + "mean": 0.0017857142624312214, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.005830903670617512, + "mean": 0.0030976677246923956, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.29315476013081415, + "best1L": 0.2098214291036129, + "mean": 0.12686012179723807, + "persist": 0.1145833343533533 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.024844720294432982, + "mean": 0.009122670628130436, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.611560827698264, + "dispersion_mean": 0.31400120074853627, + "dispersion_slope": -0.0841138324021613, + "stable_rank": 39.75286102294922, + "top8_spectral_share": 0.06829888373613358, + "condition_number": 1442.1151123046875, + "unembed_alignment": 0.0999145582318306 + }, + "corpus": { + "mean_prompt_tokens": 1527.08, + "type_token_ratio": 0.1274938662894762, + "eval_vocab_overlap": 0.6427480916030535, + "model_cross_entropy": 2.215524011904055, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.611560827698264, + "7": 0.5278771296054463, + "8": 0.45186578592102494, + "9": 0.3424866249715377, + "10": 0.27188511640134755, + "11": 0.1666228856810674, + "12": 0.09844946027181119, + "13": 0.041261775437790964 + } + }, + "100": { + "reads": { + "multihop": { + "min": 0.14979195488350733, + "best1L": 0.13453536640320504, + "mean": 0.04264909973634141, + "persist": 0.001386962564928191 + }, + "multilingual": { + "min": 0.12001450279993671, + "best1L": 0.08629441647125143, + "mean": 0.024111675464415123, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.012987012974917889, + "best1L": 0.011688311850386006, + "mean": 0.001623376621864736, + "persist": 0.0 + }, + "poetry": { + "min": 0.01457725970872811, + "best1L": 0.005830903670617512, + "mean": 0.0030976677246923956, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.29910714605024885, + "best1L": 0.21726190245577268, + "mean": 0.128162202292255, + "persist": 0.11458333116024733 + }, + "association": { + "min": 0.03571428571428571, + "best1L": 0.024844720294432982, + "mean": 0.008734471962920256, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6108778260495059, + "dispersion_mean": 0.3143331072787869, + "dispersion_slope": -0.08390952141906446, + "stable_rank": 39.20356369018555, + "top8_spectral_share": 0.06872303783893585, + "condition_number": 1501.8001708984375, + "unembed_alignment": 0.0990648865699768 + }, + "corpus": { + "mean_prompt_tokens": 1360.73, + "type_token_ratio": 0.11701807118238004, + "eval_vocab_overlap": 0.6732824427480916, + "model_cross_entropy": 2.215524011904055, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6108778260495059, + "7": 0.528094420831307, + "8": 0.4514840043408426, + "9": 0.34272320678372675, + "10": 0.27238265109221743, + "11": 0.16798439695439663, + "12": 0.09938600004779076, + "13": 0.04173235213050776 + } + }, + "150": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.04334258008748293, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12037708211158003, + "best1L": 0.08665699791163206, + "mean": 0.024428933459733213, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.011688311850386006, + "mean": 0.0017857142624312214, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.005830903670617512, + "mean": 0.003279883464399193, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3050595277122089, + "best1L": 0.21726190458450997, + "mean": 0.13467262073286942, + "persist": 0.12351190578192472 + }, + "association": { + "min": 0.04347826008285795, + "best1L": 0.0310559000021645, + "mean": 0.010675466087247645, + "persist": 0.006211180106869766 + } + }, + "operator": { + "dispersion_L0": 0.6035559965810751, + "dispersion_mean": 0.31106183257615677, + "dispersion_slope": -0.08291126388578456, + "stable_rank": 38.90498733520508, + "top8_spectral_share": 0.06871452182531357, + "condition_number": 1738.9300537109375, + "unembed_alignment": 0.0989619791507721 + }, + "corpus": { + "mean_prompt_tokens": 1263.6733333333334, + "type_token_ratio": 0.09840095805350539, + "eval_vocab_overlap": 0.7251908396946565, + "model_cross_entropy": 2.215524011904055, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6035559965810751, + "7": 0.5224424140848559, + "8": 0.4465774859827703, + "9": 0.3393836892632284, + "10": 0.26997454369903734, + "11": 0.16687282604296289, + "12": 0.09836646460887558, + "13": 0.04132124034644817 + } + }, + "200": { + "reads": { + "multihop": { + "min": 0.14979195594787598, + "best1L": 0.13314840463655336, + "mean": 0.042822468892804215, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12146482856145926, + "best1L": 0.0870195793520127, + "mean": 0.024519579085920538, + "persist": 0.002175489440560341 + }, + "order-ops": { + "min": 0.014285714099449771, + "best1L": 0.011688311850386006, + "mean": 0.0017857142624312214, + "persist": 0.0 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.005830903670617512, + "mean": 0.003279883464399193, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3035714253783226, + "best1L": 0.21875000266092165, + "mean": 0.13337053598037787, + "persist": 0.1220238119629877 + }, + "association": { + "min": 0.03881987503596714, + "best1L": 0.027950310680483068, + "mean": 0.009510869759001903, + "persist": 0.004658385179936886 + } + }, + "operator": { + "dispersion_L0": 0.6011071111137346, + "dispersion_mean": 0.30981773238997895, + "dispersion_slope": -0.08257958120201041, + "stable_rank": 38.920650482177734, + "top8_spectral_share": 0.06871134787797928, + "condition_number": 1763.9337158203125, + "unembed_alignment": 0.099746473133564 + }, + "corpus": { + "mean_prompt_tokens": 1159.06, + "type_token_ratio": 0.08913688678756923, + "eval_vocab_overlap": 0.7717557251908397, + "model_cross_entropy": 2.215524011904055, + "model_cross_entropy_n_scored_tokens": 3552, + "model_cross_entropy_convention": "t48_competence_gate: skip_first=16, token-weighted", + "model_cross_entropy_error": null + }, + "dispersion_per_layer": { + "6": 0.6011071111137346, + "7": 0.5200595666585331, + "8": 0.4449820537382939, + "9": 0.3383825395636153, + "10": 0.2688989000127502, + "11": 0.16609153010461727, + "12": 0.09781155246309159, + "13": 0.04120860546519579 + } + } + }, + "runtime_s": 1600.3, + "n_used": 200, + "n_skipped": 0 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Github_s0.json b/results/ladder410/ladder_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..a708325d66219c5044759ce8b9c44b53501605b9 --- /dev/null +++ b/results/ladder410/ladder_Github_s0.json @@ -0,0 +1,659 @@ +{ + "corpus": "Github", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.016856929237422134, + "persist": 0.006934812824640956 + }, + "multilingual": { + "min": 0.11131254530378751, + "best1L": 0.07360406113522393, + "mean": 0.030512634332158735, + "persist": 0.026105873685862337 + }, + "order-ops": { + "min": 0.08311688367809568, + "best1L": 0.08311688367809568, + "mean": 0.021678321861795018, + "persist": 0.01558441509093557 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.24702381023338862, + "best1L": 0.15476190910807677, + "mean": 0.08447802292981318, + "persist": 0.07886904730860676 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.017603755213453302, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11675126690949712, + "best1L": 0.07831762065844876, + "mean": 0.032437106394874196, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07532467506825924, + "mean": 0.018581418486844217, + "persist": 0.012987012575779642 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095628363745, + "best1L": 0.15922619349190167, + "mean": 0.09741300317857947, + "persist": 0.08779761926936251 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0011944577147785043, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.018670649966225028, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12327773869037628, + "best1L": 0.08665699937513896, + "mean": 0.034277904379580705, + "persist": 0.031544597287263186 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.0212787209810423, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618763787405, + "best1L": 0.16220238006540708, + "mean": 0.10382326080330781, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.010869565419852734, + "best1L": 0.004658385179936886, + "mean": 0.0013139034771094365, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823690840175, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12146482989192009, + "best1L": 0.08375634465898786, + "mean": 0.03352485144776957, + "persist": 0.030094271525740623 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.021478521737403104, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177952532258, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.15922618817005837, + "mean": 0.09993131937725204, + "persist": 0.09077380983425039 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017070308011690422, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.10913705639541149, + "best1L": 0.07142857102943319, + "mean": 0.030373179553342716, + "persist": 0.025018128167305673 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.08181818202137947, + "mean": 0.024675324891826937, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928603359631, + "best1L": 0.15922619242753303, + "mean": 0.08928571282220739, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017176997398824563, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11167512647807598, + "best1L": 0.07287889798837048, + "mean": 0.03101467008569411, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023876123629244312, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095415490015, + "best1L": 0.16071428837520735, + "mean": 0.09191849787852593, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01717699766491673, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11167512674416814, + "best1L": 0.07251631641494376, + "mean": 0.031014670551355396, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023676323671159998, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928582072258, + "best1L": 0.16071428837520735, + "mean": 0.09100274728345019, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01771044486667961, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.113850615386452, + "best1L": 0.0746918051902737, + "mean": 0.03187928767874837, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.023076922865584493, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709024586848, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976140805653, + "best1L": 0.16220237793666975, + "mean": 0.09581043757498264, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00011944577064631241, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07722987633730684, + "mean": 0.03226976083325488, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023076922998630574, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.278273805975914, + "best1L": 0.16220237793666975, + "mean": 0.09764194036168712, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07759245777768749, + "mean": 0.032409214946840494, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2812499978712627, + "best1L": 0.16220237793666975, + "mean": 0.09855769308550018, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07650471345654555, + "mean": 0.0322697607002088, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428837520735, + "mean": 0.09775641188025475, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06657420311655317, + "mean": 0.018137202681308345, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11747643191899572, + "best1L": 0.08085569100720542, + "mean": 0.033078596993748634, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07662337619279112, + "mean": 0.023476524245260016, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2782738038471767, + "best1L": 0.16071428837520735, + "mean": 0.09878662947033133, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01792382390704006, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07759245804377965, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.02417582429812423, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529761833803995, + "best1L": 0.16220237793666975, + "mean": 0.09592490643262863, + "persist": 0.08333333382116896 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07868020236492157, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.0792207805705922, + "best1L": 0.07792207838169166, + "mean": 0.023976024739178165, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27678571215697695, + "best1L": 0.16220237793666975, + "mean": 0.09661172330379486, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11638868546911649, + "best1L": 0.07868020236492157, + "mean": 0.03254866899390306, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023776224381955608, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.16220237793666975, + "mean": 0.0974130045090403, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07831762092454093, + "mean": 0.03246499631287796, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023876124427520802, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095223903656, + "best1L": 0.16220237793666975, + "mean": 0.09695512801408768, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + } + }, + "runtime_s": 83.4 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Github_s1.json b/results/ladder410/ladder_Github_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..a3114db92aeacb9c5aef6e59863ee6f724d4a169 --- /dev/null +++ b/results/ladder410/ladder_Github_s1.json @@ -0,0 +1,659 @@ +{ + "corpus": "Github", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06796116594757352, + "mean": 0.019417476108563796, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.1287164602960859, + "best1L": 0.09137055969664029, + "mean": 0.03536564936595304, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.05714285613170692, + "best1L": 0.053246751693742614, + "mean": 0.013586413547662752, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.005045974294521979, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28125000106436865, + "best1L": 0.16220238219414437, + "mean": 0.10290750887777124, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0005972288282854217, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06518724028553281, + "mean": 0.018884028906800916, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.12400289891021592, + "best1L": 0.08919506892561913, + "mean": 0.03475205008206623, + "persist": 0.03190717859459775 + }, + "order-ops": { + "min": 0.07142857063029494, + "best1L": 0.06883116931255374, + "mean": 0.01878121932635882, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.005382372731609004, + "persist": 0.005830903670617512 + }, + "typo": { + "min": 0.26488095202616285, + "best1L": 0.16220238219414437, + "mean": 0.0958104379741209, + "persist": 0.08630952425301075 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.08044382690318994, + "best1L": 0.06518724028553281, + "mean": 0.01899071824404278, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11820159692849432, + "best1L": 0.08266860007175378, + "mean": 0.032883360383233855, + "persist": 0.029369107713656768 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023876123895336474, + "persist": 0.016883116215467453 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.013119533258889402, + "mean": 0.004597443488559553, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333262375425, + "best1L": 0.16369047707745008, + "mean": 0.09397893731615373, + "persist": 0.08184523960309369 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06657420311655317, + "mean": 0.018884028906800916, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11965192322220121, + "best1L": 0.08556925425572055, + "mean": 0.033441177768898864, + "persist": 0.029006526805460453 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07662337725715977, + "mean": 0.021578421782968298, + "persist": 0.018181818404367993 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004821708891540766, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232142644269125, + "best1L": 0.1651785708963871, + "mean": 0.09661172250551837, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018777339220313088, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1221899933048657, + "best1L": 0.08484408898012978, + "mean": 0.03380375987450991, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.07272727281919547, + "best1L": 0.07272727281919547, + "mean": 0.019380619466703917, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28571428464991705, + "best1L": 0.1711309514939785, + "mean": 0.10428113330687795, + "persist": 0.09374999960086175 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018884028607447232, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12182740973574775, + "best1L": 0.0848440911088671, + "mean": 0.03366430549483214, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.07272727281919547, + "best1L": 0.07272727281919547, + "mean": 0.01998002005607954, + "persist": 0.015584415889212064 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2797619051166943, + "best1L": 0.16517856983201845, + "mean": 0.102106226342065, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06657420311655317, + "mean": 0.018990718027842895, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12291515405688967, + "best1L": 0.08738216145762376, + "mean": 0.03394321372200336, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.07402597500809602, + "best1L": 0.07142857063029494, + "mean": 0.01928071942113872, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28125000106436865, + "best1L": 0.1666666679084301, + "mean": 0.1023351653878178, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.01813720299729279, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1207396648824215, + "best1L": 0.08339376321860723, + "mean": 0.03377586902518358, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07532467506825924, + "mean": 0.021578421649922217, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28273809807641165, + "best1L": 0.1636904738843441, + "mean": 0.1029075091438634, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06657420311655317, + "mean": 0.018137202731200626, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11965192056127957, + "best1L": 0.08303118177822658, + "mean": 0.03360852366313338, + "persist": 0.02973168981926782 + }, + "order-ops": { + "min": 0.07792207944606032, + "best1L": 0.07662337725715977, + "mean": 0.02217782232245164, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618976661135, + "best1L": 0.16666666577969277, + "mean": 0.10313644659306322, + "persist": 0.09226190418537174 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018243892384426936, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12001450200166021, + "best1L": 0.08303118177822658, + "mean": 0.0337200873265309, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07662337619279112, + "mean": 0.022077922010794282, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16517856983201845, + "mean": 0.10176282136568002, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018243892384426936, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12037708344204086, + "best1L": 0.08375634465898786, + "mean": 0.03374797764367291, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07532467506825924, + "mean": 0.02147852160435702, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16517856983201845, + "mean": 0.10222069599798747, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.017603755213453302, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11711384834987777, + "best1L": 0.0790427838053022, + "mean": 0.03240921501336353, + "persist": 0.02719361867223467 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07532467506825924, + "mean": 0.022177821790267314, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333155938555, + "best1L": 0.16071428518210137, + "mean": 0.09409340590770755, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497065826319158, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07940536524568285, + "mean": 0.032716014488999336, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.02117882140113839, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232142644269125, + "best1L": 0.16071428198899543, + "mean": 0.0953525628096291, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017603754947361137, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07976794668606349, + "mean": 0.032660233455577066, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.021878121786617806, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095202616283, + "best1L": 0.16220238006540708, + "mean": 0.09569597124521222, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017710444600587443, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07976794668606349, + "mean": 0.03266023292339274, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.02177822120886828, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2723214253783226, + "best1L": 0.16220238006540708, + "mean": 0.09558150238756623, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01771044486667961, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11675126690949712, + "best1L": 0.08049310956682477, + "mean": 0.03282757908371942, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.021478521072172692, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27380951919725965, + "best1L": 0.16220238006540708, + "mean": 0.09569597097912005, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + } + }, + "runtime_s": 62.6 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Github_s2.json b/results/ladder410/ladder_Github_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..0ff1e9d89ee856a7261a6d6810af9190e4e53e63 --- /dev/null +++ b/results/ladder410/ladder_Github_s2.json @@ -0,0 +1,659 @@ +{ + "corpus": "Github", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.01664355046315385, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11131254423941885, + "best1L": 0.07142857302512441, + "mean": 0.030010597979915992, + "persist": 0.024292965020452226 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07142857156161751, + "mean": 0.02017981996427157, + "persist": 0.011688311318201678 + }, + "poetry": { + "min": 0.020408163645437787, + "best1L": 0.013119533258889402, + "mean": 0.00504597436104502, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333155938555, + "best1L": 0.16369047654526575, + "mean": 0.08997252743159022, + "persist": 0.07589285700981106 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07905686433826174, + "best1L": 0.06657420311655317, + "mean": 0.01856396041278328, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11856417624013764, + "best1L": 0.0790427838053022, + "mean": 0.0329670326651207, + "persist": 0.0286439446998494 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07142857156161751, + "mean": 0.021678321046887765, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.013119533658027649, + "mean": 0.004709576190050159, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16220238485506602, + "mean": 0.09809981659054756, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.017710444600587443, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11385061432208333, + "best1L": 0.07650471372263772, + "mean": 0.032325542997568846, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.0792207805705922, + "best1L": 0.07792207838169166, + "mean": 0.024175824763785516, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177886009216, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333900996615, + "best1L": 0.15773809727813518, + "mean": 0.09684065782598086, + "persist": 0.08630952358778034 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497066092411324, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.1134880328817027, + "best1L": 0.07505438742893082, + "mean": 0.03204663350645985, + "persist": 0.027556200112615312 + }, + "order-ops": { + "min": 0.08181818281965596, + "best1L": 0.0792207805705922, + "mean": 0.02337662293575704, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807474281107, + "mean": 0.004597443488559553, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095223903656, + "best1L": 0.159226194024086, + "mean": 0.09558150198842798, + "persist": 0.084821427640106 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.017497066092411324, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11203770818454879, + "best1L": 0.07614213201616492, + "mean": 0.031433033889957836, + "persist": 0.028281363658607006 + }, + "order-ops": { + "min": 0.08571428645934377, + "best1L": 0.08181818202137947, + "mean": 0.02447552453460438, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2723214307001659, + "best1L": 0.16071428305336408, + "mean": 0.09283425033624683, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497065826319158, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11457577826721328, + "best1L": 0.07759245804377965, + "mean": 0.032130306254007986, + "persist": 0.029006526539368287 + }, + "order-ops": { + "min": 0.08441558427044324, + "best1L": 0.08181818202137947, + "mean": 0.024175824597477913, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428624647005, + "mean": 0.09512362802135092, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017283686785958707, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07759245804377965, + "mean": 0.03196296075891171, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.08051948089684759, + "mean": 0.02397602450634752, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004373177886009216, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2767857164144516, + "best1L": 0.1592261932258095, + "mean": 0.0943223430908152, + "persist": 0.08184523760740246 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017603755479545465, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11530094114797455, + "best1L": 0.07722987580512251, + "mean": 0.03162826996828828, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.08051948089684759, + "mean": 0.022877122275531292, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380491154535, + "best1L": 0.16071428704474652, + "mean": 0.09500915730105979, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497066092411324, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11348803394607135, + "best1L": 0.07541696860321931, + "mean": 0.03168405166694096, + "persist": 0.02719361853918859 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.08051948089684759, + "mean": 0.023276723522160734, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976172736713, + "best1L": 0.16071428624647005, + "mean": 0.09478021931967565, + "persist": 0.08333333461944546 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497066092411324, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07795503948416029, + "mean": 0.031962961690234284, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.023476524245260016, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428624647005, + "mean": 0.09523809581462826, + "persist": 0.08333333461944546 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017710444600587443, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11530094114797455, + "best1L": 0.07831762092454093, + "mean": 0.03196296142414212, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.023376624066648737, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428624647005, + "mean": 0.09535256414008993, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017603755213453302, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07686729516301836, + "mean": 0.03190717852807471, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08051948063075542, + "best1L": 0.07792207838169166, + "mean": 0.02287712330663843, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232143389327185, + "best1L": 0.1592261932258095, + "mean": 0.09443681354501418, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01739037670527718, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11457577826721328, + "best1L": 0.07759245804377965, + "mean": 0.032018742723656554, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08051948063075542, + "best1L": 0.07792207838169166, + "mean": 0.022977022820019295, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095277122089, + "best1L": 0.1592261932258095, + "mean": 0.09420787609581437, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017283687318143035, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11348803394607135, + "best1L": 0.07650471372263772, + "mean": 0.03187928834397878, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.0231768235763801, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2708333358168602, + "best1L": 0.16071428624647005, + "mean": 0.09409340723816838, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497065826319158, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07686729516301836, + "mean": 0.031990852539560626, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.023376623135326163, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2708333358168602, + "best1L": 0.16071428624647005, + "mean": 0.0942078758297222, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017283687318143035, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07686729516301836, + "mean": 0.03199085227346846, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08051948063075542, + "best1L": 0.07792207838169166, + "mean": 0.023276722956714884, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428198899543, + "mean": 0.09455128213656801, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + } + }, + "runtime_s": 72.3 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Pile-CC_s0.json b/results/ladder410/ladder_Pile-CC_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..d9096d546cca7da31d7d74ed6717e871c0a39939 --- /dev/null +++ b/results/ladder410/ladder_Pile-CC_s0.json @@ -0,0 +1,463 @@ +{ + "corpus": "Pile-CC", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.11095700412988663, + "best1L": 0.08876560415540423, + "mean": 0.032006828380482535, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.17476432557616914, + "best1L": 0.14720812067389488, + "mean": 0.04359346189137016, + "persist": 0.036983320622571876 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.07012986977185522, + "mean": 0.016183816262387803, + "persist": 0.009090908936091832 + }, + "poetry": { + "min": 0.021865888631769588, + "best1L": 0.00728862958827189, + "mean": 0.003251850272395781, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3377976215311459, + "best1L": 0.2053571455180645, + "mean": 0.15178571401962213, + "persist": 0.16071428598037787 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180239915848, + "mean": 0.001791686484856265, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.112343965364354, + "best1L": 0.09292649051972798, + "mean": 0.032006828247436454, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1736765748688153, + "best1L": 0.14539521347199166, + "mean": 0.04395604373088905, + "persist": 0.038071065209805965 + }, + "order-ops": { + "min": 0.0753246753343514, + "best1L": 0.07012986977185522, + "mean": 0.015684315502377495, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889297, + "best1L": 0.008746355905064515, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3541666641831398, + "best1L": 0.2083333320915699, + "mean": 0.16220238033149922, + "persist": 0.16815476252564363 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180239915848, + "mean": 0.002030578109302691, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.031366691791585515, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.1729514098593167, + "best1L": 0.1475707037108285, + "mean": 0.043900261633098125, + "persist": 0.03698332022343363 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01378621397140835, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.005830903803663594, + "mean": 0.0037003811448812485, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3586309456399509, + "best1L": 0.2142857126891613, + "mean": 0.16723900900355407, + "persist": 0.1755952396030937 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.006211180239915848, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03158007003366947, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.1736765769975526, + "best1L": 0.14757070424301283, + "mean": 0.043928152482424467, + "persist": 0.037708483237241 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014485514423410808, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.00728862958827189, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3541666641831398, + "best1L": 0.20833333102720125, + "mean": 0.16369047335215978, + "persist": 0.16964285634458065 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.006211180239915848, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03104662336409092, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1715010856943471, + "best1L": 0.14503263362816401, + "mean": 0.04339822581303971, + "persist": 0.03662073891609907 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.013886113883927464, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.010204081822718893, + "mean": 0.004148912416504962, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.33928571322134565, + "best1L": 0.20684523933700152, + "mean": 0.1593406588903495, + "persist": 0.16666666657796927 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256751860891, + "mean": 0.03147338117871966, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17222624910729273, + "best1L": 0.14503263362816401, + "mean": 0.04376080725342035, + "persist": 0.03662073891609907 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01398601392949266, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.008746355905064515, + "mean": 0.004036779648491314, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20684523720826423, + "mean": 0.16048534720071725, + "persist": 0.16666666657796927 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17403915843793324, + "best1L": 0.1468455387013299, + "mean": 0.04390026216528246, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014085914241150022, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.008746355905064515, + "mean": 0.003812513979417937, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000319310595, + "best1L": 0.20386904744165285, + "mean": 0.16002747230231762, + "persist": 0.16517857063029492 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256831688541, + "mean": 0.03147338144481182, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.16860043842877662, + "best1L": 0.14213197997638158, + "mean": 0.04289619012602738, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014485514290364725, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.008746355505926269, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738134264946, + "best1L": 0.2008928544819355, + "mean": 0.1587683167308569, + "persist": 0.16517857595213822 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256831688541, + "mean": 0.03147338144481182, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1686004363000393, + "best1L": 0.14249456141676223, + "mean": 0.04295197129249573, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014285714199234332, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34226190724543165, + "best1L": 0.2038690485060215, + "mean": 0.15934066155127116, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.031580070831945965, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.16968818274991854, + "best1L": 0.14285714285714285, + "mean": 0.043147207370826175, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014185814153669136, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.2038690485060215, + "mean": 0.1600274746971471, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.03147338144481182, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17077592494232313, + "best1L": 0.1432197242975235, + "mean": 0.04323088138231209, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01438561437784561, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.008746355505926269, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.2038690485060215, + "mean": 0.15991300184811866, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 59.8 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Pile-CC_s1.json b/results/ladder410/ladder_Pile-CC_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e5d5ede8fcac7216ad03436d543d2c62bf72a7f5 --- /dev/null +++ b/results/ladder410/ladder_Pile-CC_s1.json @@ -0,0 +1,463 @@ +{ + "corpus": "Pile-CC", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.0873786418565682, + "mean": 0.0309399341765259, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.1754894863281931, + "best1L": 0.14793328515120915, + "mean": 0.043537680724901814, + "persist": 0.0373459021959986 + }, + "order-ops": { + "min": 0.0714285708963871, + "best1L": 0.06883116864732333, + "mean": 0.010989011032506824, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.018950437062552998, + "best1L": 0.005830903803663594, + "mean": 0.003251850139349699, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3586309477686882, + "best1L": 0.21428571588226727, + "mean": 0.16483516645218646, + "persist": 0.17559523907090938 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.08876560415540423, + "mean": 0.031366691791585515, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17367657646536827, + "best1L": 0.14793328515120915, + "mean": 0.04334244344915662, + "persist": 0.035895576301429956 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.07012986977185522, + "mean": 0.011788211995735765, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.00728862958827189, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000319310595, + "best1L": 0.21130952079381263, + "mean": 0.16025641054979392, + "persist": 0.16815476465438092 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09153952822089195, + "mean": 0.0315800705658538, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.172951413584607, + "best1L": 0.14793328302247183, + "mean": 0.043342444247433116, + "persist": 0.03553299539323364 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.07012986977185522, + "mean": 0.015184815407597594, + "persist": 0.009090908936091832 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.005830903803663594, + "mean": 0.003588248576436724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34672619295971735, + "best1L": 0.20238095096179418, + "mean": 0.15922618843615055, + "persist": 0.16220238405678952 + }, + "association": { + "min": 0.009316770093781608, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.08876560415540423, + "mean": 0.03126000160617488, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1729514157133443, + "best1L": 0.14865844590323313, + "mean": 0.043481898094926565, + "persist": 0.03662073878305299 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013786213838362269, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.005830903803663594, + "mean": 0.00325185020587274, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023805975914, + "best1L": 0.20535714445369585, + "mean": 0.15956959421081202, + "persist": 0.16517857063029492 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256725251675, + "mean": 0.031580070299761634, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1751269048878125, + "best1L": 0.14829586446285248, + "mean": 0.04367713576980999, + "persist": 0.03698332115475621 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.015684316034561822, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.005830903803663594, + "mean": 0.0033639829073633465, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047654526575, + "best1L": 0.19940476279173577, + "mean": 0.1603708786091634, + "persist": 0.1651785738234009 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256725251675, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17548948845693044, + "best1L": 0.1493836087839944, + "mean": 0.04376080831778901, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014885114738717675, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20089285661067283, + "mean": 0.1596840667937483, + "persist": 0.1651785738234009 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03126000187226704, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17585206776857376, + "best1L": 0.14902102947235107, + "mean": 0.043788698102746694, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01438561437784561, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.005830903803663594, + "mean": 0.0033639829738863875, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3482142889073917, + "best1L": 0.2038690485060215, + "mean": 0.16117216327360698, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.031580070831945965, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17295141624552862, + "best1L": 0.14503263149942672, + "mean": 0.04364924372306892, + "persist": 0.03662073878305299 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014885114472625511, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0038125141124640194, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.20386904637728417, + "mean": 0.159455130142825, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256725251675, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17186367139220238, + "best1L": 0.145757794380188, + "mean": 0.04364924438829933, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.01508491496289415, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.15979853405484132, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.03168676021908011, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17295141784208162, + "best1L": 0.14684554083006723, + "mean": 0.04367713523762567, + "persist": 0.03662073878305299 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.01508491496289415, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.15991300610559328, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17258883640170097, + "best1L": 0.14648295938968658, + "mean": 0.043621353805065155, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014585414468976004, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.005830903803663594, + "mean": 0.003588248576436724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.16025640815496445, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 58.6 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Pile-CC_s2.json b/results/ladder410/ladder_Pile-CC_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..bfeed215f836ee3d182f7e996e1fad52ef9f18c8 --- /dev/null +++ b/results/ladder410/ladder_Pile-CC_s2.json @@ -0,0 +1,463 @@ +{ + "corpus": "Pile-CC", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10957003970231328, + "best1L": 0.09153952901916844, + "mean": 0.03147338091262749, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1609862217945712, + "best1L": 0.13488034584692546, + "mean": 0.04113906302622387, + "persist": 0.03589557616838387 + }, + "order-ops": { + "min": 0.07792207824864558, + "best1L": 0.07532467599958181, + "mean": 0.01368631368469713, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.007288629721317973, + "mean": 0.003924646747431585, + "persist": 0.0 + }, + "typo": { + "min": 0.34821428252117975, + "best1L": 0.20535714445369585, + "mean": 0.15464743679123266, + "persist": 0.16220238405678952 + }, + "association": { + "min": 0.009316769960735525, + "best1L": 0.004658385179936886, + "mean": 0.0013139034854248166, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.03168675968689578, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17077592121703283, + "best1L": 0.14575779704110964, + "mean": 0.043230880184897354, + "persist": 0.03770848350333316 + }, + "order-ops": { + "min": 0.07662337699106761, + "best1L": 0.07272727255310331, + "mean": 0.014485514257103205, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.008746355505926269, + "mean": 0.003812513979417937, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34672619083098005, + "best1L": 0.21279761780585563, + "mean": 0.1591117217072419, + "persist": 0.1666666687067066 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0017916865181177855, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09153952928526062, + "mean": 0.03126000187226704, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.17403916056667054, + "best1L": 0.1475707037108285, + "mean": 0.043537680724901814, + "persist": 0.0377084847007479 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085914241150022, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.023323614549423968, + "best1L": 0.008746355905064515, + "mean": 0.0037003812779273304, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023827263287, + "best1L": 0.21279761993459292, + "mean": 0.16311813199094363, + "persist": 0.1741071425910507 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001911132363602519, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09153952928526062, + "mean": 0.031580070299761634, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17258883267641068, + "best1L": 0.1457577965089253, + "mean": 0.043091426470449994, + "persist": 0.036620739182191234 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013886113883927464, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.026239067981285707, + "best1L": 0.00728862958827189, + "mean": 0.003924646880477667, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3526785724929401, + "best1L": 0.20833333315593855, + "mean": 0.16151556904826844, + "persist": 0.16964285794113362 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0020305780760411707, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.03126000187226704, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.16860043417130197, + "best1L": 0.14249456141676223, + "mean": 0.04244993587157556, + "persist": 0.035532995127141476 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013286713477490204, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.33779762365988325, + "best1L": 0.20238094830087253, + "mean": 0.15670787702713693, + "persist": 0.16815476146127498 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10957003970231328, + "best1L": 0.0915395300835371, + "mean": 0.031366691791585515, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17077592494232313, + "best1L": 0.14430746861866542, + "mean": 0.04286829874451671, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013786213838362269, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20535714019622123, + "mean": 0.15831043917153562, + "persist": 0.16666666764233792 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10957003970231328, + "best1L": 0.0915395300835371, + "mean": 0.031580070299761634, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17041334137320518, + "best1L": 0.1432197242975235, + "mean": 0.04270095471292734, + "persist": 0.035895575902291706 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013886113883927464, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20535714019622123, + "mean": 0.15796703020376818, + "persist": 0.16666666551360063 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.031580070299761634, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.16968818168554986, + "best1L": 0.1439448893070221, + "mean": 0.04306353535503149, + "persist": 0.0358955767005682 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.007288629721317973, + "mean": 0.00336398310693247, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3482142889073917, + "best1L": 0.20386904531291553, + "mean": 0.1620879146669592, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.03126000213835921, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17113850638270378, + "best1L": 0.14503263362816401, + "mean": 0.04317509848624468, + "persist": 0.0358955767005682 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.007288629721317973, + "mean": 0.00336398310693247, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.20535714126058988, + "mean": 0.16220237900103843, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.031366691791585515, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17258883001548903, + "best1L": 0.14612037794930594, + "mean": 0.043286663480103016, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.007288629721317973, + "mean": 0.0034761156753769945, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.2068452350795269, + "mean": 0.1623168507856982, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.03126000213835921, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17186366926346505, + "best1L": 0.1457577965089253, + "mean": 0.04317509848624468, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.007288629721317973, + "mean": 0.0034761156753769945, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023827263287, + "best1L": 0.20535714019622123, + "mean": 0.16151556745171547, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 43.4 +} \ No newline at end of file diff --git a/results/ladder410/ladder_StackExchange_s0.json b/results/ladder410/ladder_StackExchange_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..18737b011146065f2510079b96975a5034fbef04 --- /dev/null +++ b/results/ladder410/ladder_StackExchange_s0.json @@ -0,0 +1,659 @@ +{ + "corpus": "StackExchange", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09015256645424026, + "mean": 0.030513176594727805, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16715010840977942, + "best1L": 0.1356055119207927, + "mean": 0.04348189876015697, + "persist": 0.03879622809056725 + }, + "order-ops": { + "min": 0.0740259758063725, + "best1L": 0.05454545468091965, + "mean": 0.012587412526564938, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.013119533658027649, + "best1L": 0.008746355505926269, + "mean": 0.003588248576436724, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.37500000851494925, + "best1L": 0.22023809275456838, + "mean": 0.15819597031388963, + "persist": 0.16815475959862983 + }, + "association": { + "min": 0.015527950200651373, + "best1L": 0.006211180106869766, + "mean": 0.0014333492394403688, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.0887656036232199, + "mean": 0.02965966096547033, + "persist": 0.012482662818261556 + }, + "multilingual": { + "min": 0.1718636719243867, + "best1L": 0.1395939061684268, + "mean": 0.04406760686210224, + "persist": 0.036620739315237315 + }, + "order-ops": { + "min": 0.06363636308482715, + "best1L": 0.05324675302420344, + "mean": 0.009790209887017096, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.011661807873419352, + "best1L": 0.008746355505926269, + "mean": 0.0034761157419000354, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190469094685, + "best1L": 0.2217261865735054, + "mean": 0.15831043837325914, + "persist": 0.16964285873941012 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.006211180106869766, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09292649131800447, + "mean": 0.030299797920244082, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1733139998146466, + "best1L": 0.14249455982020923, + "mean": 0.04465331602841616, + "persist": 0.03734590232904468 + }, + "order-ops": { + "min": 0.06623376746262823, + "best1L": 0.05324675302420344, + "mean": 0.009690310107544065, + "persist": 0.0 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333262375425, + "best1L": 0.22470238219414437, + "mean": 0.16002747363277844, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09292649158409663, + "mean": 0.03019310820049473, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17113850585051946, + "best1L": 0.13995648973754474, + "mean": 0.044346515488411696, + "persist": 0.03770848363637924 + }, + "order-ops": { + "min": 0.06753246752279145, + "best1L": 0.05324675302420344, + "mean": 0.010189810069277883, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.010204081822718893, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333262375425, + "best1L": 0.22172618923442705, + "mean": 0.1596840683903013, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.006211180106869766, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399342843465, + "best1L": 0.14430747234395572, + "mean": 0.04496011523795979, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010789210642022746, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3749999978712627, + "best1L": 0.22767857196075575, + "mean": 0.16197344394666807, + "persist": 0.17559523986918585 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086419079452753, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17258883054767335, + "best1L": 0.1424945630133152, + "mean": 0.04473698857639517, + "persist": 0.03770848363637924 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.011661807873419352, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3749999978712627, + "best1L": 0.2291666679084301, + "mean": 0.16082875909549849, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03019310820049473, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17150108409779413, + "best1L": 0.14140681390251433, + "mean": 0.04459753432976348, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010889111219772271, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22619047920618737, + "mean": 0.1605998133974416, + "persist": 0.17261905063475882 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.030086418680314506, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17258883267641068, + "best1L": 0.14285714232495852, + "mean": 0.04465331616146224, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.0727272726861494, + "best1L": 0.05584415593849761, + "mean": 0.010789210700230407, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246468139546254, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357110926083, + "best1L": 0.22767857302512443, + "mean": 0.15991300184811866, + "persist": 0.17559523986918585 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.029979729293180362, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17403916056667054, + "best1L": 0.14430747021521842, + "mean": 0.044820661922650676, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.07142857142857142, + "best1L": 0.05454545468091965, + "mean": 0.010689310596457549, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779648491314, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.37351190405232565, + "best1L": 0.22619047920618737, + "mean": 0.16025640975151742, + "persist": 0.17261905010257447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.02997972969231861, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.172951411988054, + "best1L": 0.14358230733445712, + "mean": 0.04468120661165033, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22470238538725035, + "mean": 0.16014194062777928, + "persist": 0.17113095468708447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.029873040172138383, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17186366766691208, + "best1L": 0.14213197731545993, + "mean": 0.0445975341967174, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010489510239234992, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22470238538725035, + "mean": 0.15979853325656482, + "persist": 0.17113095468708447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02997972909361124, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17403915630919592, + "best1L": 0.14430746915084974, + "mean": 0.04470909772706883, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010189810102539403, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246468139546254, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.15590659156441689, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418680314506, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399342843465, + "best1L": 0.14358230520571982, + "mean": 0.044681206345558167, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1563644685915538, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039639954056, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399129969732, + "best1L": 0.14394488664610045, + "mean": 0.04479277040809393, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1567078743662153, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02976634998672775, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399129969732, + "best1L": 0.14394488664610045, + "mean": 0.04479277094027826, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010189810102539403, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1569368147424289, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039639954056, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17367657274007797, + "best1L": 0.14394488664610045, + "mean": 0.04473698844334909, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713540656225, + "best1L": 0.22321428943957602, + "mean": 0.1568223461508751, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + } + }, + "runtime_s": 79.3 +} \ No newline at end of file diff --git a/results/ladder410/ladder_StackExchange_s1.json b/results/ladder410/ladder_StackExchange_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..13a6cfd15e2d6a8773a1991269c7cf4c4cccf7c6 --- /dev/null +++ b/results/ladder410/ladder_StackExchange_s1.json @@ -0,0 +1,659 @@ +{ + "corpus": "StackExchange", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.029126213198261603, + "persist": 0.011095700253333365 + }, + "multilingual": { + "min": 0.16968817742807524, + "best1L": 0.14176939906818525, + "mean": 0.04398393484630755, + "persist": 0.03734590206295252 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.011388611713690417, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.014577259974820274, + "best1L": 0.011661807607327188, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523241860525, + "best1L": 0.22767857249294007, + "mean": 0.151785708963871, + "persist": 0.16369047601308143 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0009555661319089788, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029552971278982505, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16968817955681256, + "best1L": 0.14031907064574106, + "mean": 0.04401182582867997, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.009790209920278617, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.013119533658027649, + "mean": 0.004036779648491314, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071364709309, + "best1L": 0.22619047920618737, + "mean": 0.15453297006232397, + "persist": 0.16666666844061442 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10540915360408169, + "best1L": 0.09015256725251675, + "mean": 0.02965966079916273, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399768590927, + "best1L": 0.14249455982020923, + "mean": 0.04487644308911903, + "persist": 0.03879622809056725 + }, + "order-ops": { + "min": 0.06883117037692241, + "best1L": 0.05584415593849761, + "mean": 0.009690309400736754, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011905116694315, + "best1L": 0.22767856876764977, + "mean": 0.1565934100321361, + "persist": 0.1711309514939785 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02965966079916273, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1715010872909001, + "best1L": 0.14176939693944796, + "mean": 0.044597534861947806, + "persist": 0.03879622809056725 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011905116694315, + "best1L": 0.22619047494871275, + "mean": 0.15579212669815337, + "persist": 0.16815476225955145 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02997972889404212, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1736765748688153, + "best1L": 0.14503263256379537, + "mean": 0.04509957041591406, + "persist": 0.03952139137046678 + }, + "order-ops": { + "min": 0.0727272726861494, + "best1L": 0.05584415593849761, + "mean": 0.010889110745795603, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714285182101385, + "best1L": 0.22767857302512443, + "mean": 0.15361721707241877, + "persist": 0.16815476279173577 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039174292768, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17295141411679132, + "best1L": 0.14321972589407647, + "mean": 0.044960115770144124, + "persist": 0.040246556379965374 + }, + "order-ops": { + "min": 0.07142857262598616, + "best1L": 0.05584415593849761, + "mean": 0.010389610251877457, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904584509985, + "best1L": 0.22619047920618737, + "mean": 0.15418955896581923, + "persist": 0.1696428582072258 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029979728827519075, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17077592760324478, + "best1L": 0.14140681496688298, + "mean": 0.044653315629277913, + "persist": 0.040246556379965374 + }, + "order-ops": { + "min": 0.07012987043708563, + "best1L": 0.05584415593849761, + "mean": 0.010589410609100014, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35565476013081415, + "best1L": 0.22470238325851305, + "mean": 0.15327380704028265, + "persist": 0.16666666684406145 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03019310793440257, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1689630150794983, + "best1L": 0.14068165208612168, + "mean": 0.04426284347261701, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.07142857142857142, + "best1L": 0.05454545468091965, + "mean": 0.010889110687587942, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22321428943957602, + "mean": 0.15350274635212763, + "persist": 0.16815476013081415 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02997972889404212, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1696881800889969, + "best1L": 0.14104423352650233, + "mean": 0.044402298251433034, + "persist": 0.038796229021889825 + }, + "order-ops": { + "min": 0.07142857142857142, + "best1L": 0.05454545468091965, + "mean": 0.010689310596457549, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22470238325851305, + "mean": 0.1539606223148959, + "persist": 0.16666666524750845 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039373861893, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17005076152937754, + "best1L": 0.14068165208612168, + "mean": 0.04445807995008571, + "persist": 0.03915881099445479 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010589410550892353, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22470238325851305, + "mean": 0.15361721866897174, + "persist": 0.16815476013081415 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039373861893, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16860043363911764, + "best1L": 0.14068165208612168, + "mean": 0.04429073485412768, + "persist": 0.038796229021889825 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22321428943957602, + "mean": 0.1537316877927099, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03029979732153671, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17041334296975816, + "best1L": 0.14249456141676223, + "mean": 0.04451386111655405, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.1546474370573248, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03019310793440257, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1682378500699997, + "best1L": 0.14104423352650233, + "mean": 0.04434651628668819, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010089910056974207, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15476190777761595, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16932559651987894, + "best1L": 0.14176939640726363, + "mean": 0.044430188701621125, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15453296633703367, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17041334509849548, + "best1L": 0.14176939640726363, + "mean": 0.044569644411759715, + "persist": 0.0395213919026511 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010089910056974207, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15453296580484935, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17005076365811483, + "best1L": 0.14176939640726363, + "mean": 0.04454175289720297, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010089910056974207, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15441849987421716, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + } + }, + "runtime_s": 87.0 +} \ No newline at end of file diff --git a/results/ladder410/ladder_StackExchange_s2.json b/results/ladder410/ladder_StackExchange_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..8dcb04406ae0469cba1d179399111a75c7f361c0 --- /dev/null +++ b/results/ladder410/ladder_StackExchange_s2.json @@ -0,0 +1,659 @@ +{ + "corpus": "StackExchange", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.1081830792661224, + "best1L": 0.08876560548586505, + "mean": 0.03019310820049473, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16932559545551026, + "best1L": 0.13923132632459914, + "mean": 0.04401182569563389, + "persist": 0.03843364665018661 + }, + "order-ops": { + "min": 0.08441558293998241, + "best1L": 0.05714285719607558, + "mean": 0.011888111791839557, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071471146175, + "best1L": 0.2247023768723011, + "mean": 0.15144230664840766, + "persist": 0.16369047707745008 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0009555661319089788, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09015256725251675, + "mean": 0.030513176627989327, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17041334616286413, + "best1L": 0.13959390776497976, + "mean": 0.044039717210190635, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.07922077817576272, + "best1L": 0.05454545468091965, + "mean": 0.011388611447598253, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.014577259974820274, + "best1L": 0.010204081822718893, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3690476215311459, + "best1L": 0.22321428518210137, + "mean": 0.15693681314587593, + "persist": 0.16815475959862983 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09292649158409663, + "mean": 0.03061986601512347, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16860043151038034, + "best1L": 0.13995648760880744, + "mean": 0.04367713470544134, + "persist": 0.0373459021959986 + }, + "order-ops": { + "min": 0.0662337669304439, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3690476215311459, + "best1L": 0.22470238112977572, + "mean": 0.1575091587645667, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09292649158409663, + "mean": 0.03072655513616545, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.16823785113436834, + "best1L": 0.13959390563624247, + "mean": 0.04367713483848742, + "persist": 0.0373459021959986 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010189810102539403, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3675595277122089, + "best1L": 0.22470238325851305, + "mean": 0.15636446912373816, + "persist": 0.16666666365095548 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952981744494, + "mean": 0.03029979738805975, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16860043683222362, + "best1L": 0.13995648760880744, + "mean": 0.04373291627104793, + "persist": 0.038071065076759884 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22470238325851305, + "mean": 0.15499084017106465, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09292649158409663, + "mean": 0.030406487307378223, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1693255975842476, + "best1L": 0.1395939061684268, + "mean": 0.043760807785604684, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15556318578975542, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952981744494, + "mean": 0.030193108267017772, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17005076259374619, + "best1L": 0.14031906904918806, + "mean": 0.04392815234937838, + "persist": 0.038796229021889825 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.1556776538491249, + "persist": 0.16666666471532413 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030193107734833444, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133419053895, + "best1L": 0.14031906904918806, + "mean": 0.044402298384479115, + "persist": 0.03988397454044649 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15602106228470802, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030193107734833444, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133397766522, + "best1L": 0.13995648760880744, + "mean": 0.044207062173102586, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15544871719820158, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.0300864183476993, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133397766522, + "best1L": 0.13995648760880744, + "mean": 0.04423495288938284, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15567765225257194, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02997972889404212, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1696881768958909, + "best1L": 0.13995648760880744, + "mean": 0.04406760699514832, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15556318578975542, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418414222344, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17150108409779413, + "best1L": 0.14140681337032998, + "mean": 0.0444580792848553, + "persist": 0.04097171939377274 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15670787755932128, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418480745385, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17077592121703283, + "best1L": 0.1406816504895687, + "mean": 0.044430188568575044, + "persist": 0.04060913688902344 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15636446912373816, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418480745385, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17077592334577016, + "best1L": 0.14104423192994936, + "mean": 0.04426284294043269, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15682234189340047, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16968817902462824, + "best1L": 0.14031906904918806, + "mean": 0.0441512798092195, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15670787330184663, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133419053895, + "best1L": 0.14140681337032998, + "mean": 0.04423495328852108, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15659340683903014, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + } + }, + "runtime_s": 84.4 +} \ No newline at end of file diff --git a/results/ladder410/ladder_USPTO_Backgrounds_s0.json b/results/ladder410/ladder_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..4f2e83a3694221931fad0385f5eb2dfbe8187be8 --- /dev/null +++ b/results/ladder410/ladder_USPTO_Backgrounds_s0.json @@ -0,0 +1,463 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.11095700200114932, + "best1L": 0.08183079026639462, + "mean": 0.0288061454027359, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1453952118754387, + "best1L": 0.10732414866132396, + "mean": 0.03938193852081895, + "persist": 0.03807106600808246 + }, + "order-ops": { + "min": 0.10649350618145295, + "best1L": 0.08961039049816984, + "mean": 0.01958041949962665, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.021865889563092163, + "best1L": 0.00874635563897235, + "mean": 0.004373177752963134, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3526785767504147, + "best1L": 0.1934523768723011, + "mean": 0.15934066155127116, + "persist": 0.17708333568381412 + }, + "association": { + "min": 0.017080744994538172, + "best1L": 0.006211180239915848, + "mean": 0.00203057804277965, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08321775389569146, + "mean": 0.030193107734833444, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1457577906548977, + "best1L": 0.10732415052396911, + "mean": 0.03954928374982306, + "persist": 0.03734590192990644 + }, + "order-ops": { + "min": 0.12207792060715812, + "best1L": 0.10000000255448478, + "mean": 0.02357642392494849, + "persist": 0.011688311185155596 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004036779448922191, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3630952334829739, + "best1L": 0.20386904983648232, + "mean": 0.1652930397540331, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.007763975166848728, + "mean": 0.002030578100987311, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11927877818899495, + "best1L": 0.08460471672671181, + "mean": 0.030833244922437837, + "persist": 0.020804438473922864 + }, + "multilingual": { + "min": 0.14612037156309401, + "best1L": 0.11094996652432851, + "mean": 0.039660846814513206, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.13896104267665318, + "best1L": 0.1090909093618393, + "mean": 0.02787212807951229, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004373177886009216, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3601190426519939, + "best1L": 0.20684523401515825, + "mean": 0.16597984890852654, + "persist": 0.18303571242306912 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915346403207, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.11927877818899495, + "best1L": 0.08321775469396796, + "mean": 0.030406487373901264, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14467004739812442, + "best1L": 0.10913705905633313, + "mean": 0.039437719088579924, + "persist": 0.03915881072836263 + }, + "order-ops": { + "min": 0.12467532604932785, + "best1L": 0.10259740373917989, + "mean": 0.025174824986606836, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.20535714338932717, + "mean": 0.16540750674903393, + "persist": 0.17857142777315207 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915346403207, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11789181535797459, + "best1L": 0.08183079079857894, + "mean": 0.030193107535264323, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1468455365725926, + "best1L": 0.11094996625823635, + "mean": 0.039939755973006995, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.12987013161182404, + "best1L": 0.101298702614648, + "mean": 0.026473525911569595, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004373177752963134, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3571428560784885, + "best1L": 0.19940476172736712, + "mean": 0.16185897588729858, + "persist": 0.17410714378846542 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.007763975166848728, + "mean": 0.0021500238134259625, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.11650485305913857, + "best1L": 0.08183078920202595, + "mean": 0.030193107734833444, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14757070264645986, + "best1L": 0.11203771057937827, + "mean": 0.039995537139475346, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.13376623392105103, + "best1L": 0.10389610486371177, + "mean": 0.026873127224722078, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004261045051472527, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20386904637728417, + "mean": 0.16357600688934326, + "persist": 0.17708333461944545 + }, + "association": { + "min": 0.02173912977533681, + "best1L": 0.009316770226827689, + "mean": 0.0025083611253648996, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.11650485305913857, + "best1L": 0.08183078920202595, + "mean": 0.030086418081607138, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829540678433, + "best1L": 0.11094996625823635, + "mean": 0.039856082892843654, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.12987013161182404, + "best1L": 0.101298702614648, + "mean": 0.0257742257256593, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20089285767504148, + "mean": 0.16346154042652675, + "persist": 0.17857142950275115 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.007763975166848728, + "mean": 0.0025083612085187008, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.030939933710864613, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14684554136225156, + "best1L": 0.11203771057937827, + "mean": 0.039856083158935816, + "persist": 0.039883973476077826 + }, + "order-ops": { + "min": 0.1272727277662073, + "best1L": 0.09870129876903125, + "mean": 0.025074925107349242, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833283662796, + "best1L": 0.20238095202616282, + "mean": 0.16597985343209334, + "persist": 0.18005952332168818 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.03104662336409092, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14684553976569856, + "best1L": 0.1124002920197589, + "mean": 0.03980030185942139, + "persist": 0.04060913648988519 + }, + "order-ops": { + "min": 0.1298701294830867, + "best1L": 0.10000000042574746, + "mean": 0.025674325680094107, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523454734255, + "best1L": 0.20238095202616282, + "mean": 0.16506410363529408, + "persist": 0.17708333249070815 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.03093993424304894, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1457577949123723, + "best1L": 0.11203771057937827, + "mean": 0.039632956364325116, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.1298701294830867, + "best1L": 0.10000000042574746, + "mean": 0.02547452479068722, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3675595202616283, + "best1L": 0.2053571417927742, + "mean": 0.1655219742762191, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.030833245122006962, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14612037635275296, + "best1L": 0.11167512913899762, + "mean": 0.03974451869726181, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.1285714294229235, + "best1L": 0.10000000042574746, + "mean": 0.025274725231741155, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.20684523561171123, + "mean": 0.16609431936272553, + "persist": 0.18005952225731953 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + } + }, + "runtime_s": 59.9 +} \ No newline at end of file diff --git a/results/ladder410/ladder_USPTO_Backgrounds_s1.json b/results/ladder410/ladder_USPTO_Backgrounds_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..9680cb6b4bddf78be5b3c372771d24d4af740ad4 --- /dev/null +++ b/results/ladder410/ladder_USPTO_Backgrounds_s1.json @@ -0,0 +1,463 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.12205270385103566, + "best1L": 0.08321775256523065, + "mean": 0.03126000193879008, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14974619394966535, + "best1L": 0.11276287186358656, + "mean": 0.04097171766417367, + "persist": 0.03988397454044649 + }, + "order-ops": { + "min": 0.12337662545698029, + "best1L": 0.09870130089776856, + "mean": 0.023676323671159998, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.027696792967617512, + "best1L": 0.00874635563897235, + "mean": 0.004821708891540766, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071417927742, + "best1L": 0.19940476146127498, + "mean": 0.15945513040891715, + "persist": 0.16964285913854837 + }, + "association": { + "min": 0.02018633538058826, + "best1L": 0.01086956528680665, + "mean": 0.0020305780760411707, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.12343966774642467, + "best1L": 0.08460471646061965, + "mean": 0.03126000300315874, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1497461912887437, + "best1L": 0.11348803634090084, + "mean": 0.040776482317596674, + "persist": 0.04097171779721975 + }, + "order-ops": { + "min": 0.11948051889027868, + "best1L": 0.0961038986487048, + "mean": 0.023176823775949224, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004709576190050159, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011904371636255, + "best1L": 0.20535714525197232, + "mean": 0.1656364481896162, + "persist": 0.18154761607625655 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11927877818899495, + "best1L": 0.08460471486406666, + "mean": 0.030833244922437837, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1519216825919492, + "best1L": 0.11530094407498837, + "mean": 0.04080437310039997, + "persist": 0.03915881032922438 + }, + "order-ops": { + "min": 0.1142857139930129, + "best1L": 0.09350649387176548, + "mean": 0.023076923755330166, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.023323615480746542, + "best1L": 0.00874635563897235, + "mean": 0.004036779648491314, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3526785724929401, + "best1L": 0.19345238299242087, + "mean": 0.15659340976604394, + "persist": 0.16517857129552535 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.009316770226827689, + "mean": 0.002269469650595316, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.12066574155219964, + "best1L": 0.08460471486406666, + "mean": 0.031153312751225064, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.15083393614206994, + "best1L": 0.11457578119422708, + "mean": 0.040943827746169906, + "persist": 0.03915881032922438 + }, + "order-ops": { + "min": 0.1168831153107541, + "best1L": 0.09350649506918021, + "mean": 0.023376623817187334, + "persist": 0.009090908936091832 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004261044918426445, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.19791666764233792, + "mean": 0.15716575139335223, + "persist": 0.16815476212650537 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.009316770226827689, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11650485412350722, + "best1L": 0.08183079026639462, + "mean": 0.030406487307378223, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1479332862155778, + "best1L": 0.11457578119422708, + "mean": 0.04052546440756747, + "persist": 0.03879622928798199 + }, + "order-ops": { + "min": 0.11818181736660856, + "best1L": 0.09090909055833306, + "mean": 0.023776223409056132, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714285714285715, + "best1L": 0.19196428411773273, + "mean": 0.15739468644772256, + "persist": 0.17113095508622272 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.11650485412350722, + "best1L": 0.08183079026639462, + "mean": 0.030406486509101733, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1472081212060792, + "best1L": 0.11385061565254416, + "mean": 0.0403302287948983, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.11688311517770801, + "best1L": 0.09220779274723359, + "mean": 0.02377622394124046, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004373177686440093, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35863095096179415, + "best1L": 0.19940475959862983, + "mean": 0.16037088047180856, + "persist": 0.1726190499695284 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.11650485412350722, + "best1L": 0.08183079026639462, + "mean": 0.030619865283370018, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1479332862155778, + "best1L": 0.11348803634090084, + "mean": 0.04046968211020742, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.11688311624207667, + "best1L": 0.09350649387176548, + "mean": 0.023776223542102213, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190447807312, + "best1L": 0.20089285554630415, + "mean": 0.1607142865125622, + "persist": 0.1726190499695284 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08321775389569146, + "mean": 0.03051317563014371, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14539521134325437, + "best1L": 0.11203771057937827, + "mean": 0.03980030012982232, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.12337662412651948, + "best1L": 0.09480519512934345, + "mean": 0.02447552511668099, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.026239066384732723, + "best1L": 0.00874635563897235, + "mean": 0.004485310654022864, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.20089285767504148, + "mean": 0.1641483549028635, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11373092952583518, + "best1L": 0.08183079000030245, + "mean": 0.030086418081607138, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14612037475619996, + "best1L": 0.11312545490052019, + "mean": 0.03991186472454241, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.12337662412651948, + "best1L": 0.09480519512934345, + "mean": 0.024375624405885383, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160714285714285, + "best1L": 0.1979166679084301, + "mean": 0.1624313188450677, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08321775389569146, + "mean": 0.03051317563014371, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829540678433, + "best1L": 0.11312545490052019, + "mean": 0.040051319902496677, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.12207792193761893, + "best1L": 0.09480519512934345, + "mean": 0.024275724360320185, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904903820585, + "best1L": 0.19940476385610445, + "mean": 0.16254578450960772, + "persist": 0.1770833314263395 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08183079000030245, + "mean": 0.030299796855875423, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14684553763696126, + "best1L": 0.11348803634090084, + "mean": 0.04016288250152554, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.12337662332824298, + "best1L": 0.0961038965199675, + "mean": 0.02437562445577766, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160714285714285, + "best1L": 0.19940476385610445, + "mean": 0.162660257092544, + "persist": 0.17857142737401382 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + } + }, + "runtime_s": 57.2 +} \ No newline at end of file diff --git a/results/ladder410/ladder_USPTO_Backgrounds_s2.json b/results/ladder410/ladder_USPTO_Backgrounds_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..dd86b5b46e8e9ac2315797598518e5082abdf77d --- /dev/null +++ b/results/ladder410/ladder_USPTO_Backgrounds_s2.json @@ -0,0 +1,463 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.11511789262294769, + "best1L": 0.08321775256523065, + "mean": 0.029873038841677562, + "persist": 0.020804438473922864 + }, + "multilingual": { + "min": 0.1530094269130911, + "best1L": 0.11602610695574965, + "mean": 0.04136219115129539, + "persist": 0.041334300568061214 + }, + "order-ops": { + "min": 0.11298701180411237, + "best1L": 0.09220779381160225, + "mean": 0.020379620529378632, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.03061224440378802, + "best1L": 0.01020408155662673, + "mean": 0.005045974494091102, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713859966825, + "best1L": 0.20089285927159445, + "mean": 0.16357600609106676, + "persist": 0.17559523481343473 + }, + "association": { + "min": 0.024844719895294735, + "best1L": 0.009316770359873772, + "mean": 0.003225035799135055, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.11789181722061974, + "best1L": 0.08183079133076328, + "mean": 0.030833244290468947, + "persist": 0.020804438473922864 + }, + "multilingual": { + "min": 0.15445975533553533, + "best1L": 0.1167512701026031, + "mean": 0.04094382728050862, + "persist": 0.04060913742120777 + }, + "order-ops": { + "min": 0.114285714126059, + "best1L": 0.0948051961937121, + "mean": 0.02207792202742504, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523241860525, + "best1L": 0.20238095388880797, + "mean": 0.16494963238281862, + "persist": 0.16964286020291702 + }, + "association": { + "min": 0.02173912977533681, + "best1L": 0.00776397529989481, + "mean": 0.002747252749811326, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11927877872117928, + "best1L": 0.08321775389569146, + "mean": 0.031260002737066576, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14720812067389488, + "best1L": 0.11203771057937827, + "mean": 0.0395213930335428, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.12337662386042732, + "best1L": 0.10000000149011612, + "mean": 0.024975024895476445, + "persist": 0.011688311185155596 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261045117995569, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.21428571215697698, + "mean": 0.16929945003773486, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.018633540453655378, + "best1L": 0.00776397529989481, + "mean": 0.00203057802614889, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.12066574155219964, + "best1L": 0.08183079212903976, + "mean": 0.030939934575664147, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14938360984836305, + "best1L": 0.11457578066204276, + "mean": 0.0402465554486428, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12207792273589543, + "best1L": 0.09870130036558424, + "mean": 0.023676323438329355, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004485310654022864, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3675595223903656, + "best1L": 0.20982143070016587, + "mean": 0.16861264168151788, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.00776397529989481, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.12343966774642467, + "best1L": 0.08321775469396796, + "mean": 0.030086418414222344, + "persist": 0.01664355051304613 + }, + "multilingual": { + "min": 0.14503263043505804, + "best1L": 0.10949963783579213, + "mean": 0.03988397414130824, + "persist": 0.0384336474484631 + }, + "order-ops": { + "min": 0.1272727298949446, + "best1L": 0.09870130089776856, + "mean": 0.02387612429447472, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.027696792302387103, + "best1L": 0.00874635563897235, + "mean": 0.0047095762565732, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36904762259551455, + "best1L": 0.20684523986918585, + "mean": 0.16540750887777125, + "persist": 0.17857142737401382 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.12343966774642467, + "best1L": 0.08321775469396796, + "mean": 0.030619864551616565, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14575779118708201, + "best1L": 0.1116751270102603, + "mean": 0.040051319636404514, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.12597402557730675, + "best1L": 0.09870130089776856, + "mean": 0.02397602434003992, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833304950169, + "best1L": 0.20982143176453455, + "mean": 0.16586538617100036, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.12343966774642467, + "best1L": 0.08321775469396796, + "mean": 0.030726554204842875, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14684553763696126, + "best1L": 0.11203771057937827, + "mean": 0.040134992849613936, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.12467532764588084, + "best1L": 0.09870130089776856, + "mean": 0.023276723488899215, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004597443422036511, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142857142855, + "best1L": 0.2083333358168602, + "mean": 0.16597985263381684, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.00776397529989481, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11789181589015893, + "best1L": 0.08183079026639462, + "mean": 0.02976635018629687, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14684554189443588, + "best1L": 0.11131254769861698, + "mean": 0.040302337014249394, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12207792113934245, + "best1L": 0.09480519645980426, + "mean": 0.02327672315628401, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713859966825, + "best1L": 0.1994047649204731, + "mean": 0.16426282109958784, + "persist": 0.18005952225731953 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11789181589015893, + "best1L": 0.08183079026639462, + "mean": 0.02976635018629687, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1461203790136746, + "best1L": 0.11058738375348705, + "mean": 0.03999553760513663, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12337662226387433, + "best1L": 0.09610389758433614, + "mean": 0.023876123429675187, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3630952366760799, + "best1L": 0.20238095255834715, + "mean": 0.16517857222684793, + "persist": 0.18005952438605682 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11789181589015893, + "best1L": 0.08183079212903976, + "mean": 0.030193108267017772, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14720812333481653, + "best1L": 0.11094996625823635, + "mean": 0.04010710126853415, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12077922001481056, + "best1L": 0.09610389758433614, + "mean": 0.023876123961859515, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3630952366760799, + "best1L": 0.20535714445369585, + "mean": 0.16472069839281694, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08183079212903976, + "mean": 0.030193107734833444, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14757070264645986, + "best1L": 0.11167512913899762, + "mean": 0.04021866486540863, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12077922001481056, + "best1L": 0.09610389758433614, + "mean": 0.02327672315628401, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004597443422036511, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333475249155, + "best1L": 0.20238095362271583, + "mean": 0.16483516432344913, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + } + }, + "runtime_s": 42.4 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Wikipedia_en_s0.json b/results/ladder410/ladder_Wikipedia_en_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..6c9e7434ba97fc215f2d76cbd10d5355c733c64c --- /dev/null +++ b/results/ladder410/ladder_Wikipedia_en_s0.json @@ -0,0 +1,659 @@ +{ + "corpus": "Wikipedia_en", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10402219103915351, + "best1L": 0.07905686620090689, + "mean": 0.0284860768089337, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1381435806729964, + "best1L": 0.11131254477160317, + "mean": 0.03402688706825886, + "persist": 0.02828136259423835 + }, + "order-ops": { + "min": 0.08181818095701081, + "best1L": 0.06753246752279145, + "mean": 0.02147852147131094, + "persist": 0.02207792177796364 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.005158107195581708, + "persist": 0.0 + }, + "typo": { + "min": 0.29910714498588015, + "best1L": 0.17559523561171123, + "mean": 0.12557234375604562, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0020305781425642116, + "persist": 0.001552795059978962 + } + }, + "50": { + "multihop": { + "min": 0.09986130467482976, + "best1L": 0.08044382903192725, + "mean": 0.028379387222230434, + "persist": 0.02219140050666673 + }, + "multilingual": { + "min": 0.13125452984656608, + "best1L": 0.10406091250479221, + "mean": 0.03344117830108319, + "persist": 0.026831036699669703 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.07012987083622388, + "mean": 0.019180819358942763, + "persist": 0.015584415889212064 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.005270239897072315, + "persist": 0.0 + }, + "typo": { + "min": 0.2991071439215115, + "best1L": 0.17857142697487557, + "mean": 0.1263736252273832, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.007763974900756564, + "best1L": 0.003105590119957924, + "mean": 0.001313903502055577, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.09847434264208589, + "best1L": 0.0832177536295993, + "mean": 0.02827269783509629, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13052936696580478, + "best1L": 0.10224800556898117, + "mean": 0.03377586816038404, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.07922077870794705, + "best1L": 0.068831169711692, + "mean": 0.017282717428835376, + "persist": 0.011688311318201678 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.00549450536657657, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3050595277122089, + "best1L": 0.17113095468708447, + "mean": 0.12328296580484935, + "persist": 0.12351190391927958 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.09986130467482976, + "best1L": 0.08044382903192725, + "mean": 0.028379387754414762, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13125452984656608, + "best1L": 0.10406091277088438, + "mean": 0.03397110563569835, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.07922077870794705, + "best1L": 0.068831169711692, + "mean": 0.018681318931547657, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.01020408155662673, + "mean": 0.005158107328627791, + "persist": 0.0 + }, + "typo": { + "min": 0.30357143389327185, + "best1L": 0.17410714445369585, + "mean": 0.12465659342706203, + "persist": 0.1279761925605791 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.028059319060828005, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.1298042062137808, + "best1L": 0.10224800530288901, + "mean": 0.03397110483742186, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.06493506500763553, + "mean": 0.01868131888165538, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952381236212595, + "best1L": 0.17857143016798155, + "mean": 0.12854853511920997, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02816600844796215, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13016678765416145, + "best1L": 0.10224800530288901, + "mean": 0.03394321385504944, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.018481518790524985, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004709576123527118, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3095238134264946, + "best1L": 0.17857143016798155, + "mean": 0.12774725072085857, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028272697569004128, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13125453197530337, + "best1L": 0.10333574989012309, + "mean": 0.03411055961623788, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.01868131888165538, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381236212595, + "best1L": 0.17708333422030723, + "mean": 0.12763278479022638, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028272698633372784, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1359680922968047, + "best1L": 0.10877447415675436, + "mean": 0.03486361414460199, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06493506500763553, + "mean": 0.01958041960772659, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31398809381893705, + "best1L": 0.17857143016798155, + "mean": 0.12831959500908852, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.02816600871405431, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13560551085642406, + "best1L": 0.10732414626649447, + "mean": 0.034752049416835816, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06493506500763553, + "mean": 0.019380619516596198, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.01020408155662673, + "mean": 0.005158107328627791, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31398809275456835, + "best1L": 0.17857143016798155, + "mean": 0.12774725391396455, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0010750119358168117, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.02816600871405431, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1345177654709135, + "best1L": 0.10659898338573319, + "mean": 0.03466837700190289, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.06753246858716011, + "mean": 0.019380619317027076, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.005270240030118397, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.17559523827263288, + "mean": 0.12751831619867257, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.02848607687545674, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13451776653528214, + "best1L": 0.10659898338573319, + "mean": 0.034612595037158043, + "persist": 0.028281362461192266 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.06753246858716011, + "mean": 0.019480519894776598, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.005270240030118397, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.17559523827263288, + "mean": 0.1276327815971204, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028272698633372784, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13705583661794662, + "best1L": 0.11022480045046125, + "mean": 0.03500306812514152, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06363636281873498, + "mean": 0.018981019001720206, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.0 + }, + "typo": { + "min": 0.3139880895614624, + "best1L": 0.18005952505128725, + "mean": 0.12774725444614887, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028486077141548907, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13524292888385908, + "best1L": 0.109137055597135, + "mean": 0.034891504461744, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06493506500763553, + "mean": 0.019280719471031, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.1785714291036129, + "mean": 0.1277472512530429, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13488034637910978, + "best1L": 0.10804931127599307, + "mean": 0.03480783204681107, + "persist": 0.029731690351452147 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.019080919063916162, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.12740384601056576, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1341551834983485, + "best1L": 0.10804931127599307, + "mean": 0.03475205061425056, + "persist": 0.029731690351452147 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.01898101901835097, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.1274038470749344, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13488034744347846, + "best1L": 0.10804931127599307, + "mean": 0.03477994053225432, + "persist": 0.029731690351452147 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.01898101901835097, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.1274038470749344, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + } + }, + "runtime_s": 68.1 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Wikipedia_en_s1.json b/results/ladder410/ladder_Wikipedia_en_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..3e3e1c33f06bb1a960d9da2d0cd899d0f4ad472c --- /dev/null +++ b/results/ladder410/ladder_Wikipedia_en_s1.json @@ -0,0 +1,659 @@ +{ + "corpus": "Wikipedia_en", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10263522900640965, + "best1L": 0.08460471752498831, + "mean": 0.0295529709463673, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.14503263149942672, + "best1L": 0.11312545223959855, + "mean": 0.03606292212914143, + "persist": 0.028643944832895483 + }, + "order-ops": { + "min": 0.07792207731732301, + "best1L": 0.06883116944559983, + "mean": 0.020779220952785442, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.008746355505926269, + "mean": 0.005045974427568061, + "persist": 0.0 + }, + "typo": { + "min": 0.3035714307001659, + "best1L": 0.1770833320915699, + "mean": 0.12637362336473806, + "persist": 0.1309523812628218 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.004658385179936886, + "mean": 0.0014333492727018893, + "persist": 0.001552795059978962 + } + }, + "50": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.029126212932169437, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13814358093908854, + "best1L": 0.10913705293621336, + "mean": 0.03522619491975222, + "persist": 0.028643944167665074 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.06623376719653606, + "mean": 0.02177822227323694, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.030612245202064514, + "best1L": 0.007288629721317973, + "mean": 0.004485310587499823, + "persist": 0.0 + }, + "typo": { + "min": 0.30059524306229185, + "best1L": 0.17559524040137017, + "mean": 0.12568681447633676, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.004658385179936886, + "mean": 0.0013139034854248166, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028486077141548907, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13778099896652357, + "best1L": 0.10659898285354887, + "mean": 0.034947286759104045, + "persist": 0.027918781552995955 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06753246858716011, + "mean": 0.020779220686693276, + "persist": 0.020779221185616085 + }, + "poetry": { + "min": 0.032069972184087546, + "best1L": 0.00728862958827189, + "mean": 0.004933841726077455, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.18005952771220887, + "mean": 0.12511446779327734, + "persist": 0.12351190538278647 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079079857894, + "mean": 0.02869945644800152, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1385061623794692, + "best1L": 0.10877447149583272, + "mean": 0.03528197735015835, + "persist": 0.02828136259423835 + }, + "order-ops": { + "min": 0.08051947983247894, + "best1L": 0.06753246858716011, + "mean": 0.02047952074956681, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.03498542375330414, + "best1L": 0.008746355505926269, + "mean": 0.005270240030118397, + "persist": 0.0 + }, + "typo": { + "min": 0.3035714328289032, + "best1L": 0.17708333422030723, + "mean": 0.1254578780915056, + "persist": 0.12351190431841783 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.028166008181869984, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13705583768231527, + "best1L": 0.1094996381018843, + "mean": 0.03522619491975222, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.068831169711692, + "mean": 0.019680319786337868, + "persist": 0.018181818404367993 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18005952398691857, + "mean": 0.12934981659054756, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0011944576565708434, + "persist": 0.001552795059978962 + } + }, + "175": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.028272697569004128, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13669325624193465, + "best1L": 0.11058738242302622, + "mean": 0.03511463165549295, + "persist": 0.028643944300711155 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.01998001992303346, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18005952398691857, + "mean": 0.12809065809207304, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.02848607660936458, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13741841805832727, + "best1L": 0.11022480098264557, + "mean": 0.035337758782718866, + "persist": 0.028643944300711155 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.02017982054634818, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18154761993459292, + "mean": 0.12820513200547015, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0011944576565708434, + "persist": 0.001552795059978962 + } + }, + "300": { + "multihop": { + "min": 0.0970873800771577, + "best1L": 0.07905686620090689, + "mean": 0.027845940020467554, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13306744077375957, + "best1L": 0.10696156482611384, + "mean": 0.03441735949101193, + "persist": 0.02936910797974893 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.06493506500763553, + "mean": 0.01958041955783431, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.31101190405232565, + "best1L": 0.18005952558347157, + "mean": 0.12786172117505754, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0008361203862088067, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13379260365452086, + "best1L": 0.10659898338573319, + "mean": 0.034501031772898774, + "persist": 0.02936910797974893 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06623376639825958, + "mean": 0.01958041960772659, + "persist": 0.015584415889212064 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381023338865, + "best1L": 0.18005952558347157, + "mean": 0.12786172277161054, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13379260152578354, + "best1L": 0.10659898338573319, + "mean": 0.034417358958827596, + "persist": 0.02828136259423835 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06623376639825958, + "mean": 0.019980020322171704, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381449086325, + "best1L": 0.18005952558347157, + "mean": 0.12751831247338227, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13488034584692546, + "best1L": 0.10696156482611384, + "mean": 0.03461259536977325, + "persist": 0.02864394403461899 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06623376639825958, + "mean": 0.02017981988111777, + "persist": 0.01948051926280771 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.007288629721317973, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.3080357164144516, + "best1L": 0.17857142963579722, + "mean": 0.12694597191044263, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.0970873800771577, + "best1L": 0.08044382796755858, + "mean": 0.027845940286559717, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1337926025901522, + "best1L": 0.10696156482611384, + "mean": 0.03447314092357244, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080918997393122, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30505952558347155, + "best1L": 0.17708333315593855, + "mean": 0.12637362868658134, + "persist": 0.1264880955485361 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.0970873800771577, + "best1L": 0.08044382796755858, + "mean": 0.027845940286559717, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1341551819017955, + "best1L": 0.10696156482611384, + "mean": 0.03458470445392387, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080919263485287, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952381023338865, + "best1L": 0.17559523827263288, + "mean": 0.12797619295971735, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.09847434290817805, + "best1L": 0.08044382796755858, + "mean": 0.027845940286559717, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13488034584692546, + "best1L": 0.10732414626649447, + "mean": 0.03461259517020413, + "persist": 0.028281362461192266 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080919263485287, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004709575990481036, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.17559523827263288, + "mean": 0.1279761895005192, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.028059319060828005, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1352429272873061, + "best1L": 0.10732414626649447, + "mean": 0.03466837660276464, + "persist": 0.028281362461192266 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080919263485287, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.17708333422030723, + "mean": 0.12751831726304122, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.028059319060828005, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1341551819017955, + "best1L": 0.10659898285354887, + "mean": 0.03461259517020413, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.06363636388310365, + "mean": 0.01928071940450796, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30505952345473425, + "best1L": 0.17708333422030723, + "mean": 0.12728937795119627, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + } + }, + "runtime_s": 74.2 +} \ No newline at end of file diff --git a/results/ladder410/ladder_Wikipedia_en_s2.json b/results/ladder410/ladder_Wikipedia_en_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..a0d258cfcc27add6809fa042d358d6ce38c460d5 --- /dev/null +++ b/results/ladder410/ladder_Wikipedia_en_s2.json @@ -0,0 +1,463 @@ +{ + "corpus": "Wikipedia_en", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cuda", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079079857894, + "mean": 0.02869945584929415, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1421319775815521, + "best1L": 0.11312545250569071, + "mean": 0.03592346774946366, + "persist": 0.031544597154217105 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06363636281873498, + "mean": 0.020879121663581048, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30059523986918585, + "best1L": 0.1800595261156559, + "mean": 0.12339743519467967, + "persist": 0.1220238082376974 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028806145302951336, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1359680893697909, + "best1L": 0.10841189165200506, + "mean": 0.03491939544411642, + "persist": 0.029369107447564602 + }, + "order-ops": { + "min": 0.08051947983247894, + "best1L": 0.07012987083622388, + "mean": 0.020079919968598654, + "persist": 0.01948051926280771 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.0 + }, + "typo": { + "min": 0.3020833336881229, + "best1L": 0.17857142803924425, + "mean": 0.1250000007982765, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "75": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.0279526294076017, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13161711155303887, + "best1L": 0.10442349474344935, + "mean": 0.03388743262205805, + "persist": 0.027556199580430984 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.018781218977112855, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.01020408155662673, + "mean": 0.005158107195581708, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3020833304950169, + "best1L": 0.17261904890515975, + "mean": 0.12396978054727827, + "persist": 0.12202380943511214 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028166008980146477, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13451776653528214, + "best1L": 0.10804931021162442, + "mean": 0.03450103163985269, + "persist": 0.02973169008535998 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.019380619250504032, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.01020408155662673, + "mean": 0.005158107195581708, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3005952388048172, + "best1L": 0.1755952350795269, + "mean": 0.12396977815244879, + "persist": 0.12202381089861904 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "150": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08044382796755858, + "mean": 0.028059318528643677, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1330674391772066, + "best1L": 0.10659898338573319, + "mean": 0.03425001412896173, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06753246858716011, + "mean": 0.019080919113808444, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.00728862958827189, + "mean": 0.00515810726210475, + "persist": 0.0 + }, + "typo": { + "min": 0.3124999978712627, + "best1L": 0.1770833320915699, + "mean": 0.12797619083098002, + "persist": 0.12351190471755606 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "175": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.028166007915777818, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.13343001848884992, + "best1L": 0.10696156482611384, + "mean": 0.034194232164216895, + "persist": 0.027556199580430984 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06753246858716011, + "mean": 0.01898101906824325, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.31101190405232565, + "best1L": 0.17559523933700152, + "mean": 0.12809065729379654, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0013139034272171557, + "persist": 0.001552795059978962 + } + }, + "200": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02837938695613827, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13379260205796786, + "best1L": 0.10659898338573319, + "mean": 0.034222122614404986, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06753246858716011, + "mean": 0.01918081915937364, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30803571215697695, + "best1L": 0.17261904637728417, + "mean": 0.12614468804427556, + "persist": 0.12202380664114441 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "300": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08044382796755858, + "mean": 0.028272697302911962, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13488034425037249, + "best1L": 0.10732414679867881, + "mean": 0.03438946810950126, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.019780219698856984, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.1785714291036129, + "mean": 0.1287774701735803, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.0279526294076017, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13560550713113376, + "best1L": 0.10804930967944008, + "mean": 0.034501032172037024, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.019480519562161396, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.1785714291036129, + "mean": 0.1279761895005192, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0008361203862088067, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02805931879473584, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13451776280999184, + "best1L": 0.10732414573431015, + "mean": 0.034389468508639505, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.019380619250504032, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.17559523827263288, + "mean": 0.1279761895005192, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02816600844796215, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13451776280999184, + "best1L": 0.10768672823905945, + "mean": 0.034333686410848586, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.01958041960772659, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004709575990481036, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.17559523827263288, + "mean": 0.12763278585459506, + "persist": 0.12351190471755606 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + } + }, + "runtime_s": 63.7 +} \ No newline at end of file diff --git a/results/ladder410_cpu/e51_interaction_cpu.json b/results/ladder410_cpu/e51_interaction_cpu.json new file mode 100644 index 0000000000000000000000000000000000000000..bde08d0361fccb557c67018e11972fec93bd7270 --- /dev/null +++ b/results/ladder410_cpu/e51_interaction_cpu.json @@ -0,0 +1,753 @@ +{ + "experiment": "E51 \u2014 the corpus x concept-set interaction decomposition, STORED", + "why": "the programme's headline interaction numbers (6.8% / 1.9% / 4614x at 410M, 9.2% / 2.2% at 1B) appear in six documents including the paper abstract and in NO results JSON; no committed script computed them. This stores the decomposition and reports whether it reproduces them.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json" + ], + "aggregation": "persist", + "n_min": 75, + "decomposition": "balanced two-factor (eval set, corpus) with seed as the replication unit, after averaging over the NESTED N axis (SPINE A1.1)", + "scales": { + "410m": { + "admitted5": { + "grand_mean": 0.041562782583838484, + "SS_total": 0.2292560600953523, + "frac_set_main": 0.9127840707082612, + "frac_corpus_main": 0.019063561844296986, + "frac_interaction": 0.06780610811206689, + "frac_residual_seed": 0.0003462593353752108, + "SS_set": 0.20926127976837341, + "SS_corpus": 0.004370437079807615, + "SS_interaction": 0.01554496119617196, + "SS_residual_seed": 7.938205099935607e-05, + "interaction_over_noise_floor": 195.82463542417236, + "candidate_noise_floors": { + "seed_x_set": 2765.8590412556186, + "seed_x_corpus": 2547.501089619223, + "seed_main": 10207.27242673298, + "three_way_residual": 235.04261880327857, + "full_two_factor_residual": 195.82463542417236, + "mean_squares_F": 611.9519857005386 + }, + "SS_seed_main": 1.522929980340243e-06, + "SS_seed_x_set": 5.620301311203121e-06, + "SS_seed_x_corpus": 6.102043001871876e-06, + "SS_three_way_residual": 6.613677670594064e-05, + "interaction_over_corpus_main": 3.5568436090735904, + "per_corpus_main": { + "Github": -0.01347872879030922, + "Pile-CC": 0.004152965700311567, + "StackExchange": 0.004111093936103677, + "USPTO_Backgrounds": 0.00817248226289137, + "Wikipedia_en": -0.002957813108997391 + }, + "per_set_main": { + "multihop": -0.026766313765710467, + "multilingual": -0.007188130629855016, + "order-ops": -0.03100069919284306, + "poetry": -0.038628048659671864, + "typo": 0.10358319224808042 + }, + "interaction_cells": { + "Github|multihop": 0.002347803893777131, + "Github|multilingual": 0.007359541421481611, + "Github|order-ops": 0.021902421601496788, + "Github|poetry": 0.014917172619105731, + "Github|typo": -0.04652693953586129, + "Pile-CC|multihop": -0.002049038819870147, + "Pile-CC|multilingual": -0.002014310009994358, + "Pile-CC|order-ops": -0.009327843746568837, + "Pile-CC|poetry": -0.005629973706823811, + "Pile-CC|typo": 0.019021166283257146, + "StackExchange|multihop": -0.005004914186737208, + "StackExchange|multilingual": 0.0005263050882960973, + "StackExchange|order-ops": -0.012137617725802928, + "StackExchange|poetry": -0.002742065627195464, + "StackExchange|typo": 0.019358292451439496, + "USPTO_Backgrounds|multihop": -0.0039624271219566715, + "USPTO_Backgrounds|multilingual": -0.0031063158001021426, + "USPTO_Backgrounds|order-ops": -0.01002845712983172, + "USPTO_Backgrounds|poetry": -0.007273936922114997, + "USPTO_Backgrounds|typo": 0.024371136974005517, + "Wikipedia_en|multihop": 0.008668576234786889, + "Wikipedia_en|multilingual": -0.0027652206996812007, + "Wikipedia_en|order-ops": 0.009591497000706682, + "Wikipedia_en|poetry": 0.0007288036370285272, + "Wikipedia_en|typo": -0.016223656172840918 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06780610811206689, + "null2_within_cell_mean": 0.02346084723443413, + "null2_max": 0.05845004448303806, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.06780610811206689, + "fires": null + }, + "C4_F_test": { + "F": 611.9519857005386, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03464578503101518, + "SS_total": 0.25078671495492033, + "frac_set_main": 0.920269897970682, + "frac_corpus_main": 0.014504644672843087, + "frac_interaction": 0.06490793528785861, + "frac_residual_seed": 0.00031752206861642384, + "SS_set": 0.23079146458396704, + "SS_corpus": 0.003637572189090703, + "SS_interaction": 0.016278047865348613, + "SS_residual_seed": 7.963031651400374e-05, + "interaction_over_noise_floor": 204.42023312171523, + "candidate_noise_floors": { + "seed_x_set": 2661.409245530769, + "seed_x_corpus": 3356.186966508041, + "seed_main": 15120.457316788837, + "three_way_residual": 240.84487579128432, + "full_two_factor_residual": 204.42023312171523, + "mean_squares_F": 613.2606993651457 + }, + "SS_seed_main": 1.0765579059090003e-06, + "SS_seed_x_set": 6.1163264885638645e-06, + "SS_seed_x_corpus": 4.850161218010204e-06, + "SS_three_way_residual": 6.758727090152059e-05, + "interaction_over_corpus_main": 4.474975895782207, + "per_corpus_main": { + "Github": -0.011242406869740797, + "Pile-CC": 0.0034506718724431965, + "StackExchange": 0.003415778735603288, + "USPTO_Backgrounds": 0.006800269007926366, + "Wikipedia_en": -0.002424312746232067 + }, + "per_set_main": { + "multihop": -0.019849316212887165, + "multilingual": -0.00027113307703171413, + "order-ops": -0.02408370164001976, + "poetry": -0.03171105110684856, + "typo": 0.11050018980090373, + "association": -0.03458498776411653 + }, + "interaction_cells": { + "Github|multihop": 0.00011148197320871162, + "Github|multilingual": 0.005123219500913184, + "Github|order-ops": 0.01966609968092836, + "Github|poetry": 0.012680850698537312, + "Github|typo": -0.048763261456429716, + "Github|association": 0.011181609602842146, + "Pile-CC|multihop": -0.0013467449920017763, + "Pile-CC|multilingual": -0.001312016182125987, + "Pile-CC|order-ops": -0.008625549918700466, + "Pile-CC|poetry": -0.0049276798789554405, + "Pile-CC|typo": 0.01972346011112551, + "Pile-CC|association": -0.003511469139341847, + "StackExchange|multihop": -0.004309598986236819, + "StackExchange|multilingual": 0.0012216202887964864, + "StackExchange|order-ops": -0.011442302525302542, + "StackExchange|poetry": -0.002046750426695075, + "StackExchange|typo": 0.020053607651939878, + "StackExchange|association": -0.003476576002501938, + "USPTO_Backgrounds|multihop": -0.00259021386699166, + "USPTO_Backgrounds|multilingual": -0.001734102545137138, + "USPTO_Backgrounds|order-ops": -0.008656243874866708, + "USPTO_Backgrounds|poetry": -0.005901723667149993, + "USPTO_Backgrounds|typo": 0.025743350228970514, + "USPTO_Backgrounds|association": -0.006861066274825016, + "Wikipedia_en|multihop": 0.008135075872021572, + "Wikipedia_en|multilingual": -0.0032987210624465246, + "Wikipedia_en|order-ops": 0.009057996637941358, + "Wikipedia_en|poetry": 0.00019530327426320326, + "Wikipedia_en|typo": -0.01675715653560625, + "Wikipedia_en|association": 0.0026675018138266546 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06490793528785861, + "null2_within_cell_mean": 0.022186166756653344, + "null2_max": 0.06209373231233767, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.06490793528785861, + "fires": null + }, + "C4_F_test": { + "F": 613.2606993651457, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + }, + "1b": { + "admitted5": { + "grand_mean": 0.04294701039968502, + "SS_total": 0.5877733083330603, + "frac_set_main": 0.8857950497345805, + "frac_corpus_main": 0.022117202251311613, + "frac_interaction": 0.09158325214755474, + "frac_residual_seed": 0.0005044958665530746, + "SS_set": 0.520646686887542, + "SS_corpus": 0.012999901138324835, + "SS_interaction": 0.053830191102669095, + "SS_residual_seed": 0.00029652920452425476, + "interaction_over_noise_floor": 181.5341972438537, + "candidate_noise_floors": { + "seed_x_set": 701.2113462326924, + "seed_x_corpus": 1300.55822013104, + "seed_main": 3666.99058665031, + "three_way_residual": 328.85037669047756, + "full_two_factor_residual": 181.5341972438537, + "mean_squares_F": 567.2943663870428 + }, + "SS_seed_main": 1.4679664381642555e-05, + "SS_seed_x_set": 7.676742738387734e-05, + "SS_seed_x_corpus": 4.139006641105643e-05, + "SS_three_way_residual": 0.00016369204634767822, + "interaction_over_corpus_main": 4.1408154208167804, + "per_corpus_main": { + "Github": -0.015690614938115084, + "Pile-CC": 0.0030669355334802736, + "StackExchange": 0.015805140123480846, + "USPTO_Backgrounds": 0.011754592504973217, + "Wikipedia_en": -0.014936053223819253 + }, + "per_set_main": { + "multihop": -0.04065852216755351, + "multilingual": -0.04060835923999548, + "order-ops": -0.04294701039968502, + "poetry": -0.04241251089654508, + "typo": 0.1666264027037791 + }, + "interaction_cells": { + "Github|multihop": 0.01617605183583995, + "Github|multilingual": 0.01740079140379315, + "Github|order-ops": 0.015690614938115084, + "Github|poetry": 0.015399069754584209, + "Github|typo": -0.06466652793233241, + "Pile-CC|multihop": -0.005239843551867775, + "Pile-CC|multilingual": -0.003502033432679519, + "Pile-CC|order-ops": -0.0030669355334802736, + "Pile-CC|poetry": -0.003601435036620214, + "Pile-CC|typo": 0.01541024755464776, + "StackExchange|multihop": -0.01531970322575598, + "StackExchange|multilingual": -0.015847441318134474, + "StackExchange|order-ops": -0.015805140123480846, + "StackExchange|poetry": -0.016339639626620786, + "StackExchange|typo": 0.06331192429399204, + "USPTO_Backgrounds|multihop": -0.009882193042320159, + "USPTO_Backgrounds|multilingual": -0.012975283813263697, + "USPTO_Backgrounds|order-ops": -0.011754592504973217, + "USPTO_Backgrounds|poetry": -0.012289092008113157, + "USPTO_Backgrounds|typo": 0.046901161368670195, + "Wikipedia_en|multihop": 0.014265687984103964, + "Wikipedia_en|multilingual": 0.014923967160284518, + "Wikipedia_en|order-ops": 0.014936053223819253, + "Wikipedia_en|poetry": 0.01683109691676994, + "Wikipedia_en|typo": -0.06095680528497768 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.09158325214755474, + "null2_within_cell_mean": 0.029931755338755018, + "null2_max": 0.08481827845754085, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.09158325214755474, + "fires": null + }, + "C4_F_test": { + "F": 567.2943663870428, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03645773986613171, + "SS_total": 0.6067938204847383, + "frac_set_main": 0.8892583495369545, + "frac_corpus_main": 0.018617391676965456, + "frac_interaction": 0.0916251460361479, + "frac_residual_seed": 0.000499112749932075, + "SS_set": 0.5395964713134814, + "SS_corpus": 0.011296918223126638, + "SS_interaction": 0.055597572415746255, + "SS_residual_seed": 0.0003028585323839276, + "interaction_over_noise_floor": 183.5760477940451, + "candidate_noise_floors": { + "seed_x_set": 696.3305263109522, + "seed_x_corpus": 1715.3652591502535, + "seed_main": 4592.651780597178, + "three_way_residual": 311.4751704961482, + "full_two_factor_residual": 183.5760477940451, + "mean_squares_F": 550.7281433821353 + }, + "SS_seed_main": 1.2105767010386526e-05, + "SS_seed_x_set": 7.984365228147229e-05, + "SS_seed_x_corpus": 3.241150659847677e-05, + "SS_three_way_residual": 0.00017849760649359299, + "interaction_over_corpus_main": 4.921481356032917, + "per_corpus_main": { + "Github": -0.013744076981490095, + "Pile-CC": 0.002771445592894911, + "StackExchange": 0.013257216481447578, + "USPTO_Backgrounds": 0.01009742612987461, + "Wikipedia_en": -0.012382011222727005 + }, + "per_set_main": { + "multihop": -0.0341692516340002, + "multilingual": -0.034119088706442174, + "order-ops": -0.03645773986613171, + "poetry": -0.03592324036299177, + "typo": 0.1731156732373324, + "association": -0.03244635266776655 + }, + "interaction_cells": { + "Github|multihop": 0.014229513879214964, + "Github|multilingual": 0.015454253447168165, + "Github|order-ops": 0.013744076981490091, + "Github|poetry": 0.013452531797959216, + "Github|typo": -0.06661306588895739, + "Github|association": 0.009732689783124929, + "Pile-CC|multihop": -0.004944353611282412, + "Pile-CC|multilingual": -0.0032065434920941494, + "Pile-CC|order-ops": -0.002771445592894911, + "Pile-CC|poetry": -0.0033059450960348513, + "Pile-CC|typo": 0.01570573749523313, + "Pile-CC|association": -0.0014774497029268058, + "StackExchange|multihop": -0.012771779583722713, + "StackExchange|multilingual": -0.0132995176761012, + "StackExchange|order-ops": -0.013257216481447578, + "StackExchange|poetry": -0.013791715984587519, + "StackExchange|typo": 0.06585984793602533, + "StackExchange|association": -0.012739618210166344, + "USPTO_Backgrounds|multihop": -0.008225026667221552, + "USPTO_Backgrounds|multilingual": -0.011318117438165083, + "USPTO_Backgrounds|order-ops": -0.01009742612987461, + "USPTO_Backgrounds|poetry": -0.010631925633014551, + "USPTO_Backgrounds|typo": 0.04855832774376881, + "USPTO_Backgrounds|association": -0.008285831875493012, + "Wikipedia_en|multihop": 0.01171164598301172, + "Wikipedia_en|multilingual": 0.012369925159192274, + "Wikipedia_en|order-ops": 0.012382011222727002, + "Wikipedia_en|poetry": 0.014277054915677698, + "Wikipedia_en|typo": -0.06351084728606993, + "Wikipedia_en|association": 0.012770210005461219 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.0916251460361479, + "null2_within_cell_mean": 0.030780725304871703, + "null2_max": 0.08516371461203105, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.0916251460361479, + "fires": null + }, + "C4_F_test": { + "F": 550.7281433821353, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + } + }, + "ladder1b_dir": "results/ladder1b", + "ladder410_dir": "results/ladder410_cpu", + "reproduces_the_prose": { + "410m": { + "claimed": { + "interaction": 6.8, + "corpus_main": 1.9, + "set_main": 91.3, + "noise_floor": 4614 + }, + "recomputed": { + "interaction": 6.780610811206689, + "corpus_main": 1.9063561844296986, + "set_main": 91.27840707082612, + "noise_floor": 195.82463542417236 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + }, + "1b": { + "claimed": { + "interaction": 9.2, + "corpus_main": 2.2, + "set_main": 88.6 + }, + "recomputed": { + "interaction": 9.158325214755473, + "corpus_main": 2.2117202251311614, + "set_main": 88.57950497345804, + "noise_floor": 181.5341972438537 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + } + }, + "noise_floor_4614_audit": { + "claimed": 4614, + "claimed_where": "PLAN.tex:52 and :270, HANDOFF.md:63 \u2014 described as 'the seed x set noise floor'. In no results file.", + "candidates_computed": { + "seed_x_set": 2765.8590412556186, + "seed_x_corpus": 2547.501089619223, + "seed_main": 10207.27242673298, + "three_way_residual": 235.04261880327857, + "full_two_factor_residual": 195.82463542417236, + "mean_squares_F": 611.9519857005386 + }, + "any_candidate_within_2pct": {}, + "reproduces": false, + "reading": "The six VARIANCE FRACTIONS reproduce exactly at both scales, so the decomposition specified in this file is the one that was used. The 4614x figure does NOT follow from it under any of six natural denominators. Cite a named, stored ratio from candidate_noise_floors instead, or drop the multiplier \u2014 do not restate 4614x." + }, + "VERDICT": "REPRODUCES \u2014 the headline interaction numbers now have a results file and match the prose to within 0.1 percentage points. || BUT the '4614x noise floor' does NOT reproduce under any natural denominator (nearest candidates: seed x set 2768x, seed x corpus 2548x, seed main 10268x, three-way residual 235x). The multiplier is unsupported and must not be restated until the operator names the intended denominator.", + "provenance": { + "schema_version": 1, + "experiment": "E51", + "utc": "2026-08-20T09:00:58+00:00", + "script": { + "path": "experiments/t51_interaction_variance.py", + "sha256": "66832a320192ddaaf2a5f0a550bf563078666cfd11c6bb64f3e1e0c5891ed7d4", + "n_lines": 327 + }, + "git": { + "commit": "b197ac445bf89d5b778d5e16ce9d1c325c09473e", + "commit_short": "b197ac445bf8", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/ladder410_cpu/", + "results/r3_close_d2.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "aee92baa8be53162e66b7c00eec04f9286b40c2afaf4dbb2dbe8c1782597cc28", + "describe": "b197ac4-dirty" + }, + "argv": [ + "experiments/t51_interaction_variance.py", + "--ladder410-dir", + "results/ladder410_cpu", + "--out", + "results/ladder410_cpu/e51_interaction_cpu.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410_cpu/ladder_Github_s0.json", + "sha256": "4a6e725faf9298317522b45e07e59cccf642fce50a8b5e00c8f42ac8babf782c", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Github_s1.json", + "sha256": "800a48a767c095a5643413d8693e0a94e0b871915df8445b75f2df552f7f648d", + "bytes": 15765, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Github_s2.json", + "sha256": "2efceaa5838a92c5707600a2d680d0411d485064a816c61f4536f0364a4cb1c4", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Pile-CC_s0.json", + "sha256": "4486882a4bf78c6f75e66c8c7e17b56ba23b746da708a09474fe18e9faf4d82d", + "bytes": 10906, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Pile-CC_s1.json", + "sha256": "f288e7b11b2ab2837d713239c3644568e4071ea1ebd326baf276ab9acf2c89b9", + "bytes": 10918, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Pile-CC_s2.json", + "sha256": "766eda99f7f2ea462a73ad816ad44029508d8ea7b6279561bdd033638fc70984", + "bytes": 10909, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_StackExchange_s0.json", + "sha256": "3922aed1bba3b34bc29cbd8e12866108980417e97c02a16ba9d70d63bb1946c5", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_StackExchange_s1.json", + "sha256": "09a2c3a17459e3abc354f0774267ea3747782657616c8f9f5dec1de889f9012c", + "bytes": 15791, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_StackExchange_s2.json", + "sha256": "69c734ca706ba24daabe14ada761252363afbc676b1948cd03d09ad6d2509002", + "bytes": 15781, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_USPTO_Backgrounds_s0.json", + "sha256": "dca4188fad0ee8ad338f3b0b21e493c2bb4de41f0257eb3c65c3fbddd31a4bae", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_USPTO_Backgrounds_s1.json", + "sha256": "2dd8d1c35a0f4f8ce4425f28997662fdf166344de413b51c51da02786b3a2f89", + "bytes": 10900, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_USPTO_Backgrounds_s2.json", + "sha256": "4c27aca6932a25a0ec89ce00fca4709fa00ada0f4360fdde2f19cc0039e1430b", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Wikipedia_en_s0.json", + "sha256": "df4f2d634f7a662598cd49999b5a355203684b961c36ea78ff679ca20891a7de", + "bytes": 15637, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Wikipedia_en_s1.json", + "sha256": "d8b27dbb61408fa4b77a5dfc9f4303bd951b958797566eec40e0f66fdf06537f", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Wikipedia_en_s2.json", + "sha256": "97fde88911ffbc01634df8d0df6e59357efa5e09a4ff2f9ffac3e7d01d3ddd20", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "e22764e0521f0412551dce76560830ff59f177b5a3b26e88b85bc8611ee5d38f" + } +} \ No newline at end of file diff --git a/results/ladder410_cpu/e53_ladder_summary_cpu.json b/results/ladder410_cpu/e53_ladder_summary_cpu.json new file mode 100644 index 0000000000000000000000000000000000000000..8716be62ef25500a24d4b3e5fee4f3cda295b0a8 --- /dev/null +++ b/results/ladder410_cpu/e53_ladder_summary_cpu.json @@ -0,0 +1,1406 @@ +{ + "experiment": "E53 \u2014 the ladder numbers the paper states and no file contains", + "why": "EXPERIMENT_VALIDITY_AUDIT v1 section 5 specified this script and it was never written (lab health H10 flagged it); v2 section 3.1 then found the paper's N-ladder sentence is contradicted by the file it cites.", + "recomputes_not_remeasures": true, + "ladder410_dir": "results/ladder410_cpu", + "inputs": [ + "results/ladder410_cpu/*.json", + "results/ladder1b/*.json", + "results/e28_*_410m_n*_s*.pt (filenames only)", + "results/e36_qladder_410m.json", + "results/e54_aggregation_audit.json" + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "asymptote_n_min": 75, + "grid_shape": { + "per_cell": { + "Github|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Github|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Github|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Wikipedia_en|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Wikipedia_en|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Wikipedia_en|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "StackExchange|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "StackExchange|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "StackExchange|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "n_fitted_lenses": 6, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_reconstructed": 16, + "max_N": 800, + "reaches_800": true, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Pile-CC|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Pile-CC|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "Pile-CC|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "USPTO_Backgrounds|s0": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "USPTO_Backgrounds|s1": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + }, + "USPTO_Backgrounds|s2": { + "fitted_blocks": [ + 25, + 50, + 100, + 200, + 400 + ], + "n_fitted_lenses": 5, + "reconstructed_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_reconstructed": 11, + "max_N": 400, + "reaches_800": false, + "C2_reconstruction_matches_fitted_blocks": true + } + }, + "n_cells": 15, + "n_cells_reaching_N800": 8, + "n_cells_stopping_at_N400": 7, + "total_fitted_lenses_410m": 83, + "C2_every_reconstruction_matches_its_fitted_blocks": true, + "PAPER_SAYS": "5 corpora x 3 seed blocks x 16 values of N in [25,800]", + "WHAT_IS_ON_DISK": "83 fitted lenses across 15 corpus-seed cells; 8 cells reach N=800 with 16 reconstructed values, 7 stop at N=400 with 11. The N values are consecutive-run sums of disjoint fitted blocks (e28_ladder_410m.py reconstruct()), not independent fits." + }, + "by_scale": { + "410m": { + "persist": { + "asymptote": { + "Github": 0.028084053793529264, + "Wikipedia_en": 0.03860496947484109, + "StackExchange": 0.04567387651994216, + "Pile-CC": 0.04571574828415005, + "USPTO_Backgrounds": 0.049735264846729854 + }, + "seed_sd": { + "Github": 0.0003725015802729375, + "Wikipedia_en": 0.0004975124529386809, + "StackExchange": 0.0002996455447132135, + "Pile-CC": 0.00018354240934833726, + "USPTO_Backgrounds": 0.0005027384624140496 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "StackExchange": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Pile-CC": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ] + }, + "asymptote_grids_identical_across_corpora": false, + "spread": 0.02165121105320059, + "pooled_seed_sd": 0.0003711880899374438, + "pooled_seed_sd_rms": 0.00039051179194006747, + "spread_over_seed_sd": 58.32948750281687 + }, + "min": { + "asymptote": { + "Github": 0.11288843376397276, + "Wikipedia_en": 0.13042415689524947, + "StackExchange": 0.14538949898881268, + "Pile-CC": 0.1449443925970367, + "USPTO_Backgrounds": 0.15532431357377577 + }, + "seed_sd": { + "Github": 0.00034988870729241053, + "Wikipedia_en": 0.00011214238425683794, + "StackExchange": 0.0012802615956001497, + "Pile-CC": 0.00042848223307768936, + "USPTO_Backgrounds": 0.0016506736873259887 + }, + "order_best_to_worst": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "StackExchange": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "Pile-CC": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ] + }, + "asymptote_grids_identical_across_corpora": false, + "spread": 0.04243587980980301, + "pooled_seed_sd": 0.0007642897215106153, + "pooled_seed_sd_rms": 0.0009677177709538896, + "spread_over_seed_sd": 55.52328994550481 + } + }, + "1b": { + "persist": { + "asymptote": { + "Github": 0.027256395461569938, + "Wikipedia_en": 0.02801095717586577, + "StackExchange": 0.05875215052316586, + "Pile-CC": 0.046013945933165296, + "USPTO_Backgrounds": 0.05470160290465824 + }, + "seed_sd": { + "Github": 0.0013762403288060878, + "Wikipedia_en": 0.00020285804501715254, + "StackExchange": 0.0007357119491078886, + "Pile-CC": 0.00039438237251602294, + "USPTO_Backgrounds": 0.0017248115034397009 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 200 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 200 + ], + "StackExchange": [ + 75, + 100, + 150, + 200 + ], + "Pile-CC": [ + 75, + 100, + 150, + 200 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 200 + ] + }, + "asymptote_grids_identical_across_corpora": true, + "spread": 0.03149575506159592, + "pooled_seed_sd": 0.0008868008397773706, + "pooled_seed_sd_rms": 0.0010589592134987906, + "spread_over_seed_sd": 35.51615385197748 + }, + "min": { + "asymptote": { + "Github": 0.13184866054604452, + "Wikipedia_en": 0.12168779909788144, + "StackExchange": 0.1760720702420388, + "Pile-CC": 0.14951401859566216, + "USPTO_Backgrounds": 0.16125627207968915 + }, + "seed_sd": { + "Github": 0.0011745417158351128, + "Wikipedia_en": 0.0006643603940748279, + "StackExchange": 0.000781853055555403, + "Pile-CC": 0.0016589081835269132, + "USPTO_Backgrounds": 0.0026333882425492983 + }, + "order_best_to_worst": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "asymptote_N_grid_per_corpus": { + "Github": [ + 75, + 100, + 150, + 200 + ], + "Wikipedia_en": [ + 75, + 100, + 150, + 200 + ], + "StackExchange": [ + 75, + 100, + 150, + 200 + ], + "Pile-CC": [ + 75, + 100, + 150, + 200 + ], + "USPTO_Backgrounds": [ + 75, + 100, + 150, + 200 + ] + }, + "asymptote_grids_identical_across_corpora": true, + "spread": 0.054384271144157365, + "pooled_seed_sd": 0.001382610318308311, + "pooled_seed_sd_rms": 0.001556851127969718, + "spread_over_seed_sd": 39.33448956948266 + } + } + }, + "cross_scale": { + "persist": { + "order_410m": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "order_1b": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "spearman_of_asymptotes": 0.7, + "n": 5, + "worst_corpus_410m": "Github", + "worst_corpus_1b": "Github", + "same_worst_corpus": true, + "note": "n=5, so rho=+0.7 is p~0.19 and rho=+0.8 is p~0.10. 'The corpus effect replicates at 1B' is a claim that corpora DIFFER at both scales, which holds; it is not a claim that the SAME corpora are good, which this does not establish." + }, + "min": { + "order_410m": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "order_1b": [ + "StackExchange", + "USPTO_Backgrounds", + "Pile-CC", + "Github", + "Wikipedia_en" + ], + "spearman_of_asymptotes": 0.8, + "n": 5, + "worst_corpus_410m": "Github", + "worst_corpus_1b": "Wikipedia_en", + "same_worst_corpus": false, + "note": "n=5, so rho=+0.7 is p~0.19 and rho=+0.8 is p~0.10. 'The corpus effect replicates at 1B' is a claim that corpora DIFFER at both scales, which holds; it is not a claim that the SAME corpora are good, which this does not establish." + } + }, + "bands_declared_vs_used": { + "410m": { + "n_layers": 24, + "L38_L92": [ + 9, + 22 + ], + "target_layer_index": 22, + "declared_by_the_papers_own_rule": [ + 9, + 21 + ], + "used_in_the_ladder": [ + 9, + 21 + ], + "matches": true + }, + "1b": { + "n_layers": 16, + "L38_L92": [ + 6, + 15 + ], + "target_layer_index": 14, + "declared_by_the_papers_own_rule": [ + 6, + 13 + ], + "used_in_the_ladder": [ + 5, + 13 + ], + "matches": false + } + }, + "e36_containment_slopes": { + "persist": { + "logit": { + "slope_linear_containment": -0.003720899629554985, + "slope_log10_containment": -0.0005296062512720639, + "spearman": -0.2727272727272727, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.005342012891958096, + "StackExchange": -0.0033105996835403307, + "Wikipedia_en": -0.004159231116162598, + "USPTO_Backgrounds": -0.0034842593960081234, + "Github": -0.002234113859375035, + "CONTROL_PubMed_2023": -0.0037184028182850903, + "OOD_Wikipedia_2023": -0.0036969276410433907, + "TRAP_FineWeb": -0.002840794828560368, + "OOD_News_2024": -0.002472647958510993, + "OOD_arXiv_2023": -0.00535538400064562, + "OOD_CommonPile": -0.004356052515853489 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0008318596283442356, + "StackExchange": -0.000389681627275286, + "Wikipedia_en": -0.0005771961173907728, + "USPTO_Backgrounds": -0.00042768514003331125, + "Github": -0.00015929190525716992, + "CONTROL_PubMed_2023": -0.0005051741014559277, + "OOD_Wikipedia_2023": -0.0005653796365463086, + "TRAP_FineWeb": -0.0005079235074284086, + "OOD_News_2024": 9.839815919873245e-06, + "OOD_arXiv_2023": -0.0015902638125509932, + "OOD_CommonPile": -0.000617386511644501 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 1 + }, + "J|Github": { + "slope_linear_containment": -0.0014550596366756292, + "slope_log10_containment": -1.0939804428585397e-06, + "spearman": -0.19090909090909092, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.001561401546835692, + "StackExchange": -0.0005760183119989624, + "Wikipedia_en": -0.001992270380004623, + "USPTO_Backgrounds": -0.0017519561244663739, + "Github": -0.0008909328785794681, + "CONTROL_PubMed_2023": -0.0016397555863833476, + "OOD_Wikipedia_2023": -0.0009673012084557685, + "TRAP_FineWeb": -0.0011688572755800576, + "OOD_News_2024": -0.0014455593119102068, + "OOD_arXiv_2023": -0.0019401073419784616, + "OOD_CommonPile": -0.0021622055526610198 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 7.386689134070697e-06, + "StackExchange": 0.00022555885965485492, + "Wikipedia_en": -8.869649789138863e-05, + "USPTO_Backgrounds": -3.429243288848448e-05, + "Github": 0.0001519590333369196, + "CONTROL_PubMed_2023": -4.069669530389137e-05, + "OOD_Wikipedia_2023": -9.637614420813418e-05, + "TRAP_FineWeb": 6.554606456226265e-06, + "OOD_News_2024": 9.277410192215612e-05, + "OOD_arXiv_2023": -0.00015958374768649015, + "OOD_CommonPile": -0.00013358642358098752 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 5 + }, + "J|Wikipedia_en": { + "slope_linear_containment": -0.0009354827291769033, + "slope_log10_containment": 0.0002869798762885821, + "spearman": 0.06363636363636363, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0016945220408105522, + "StackExchange": 0.0007790027011743094, + "Wikipedia_en": -0.001943307383106595, + "USPTO_Backgrounds": -0.0014773995230767255, + "Github": 0.0006114770190120221, + "CONTROL_PubMed_2023": -0.001345574896548144, + "OOD_Wikipedia_2023": -0.00013865249525168055, + "TRAP_FineWeb": -0.0008008900076050542, + "OOD_News_2024": -0.00043233948548224637, + "OOD_arXiv_2023": -0.0017250513630531405, + "OOD_CommonPile": -0.00225182718285068 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0001587231312785397, + "StackExchange": 0.0007053078107235931, + "Wikipedia_en": 0.00010082406161996977, + "USPTO_Backgrounds": 0.00020565756093412292, + "Github": 0.0006557206071300077, + "CONTROL_PubMed_2023": 0.00018259944505365897, + "OOD_Wikipedia_2023": 0.00014469565594808258, + "TRAP_FineWeb": 0.00029123958467279065, + "OOD_News_2024": 0.000590124287887275, + "OOD_arXiv_2023": 5.458070632956026e-05, + "OOD_CommonPile": -6.767144880380829e-07 + }, + "n_loo_sign_flips_linear": 2, + "n_loo_sign_flips_log10": 1 + }, + "J|StackExchange": { + "slope_linear_containment": 0.0002240976423130857, + "slope_log10_containment": 0.00043591080099479097, + "spearman": 0.2818181818181818, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": 9.734162192914718e-05, + "StackExchange": 0.0013572724997980054, + "Wikipedia_en": -0.00024598000465200266, + "USPTO_Backgrounds": -0.00032751857982107153, + "Github": 0.000645747151939162, + "CONTROL_PubMed_2023": 0.00014738936959018587, + "OOD_Wikipedia_2023": 0.0008552523402857944, + "TRAP_FineWeb": 0.00026812738983809003, + "OOD_News_2024": 0.00010939484488637846, + "OOD_arXiv_2023": -0.0003280293027039663, + "OOD_CommonPile": -0.00022378417326704018 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0004285719021829549, + "StackExchange": 0.0007086432089750901, + "Wikipedia_en": 0.0003522684629005442, + "USPTO_Backgrounds": 0.00033586356876891434, + "Github": 0.0005465238526106352, + "CONTROL_PubMed_2023": 0.00043017554862703327, + "OOD_Wikipedia_2023": 0.0003323733409809342, + "TRAP_FineWeb": 0.00043633093680939597, + "OOD_News_2024": 0.0004872682870979624, + "OOD_arXiv_2023": 0.00033084731233546635, + "OOD_CommonPile": 0.0003710199870615957 + }, + "n_loo_sign_flips_linear": 4, + "n_loo_sign_flips_log10": 0 + }, + "J|Pile-CC": { + "slope_linear_containment": -0.0011535435993366304, + "slope_log10_containment": 0.00013903741903277346, + "spearman": -0.00909090909090909, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0018561479020940102, + "StackExchange": 0.0003276404302305153, + "Wikipedia_en": -0.0022811577240861203, + "USPTO_Backgrounds": -0.0015056028936971267, + "Github": 0.00014389080343519365, + "CONTROL_PubMed_2023": -0.0013506750771647544, + "OOD_Wikipedia_2023": -0.0005025777305976444, + "TRAP_FineWeb": -0.0010021430954018857, + "OOD_News_2024": -0.0007940013046808708, + "OOD_arXiv_2023": -0.0018366231525598165, + "OOD_CommonPile": -0.0021429228225167247 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 1.9282762995631993e-05, + "StackExchange": 0.000501527682090862, + "Wikipedia_en": -7.739278687463401e-05, + "USPTO_Backgrounds": 9.533123564495353e-05, + "Github": 0.00045059330345505465, + "CONTROL_PubMed_2023": 9.841175561313877e-05, + "OOD_Wikipedia_2023": 1.894809683383725e-05, + "TRAP_FineWeb": 0.0001439618487261618, + "OOD_News_2024": 0.00036998076199487877, + "OOD_arXiv_2023": -9.674744982390889e-05, + "OOD_CommonPile": -6.733770272352924e-05 + }, + "n_loo_sign_flips_linear": 2, + "n_loo_sign_flips_log10": 3 + }, + "J|USPTO_Backgrounds": { + "slope_linear_containment": -0.000885040945743085, + "slope_log10_containment": 0.0002350879878021014, + "spearman": 0.20909090909090908, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0014123147101777196, + "StackExchange": 0.00024297344343503642, + "Wikipedia_en": -0.001286481013089218, + "USPTO_Backgrounds": -0.0014672707299009076, + "Github": 7.746754680233447e-05, + "CONTROL_PubMed_2023": -0.0010450754171928793, + "OOD_Wikipedia_2023": -0.00014922575853216864, + "TRAP_FineWeb": -0.0007833716787412385, + "OOD_News_2024": -0.0010267371441971883, + "OOD_arXiv_2023": -0.0015151819788109704, + "OOD_CommonPile": -0.0014965239401716014 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.00015317413921890838, + "StackExchange": 0.0005186653686300515, + "Wikipedia_en": 0.0001788193096829291, + "USPTO_Backgrounds": 0.00014089125945786622, + "Github": 0.00047451609661586626, + "CONTROL_PubMed_2023": 0.00020754413654669994, + "OOD_Wikipedia_2023": 0.00010314967326905822, + "TRAP_FineWeb": 0.0002386930570768142, + "OOD_News_2024": 0.0002977371626300231, + "OOD_arXiv_2023": 7.147629354977188e-05, + "OOD_CommonPile": 0.00014381364955460372 + }, + "n_loo_sign_flips_linear": 2, + "n_loo_sign_flips_log10": 0 + }, + "_summary": { + "n_rungs": 11, + "max_abs_spearman": 0.2818181818181818, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_linear": 3, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_log10": 1 + } + }, + "min": { + "logit": { + "slope_linear_containment": -0.0010699323813758122, + "slope_log10_containment": 0.0009282505416221876, + "spearman": 0.00909090909090909, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0031277114593503412, + "StackExchange": -0.0005606867995279083, + "Wikipedia_en": -0.0018411056179029237, + "USPTO_Backgrounds": -0.00022972811415334008, + "Github": 0.00048355835055198017, + "CONTROL_PubMed_2023": -0.0005893318825439976, + "OOD_Wikipedia_2023": 0.00013717442831089735, + "TRAP_FineWeb": 0.0009480040767784513, + "OOD_News_2024": -0.0011114299140829592, + "OOD_arXiv_2023": -0.002789304049721013, + "OOD_CommonPile": -0.0033985900411396585 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0005572752682302916, + "StackExchange": 0.001117413951950436, + "Wikipedia_en": 0.0008339949157585983, + "USPTO_Backgrounds": 0.0011881567350198682, + "Github": 0.0013385498123661536, + "CONTROL_PubMed_2023": 0.0011432343029709941, + "OOD_Wikipedia_2023": 0.0007217489988221277, + "TRAP_FineWeb": 0.0009630678891223896, + "OOD_News_2024": 0.0012048674710092267, + "OOD_arXiv_2023": 0.0005171717376496735, + "OOD_CommonPile": 0.00046510911732908856 + }, + "n_loo_sign_flips_linear": 3, + "n_loo_sign_flips_log10": 0 + }, + "J|Github": { + "slope_linear_containment": -0.0035714446311215964, + "slope_log10_containment": -0.00029611400443362804, + "spearman": -0.09090909090909091, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0052687368331600405, + "StackExchange": -0.001960639289563647, + "Wikipedia_en": -0.0034644178667903418, + "USPTO_Backgrounds": -0.0048044144955547655, + "Github": -0.002052368335791041, + "CONTROL_PubMed_2023": -0.0035556823516738945, + "OOD_Wikipedia_2023": -0.002989236397777608, + "TRAP_FineWeb": -0.0026493610346831294, + "OOD_News_2024": -0.004848794207343968, + "OOD_arXiv_2023": -0.004352390461459243, + "OOD_CommonPile": -0.00348374819833562 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0006047413787830261, + "StackExchange": 0.00012156774546108781, + "Wikipedia_en": -0.0002130582540511553, + "USPTO_Backgrounds": -0.0005044491607809716, + "Github": 9.133692692012993e-05, + "CONTROL_PubMed_2023": -0.0002577355308012017, + "OOD_Wikipedia_2023": -0.00042586913211386726, + "TRAP_FineWeb": -0.00027397588588984654, + "OOD_News_2024": -0.0005900652750617479, + "OOD_arXiv_2023": -0.0006067527223211215, + "OOD_CommonPile": -0.00013541611648333051 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 2 + }, + "J|Wikipedia_en": { + "slope_linear_containment": -0.004603344091202535, + "slope_log10_containment": -0.000310141050106287, + "spearman": 0.05454545454545454, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.007037656959177343, + "StackExchange": -0.00033718111815656036, + "Wikipedia_en": -0.005967997333887068, + "USPTO_Backgrounds": -0.005772717284704184, + "Github": -0.0016207231641492746, + "CONTROL_PubMed_2023": -0.005800630286563434, + "OOD_Wikipedia_2023": -0.003278371513152543, + "TRAP_FineWeb": -0.003957832832162028, + "OOD_News_2024": -0.004794875269979331, + "OOD_arXiv_2023": -0.00468132498800471, + "OOD_CommonPile": -0.007606565142089651 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0007574823472158516, + "StackExchange": 0.0007204277251684816, + "Wikipedia_en": -0.0005308878981940796, + "USPTO_Backgrounds": -0.00048300317363772916, + "Github": 0.0004128726742000434, + "CONTROL_PubMed_2023": -0.0006472133229897882, + "OOD_Wikipedia_2023": -0.0005774356935193184, + "TRAP_FineWeb": -0.00029021934148038225, + "OOD_News_2024": -0.00016452947655907251, + "OOD_arXiv_2023": -4.779378759709274e-06, + "OOD_CommonPile": -0.0010117930336119925 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 2 + }, + "J|StackExchange": { + "slope_linear_containment": -0.0035589979944138225, + "slope_log10_containment": -0.00028825210444100556, + "spearman": 0.00909090909090909, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.005354689643722768, + "StackExchange": -7.861127536555136e-05, + "Wikipedia_en": -0.00400213687233838, + "USPTO_Backgrounds": -0.004980725535569202, + "Github": -0.0018976731424252432, + "CONTROL_PubMed_2023": -0.00422844192966735, + "OOD_Wikipedia_2023": -0.0023323168584560294, + "TRAP_FineWeb": -0.0033621771436820732, + "OOD_News_2024": -0.0057165041295091585, + "OOD_arXiv_2023": -0.00295331038636373, + "OOD_CommonPile": -0.004445761072997944 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0006181090101206631, + "StackExchange": 0.0005463290236336597, + "Wikipedia_en": -0.00032653417926189267, + "USPTO_Backgrounds": -0.0005375644780560232, + "Github": 0.00012984701853845188, + "CONTROL_PubMed_2023": -0.0004691915081592516, + "OOD_Wikipedia_2023": -0.0005311457658765471, + "TRAP_FineWeb": -0.0002776836972149278, + "OOD_News_2024": -0.0008879258086593867, + "OOD_arXiv_2023": 0.00038247464336330864, + "OOD_CommonPile": -0.00041503279685644066 + }, + "n_loo_sign_flips_linear": 0, + "n_loo_sign_flips_log10": 3 + }, + "J|Pile-CC": { + "slope_linear_containment": -0.004014659692830054, + "slope_log10_containment": -0.00033377197965353794, + "spearman": -0.01818181818181818, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.006127892387626296, + "StackExchange": 2.76804226033701e-05, + "Wikipedia_en": -0.005243503813823139, + "USPTO_Backgrounds": -0.0047696458834969166, + "Github": -0.002156511734341, + "CONTROL_PubMed_2023": -0.005012628579589455, + "OOD_Wikipedia_2023": -0.003101921604069063, + "TRAP_FineWeb": -0.003264071432890173, + "OOD_News_2024": -0.004697706531445889, + "OOD_arXiv_2023": -0.004054662748765874, + "OOD_CommonPile": -0.005922931885080463 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": -0.0007254269721705388, + "StackExchange": 0.0006331040930860547, + "Wikipedia_en": -0.0005384660407427376, + "USPTO_Backgrounds": -0.00043042208791283605, + "Github": 0.0001339564204317549, + "CONTROL_PubMed_2023": -0.0006161089168561895, + "OOD_Wikipedia_2023": -0.000525059227267405, + "TRAP_FineWeb": -0.0003134391082028514, + "OOD_News_2024": -0.0004019695039494006, + "OOD_arXiv_2023": -9.308719321018015e-05, + "OOD_CommonPile": -0.0007469416250329605 + }, + "n_loo_sign_flips_linear": 1, + "n_loo_sign_flips_log10": 2 + }, + "J|USPTO_Backgrounds": { + "slope_linear_containment": -4.247891056195874e-05, + "slope_log10_containment": 0.0010588073618437773, + "spearman": 0.15454545454545454, + "loo_slope_linear_by_dropped_rung": { + "Pile-CC": -0.0026059268745792716, + "StackExchange": 0.002348159889371986, + "Wikipedia_en": -0.0007002231730581452, + "USPTO_Backgrounds": -0.0003369993638281012, + "Github": 0.0026729119488506235, + "CONTROL_PubMed_2023": -0.0006377075981444336, + "OOD_Wikipedia_2023": 0.0013686035810469748, + "TRAP_FineWeb": 0.0016655550887958454, + "OOD_News_2024": -0.0005666647124356315, + "OOD_arXiv_2023": 4.532180181529704e-05, + "OOD_CommonPile": -0.004017114631374387 + }, + "loo_slope_log10_by_dropped_rung": { + "Pile-CC": 0.0005629508738604782, + "StackExchange": 0.0016530366826223108, + "Wikipedia_en": 0.0009751134880302337, + "USPTO_Backgrounds": 0.0010564825770613415, + "Github": 0.0017027487188217658, + "CONTROL_PubMed_2023": 0.0009217765107029212, + "OOD_Wikipedia_2023": 0.0008249363625129528, + "TRAP_FineWeb": 0.0010865656480485888, + "OOD_News_2024": 0.001134563899364181, + "OOD_arXiv_2023": 0.0018849705502802568, + "OOD_CommonPile": 7.827160798897051e-05 + }, + "n_loo_sign_flips_linear": 5, + "n_loo_sign_flips_log10": 0 + }, + "_summary": { + "n_rungs": 11, + "max_abs_spearman": 0.15454545454545454, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_linear": 1, + "n_fitted_arms_flipping_sign_when_github_rung_dropped_log10": 4 + } + } + }, + "control_C1_agrees_with_e54": { + "n_quantities_checked": 24, + "max_abs_diff": 0.14555705543928354, + "tolerance": 1e-12, + "fires": false, + "reference": "results/e54_aggregation_audit.json" + }, + "provenance": { + "schema_version": 1, + "experiment": "E53", + "utc": "2026-08-20T09:01:07+00:00", + "script": { + "path": "experiments/t53_ladder_summary.py", + "sha256": "48652340444f107ec57fc3f06da61c1b968feab8618e9c3638b6880db6e7f62d", + "n_lines": 308 + }, + "git": { + "commit": "b197ac445bf89d5b778d5e16ce9d1c325c09473e", + "commit_short": "b197ac445bf8", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/ladder410_cpu/", + "results/r3_close_d2.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "aee92baa8be53162e66b7c00eec04f9286b40c2afaf4dbb2dbe8c1782597cc28", + "describe": "b197ac4-dirty" + }, + "argv": [ + "experiments/t53_ladder_summary.py", + "--ladder410-dir", + "results/ladder410_cpu", + "--out", + "results/ladder410_cpu/e53_ladder_summary_cpu.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e54_aggregation_audit.json", + "sha256": "886885b41660dd01808aa626d5f87dddf6d5402f2c9b2f8241ac4470df82276b", + "bytes": 69079, + "exists": true + }, + { + "path": "results/e36_qladder_410m.json", + "sha256": "d0cc7b6381ac515f6d53401705e246d8c20c67884cda25e4b1d35d5312f45153", + "bytes": 80656, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "3b92d96704a1287abaade52db9f69c819e4ca66f526acc0e2f69456adb5d4a2a" + } +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Github_s0.json b/results/ladder410_cpu/ladder_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..f98eb7b9ffefc1ccbace750c5019de28cd007ec5 --- /dev/null +++ b/results/ladder410_cpu/ladder_Github_s0.json @@ -0,0 +1,659 @@ +{ + "corpus": "Github", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.016856929237422134, + "persist": 0.006934812824640956 + }, + "multilingual": { + "min": 0.11131254530378751, + "best1L": 0.07360406113522393, + "mean": 0.03054052504843899, + "persist": 0.026468455392335142 + }, + "order-ops": { + "min": 0.08311688367809568, + "best1L": 0.08311688367809568, + "mean": 0.021678321861795018, + "persist": 0.01558441509093557 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.24702381023338862, + "best1L": 0.15476190910807677, + "mean": 0.08447802292981318, + "persist": 0.07886904730860676 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.017603755213453302, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11675126690949712, + "best1L": 0.07831762065844876, + "mean": 0.032437106394874196, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07532467506825924, + "mean": 0.018581418486844217, + "persist": 0.012987012575779642 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095628363745, + "best1L": 0.15922619349190167, + "mean": 0.09741300317857947, + "persist": 0.08779761926936251 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0011944577147785043, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.018670649966225028, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12327773869037628, + "best1L": 0.08665699937513896, + "mean": 0.034277904379580705, + "persist": 0.031544597287263186 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.0212787209810423, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618763787405, + "best1L": 0.16220238006540708, + "mean": 0.10382326080330781, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.010869565419852734, + "best1L": 0.004658385179936886, + "mean": 0.0013139034771094365, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823690840175, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12146482989192009, + "best1L": 0.08375634465898786, + "mean": 0.03352485144776957, + "persist": 0.030094271525740623 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.021478521737403104, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177952532258, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.15922618817005837, + "mean": 0.09993131937725204, + "persist": 0.09077380983425039 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017070308011690422, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.10913705639541149, + "best1L": 0.07142857102943319, + "mean": 0.030373179553342716, + "persist": 0.025018128167305673 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.08181818202137947, + "mean": 0.024675324891826937, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928603359631, + "best1L": 0.15922619242753303, + "mean": 0.08928571282220739, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017176997398824563, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11167512647807598, + "best1L": 0.07287889798837048, + "mean": 0.03101467008569411, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023876123629244312, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095415490015, + "best1L": 0.16071428837520735, + "mean": 0.09191849787852593, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01717699766491673, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11167512674416814, + "best1L": 0.07251631641494376, + "mean": 0.031014670551355396, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023676323671159998, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928582072258, + "best1L": 0.16071428837520735, + "mean": 0.09100274728345019, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01771044486667961, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.113850615386452, + "best1L": 0.0746918051902737, + "mean": 0.03187928767874837, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.023076922865584493, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709024586848, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976140805653, + "best1L": 0.16220237793666975, + "mean": 0.09581043757498264, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00011944577064631241, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07722987633730684, + "mean": 0.03226976083325488, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023076922998630574, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.278273805975914, + "best1L": 0.16220237793666975, + "mean": 0.09764194036168712, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07759245777768749, + "mean": 0.032409214946840494, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2812499978712627, + "best1L": 0.16220237793666975, + "mean": 0.09855769308550018, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07650471345654555, + "mean": 0.0322697607002088, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428837520735, + "mean": 0.09775641188025475, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06657420311655317, + "mean": 0.018137202681308345, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11747643191899572, + "best1L": 0.08085569100720542, + "mean": 0.033078596993748634, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07662337619279112, + "mean": 0.023476524245260016, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2782738038471767, + "best1L": 0.16071428837520735, + "mean": 0.09878662947033133, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01792382390704006, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07759245804377965, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.02417582429812423, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529761833803995, + "best1L": 0.16220237793666975, + "mean": 0.09592490643262863, + "persist": 0.08333333382116896 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07868020236492157, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.0792207805705922, + "best1L": 0.07792207838169166, + "mean": 0.023976024739178165, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27678571215697695, + "best1L": 0.16220237793666975, + "mean": 0.09661172330379486, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11638868546911649, + "best1L": 0.07868020236492157, + "mean": 0.03254866899390306, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023776224381955608, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.16220237793666975, + "mean": 0.0974130045090403, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07831762092454093, + "mean": 0.03246499631287796, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023876124427520802, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095223903656, + "best1L": 0.16220237793666975, + "mean": 0.09695512801408768, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + } + }, + "runtime_s": 1604.6 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Github_s1.json b/results/ladder410_cpu/ladder_Github_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..2abe3871e00d7c19032dcaaeb9e4e0d56da19f93 --- /dev/null +++ b/results/ladder410_cpu/ladder_Github_s1.json @@ -0,0 +1,659 @@ +{ + "corpus": "Github", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06796116594757352, + "mean": 0.019417476108563796, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.1287164602960859, + "best1L": 0.09137055969664029, + "mean": 0.03536564936595304, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.05714285613170692, + "best1L": 0.053246751693742614, + "mean": 0.013586413547662752, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.005045974294521979, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28125000106436865, + "best1L": 0.16220238006540708, + "mean": 0.10267856956592628, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0005972288282854217, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06518724028553281, + "mean": 0.018884028906800916, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.12400289891021592, + "best1L": 0.08919506892561913, + "mean": 0.03475205008206623, + "persist": 0.03190717859459775 + }, + "order-ops": { + "min": 0.07142857063029494, + "best1L": 0.06883116931255374, + "mean": 0.01878121932635882, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.005382372731609004, + "persist": 0.005830903670617512 + }, + "typo": { + "min": 0.26488095202616285, + "best1L": 0.16220238219414437, + "mean": 0.0958104379741209, + "persist": 0.08630952425301075 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.08044382690318994, + "best1L": 0.06518724028553281, + "mean": 0.01899071824404278, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11820159692849432, + "best1L": 0.08266860007175378, + "mean": 0.032883360383233855, + "persist": 0.029369107713656768 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023876123895336474, + "persist": 0.016883116215467453 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.013119533258889402, + "mean": 0.004597443488559553, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333262375425, + "best1L": 0.16369047707745008, + "mean": 0.09397893731615373, + "persist": 0.08184523960309369 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06657420311655317, + "mean": 0.018884028906800916, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11965192322220121, + "best1L": 0.08556925425572055, + "mean": 0.03346906848517912, + "persist": 0.029006526805460453 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07662337725715977, + "mean": 0.021578421782968298, + "persist": 0.018181818404367993 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004821708891540766, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232142644269125, + "best1L": 0.1651785708963871, + "mean": 0.09661172250551837, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018777339220313088, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1221899933048657, + "best1L": 0.08484408898012978, + "mean": 0.03380375987450991, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.07272727281919547, + "best1L": 0.07272727281919547, + "mean": 0.019380619466703917, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28571428464991705, + "best1L": 0.1711309514939785, + "mean": 0.10428113330687795, + "persist": 0.09374999960086175 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018884028607447232, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12182740973574775, + "best1L": 0.0848440911088671, + "mean": 0.03366430549483214, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.07272727281919547, + "best1L": 0.07272727281919547, + "mean": 0.01998002005607954, + "persist": 0.015584415889212064 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2797619051166943, + "best1L": 0.16517856983201845, + "mean": 0.102106226342065, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07905686513653823, + "best1L": 0.06657420311655317, + "mean": 0.018990718027842895, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12291515405688967, + "best1L": 0.08738216145762376, + "mean": 0.03394321372200336, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.07402597500809602, + "best1L": 0.07142857063029494, + "mean": 0.01928071942113872, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28125000106436865, + "best1L": 0.1666666679084301, + "mean": 0.1023351653878178, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.01813720299729279, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1207396648824215, + "best1L": 0.08339376321860723, + "mean": 0.03377586902518358, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07532467506825924, + "mean": 0.021578421649922217, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28273809807641165, + "best1L": 0.1636904738843441, + "mean": 0.1029075091438634, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06657420311655317, + "mean": 0.018137202731200626, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11965192056127957, + "best1L": 0.08303118177822658, + "mean": 0.03360852366313338, + "persist": 0.02973168981926782 + }, + "order-ops": { + "min": 0.07792207944606032, + "best1L": 0.07662337725715977, + "mean": 0.02217782232245164, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618976661135, + "best1L": 0.16666666577969277, + "mean": 0.10313644659306322, + "persist": 0.09226190418537174 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018243892384426936, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12001450200166021, + "best1L": 0.08303118177822658, + "mean": 0.0337200873265309, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07662337619279112, + "mean": 0.022077922010794282, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16517856983201845, + "mean": 0.10164835224194187, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.018243892384426936, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.12037708344204086, + "best1L": 0.08375634465898786, + "mean": 0.03374797764367291, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07532467506825924, + "mean": 0.02147852160435702, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16517856983201845, + "mean": 0.10222069599798747, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.017603755213453302, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11711384834987777, + "best1L": 0.0790427838053022, + "mean": 0.03240921501336353, + "persist": 0.02719361867223467 + }, + "order-ops": { + "min": 0.07662337725715977, + "best1L": 0.07532467506825924, + "mean": 0.022177821790267314, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333155938555, + "best1L": 0.16071428518210137, + "mean": 0.09409340590770755, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497065826319158, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07940536524568285, + "mean": 0.03274390573746392, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.02117882140113839, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232142644269125, + "best1L": 0.16071428198899543, + "mean": 0.0953525628096291, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017603754947361137, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07976794668606349, + "mean": 0.032660233455577066, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.021878121786617806, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095202616283, + "best1L": 0.16220238006540708, + "mean": 0.09569597124521222, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017710444600587443, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07976794668606349, + "mean": 0.03266023292339274, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.02177822120886828, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2723214253783226, + "best1L": 0.16220238006540708, + "mean": 0.09558150238756623, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01771044486667961, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11675126690949712, + "best1L": 0.08049310956682477, + "mean": 0.03285546979999968, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.021478521072172692, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27380951919725965, + "best1L": 0.16220238006540708, + "mean": 0.09569597097912005, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + } + }, + "runtime_s": 1599.3 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Github_s2.json b/results/ladder410_cpu/ladder_Github_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..cb9877c84ae5af2f0198ef0e06d3c6b7e5609ce4 --- /dev/null +++ b/results/ladder410_cpu/ladder_Github_s2.json @@ -0,0 +1,659 @@ +{ + "corpus": "Github", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.016536861076019704, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11131254423941885, + "best1L": 0.07142857302512441, + "mean": 0.030010597979915992, + "persist": 0.024292965020452226 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07142857156161751, + "mean": 0.02017981996427157, + "persist": 0.011688311318201678 + }, + "poetry": { + "min": 0.020408163645437787, + "best1L": 0.013119533258889402, + "mean": 0.00504597436104502, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333155938555, + "best1L": 0.16369047654526575, + "mean": 0.08997252743159022, + "persist": 0.07589285700981106 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07905686433826174, + "best1L": 0.06657420311655317, + "mean": 0.01856396041278328, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11856417624013764, + "best1L": 0.0790427838053022, + "mean": 0.0329670326651207, + "persist": 0.0286439446998494 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07142857156161751, + "mean": 0.021678321046887765, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.013119533658027649, + "mean": 0.004709576190050159, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16220238485506602, + "mean": 0.09809981659054756, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.017710444600587443, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11385061432208333, + "best1L": 0.07650471372263772, + "mean": 0.0323534337138491, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.0792207805705922, + "best1L": 0.07792207838169166, + "mean": 0.024175824763785516, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177886009216, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333900996615, + "best1L": 0.15773809727813518, + "mean": 0.09684065782598086, + "persist": 0.08630952358778034 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497066092411324, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.1134880328817027, + "best1L": 0.07505438742893082, + "mean": 0.03204663350645985, + "persist": 0.027556200112615312 + }, + "order-ops": { + "min": 0.08181818281965596, + "best1L": 0.0792207805705922, + "mean": 0.02337662293575704, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807474281107, + "mean": 0.004597443488559553, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529761833803995, + "best1L": 0.16071428890739167, + "mean": 0.0956959705799818, + "persist": 0.084821427640106 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.017497066092411324, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11203770818454879, + "best1L": 0.07614213201616492, + "mean": 0.031433033889957836, + "persist": 0.028281363658607006 + }, + "order-ops": { + "min": 0.08571428645934377, + "best1L": 0.08181818202137947, + "mean": 0.02447552453460438, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2723214307001659, + "best1L": 0.16071428305336408, + "mean": 0.09283425033624683, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497065826319158, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11457577826721328, + "best1L": 0.07759245804377965, + "mean": 0.032130306254007986, + "persist": 0.029006526539368287 + }, + "order-ops": { + "min": 0.08441558427044324, + "best1L": 0.08181818202137947, + "mean": 0.024175824597477913, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428624647005, + "mean": 0.09512362802135092, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017283686785958707, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07759245804377965, + "mean": 0.03196296075891171, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.08051948089684759, + "mean": 0.02397602450634752, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004373177886009216, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2767857164144516, + "best1L": 0.1592261932258095, + "mean": 0.0943223430908152, + "persist": 0.08184523760740246 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017603755479545465, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11530094114797455, + "best1L": 0.07722987580512251, + "mean": 0.03162826996828828, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.08051948089684759, + "mean": 0.022877122275531292, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380491154535, + "best1L": 0.16071428704474652, + "mean": 0.09500915730105979, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017603755213453302, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11348803394607135, + "best1L": 0.07541696860321931, + "mean": 0.03168405166694096, + "persist": 0.02719361853918859 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.08051948089684759, + "mean": 0.023276723522160734, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976172736713, + "best1L": 0.16071428624647005, + "mean": 0.09478021931967565, + "persist": 0.08333333461944546 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497066092411324, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07795503948416029, + "mean": 0.031962961690234284, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.023476524245260016, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428624647005, + "mean": 0.09523809581462826, + "persist": 0.08333333461944546 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017710444600587443, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11530094114797455, + "best1L": 0.07831762092454093, + "mean": 0.03196296142414212, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.023376624066648737, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428624647005, + "mean": 0.09535256414008993, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017603755213453302, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.113850615386452, + "best1L": 0.07650471372263772, + "mean": 0.03190717879416687, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08051948063075542, + "best1L": 0.07792207838169166, + "mean": 0.02287712330663843, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232143389327185, + "best1L": 0.1592261932258095, + "mean": 0.09443681354501418, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01739037670527718, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11457577826721328, + "best1L": 0.07759245804377965, + "mean": 0.032018742723656554, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08051948063075542, + "best1L": 0.07792207838169166, + "mean": 0.022977022820019295, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095277122089, + "best1L": 0.1592261932258095, + "mean": 0.09420787609581437, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017283687318143035, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11348803394607135, + "best1L": 0.07650471372263772, + "mean": 0.03187928834397878, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.0231768235763801, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2708333358168602, + "best1L": 0.16071428624647005, + "mean": 0.09409340723816838, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017497065826319158, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07686729516301836, + "mean": 0.031990852539560626, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08181818175528731, + "best1L": 0.07922077950622354, + "mean": 0.023376623135326163, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2708333358168602, + "best1L": 0.16071428624647005, + "mean": 0.0942078758297222, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017283687318143035, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07686729516301836, + "mean": 0.03199085227346846, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.08051948063075542, + "best1L": 0.07792207838169166, + "mean": 0.023276722956714884, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428198899543, + "mean": 0.09455128213656801, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + } + }, + "runtime_s": 1590.2 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Pile-CC_s0.json b/results/ladder410_cpu/ladder_Pile-CC_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..f7e3d5a04c73a3f3baba5c758dd917160850ac0d --- /dev/null +++ b/results/ladder410_cpu/ladder_Pile-CC_s0.json @@ -0,0 +1,463 @@ +{ + "corpus": "Pile-CC", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.11095700412988663, + "best1L": 0.08876560415540423, + "mean": 0.032006828380482535, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.17476432557616914, + "best1L": 0.14720812067389488, + "mean": 0.04359346189137016, + "persist": 0.036983320622571876 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.07012986977185522, + "mean": 0.016183816262387803, + "persist": 0.009090908936091832 + }, + "poetry": { + "min": 0.021865888631769588, + "best1L": 0.00728862958827189, + "mean": 0.003251850272395781, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3377976215311459, + "best1L": 0.2053571455180645, + "mean": 0.15178571401962213, + "persist": 0.16071428598037787 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180239915848, + "mean": 0.001791686484856265, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.112343965364354, + "best1L": 0.09292649051972798, + "mean": 0.032006828247436454, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17331399555717195, + "best1L": 0.14503263416034834, + "mean": 0.04392815301460879, + "persist": 0.038071065209805965 + }, + "order-ops": { + "min": 0.0753246753343514, + "best1L": 0.07012986977185522, + "mean": 0.015684315502377495, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889297, + "best1L": 0.008746355905064515, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3541666641831398, + "best1L": 0.2083333320915699, + "mean": 0.16220238033149922, + "persist": 0.16815476252564363 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180239915848, + "mean": 0.002030578109302691, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.031366691791585515, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.1729514098593167, + "best1L": 0.1475707037108285, + "mean": 0.043900261633098125, + "persist": 0.03698332022343363 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01378621397140835, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.005830903803663594, + "mean": 0.0037003811448812485, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3586309456399509, + "best1L": 0.2142857126891613, + "mean": 0.16723900900355407, + "persist": 0.1755952396030937 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.006211180239915848, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03158007003366947, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.17403915843793324, + "best1L": 0.14793328568339348, + "mean": 0.0439560433317508, + "persist": 0.037708483237241 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014485514423410808, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.00728862958827189, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3541666641831398, + "best1L": 0.20833333102720125, + "mean": 0.16369047335215978, + "persist": 0.16964285634458065 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.006211180239915848, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03104662336409092, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1715010856943471, + "best1L": 0.14503263362816401, + "mean": 0.04339822581303971, + "persist": 0.03662073891609907 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.013886113883927464, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.010204081822718893, + "mean": 0.004148912416504962, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.33928571322134565, + "best1L": 0.20684523933700152, + "mean": 0.1593406588903495, + "persist": 0.16666666657796927 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256751860891, + "mean": 0.03147338117871966, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17222624910729273, + "best1L": 0.14503263362816401, + "mean": 0.04376080725342035, + "persist": 0.03662073891609907 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01398601392949266, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.008746355905064515, + "mean": 0.004036779648491314, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20684523720826423, + "mean": 0.16048534720071725, + "persist": 0.16666666657796927 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17403915843793324, + "best1L": 0.1468455387013299, + "mean": 0.04390026216528246, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014085914241150022, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.008746355905064515, + "mean": 0.003812513979417937, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000319310595, + "best1L": 0.20386904744165285, + "mean": 0.16002747230231762, + "persist": 0.16517857063029492 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256831688541, + "mean": 0.03147338144481182, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.16860043842877662, + "best1L": 0.14213197997638158, + "mean": 0.042868299409747124, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014485514290364725, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.008746355505926269, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738134264946, + "best1L": 0.2008928544819355, + "mean": 0.1587683167308569, + "persist": 0.16517857595213822 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256831688541, + "mean": 0.03147338144481182, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1686004363000393, + "best1L": 0.14249456141676223, + "mean": 0.04297986200877598, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014285714199234332, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34226190724543165, + "best1L": 0.2038690485060215, + "mean": 0.15934066155127116, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.031580070831945965, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.16968818274991854, + "best1L": 0.14285714285714285, + "mean": 0.043147207370826175, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.014185814153669136, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.2038690485060215, + "mean": 0.1600274746971471, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.03147338144481182, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17077592494232313, + "best1L": 0.1432197242975235, + "mean": 0.04323088138231209, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01438561437784561, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.008746355505926269, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.2038690485060215, + "mean": 0.15991300184811866, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 1598.8 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Pile-CC_s1.json b/results/ladder410_cpu/ladder_Pile-CC_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e97a897568a8f8ab0ce268389f8bb8a52b4c644d --- /dev/null +++ b/results/ladder410_cpu/ladder_Pile-CC_s1.json @@ -0,0 +1,463 @@ +{ + "corpus": "Pile-CC", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.0873786418565682, + "mean": 0.0309399341765259, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.1754894863281931, + "best1L": 0.14793328515120915, + "mean": 0.043509790008621554, + "persist": 0.0373459021959986 + }, + "order-ops": { + "min": 0.0714285708963871, + "best1L": 0.06883116864732333, + "mean": 0.010989011032506824, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.018950437062552998, + "best1L": 0.005830903803663594, + "mean": 0.003251850139349699, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3586309477686882, + "best1L": 0.21428571588226727, + "mean": 0.16483516645218646, + "persist": 0.17559523907090938 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.08876560415540423, + "mean": 0.031366691791585515, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17367657646536827, + "best1L": 0.14793328515120915, + "mean": 0.04337033522980554, + "persist": 0.035895576301429956 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.07012986977185522, + "mean": 0.011788211995735765, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.00728862958827189, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000319310595, + "best1L": 0.21130952079381263, + "mean": 0.16037087914134776, + "persist": 0.16964286006987095 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09153952822089195, + "mean": 0.0315800705658538, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.172951413584607, + "best1L": 0.14793328302247183, + "mean": 0.043342444247433116, + "persist": 0.03553299539323364 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.07012986977185522, + "mean": 0.015184815407597594, + "persist": 0.009090908936091832 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.005830903803663594, + "mean": 0.003588248576436724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34672619295971735, + "best1L": 0.20238095096179418, + "mean": 0.15922618843615055, + "persist": 0.16220238405678952 + }, + "association": { + "min": 0.009316770093781608, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.08876560415540423, + "mean": 0.03126000160617488, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1729514157133443, + "best1L": 0.14865844590323313, + "mean": 0.04350978934339115, + "persist": 0.03662073878305299 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013786213838362269, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.005830903803663594, + "mean": 0.00325185020587274, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023805975914, + "best1L": 0.20535714445369585, + "mean": 0.15956959421081202, + "persist": 0.16517857063029492 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256725251675, + "mean": 0.031580070299761634, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.17548948845693044, + "best1L": 0.14829586446285248, + "mean": 0.04370502648609025, + "persist": 0.03698332115475621 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.015684316034561822, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.005830903803663594, + "mean": 0.0033639829073633465, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047654526575, + "best1L": 0.19940476279173577, + "mean": 0.1603708786091634, + "persist": 0.1651785738234009 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256725251675, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17548948845693044, + "best1L": 0.1493836087839944, + "mean": 0.04376080831778901, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014885114738717675, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20089285661067283, + "mean": 0.1596840667937483, + "persist": 0.1651785738234009 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03126000187226704, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17585206776857376, + "best1L": 0.14902102947235107, + "mean": 0.043788698102746694, + "persist": 0.03625815734267235 + }, + "order-ops": { + "min": 0.07402597420981952, + "best1L": 0.07012986977185522, + "mean": 0.01438561437784561, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.005830903803663594, + "mean": 0.0033639829738863875, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3482142889073917, + "best1L": 0.2038690485060215, + "mean": 0.16117216327360698, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256725251675, + "mean": 0.03136669152549335, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17295141624552862, + "best1L": 0.14503263149942672, + "mean": 0.04364924372306892, + "persist": 0.03662073878305299 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014885114472625511, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0038125141124640194, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.20386904637728417, + "mean": 0.159455130142825, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.09015256725251675, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17186367139220238, + "best1L": 0.145757794380188, + "mean": 0.04364924438829933, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.01508491496289415, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.15979853405484132, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.03168676021908011, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17295141784208162, + "best1L": 0.14684554083006723, + "mean": 0.04367713523762567, + "persist": 0.03662073878305299 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.01508491496289415, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.15979853751403944, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.03136669152549335, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17258883640170097, + "best1L": 0.14648295938968658, + "mean": 0.043621353805065155, + "persist": 0.036983320356479714 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014585414468976004, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.005830903803663594, + "mean": 0.003588248576436724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.16025640815496445, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 1618.1 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Pile-CC_s2.json b/results/ladder410_cpu/ladder_Pile-CC_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..76dba257ad58ecde55cc3baf61e2f608b3374e09 --- /dev/null +++ b/results/ladder410_cpu/ladder_Pile-CC_s2.json @@ -0,0 +1,463 @@ +{ + "corpus": "Pile-CC", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10957003970231328, + "best1L": 0.09153952901916844, + "mean": 0.03147338091262749, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1609862217945712, + "best1L": 0.13488034584692546, + "mean": 0.04113906302622387, + "persist": 0.03589557616838387 + }, + "order-ops": { + "min": 0.07792207824864558, + "best1L": 0.07532467599958181, + "mean": 0.01368631368469713, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.007288629721317973, + "mean": 0.003924646747431585, + "persist": 0.0 + }, + "typo": { + "min": 0.34821428252117975, + "best1L": 0.20535714445369585, + "mean": 0.15464743679123266, + "persist": 0.16220238405678952 + }, + "association": { + "min": 0.009316769960735525, + "best1L": 0.004658385179936886, + "mean": 0.0013139034854248166, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.03168675968689578, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17077592121703283, + "best1L": 0.14575779704110964, + "mean": 0.043230880184897354, + "persist": 0.03770848350333316 + }, + "order-ops": { + "min": 0.07662337699106761, + "best1L": 0.07272727255310331, + "mean": 0.014485514257103205, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.008746355505926269, + "mean": 0.003812513979417937, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34672619083098005, + "best1L": 0.21279761780585563, + "mean": 0.15899725311568805, + "persist": 0.1666666687067066 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0017916865181177855, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09153952928526062, + "mean": 0.03126000187226704, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.17403916056667054, + "best1L": 0.1475707037108285, + "mean": 0.043537680724901814, + "persist": 0.0377084847007479 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085914241150022, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.023323614549423968, + "best1L": 0.008746355905064515, + "mean": 0.0037003812779273304, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023827263287, + "best1L": 0.21279761993459292, + "mean": 0.16311813199094363, + "persist": 0.1741071425910507 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001911132363602519, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09153952928526062, + "mean": 0.031580070299761634, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17258883267641068, + "best1L": 0.1457577965089253, + "mean": 0.043091426470449994, + "persist": 0.036620739182191234 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013886113883927464, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.026239067981285707, + "best1L": 0.00728862958827189, + "mean": 0.003924646880477667, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3526785724929401, + "best1L": 0.20982142697487557, + "mean": 0.16163003976855958, + "persist": 0.16964285794113362 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0020305780760411707, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11095700253333364, + "best1L": 0.0915395300835371, + "mean": 0.03126000187226704, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.16860043417130197, + "best1L": 0.14249456141676223, + "mean": 0.04244993587157556, + "persist": 0.035532995127141476 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013286713477490204, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.33779762365988325, + "best1L": 0.20238094830087253, + "mean": 0.15670787702713693, + "persist": 0.16815476146127498 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10957003970231328, + "best1L": 0.0915395300835371, + "mean": 0.031366691791585515, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17077592494232313, + "best1L": 0.14430746861866542, + "mean": 0.04286829874451671, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013786213838362269, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20535714019622123, + "mean": 0.15831043917153562, + "persist": 0.16666666764233792 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10957003970231328, + "best1L": 0.0915395300835371, + "mean": 0.031580070299761634, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.17041334137320518, + "best1L": 0.1432197242975235, + "mean": 0.04270095471292734, + "persist": 0.035895575902291706 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.013886113883927464, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20535714019622123, + "mean": 0.15785256108003004, + "persist": 0.16666666551360063 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10957004023449761, + "best1L": 0.09015256645424026, + "mean": 0.031580070299761634, + "persist": 0.018030513344066485 + }, + "multilingual": { + "min": 0.16968818168554986, + "best1L": 0.1439448893070221, + "mean": 0.04306353535503149, + "persist": 0.0358955767005682 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.007288629721317973, + "mean": 0.00336398310693247, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3482142889073917, + "best1L": 0.20386904531291553, + "mean": 0.1620879146669592, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.03126000213835921, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17113850638270378, + "best1L": 0.14503263362816401, + "mean": 0.04317509848624468, + "persist": 0.0358955767005682 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.007288629721317973, + "mean": 0.00336398310693247, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.20535714126058988, + "mean": 0.16220237900103843, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.031366691791585515, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17258883001548903, + "best1L": 0.14612037794930594, + "mean": 0.043286663480103016, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.007288629721317973, + "mean": 0.0034761156753769945, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.2068452350795269, + "mean": 0.1623168507856982, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10818307740347725, + "best1L": 0.09015256645424026, + "mean": 0.03126000213835921, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.17186366926346505, + "best1L": 0.1457577965089253, + "mean": 0.04317509848624468, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.07532467426998275, + "best1L": 0.07012986977185522, + "mean": 0.014085913975057858, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.007288629721317973, + "mean": 0.0034761156753769945, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023827263287, + "best1L": 0.20535714019622123, + "mean": 0.16151556745171547, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 1638.2 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_StackExchange_s0.json b/results/ladder410_cpu/ladder_StackExchange_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..d2f8977ed743534a53377df15147daa059505c61 --- /dev/null +++ b/results/ladder410_cpu/ladder_StackExchange_s0.json @@ -0,0 +1,659 @@ +{ + "corpus": "StackExchange", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09015256645424026, + "mean": 0.030513176594727805, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16715010840977942, + "best1L": 0.1356055119207927, + "mean": 0.04348189876015697, + "persist": 0.03879622809056725 + }, + "order-ops": { + "min": 0.0740259758063725, + "best1L": 0.05454545468091965, + "mean": 0.012587412526564938, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.013119533658027649, + "best1L": 0.008746355505926269, + "mean": 0.003588248576436724, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.37500000851494925, + "best1L": 0.22023809275456838, + "mean": 0.15819597031388963, + "persist": 0.16815475959862983 + }, + "association": { + "min": 0.015527950200651373, + "best1L": 0.006211180106869766, + "mean": 0.0014333492394403688, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.0887656036232199, + "mean": 0.02965966096547033, + "persist": 0.012482662818261556 + }, + "multilingual": { + "min": 0.1718636719243867, + "best1L": 0.1395939061684268, + "mean": 0.04406760686210224, + "persist": 0.036620739315237315 + }, + "order-ops": { + "min": 0.06363636308482715, + "best1L": 0.05324675302420344, + "mean": 0.009790209887017096, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.011661807873419352, + "best1L": 0.008746355505926269, + "mean": 0.0034761157419000354, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190469094685, + "best1L": 0.2217261865735054, + "mean": 0.15831043837325914, + "persist": 0.16964285873941012 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.006211180106869766, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09292649131800447, + "mean": 0.030299797920244082, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1733139998146466, + "best1L": 0.14249455982020923, + "mean": 0.04465331602841616, + "persist": 0.03734590232904468 + }, + "order-ops": { + "min": 0.06623376746262823, + "best1L": 0.05324675302420344, + "mean": 0.009690310107544065, + "persist": 0.0 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333262375425, + "best1L": 0.22470238219414437, + "mean": 0.16002747363277844, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09153952901916844, + "mean": 0.030086418813360587, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17113850585051946, + "best1L": 0.13995648973754474, + "mean": 0.044346515488411696, + "persist": 0.03770848363637924 + }, + "order-ops": { + "min": 0.06753246752279145, + "best1L": 0.05324675302420344, + "mean": 0.010189810069277883, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.010204081822718893, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333262375425, + "best1L": 0.22172618923442705, + "mean": 0.1596840683903013, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.006211180106869766, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399342843465, + "best1L": 0.14430747234395572, + "mean": 0.04496011523795979, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010789210642022746, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3749999978712627, + "best1L": 0.22767857196075575, + "mean": 0.16197344394666807, + "persist": 0.17559523986918585 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086419079452753, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17258883054767335, + "best1L": 0.1424945630133152, + "mean": 0.04473698857639517, + "persist": 0.03770848363637924 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.011661807873419352, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3749999978712627, + "best1L": 0.2291666679084301, + "mean": 0.16082875909549849, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03019310820049473, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17150108409779413, + "best1L": 0.14140681390251433, + "mean": 0.04456964361348322, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010889111219772271, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22619047920618737, + "mean": 0.1605998133974416, + "persist": 0.17261905063475882 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.030086418680314506, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17258883267641068, + "best1L": 0.14285714232495852, + "mean": 0.044625425445181985, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.0727272726861494, + "best1L": 0.05584415593849761, + "mean": 0.010789210700230407, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246468139546254, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357110926083, + "best1L": 0.22767857302512443, + "mean": 0.15991300184811866, + "persist": 0.17559523986918585 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.029979729293180362, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17403916056667054, + "best1L": 0.14430747021521842, + "mean": 0.044820661922650676, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.07142857142857142, + "best1L": 0.05454545468091965, + "mean": 0.010689310596457549, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779648491314, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.37351190405232565, + "best1L": 0.22619047920618737, + "mean": 0.16025640975151742, + "persist": 0.17261905010257447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.02997972969231861, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.172951411988054, + "best1L": 0.14358230733445712, + "mean": 0.04468120661165033, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22470238538725035, + "mean": 0.16014194062777928, + "persist": 0.17113095468708447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.029873040172138383, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17186366766691208, + "best1L": 0.14213197731545993, + "mean": 0.0445975341967174, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010489510239234992, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22470238538725035, + "mean": 0.15979853325656482, + "persist": 0.17113095468708447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039440384934, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17403915630919592, + "best1L": 0.14430746915084974, + "mean": 0.04470909772706883, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010189810102539403, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246468139546254, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.15602106228470802, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418680314506, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399342843465, + "best1L": 0.14358230520571982, + "mean": 0.044653315629277913, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1563644685915538, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039639954056, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399129969732, + "best1L": 0.14394488664610045, + "mean": 0.04479277040809393, + "persist": 0.039158809397901805 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1567078743662153, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02976634998672775, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399129969732, + "best1L": 0.14394488664610045, + "mean": 0.04479277094027826, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010189810102539403, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1569368147424289, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02976634998672775, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17367657274007797, + "best1L": 0.14394488664610045, + "mean": 0.04473698844334909, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713540656225, + "best1L": 0.22321428943957602, + "mean": 0.1568223461508751, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + } + }, + "runtime_s": 2353.3 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_StackExchange_s1.json b/results/ladder410_cpu/ladder_StackExchange_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..63af413eaa21252097a050d27dc6dfd6958f745a --- /dev/null +++ b/results/ladder410_cpu/ladder_StackExchange_s1.json @@ -0,0 +1,659 @@ +{ + "corpus": "StackExchange", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952901916844, + "mean": 0.029126213198261603, + "persist": 0.011095700253333365 + }, + "multilingual": { + "min": 0.16968817742807524, + "best1L": 0.14176939906818525, + "mean": 0.04398393484630755, + "persist": 0.03734590206295252 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.011388611713690417, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.014577259974820274, + "best1L": 0.011661807607327188, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523241860525, + "best1L": 0.22767857249294007, + "mean": 0.151785708963871, + "persist": 0.16369047601308143 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0009555661319089788, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029552971278982505, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16968817955681256, + "best1L": 0.14031907064574106, + "mean": 0.043983935112399716, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.009790209920278617, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.013119533658027649, + "mean": 0.004036779648491314, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071364709309, + "best1L": 0.22619047920618737, + "mean": 0.15453297006232397, + "persist": 0.16666666844061442 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10540915360408169, + "best1L": 0.09015256725251675, + "mean": 0.02965966079916273, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17331399768590927, + "best1L": 0.14249455982020923, + "mean": 0.04487644308911903, + "persist": 0.03879622809056725 + }, + "order-ops": { + "min": 0.06883117037692241, + "best1L": 0.05584415593849761, + "mean": 0.009690309400736754, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011905116694315, + "best1L": 0.22767856876764977, + "mean": 0.1565934100321361, + "persist": 0.1711309514939785 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02965966079916273, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1715010872909001, + "best1L": 0.14176939693944796, + "mean": 0.044597534861947806, + "persist": 0.03879622809056725 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011905116694315, + "best1L": 0.22619047494871275, + "mean": 0.15579212669815337, + "persist": 0.16815476225955145 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02997972889404212, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1736765748688153, + "best1L": 0.14503263256379537, + "mean": 0.04509957041591406, + "persist": 0.03952139137046678 + }, + "order-ops": { + "min": 0.0727272726861494, + "best1L": 0.05584415593849761, + "mean": 0.010889110745795603, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714285182101385, + "best1L": 0.22767857302512443, + "mean": 0.15361721707241877, + "persist": 0.16815476279173577 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039174292768, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17295141411679132, + "best1L": 0.14321972589407647, + "mean": 0.044960115770144124, + "persist": 0.040246556379965374 + }, + "order-ops": { + "min": 0.07142857262598616, + "best1L": 0.05584415593849761, + "mean": 0.010389610251877457, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904584509985, + "best1L": 0.22619047920618737, + "mean": 0.15418955896581923, + "persist": 0.1696428582072258 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029979728827519075, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17077592760324478, + "best1L": 0.14140681496688298, + "mean": 0.044653315629277913, + "persist": 0.040246556379965374 + }, + "order-ops": { + "min": 0.07012987043708563, + "best1L": 0.05584415593849761, + "mean": 0.010589410609100014, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35565476013081415, + "best1L": 0.22470238325851305, + "mean": 0.15327380704028265, + "persist": 0.16666666684406145 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03019310793440257, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1689630150794983, + "best1L": 0.14068165208612168, + "mean": 0.04426284347261701, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.07142857142857142, + "best1L": 0.05454545468091965, + "mean": 0.010889110687587942, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22321428943957602, + "mean": 0.15350274635212763, + "persist": 0.16815476013081415 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02997972889404212, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1696881800889969, + "best1L": 0.14104423352650233, + "mean": 0.044402298251433034, + "persist": 0.038796229021889825 + }, + "order-ops": { + "min": 0.07142857142857142, + "best1L": 0.05454545468091965, + "mean": 0.010689310596457549, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22470238325851305, + "mean": 0.1539606223148959, + "persist": 0.16666666524750845 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039373861893, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17005076152937754, + "best1L": 0.14068165208612168, + "mean": 0.04445807995008571, + "persist": 0.03915881099445479 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010589410550892353, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22470238325851305, + "mean": 0.15361721866897174, + "persist": 0.16815476013081415 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.029873039373861893, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16860043363911764, + "best1L": 0.14068165208612168, + "mean": 0.04429073485412768, + "persist": 0.038796229021889825 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.010489510505327157, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22321428943957602, + "mean": 0.1537316877927099, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03029979732153671, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17041334296975816, + "best1L": 0.14249456141676223, + "mean": 0.04451386111655405, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.1546474370573248, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.03019310793440257, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1682378500699997, + "best1L": 0.14104423352650233, + "mean": 0.04434651628668819, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010089910056974207, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.1546474370573248, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16932559651987894, + "best1L": 0.14176939640726363, + "mean": 0.044430188701621125, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15453296633703367, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17041334509849548, + "best1L": 0.14176939640726363, + "mean": 0.044569644411759715, + "persist": 0.0395213919026511 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010089910056974207, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15453296580484935, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17005076365811483, + "best1L": 0.14176939640726363, + "mean": 0.044513862180922716, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010089910056974207, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15441849987421716, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + } + }, + "runtime_s": 2894.6 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_StackExchange_s2.json b/results/ladder410_cpu/ladder_StackExchange_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..e387000d0b0cb6c5942dc7608152f872da7a745c --- /dev/null +++ b/results/ladder410_cpu/ladder_StackExchange_s2.json @@ -0,0 +1,659 @@ +{ + "corpus": "StackExchange", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.1081830792661224, + "best1L": 0.08876560548586505, + "mean": 0.03019310820049473, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16932559545551026, + "best1L": 0.13923132632459914, + "mean": 0.04401182569563389, + "persist": 0.03843364665018661 + }, + "order-ops": { + "min": 0.08441558293998241, + "best1L": 0.05714285719607558, + "mean": 0.011888111791839557, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071471146175, + "best1L": 0.2247023768723011, + "mean": 0.15144230664840766, + "persist": 0.16369047707745008 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0009555661319089788, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09015256725251675, + "mean": 0.030513176627989327, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17041334616286413, + "best1L": 0.13959390776497976, + "mean": 0.044039717210190635, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.07922077817576272, + "best1L": 0.05454545468091965, + "mean": 0.011388611447598253, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.014577259974820274, + "best1L": 0.010204081822718893, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3690476215311459, + "best1L": 0.22321428518210137, + "mean": 0.15693681314587593, + "persist": 0.16815475959862983 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09292649158409663, + "mean": 0.03061986601512347, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16860043151038034, + "best1L": 0.13995648760880744, + "mean": 0.04367713470544134, + "persist": 0.0373459021959986 + }, + "order-ops": { + "min": 0.0662337669304439, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3690476215311459, + "best1L": 0.22470238112977572, + "mean": 0.1575091587645667, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10818308033049107, + "best1L": 0.09292649158409663, + "mean": 0.03072655513616545, + "persist": 0.0152565882142101 + }, + "multilingual": { + "min": 0.16823785113436834, + "best1L": 0.13959390563624247, + "mean": 0.04370502582085984, + "persist": 0.0373459021959986 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.010189810102539403, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3675595277122089, + "best1L": 0.22470238325851305, + "mean": 0.15636446912373816, + "persist": 0.16666666365095548 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952981744494, + "mean": 0.03029979738805975, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16860043683222362, + "best1L": 0.13995648760880744, + "mean": 0.04373291627104793, + "persist": 0.038071065076759884 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22470238325851305, + "mean": 0.1548763715795108, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09292649158409663, + "mean": 0.030406487307378223, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1693255975842476, + "best1L": 0.1395939061684268, + "mean": 0.043760807785604684, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.009890109965843814, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15556318578975542, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09153952981744494, + "mean": 0.030086418613791466, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17005076259374619, + "best1L": 0.14031906904918806, + "mean": 0.04392815234937838, + "persist": 0.038796229021889825 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.1556776538491249, + "persist": 0.16666666471532413 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030193107734833444, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133419053895, + "best1L": 0.14031906904918806, + "mean": 0.044402298384479115, + "persist": 0.03988397454044649 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15602106228470802, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030193107734833444, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133397766522, + "best1L": 0.13995648760880744, + "mean": 0.044207062173102586, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15544871719820158, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.0300864183476993, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133397766522, + "best1L": 0.13995648760880744, + "mean": 0.04423495288938284, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15567765225257194, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.02997972889404212, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1696881768958909, + "best1L": 0.13995648760880744, + "mean": 0.04403971627886806, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06753246911934443, + "best1L": 0.05454545468091965, + "mean": 0.00999001001140901, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15556318578975542, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418414222344, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17150108409779413, + "best1L": 0.14140681337032998, + "mean": 0.0444580792848553, + "persist": 0.04097171939377274 + }, + "order-ops": { + "min": 0.0701298713684082, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15670787755932128, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418480745385, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17077592121703283, + "best1L": 0.1406816504895687, + "mean": 0.044402297852294784, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15636446912373816, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418480745385, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.17077592334577016, + "best1L": 0.14104423192994936, + "mean": 0.04426284294043269, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.010389610193669796, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15682234189340047, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.16968817902462824, + "best1L": 0.14031906904918806, + "mean": 0.04417917052549975, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15670787330184663, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10679611749947071, + "best1L": 0.09015256725251675, + "mean": 0.030086418547268425, + "persist": 0.013869625649281911 + }, + "multilingual": { + "min": 0.1704133419053895, + "best1L": 0.14140681337032998, + "mean": 0.04423495328852108, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.06883116917950767, + "best1L": 0.05454545468091965, + "mean": 0.0102897101481046, + "persist": 0.0025974025151559283 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15659340683903014, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + } + }, + "runtime_s": 2915.4 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_USPTO_Backgrounds_s0.json b/results/ladder410_cpu/ladder_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..f91aa24d1b33f7a21c5d1fa574df9fdf71fa61b6 --- /dev/null +++ b/results/ladder410_cpu/ladder_USPTO_Backgrounds_s0.json @@ -0,0 +1,463 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.11095700200114932, + "best1L": 0.08183079026639462, + "mean": 0.0288061454027359, + "persist": 0.016643550779138292 + }, + "multilingual": { + "min": 0.1453952118754387, + "best1L": 0.10732414866132396, + "mean": 0.03938193852081895, + "persist": 0.03807106600808246 + }, + "order-ops": { + "min": 0.10649350618145295, + "best1L": 0.08961039049816984, + "mean": 0.01958041949962665, + "persist": 0.0038961039057799746 + }, + "poetry": { + "min": 0.021865889563092163, + "best1L": 0.00874635563897235, + "mean": 0.004373177752963134, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3526785767504147, + "best1L": 0.1934523768723011, + "mean": 0.15934066155127116, + "persist": 0.17708333568381412 + }, + "association": { + "min": 0.017080744994538172, + "best1L": 0.006211180239915848, + "mean": 0.00203057804277965, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08321775389569146, + "mean": 0.030193107734833444, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1457577906548977, + "best1L": 0.10732415052396911, + "mean": 0.03954928374982306, + "persist": 0.03734590192990644 + }, + "order-ops": { + "min": 0.12207792060715812, + "best1L": 0.10000000255448478, + "mean": 0.02357642392494849, + "persist": 0.011688311185155596 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004036779448922191, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3630952334829739, + "best1L": 0.20386904983648232, + "mean": 0.1652930397540331, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.007763975166848728, + "mean": 0.002030578100987311, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11927877818899495, + "best1L": 0.08460471672671181, + "mean": 0.030833244922437837, + "persist": 0.020804438473922864 + }, + "multilingual": { + "min": 0.14612037156309401, + "best1L": 0.11094996652432851, + "mean": 0.039660846814513206, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.13896104267665318, + "best1L": 0.1090909093618393, + "mean": 0.02787212807951229, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004373177886009216, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3601190426519939, + "best1L": 0.20684523401515825, + "mean": 0.16597984890852654, + "persist": 0.18303571242306912 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915346403207, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.11927877818899495, + "best1L": 0.08321775469396796, + "mean": 0.030406487373901264, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14467004739812442, + "best1L": 0.10913705905633313, + "mean": 0.039437719088579924, + "persist": 0.03915881072836263 + }, + "order-ops": { + "min": 0.12467532604932785, + "best1L": 0.10259740373917989, + "mean": 0.025174824986606836, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.20535714338932717, + "mean": 0.16540750674903393, + "persist": 0.17857142777315207 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915346403207, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11789181535797459, + "best1L": 0.08183079079857894, + "mean": 0.030193107535264323, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1468455365725926, + "best1L": 0.11094996625823635, + "mean": 0.039967646689287255, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.12987013161182404, + "best1L": 0.101298702614648, + "mean": 0.026473525911569595, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004373177752963134, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3571428560784885, + "best1L": 0.19940476172736712, + "mean": 0.16185897588729858, + "persist": 0.17410714378846542 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.007763975166848728, + "mean": 0.0021500238134259625, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.11650485305913857, + "best1L": 0.08183078920202595, + "mean": 0.030193107734833444, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14720812014171056, + "best1L": 0.11167512913899762, + "mean": 0.03996764615710292, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.13376623392105103, + "best1L": 0.10389610486371177, + "mean": 0.026873127224722078, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004261045051472527, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20386904637728417, + "mean": 0.16357600688934326, + "persist": 0.17708333461944545 + }, + "association": { + "min": 0.02173912977533681, + "best1L": 0.009316770226827689, + "mean": 0.0025083611253648996, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.11650485305913857, + "best1L": 0.08183078920202595, + "mean": 0.030086418081607138, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829540678433, + "best1L": 0.11094996625823635, + "mean": 0.039883973609123914, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.12987013161182404, + "best1L": 0.101298702614648, + "mean": 0.0257742257256593, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20089285767504148, + "mean": 0.16346154042652675, + "persist": 0.17857142950275115 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.007763975166848728, + "mean": 0.0025083612085187008, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.030939933710864613, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14684554136225156, + "best1L": 0.11203771057937827, + "mean": 0.039856083158935816, + "persist": 0.039883973476077826 + }, + "order-ops": { + "min": 0.1272727277662073, + "best1L": 0.09870129876903125, + "mean": 0.025074925107349242, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833283662796, + "best1L": 0.20238095202616282, + "mean": 0.16597985343209334, + "persist": 0.18005952332168818 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.03104662336409092, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829577931336, + "best1L": 0.11203771057937827, + "mean": 0.03977241101009505, + "persist": 0.04060913648988519 + }, + "order-ops": { + "min": 0.1298701294830867, + "best1L": 0.10000000042574746, + "mean": 0.025674325680094107, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523454734255, + "best1L": 0.20238095202616282, + "mean": 0.16506410363529408, + "persist": 0.17708333249070815 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.03093993424304894, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1457577949123723, + "best1L": 0.11203771057937827, + "mean": 0.039632956364325116, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.1298701294830867, + "best1L": 0.10000000042574746, + "mean": 0.02547452479068722, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3675595202616283, + "best1L": 0.2053571417927742, + "mean": 0.1655219742762191, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08460471592843533, + "mean": 0.030833245122006962, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14612037635275296, + "best1L": 0.11167512913899762, + "mean": 0.03974451869726181, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.1285714294229235, + "best1L": 0.10000000042574746, + "mean": 0.025274725231741155, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.20684523561171123, + "mean": 0.16609431936272553, + "persist": 0.18005952225731953 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + } + }, + "runtime_s": 1754.8 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_USPTO_Backgrounds_s1.json b/results/ladder410_cpu/ladder_USPTO_Backgrounds_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..039530fbe686e52a6894fc3e1863cd7aa05d9fab --- /dev/null +++ b/results/ladder410_cpu/ladder_USPTO_Backgrounds_s1.json @@ -0,0 +1,463 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.12205270385103566, + "best1L": 0.08321775256523065, + "mean": 0.03126000193879008, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14974619394966535, + "best1L": 0.11276287186358656, + "mean": 0.04097171766417367, + "persist": 0.03988397454044649 + }, + "order-ops": { + "min": 0.12337662545698029, + "best1L": 0.09870130089776856, + "mean": 0.023676323671159998, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.027696792967617512, + "best1L": 0.00874635563897235, + "mean": 0.004821708891540766, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071417927742, + "best1L": 0.19940476146127498, + "mean": 0.15945513040891715, + "persist": 0.16964285913854837 + }, + "association": { + "min": 0.02018633538058826, + "best1L": 0.01086956528680665, + "mean": 0.0020305780760411707, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.12343966774642467, + "best1L": 0.08460471646061965, + "mean": 0.03126000300315874, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1497461912887437, + "best1L": 0.11348803634090084, + "mean": 0.040776482317596674, + "persist": 0.04097171779721975 + }, + "order-ops": { + "min": 0.11948051889027868, + "best1L": 0.0961038986487048, + "mean": 0.023176823775949224, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004709576190050159, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011904371636255, + "best1L": 0.20535714525197232, + "mean": 0.1656364481896162, + "persist": 0.18154761607625655 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11927877818899495, + "best1L": 0.08460471486406666, + "mean": 0.030833244922437837, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1519216825919492, + "best1L": 0.11530094407498837, + "mean": 0.04080437310039997, + "persist": 0.03915881032922438 + }, + "order-ops": { + "min": 0.1142857139930129, + "best1L": 0.09350649387176548, + "mean": 0.023076923755330166, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.023323615480746542, + "best1L": 0.00874635563897235, + "mean": 0.004036779648491314, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3526785724929401, + "best1L": 0.19345238299242087, + "mean": 0.1564789390457528, + "persist": 0.16517857129552535 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.009316770226827689, + "mean": 0.002269469650595316, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.12066574155219964, + "best1L": 0.08460471486406666, + "mean": 0.031153312751225064, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.15083393614206994, + "best1L": 0.11457578119422708, + "mean": 0.040943827746169906, + "persist": 0.03915881032922438 + }, + "order-ops": { + "min": 0.1168831153107541, + "best1L": 0.09350649506918021, + "mean": 0.023376623817187334, + "persist": 0.009090908936091832 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004261044918426445, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.19791666764233792, + "mean": 0.15716575139335223, + "persist": 0.16815476212650537 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.009316770226827689, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.11650485412350722, + "best1L": 0.08183079026639462, + "mean": 0.030406487307378223, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1479332862155778, + "best1L": 0.11457578119422708, + "mean": 0.04052546440756747, + "persist": 0.03879622928798199 + }, + "order-ops": { + "min": 0.11818181736660856, + "best1L": 0.09090909055833306, + "mean": 0.023776223409056132, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714285714285715, + "best1L": 0.19196428411773273, + "mean": 0.15739468644772256, + "persist": 0.17113095508622272 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.11650485412350722, + "best1L": 0.08183079026639462, + "mean": 0.03029979738805975, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1472081212060792, + "best1L": 0.11385061565254416, + "mean": 0.0403302287948983, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.11688311517770801, + "best1L": 0.09220779274723359, + "mean": 0.02377622394124046, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004373177686440093, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35863095096179415, + "best1L": 0.19940475959862983, + "mean": 0.16037088047180856, + "persist": 0.1726190499695284 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.11650485412350722, + "best1L": 0.08183079026639462, + "mean": 0.030619865283370018, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1479332862155778, + "best1L": 0.11348803634090084, + "mean": 0.04046968211020742, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.11688311624207667, + "best1L": 0.09350649387176548, + "mean": 0.023776223542102213, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190447807312, + "best1L": 0.20089285554630415, + "mean": 0.1607142865125622, + "persist": 0.1726190499695284 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08321775389569146, + "mean": 0.03051317563014371, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14539521134325437, + "best1L": 0.11203771057937827, + "mean": 0.03980030012982232, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.12337662412651948, + "best1L": 0.09480519512934345, + "mean": 0.02447552511668099, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.026239066384732723, + "best1L": 0.00874635563897235, + "mean": 0.004485310654022864, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.20089285767504148, + "mean": 0.1641483549028635, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11373092952583518, + "best1L": 0.08183079000030245, + "mean": 0.030086418081607138, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14612037475619996, + "best1L": 0.11312545490052019, + "mean": 0.03991186472454241, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.12337662412651948, + "best1L": 0.09480519512934345, + "mean": 0.024375624405885383, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160714285714285, + "best1L": 0.1979166679084301, + "mean": 0.1624313188450677, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08321775389569146, + "mean": 0.03051317563014371, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.1464829540678433, + "best1L": 0.11312545490052019, + "mean": 0.040051319902496677, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.12207792193761893, + "best1L": 0.09480519512934345, + "mean": 0.024275724360320185, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904903820585, + "best1L": 0.19940476385610445, + "mean": 0.16254578450960772, + "persist": 0.1770833314263395 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11511789129248687, + "best1L": 0.08183079000030245, + "mean": 0.030299796855875423, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14684553763696126, + "best1L": 0.11348803634090084, + "mean": 0.04016288250152554, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.12337662332824298, + "best1L": 0.0961038965199675, + "mean": 0.02437562445577766, + "persist": 0.010389610060623713 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160714285714285, + "best1L": 0.19940476385610445, + "mean": 0.162660257092544, + "persist": 0.17857142737401382 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + } + }, + "runtime_s": 1929.0 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_USPTO_Backgrounds_s2.json b/results/ladder410_cpu/ladder_USPTO_Backgrounds_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..12864fed3e8c1dca3c0c80fccebb2df4c1d45e5d --- /dev/null +++ b/results/ladder410_cpu/ladder_USPTO_Backgrounds_s2.json @@ -0,0 +1,463 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.11511789262294769, + "best1L": 0.08321775256523065, + "mean": 0.029873038841677562, + "persist": 0.020804438473922864 + }, + "multilingual": { + "min": 0.1530094269130911, + "best1L": 0.11602610695574965, + "mean": 0.04136219115129539, + "persist": 0.041334300568061214 + }, + "order-ops": { + "min": 0.11298701180411237, + "best1L": 0.09220779381160225, + "mean": 0.020279720483813435, + "persist": 0.0012987012575779641 + }, + "poetry": { + "min": 0.03061224440378802, + "best1L": 0.01020408155662673, + "mean": 0.005045974494091102, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713859966825, + "best1L": 0.20089285927159445, + "mean": 0.16357600609106676, + "persist": 0.17559523481343473 + }, + "association": { + "min": 0.024844719895294735, + "best1L": 0.009316770359873772, + "mean": 0.003225035799135055, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.11789181722061974, + "best1L": 0.08183079133076328, + "mean": 0.030833244290468947, + "persist": 0.020804438473922864 + }, + "multilingual": { + "min": 0.15445975533553533, + "best1L": 0.1167512701026031, + "mean": 0.04094382728050862, + "persist": 0.04060913742120777 + }, + "order-ops": { + "min": 0.114285714126059, + "best1L": 0.0948051961937121, + "mean": 0.02207792202742504, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523241860525, + "best1L": 0.20238095388880797, + "mean": 0.16494963238281862, + "persist": 0.16964286020291702 + }, + "association": { + "min": 0.02173912977533681, + "best1L": 0.00776397529989481, + "mean": 0.002747252749811326, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.11927877872117928, + "best1L": 0.08321775389569146, + "mean": 0.031260002737066576, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14720812067389488, + "best1L": 0.11203771057937827, + "mean": 0.0395213930335428, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.12337662386042732, + "best1L": 0.10000000149011612, + "mean": 0.024975024895476445, + "persist": 0.011688311185155596 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261045117995569, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.21428571215697698, + "mean": 0.16929945003773486, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.018633540453655378, + "best1L": 0.00776397529989481, + "mean": 0.00203057802614889, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.12066574155219964, + "best1L": 0.08183079212903976, + "mean": 0.030939934575664147, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14938360984836305, + "best1L": 0.11457578066204276, + "mean": 0.0402465554486428, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12207792273589543, + "best1L": 0.09870130036558424, + "mean": 0.023676323438329355, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004485310654022864, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3675595223903656, + "best1L": 0.20982143070016587, + "mean": 0.16861264168151788, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.00776397529989481, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.12343966774642467, + "best1L": 0.08321775469396796, + "mean": 0.030086418414222344, + "persist": 0.01664355051304613 + }, + "multilingual": { + "min": 0.14503263043505804, + "best1L": 0.10949963783579213, + "mean": 0.039856083425027986, + "persist": 0.0384336474484631 + }, + "order-ops": { + "min": 0.1272727298949446, + "best1L": 0.09870130089776856, + "mean": 0.02387612429447472, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.027696792302387103, + "best1L": 0.00874635563897235, + "mean": 0.0047095762565732, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36904762259551455, + "best1L": 0.20684523986918585, + "mean": 0.16540750887777125, + "persist": 0.17857142737401382 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.12343966774642467, + "best1L": 0.08321775469396796, + "mean": 0.030619864551616565, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14575779118708201, + "best1L": 0.1116751270102603, + "mean": 0.040051319636404514, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.12597402557730675, + "best1L": 0.09870130089776856, + "mean": 0.02397602434003992, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833304950169, + "best1L": 0.20982143176453455, + "mean": 0.16586538617100036, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.12205270544758864, + "best1L": 0.08183079212903976, + "mean": 0.03061986475118569, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14684553763696126, + "best1L": 0.11203771057937827, + "mean": 0.040134992849613936, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.12467532764588084, + "best1L": 0.09870130089776856, + "mean": 0.023276723488899215, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004597443422036511, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142857142855, + "best1L": 0.2083333358168602, + "mean": 0.16597985263381684, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.00776397529989481, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.11789181589015893, + "best1L": 0.08183079026639462, + "mean": 0.02976635018629687, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.14684554189443588, + "best1L": 0.11131254769861698, + "mean": 0.040302337014249394, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12207792113934245, + "best1L": 0.09480519645980426, + "mean": 0.02327672315628401, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713859966825, + "best1L": 0.1994047649204731, + "mean": 0.16426282109958784, + "persist": 0.18005952225731953 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.11789181589015893, + "best1L": 0.08183079026639462, + "mean": 0.02976635018629687, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1461203790136746, + "best1L": 0.11058738375348705, + "mean": 0.03999553760513663, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12337662226387433, + "best1L": 0.09610389758433614, + "mean": 0.023876123429675187, + "persist": 0.006493506687028068 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3630952366760799, + "best1L": 0.20238095255834715, + "mean": 0.16517857222684793, + "persist": 0.18005952438605682 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.11789181589015893, + "best1L": 0.08183079212903976, + "mean": 0.030193108267017772, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14720812333481653, + "best1L": 0.11094996625823635, + "mean": 0.04010710126853415, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12077922001481056, + "best1L": 0.09610389758433614, + "mean": 0.023876123961859515, + "persist": 0.007792207811559949 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3630952366760799, + "best1L": 0.20535714445369585, + "mean": 0.16472069839281694, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.11927877978554793, + "best1L": 0.08183079212903976, + "mean": 0.030193107734833444, + "persist": 0.019417475908994675 + }, + "multilingual": { + "min": 0.14757070264645986, + "best1L": 0.11167512913899762, + "mean": 0.04021866486540863, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.12077922001481056, + "best1L": 0.09610389758433614, + "mean": 0.02327672315628401, + "persist": 0.005194805030311857 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004597443422036511, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333475249155, + "best1L": 0.20238095362271583, + "mean": 0.16483516432344913, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + } + }, + "runtime_s": 1891.5 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Wikipedia_en_s0.json b/results/ladder410_cpu/ladder_Wikipedia_en_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..a72b9b260990020a66bfaab0e3ed6487afee382b --- /dev/null +++ b/results/ladder410_cpu/ladder_Wikipedia_en_s0.json @@ -0,0 +1,659 @@ +{ + "corpus": "Wikipedia_en", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10402219103915351, + "best1L": 0.07905686620090689, + "mean": 0.0284860768089337, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1381435806729964, + "best1L": 0.11131254477160317, + "mean": 0.03402688706825886, + "persist": 0.02828136259423835 + }, + "order-ops": { + "min": 0.08181818095701081, + "best1L": 0.06753246752279145, + "mean": 0.02147852147131094, + "persist": 0.02207792177796364 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.005158107195581708, + "persist": 0.0 + }, + "typo": { + "min": 0.29910714498588015, + "best1L": 0.17559523561171123, + "mean": 0.12557234375604562, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0020305781425642116, + "persist": 0.001552795059978962 + } + }, + "50": { + "multihop": { + "min": 0.09986130467482976, + "best1L": 0.08044382903192725, + "mean": 0.028379387222230434, + "persist": 0.02219140050666673 + }, + "multilingual": { + "min": 0.13125452984656608, + "best1L": 0.10406091250479221, + "mean": 0.03344117830108319, + "persist": 0.026831036699669703 + }, + "order-ops": { + "min": 0.08311688314591135, + "best1L": 0.07012987083622388, + "mean": 0.019180819358942763, + "persist": 0.015584415889212064 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.005270239897072315, + "persist": 0.0 + }, + "typo": { + "min": 0.2991071439215115, + "best1L": 0.17857142697487557, + "mean": 0.1263736252273832, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.007763974900756564, + "best1L": 0.003105590119957924, + "mean": 0.001313903502055577, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.09847434264208589, + "best1L": 0.0832177536295993, + "mean": 0.02827269783509629, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13052936696580478, + "best1L": 0.10224800556898117, + "mean": 0.03377586816038404, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.07922077870794705, + "best1L": 0.068831169711692, + "mean": 0.017282717428835376, + "persist": 0.011688311318201678 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.00549450536657657, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3050595277122089, + "best1L": 0.17113095468708447, + "mean": 0.12328296580484935, + "persist": 0.12351190391927958 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.09986130467482976, + "best1L": 0.08044382903192725, + "mean": 0.028379387754414762, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.1316171112869467, + "best1L": 0.10406091277088438, + "mean": 0.03399899635197861, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.07922077870794705, + "best1L": 0.068831169711692, + "mean": 0.018681318931547657, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.01020408155662673, + "mean": 0.005158107328627791, + "persist": 0.0 + }, + "typo": { + "min": 0.30357143389327185, + "best1L": 0.17410714445369585, + "mean": 0.12465659342706203, + "persist": 0.1279761925605791 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.028059319060828005, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.1298042062137808, + "best1L": 0.10224800530288901, + "mean": 0.03397110483742186, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.06493506500763553, + "mean": 0.01868131888165538, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952381236212595, + "best1L": 0.17857143016798155, + "mean": 0.12854853511920997, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02816600844796215, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13016678765416145, + "best1L": 0.10224800530288901, + "mean": 0.03394321385504944, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.018481518790524985, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004709576123527118, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3095238134264946, + "best1L": 0.17857143016798155, + "mean": 0.12774725072085857, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028272697569004128, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13125453197530337, + "best1L": 0.10333574989012309, + "mean": 0.03411055961623788, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.01868131888165538, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381236212595, + "best1L": 0.17708333422030723, + "mean": 0.12763278479022638, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028272698633372784, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1359680922968047, + "best1L": 0.10877447415675436, + "mean": 0.03486361414460199, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06493506500763553, + "mean": 0.01958041960772659, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31398809381893705, + "best1L": 0.17857143016798155, + "mean": 0.12831959500908852, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.02816600871405431, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13560551085642406, + "best1L": 0.10732414626649447, + "mean": 0.034752049416835816, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06493506500763553, + "mean": 0.019380619516596198, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.01020408155662673, + "mean": 0.005158107328627791, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31398809275456835, + "best1L": 0.17857143016798155, + "mean": 0.12774725391396455, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0010750119358168117, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.02816600871405431, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1345177654709135, + "best1L": 0.10659898338573319, + "mean": 0.03466837700190289, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.06753246858716011, + "mean": 0.019380619317027076, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.005270240030118397, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.17559523827263288, + "mean": 0.12751831619867257, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.02848607687545674, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13451776653528214, + "best1L": 0.10659898338573319, + "mean": 0.034612595037158043, + "persist": 0.028281362461192266 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.06753246858716011, + "mean": 0.019480519894776598, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.005270240030118397, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.17559523827263288, + "mean": 0.1276327815971204, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028272698633372784, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13705583661794662, + "best1L": 0.11022480045046125, + "mean": 0.035030959373606105, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06363636281873498, + "mean": 0.018981019001720206, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.0 + }, + "typo": { + "min": 0.3139880895614624, + "best1L": 0.18005952505128725, + "mean": 0.12774725444614887, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028486077141548907, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13524292888385908, + "best1L": 0.109137055597135, + "mean": 0.034891504461744, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06493506500763553, + "mean": 0.019280719471031, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.1785714291036129, + "mean": 0.1277472512530429, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13488034637910978, + "best1L": 0.10804931127599307, + "mean": 0.03480783204681107, + "persist": 0.029731690351452147 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.019080919063916162, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.12740384601056576, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1341551834983485, + "best1L": 0.10804931127599307, + "mean": 0.03475205061425056, + "persist": 0.029731690351452147 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.01898101901835097, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.1274038470749344, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13488034744347846, + "best1L": 0.10804931127599307, + "mean": 0.03477994053225432, + "persist": 0.029731690351452147 + }, + "order-ops": { + "min": 0.07662337619279112, + "best1L": 0.06493506500763553, + "mean": 0.01898101901835097, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.1274038470749344, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + } + }, + "runtime_s": 2927.5 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Wikipedia_en_s1.json b/results/ladder410_cpu/ladder_Wikipedia_en_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..0d2e890c9a79d525fce92d13c1666ba94a8af688 --- /dev/null +++ b/results/ladder410_cpu/ladder_Wikipedia_en_s1.json @@ -0,0 +1,659 @@ +{ + "corpus": "Wikipedia_en", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10263522900640965, + "best1L": 0.08460471752498831, + "mean": 0.0295529709463673, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.14467005005904607, + "best1L": 0.1127628707992179, + "mean": 0.03603503088067685, + "persist": 0.028643944832895483 + }, + "order-ops": { + "min": 0.07792207731732301, + "best1L": 0.06883116944559983, + "mean": 0.020779220952785442, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.008746355505926269, + "mean": 0.005045974427568061, + "persist": 0.0 + }, + "typo": { + "min": 0.3035714307001659, + "best1L": 0.1770833320915699, + "mean": 0.12637362336473806, + "persist": 0.1309523812628218 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.004658385179936886, + "mean": 0.0014333492727018893, + "persist": 0.001552795059978962 + } + }, + "50": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.029126212932169437, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13814358093908854, + "best1L": 0.10913705293621336, + "mean": 0.03522619491975222, + "persist": 0.028643944167665074 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.06623376719653606, + "mean": 0.02177822227323694, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.030612245202064514, + "best1L": 0.007288629721317973, + "mean": 0.004485310587499823, + "persist": 0.0 + }, + "typo": { + "min": 0.30059524306229185, + "best1L": 0.17559524040137017, + "mean": 0.12568681447633676, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.004658385179936886, + "mean": 0.0013139034854248166, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028486077141548907, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13778099896652357, + "best1L": 0.10659898285354887, + "mean": 0.034947286759104045, + "persist": 0.027918781552995955 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06753246858716011, + "mean": 0.020779220686693276, + "persist": 0.020779221185616085 + }, + "poetry": { + "min": 0.032069972184087546, + "best1L": 0.00728862958827189, + "mean": 0.004933841726077455, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.18005952771220887, + "mean": 0.12511446779327734, + "persist": 0.12351190538278647 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079079857894, + "mean": 0.02869945644800152, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1385061623794692, + "best1L": 0.10877447149583272, + "mean": 0.03528197735015835, + "persist": 0.02828136259423835 + }, + "order-ops": { + "min": 0.08051947983247894, + "best1L": 0.06753246858716011, + "mean": 0.02047952074956681, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.03498542375330414, + "best1L": 0.008746355505926269, + "mean": 0.005270240030118397, + "persist": 0.0 + }, + "typo": { + "min": 0.3035714328289032, + "best1L": 0.17708333422030723, + "mean": 0.1254578780915056, + "persist": 0.12351190431841783 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.028166008181869984, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13705583768231527, + "best1L": 0.1094996381018843, + "mean": 0.03522619491975222, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.068831169711692, + "mean": 0.019680319786337868, + "persist": 0.018181818404367993 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18005952398691857, + "mean": 0.12934981659054756, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0011944576565708434, + "persist": 0.001552795059978962 + } + }, + "175": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.028272697569004128, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13669325624193465, + "best1L": 0.11058738242302622, + "mean": 0.03511463165549295, + "persist": 0.028643944300711155 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.01998001992303346, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18005952398691857, + "mean": 0.12809065809207304, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079079857894, + "mean": 0.02848607660936458, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13741841805832727, + "best1L": 0.11022480098264557, + "mean": 0.035337758782718866, + "persist": 0.028643944300711155 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.02017982054634818, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18154761993459292, + "mean": 0.12820513200547015, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0011944576565708434, + "persist": 0.001552795059978962 + } + }, + "300": { + "multihop": { + "min": 0.0970873800771577, + "best1L": 0.07905686620090689, + "mean": 0.027845940020467554, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13306744077375957, + "best1L": 0.10696156482611384, + "mean": 0.03441735949101193, + "persist": 0.02936910797974893 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.06493506500763553, + "mean": 0.01958041955783431, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.31101190405232565, + "best1L": 0.18005952558347157, + "mean": 0.12786172117505754, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0008361203862088067, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13379260365452086, + "best1L": 0.10659898338573319, + "mean": 0.034501031772898774, + "persist": 0.02936910797974893 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06623376639825958, + "mean": 0.01958041960772659, + "persist": 0.015584415889212064 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381023338865, + "best1L": 0.18005952558347157, + "mean": 0.12786172277161054, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13379260152578354, + "best1L": 0.10659898338573319, + "mean": 0.034417358958827596, + "persist": 0.02828136259423835 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06623376639825958, + "mean": 0.019980020322171704, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381449086325, + "best1L": 0.18005952558347157, + "mean": 0.12751831247338227, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028379387754414762, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13488034584692546, + "best1L": 0.10696156482611384, + "mean": 0.03461259536977325, + "persist": 0.02864394403461899 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06623376639825958, + "mean": 0.02017981988111777, + "persist": 0.01948051926280771 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.007288629721317973, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.3080357164144516, + "best1L": 0.17857142963579722, + "mean": 0.12694597191044263, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.0970873800771577, + "best1L": 0.08044382796755858, + "mean": 0.027845940286559717, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1337926025901522, + "best1L": 0.10696156482611384, + "mean": 0.03447314092357244, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080918997393122, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30505952558347155, + "best1L": 0.17708333315593855, + "mean": 0.12637362868658134, + "persist": 0.1264880955485361 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.0970873800771577, + "best1L": 0.08044382796755858, + "mean": 0.027845940286559717, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1341551819017955, + "best1L": 0.10696156482611384, + "mean": 0.03458470445392387, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080919263485287, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952381023338865, + "best1L": 0.17559523827263288, + "mean": 0.12797619295971735, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.09847434290817805, + "best1L": 0.08044382796755858, + "mean": 0.027845940286559717, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.13488034584692546, + "best1L": 0.10732414626649447, + "mean": 0.03461259517020413, + "persist": 0.028281362461192266 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080919263485287, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004709575990481036, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.17559523827263288, + "mean": 0.1279761895005192, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.028059319060828005, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1352429272873061, + "best1L": 0.10732414626649447, + "mean": 0.03466837660276464, + "persist": 0.028281362461192266 + }, + "order-ops": { + "min": 0.07532467400389058, + "best1L": 0.06363636388310365, + "mean": 0.019080919263485287, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.17708333422030723, + "mean": 0.12751831726304122, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.028059319060828005, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1341551819017955, + "best1L": 0.10659898285354887, + "mean": 0.03458470392173955, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.07662337539451462, + "best1L": 0.06363636388310365, + "mean": 0.01928071940450796, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30505952345473425, + "best1L": 0.17708333422030723, + "mean": 0.12728937795119627, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + } + }, + "runtime_s": 2809.6 +} \ No newline at end of file diff --git a/results/ladder410_cpu/ladder_Wikipedia_en_s2.json b/results/ladder410_cpu/ladder_Wikipedia_en_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..32b65b03e1f02531ffd3801578ce998075362867 --- /dev/null +++ b/results/ladder410_cpu/ladder_Wikipedia_en_s2.json @@ -0,0 +1,463 @@ +{ + "corpus": "Wikipedia_en", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079079857894, + "mean": 0.02869945584929415, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.1421319775815521, + "best1L": 0.11312545250569071, + "mean": 0.03592346774946366, + "persist": 0.031544597154217105 + }, + "order-ops": { + "min": 0.08051948089684759, + "best1L": 0.06363636281873498, + "mean": 0.020879121663581048, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30059523986918585, + "best1L": 0.1800595261156559, + "mean": 0.12339743519467967, + "persist": 0.1220238082376974 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028806145302951336, + "persist": 0.02219140077275889 + }, + "multilingual": { + "min": 0.1359680893697909, + "best1L": 0.10841189165200506, + "mean": 0.03491939544411642, + "persist": 0.029369107447564602 + }, + "order-ops": { + "min": 0.08051947983247894, + "best1L": 0.07012987083622388, + "mean": 0.020079919968598654, + "persist": 0.01948051926280771 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.0 + }, + "typo": { + "min": 0.3020833336881229, + "best1L": 0.17857142803924425, + "mean": 0.1250000007982765, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "75": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.0279526294076017, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13161711155303887, + "best1L": 0.10442349474344935, + "mean": 0.03388743262205805, + "persist": 0.027556199580430984 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.018681318931547657, + "persist": 0.014285714631634099 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.01020408155662673, + "mean": 0.005158107195581708, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3020833304950169, + "best1L": 0.17261904890515975, + "mean": 0.12396978054727827, + "persist": 0.12202380943511214 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.10124826723975795, + "best1L": 0.08183079026639462, + "mean": 0.028059319593012333, + "persist": 0.020804438207830702 + }, + "multilingual": { + "min": 0.13451776653528214, + "best1L": 0.10804931021162442, + "mean": 0.03450103163985269, + "persist": 0.02973169008535998 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.068831169711692, + "mean": 0.019380619250504032, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.01020408155662673, + "mean": 0.005158107195581708, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3005952388048172, + "best1L": 0.1755952350795269, + "mean": 0.12396977815244879, + "persist": 0.12202381089861904 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "150": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08044382796755858, + "mean": 0.028059318528643677, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.1330674391772066, + "best1L": 0.10659898338573319, + "mean": 0.03425001412896173, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06753246858716011, + "mean": 0.019080919113808444, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.00728862958827189, + "mean": 0.00515810726210475, + "persist": 0.0 + }, + "typo": { + "min": 0.3124999978712627, + "best1L": 0.1770833320915699, + "mean": 0.12797619083098002, + "persist": 0.12351190471755606 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "175": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.028166007915777818, + "persist": 0.01803051307797432 + }, + "multilingual": { + "min": 0.13343001848884992, + "best1L": 0.10696156482611384, + "mean": 0.034194232164216895, + "persist": 0.027556199580430984 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06753246858716011, + "mean": 0.01898101906824325, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.31101190405232565, + "best1L": 0.17559523933700152, + "mean": 0.12809065729379654, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0013139034272171557, + "persist": 0.001552795059978962 + } + }, + "200": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02837938695613827, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13379260205796786, + "best1L": 0.10659898338573319, + "mean": 0.034222122614404986, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.0792207797723157, + "best1L": 0.06753246858716011, + "mean": 0.01918081915937364, + "persist": 0.016883117013743947 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30803571215697695, + "best1L": 0.17261904637728417, + "mean": 0.12614468804427556, + "persist": 0.12202380664114441 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "300": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08044382796755858, + "mean": 0.028272697302911962, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13488034425037249, + "best1L": 0.10732414679867881, + "mean": 0.03438946810950126, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.019780219698856984, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.1785714291036129, + "mean": 0.1287774701735803, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.0279526294076017, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13560550713113376, + "best1L": 0.10804930967944008, + "mean": 0.034501032172037024, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.019480519562161396, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.1785714291036129, + "mean": 0.1279761895005192, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0008361203862088067, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02805931879473584, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13451776280999184, + "best1L": 0.10732414573431015, + "mean": 0.034389468508639505, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.019380619250504032, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.17559523827263288, + "mean": 0.1279761895005192, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.09986130547310625, + "best1L": 0.08183079026639462, + "mean": 0.02816600844796215, + "persist": 0.01941747564290251 + }, + "multilingual": { + "min": 0.13451776280999184, + "best1L": 0.10768672823905945, + "mean": 0.034333686410848586, + "persist": 0.02900652600718396 + }, + "order-ops": { + "min": 0.07792207758341517, + "best1L": 0.06623376639825958, + "mean": 0.01958041960772659, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004709575990481036, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.17559523827263288, + "mean": 0.12763278585459506, + "persist": 0.12351190471755606 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + } + }, + "runtime_s": 978.9 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rescore/ladder_Github_s0.json b/results/ladder410_cpu_rescore/ladder_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..409e748ea30aa27e2df29dcf3a8c13877de129de --- /dev/null +++ b/results/ladder410_cpu_rescore/ladder_Github_s0.json @@ -0,0 +1,659 @@ +{ + "corpus": "Github", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.016856929237422134, + "persist": 0.006934812824640956 + }, + "multilingual": { + "min": 0.11131254530378751, + "best1L": 0.07360406113522393, + "mean": 0.03054052504843899, + "persist": 0.026468455392335142 + }, + "order-ops": { + "min": 0.08311688367809568, + "best1L": 0.08311688367809568, + "mean": 0.021678321861795018, + "persist": 0.01558441509093557 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.24702381023338862, + "best1L": 0.15476190910807677, + "mean": 0.08447802292981318, + "persist": 0.07886904730860676 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.017603755213453302, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11675126690949712, + "best1L": 0.07831762065844876, + "mean": 0.032437106394874196, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07532467506825924, + "mean": 0.018581418486844217, + "persist": 0.012987012575779642 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095628363745, + "best1L": 0.15922619349190167, + "mean": 0.09741300317857947, + "persist": 0.08779761926936251 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0011944577147785043, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.018670649966225028, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12327773869037628, + "best1L": 0.08665699937513896, + "mean": 0.034277904379580705, + "persist": 0.031544597287263186 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.0212787209810423, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618763787405, + "best1L": 0.16220238006540708, + "mean": 0.10382326080330781, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.010869565419852734, + "best1L": 0.004658385179936886, + "mean": 0.0013139034771094365, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823690840175, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12146482989192009, + "best1L": 0.08375634465898786, + "mean": 0.03352485144776957, + "persist": 0.030094271525740623 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.021478521737403104, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177952532258, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.15922618817005837, + "mean": 0.09993131937725204, + "persist": 0.09077380983425039 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017070308011690422, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.10913705639541149, + "best1L": 0.07142857102943319, + "mean": 0.030373179553342716, + "persist": 0.025018128167305673 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.08181818202137947, + "mean": 0.024675324891826937, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928603359631, + "best1L": 0.15922619242753303, + "mean": 0.08928571282220739, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017176997398824563, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11167512647807598, + "best1L": 0.07287889798837048, + "mean": 0.03101467008569411, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023876123629244312, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095415490015, + "best1L": 0.16071428837520735, + "mean": 0.09191849787852593, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01717699766491673, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11167512674416814, + "best1L": 0.07251631641494376, + "mean": 0.031014670551355396, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023676323671159998, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928582072258, + "best1L": 0.16071428837520735, + "mean": 0.09100274728345019, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01771044486667961, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.113850615386452, + "best1L": 0.0746918051902737, + "mean": 0.03187928767874837, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.023076922865584493, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709024586848, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976140805653, + "best1L": 0.16220237793666975, + "mean": 0.09581043757498264, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00011944577064631241, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07722987633730684, + "mean": 0.03226976083325488, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023076922998630574, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.278273805975914, + "best1L": 0.16220237793666975, + "mean": 0.09764194036168712, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07759245777768749, + "mean": 0.032409214946840494, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2812499978712627, + "best1L": 0.16220237793666975, + "mean": 0.09855769308550018, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07650471345654555, + "mean": 0.0322697607002088, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428837520735, + "mean": 0.09775641188025475, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06657420311655317, + "mean": 0.018137202681308345, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11747643191899572, + "best1L": 0.08085569100720542, + "mean": 0.033078596993748634, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07662337619279112, + "mean": 0.023476524245260016, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2782738038471767, + "best1L": 0.16071428837520735, + "mean": 0.09878662947033133, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01792382390704006, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07759245804377965, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.02417582429812423, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529761833803995, + "best1L": 0.16220237793666975, + "mean": 0.09592490643262863, + "persist": 0.08333333382116896 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07868020236492157, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.0792207805705922, + "best1L": 0.07792207838169166, + "mean": 0.023976024739178165, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27678571215697695, + "best1L": 0.16220237793666975, + "mean": 0.09661172330379486, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11638868546911649, + "best1L": 0.07868020236492157, + "mean": 0.03254866899390306, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023776224381955608, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.16220237793666975, + "mean": 0.0974130045090403, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07831762092454093, + "mean": 0.03246499631287796, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023876124427520802, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095223903656, + "best1L": 0.16220237793666975, + "mean": 0.09695512801408768, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + } + }, + "runtime_s": 240.1 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Github_s0.json b/results/ladder410_cpu_rstrip/ladder_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..f963386c2d6a7646b09e6a6f7cc0a8a33ef40572 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Github_s0.json @@ -0,0 +1,661 @@ +{ + "corpus": "Github", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.1789181706096445, + "best1L": 0.1303744800388813, + "mean": 0.044596181118062565, + "persist": 0.02912621306521552 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07614213228225708, + "mean": 0.03098677970202906, + "persist": 0.026468455392335142 + }, + "order-ops": { + "min": 0.5909090851034436, + "best1L": 0.3662337708686079, + "mean": 0.32657341978379656, + "persist": 0.36363636010459494 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.24702381023338862, + "best1L": 0.15476190910807677, + "mean": 0.08447802292981318, + "persist": 0.07886904730860676 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.18307905750615255, + "best1L": 0.13592232896813325, + "mean": 0.04576976417696902, + "persist": 0.02773925050028733 + }, + "multilingual": { + "min": 0.12001450413039752, + "best1L": 0.08158085362187453, + "mean": 0.0330228156942342, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.6025974048035485, + "best1L": 0.366233764482396, + "mean": 0.32847152277827263, + "persist": 0.35974026205284254 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095628363745, + "best1L": 0.15922619349190167, + "mean": 0.09741300317857947, + "persist": 0.08779761926936251 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0011944577147785043, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.17891817220619746, + "best1L": 0.13592232896813325, + "mean": 0.04683665831440261, + "persist": 0.030513175896235874 + }, + "multilingual": { + "min": 0.12654097378253937, + "best1L": 0.08919506945780345, + "mean": 0.034891504395220964, + "persist": 0.031544597287263186 + }, + "order-ops": { + "min": 0.6012986983571734, + "best1L": 0.3662337729973452, + "mean": 0.3197802220072065, + "persist": 0.35064934673053877 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618763787405, + "best1L": 0.16220238006540708, + "mean": 0.10382326080330781, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.010869565419852734, + "best1L": 0.004658385179936886, + "mean": 0.0013139034771094365, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.17614424707634108, + "best1L": 0.13453536720148154, + "mean": 0.04619652172550559, + "persist": 0.027739251298563822 + }, + "multilingual": { + "min": 0.12400289784584727, + "best1L": 0.08665699831077031, + "mean": 0.03408266896648066, + "persist": 0.030094271525740623 + }, + "order-ops": { + "min": 0.5974025960479464, + "best1L": 0.3675324666712965, + "mean": 0.3235764184168407, + "persist": 0.35454545116850306 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177952532258, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.15922618817005837, + "mean": 0.09993131937725204, + "persist": 0.09077380983425039 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.18030513290848052, + "best1L": 0.13176144286990166, + "mean": 0.04448949113222105, + "persist": 0.02219140050666673 + }, + "multilingual": { + "min": 0.11131254716643266, + "best1L": 0.07360405967171703, + "mean": 0.030819433674748455, + "persist": 0.025018128167305673 + }, + "order-ops": { + "min": 0.6064935049840382, + "best1L": 0.3675324719931398, + "mean": 0.3249750291662557, + "persist": 0.355844149099929 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928603359631, + "best1L": 0.15922619242753303, + "mean": 0.08928571282220739, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.1761442476085254, + "best1L": 0.13176144286990166, + "mean": 0.04448949113222105, + "persist": 0.024965325902615274 + }, + "multilingual": { + "min": 0.11385061724909715, + "best1L": 0.07541696807103497, + "mean": 0.03146092473928418, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.6090909051043647, + "best1L": 0.368831172053303, + "mean": 0.32747252764446394, + "persist": 0.35584414697119165 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095415490015, + "best1L": 0.16071428837520735, + "mean": 0.09191849787852593, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.17891817167401314, + "best1L": 0.13453536640320504, + "mean": 0.04480955936014652, + "persist": 0.02635228873363563 + }, + "multilingual": { + "min": 0.11385061751518931, + "best1L": 0.07469180505722761, + "mean": 0.03143303342429655, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.6077922050442014, + "best1L": 0.368831172053303, + "mean": 0.3280719241925648, + "persist": 0.35974025779536795 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928582072258, + "best1L": 0.16071428837520735, + "mean": 0.09100274728345019, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.18862690776586533, + "best1L": 0.13453536640320504, + "mean": 0.04726341639512351, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11602610402873584, + "best1L": 0.07759245671331882, + "mean": 0.03232554180015411, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.601298702614648, + "best1L": 0.36883117045675007, + "mean": 0.32697303167411257, + "persist": 0.35714285767504145 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709024586848, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976140805653, + "best1L": 0.16220237793666975, + "mean": 0.09581043757498264, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00011944577064631241, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.19417475589684077, + "best1L": 0.13592232816985675, + "mean": 0.04769017394366009, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11675126690949712, + "best1L": 0.07976794641997133, + "mean": 0.03271601548684495, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.6025974026748112, + "best1L": 0.3675324688000338, + "mean": 0.32677322519677027, + "persist": 0.35714285767504145 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.278273805975914, + "best1L": 0.16220237793666975, + "mean": 0.09764194036168712, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.19001387059688568, + "best1L": 0.13453536640320504, + "mean": 0.047156726741897206, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11747642979025841, + "best1L": 0.08013052786035198, + "mean": 0.03288336031671081, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.6025974026748112, + "best1L": 0.3675324688000338, + "mean": 0.32667331929717747, + "persist": 0.35714285767504145 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2812499978712627, + "best1L": 0.16220237793666975, + "mean": 0.09855769308550018, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.18862690670149668, + "best1L": 0.13453536640320504, + "mean": 0.04683665884658694, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11747642979025841, + "best1L": 0.07940536497959069, + "mean": 0.03277179678635938, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.601298702614648, + "best1L": 0.3662337698042393, + "mean": 0.3270729258656502, + "persist": 0.35974026205284254 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428837520735, + "mean": 0.09775641188025475, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.19417475483247212, + "best1L": 0.14008321639682567, + "mean": 0.04875706781500152, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11965192269001688, + "best1L": 0.08303118177822658, + "mean": 0.033552741299250295, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.5961038917303085, + "best1L": 0.3675324703965868, + "mean": 0.32447552414877073, + "persist": 0.35324675110833986 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2782738038471767, + "best1L": 0.16071428837520735, + "mean": 0.09878662947033133, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.1914008312991687, + "best1L": 0.13730928993650846, + "mean": 0.047476795435483964, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11820159267101969, + "best1L": 0.08049310956682477, + "mean": 0.032994923647493124, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.5987012961081096, + "best1L": 0.3675324703965868, + "mean": 0.32537461976919857, + "persist": 0.35194804785507067 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529761833803995, + "best1L": 0.16220237793666975, + "mean": 0.09592490643262863, + "persist": 0.08333333382116896 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.19417475589684077, + "best1L": 0.13730928993650846, + "mean": 0.047796863596886396, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11820159267101969, + "best1L": 0.0815808538879667, + "mean": 0.032994923647493124, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.5987013003655842, + "best1L": 0.3675324703965868, + "mean": 0.32487512326666285, + "persist": 0.354545457554715 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27678571215697695, + "best1L": 0.16220237793666975, + "mean": 0.09661172330379486, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.19278779413018907, + "best1L": 0.13869625250143663, + "mean": 0.047583484556525946, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11892675555178098, + "best1L": 0.0815808538879667, + "mean": 0.03305070508005364, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.6000000004257474, + "best1L": 0.3675324703965868, + "mean": 0.3256743204380785, + "persist": 0.35454544903976576 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.16220237793666975, + "mean": 0.0974130045090403, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.19001386953251703, + "best1L": 0.13730928993650846, + "mean": 0.04737010578225766, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11820159267101969, + "best1L": 0.08121827244758606, + "mean": 0.03296703239902854, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.5987013003655842, + "best1L": 0.3675324703965868, + "mean": 0.3254745246044227, + "persist": 0.35584415761487825 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095223903656, + "best1L": 0.16220237793666975, + "mean": 0.09695512801408768, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + } + }, + "runtime_s": 1535.8 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Github_s1.json b/results/ladder410_cpu_rstrip/ladder_Github_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..1364fab300d23bdefeee4e0d419bfa43f274d22a --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Github_s1.json @@ -0,0 +1,661 @@ +{ + "corpus": "Github", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.1969486828893423, + "best1L": 0.13869625596063478, + "mean": 0.050784167767103226, + "persist": 0.030513175896235874 + }, + "multilingual": { + "min": 0.13197969325951167, + "best1L": 0.09390862977930478, + "mean": 0.035867684919919283, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.557142857994352, + "best1L": 0.3636363611689636, + "mean": 0.30679320224693846, + "persist": 0.344155845365354 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.005045974294521979, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28125000106436865, + "best1L": 0.16220238006540708, + "mean": 0.10267856956592628, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0005972288282854217, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.20527045481971332, + "best1L": 0.13869625250143663, + "mean": 0.05238450824150017, + "persist": 0.03328710129218442 + }, + "multilingual": { + "min": 0.1269035546907357, + "best1L": 0.09173314007265228, + "mean": 0.035254086168216806, + "persist": 0.03190717859459775 + }, + "order-ops": { + "min": 0.5935065001249313, + "best1L": 0.3701298673238073, + "mean": 0.3280719316431454, + "persist": 0.36233767068811823 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.005382372731609004, + "persist": 0.005830903670617512 + }, + "typo": { + "min": 0.26488095202616285, + "best1L": 0.16220238219414437, + "mean": 0.0958104379741209, + "persist": 0.08630952425301075 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.1997226034956319, + "best1L": 0.14008321852556296, + "mean": 0.05270457746727126, + "persist": 0.030513175896235874 + }, + "multilingual": { + "min": 0.12182740920356341, + "best1L": 0.08556925372353621, + "mean": 0.03349695986668978, + "persist": 0.029369107713656768 + }, + "order-ops": { + "min": 0.5974026045628956, + "best1L": 0.37402597601924625, + "mean": 0.3332667372056416, + "persist": 0.3636363632977009 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.013119533258889402, + "mean": 0.004597443488559553, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333262375425, + "best1L": 0.16369047707745008, + "mean": 0.09397893731615373, + "persist": 0.08184523960309369 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.20249653341514723, + "best1L": 0.142857146316341, + "mean": 0.05249119849343385, + "persist": 0.030513175896235874 + }, + "multilingual": { + "min": 0.12291515405688967, + "best1L": 0.08810732540275369, + "mean": 0.03402688706825886, + "persist": 0.029006526805460453 + }, + "order-ops": { + "min": 0.5896104020731789, + "best1L": 0.3701298715812819, + "mean": 0.3272727227636746, + "persist": 0.3636363632977009 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004821708891540766, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232142644269125, + "best1L": 0.1651785708963871, + "mean": 0.09661172250551837, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.18723994599921362, + "best1L": 0.13730929286352225, + "mean": 0.04886375780084303, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.12436548194714955, + "best1L": 0.08738216225590024, + "mean": 0.034277905244380236, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.589610395686967, + "best1L": 0.36753247146095547, + "mean": 0.3227772223097937, + "persist": 0.3584415524133614 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28571428464991705, + "best1L": 0.1711309514939785, + "mean": 0.10428113330687795, + "persist": 0.09374999960086175 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.1914008312991687, + "best1L": 0.13869625463017396, + "mean": 0.050037341325410774, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.12400289837803159, + "best1L": 0.0873821611915316, + "mean": 0.03422212354572756, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.6000000004257474, + "best1L": 0.3688311715211187, + "mean": 0.3263736202248505, + "persist": 0.3610389621130058 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2797619051166943, + "best1L": 0.16517856983201845, + "mean": 0.102106226342065, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.19278779359800474, + "best1L": 0.13869625463017396, + "mean": 0.05035740961985929, + "persist": 0.029126214129584178 + }, + "multilingual": { + "min": 0.12545322413955415, + "best1L": 0.0899202326046569, + "mean": 0.034528922489179034, + "persist": 0.030094271259648458 + }, + "order-ops": { + "min": 0.5987013003655842, + "best1L": 0.3688311715211187, + "mean": 0.3244755230844021, + "persist": 0.35844156092831064 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28125000106436865, + "best1L": 0.1666666679084301, + "mean": 0.1023351653878178, + "persist": 0.09226190365318741 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.18446602246591023, + "best1L": 0.13592233109687055, + "mean": 0.04801024104069386, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.1232777370938233, + "best1L": 0.08593183543000903, + "mean": 0.034277905111334155, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.6000000004257474, + "best1L": 0.3688311699245657, + "mean": 0.32637361916048185, + "persist": 0.35844156199267935 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28273809807641165, + "best1L": 0.1636904738843441, + "mean": 0.1029075091438634, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.18446602246591023, + "best1L": 0.13730929286352225, + "mean": 0.04833031106474144, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.12218999277268137, + "best1L": 0.08556925398962838, + "mean": 0.03411055974928396, + "persist": 0.02973168981926782 + }, + "order-ops": { + "min": 0.5987013046230588, + "best1L": 0.37012986998472897, + "mean": 0.32717282165374073, + "persist": 0.3610389621130058 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618976661135, + "best1L": 0.16666666577969277, + "mean": 0.10313644659306322, + "persist": 0.09226190418537174 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.19001386953251703, + "best1L": 0.13869625463017396, + "mean": 0.04907713544421962, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.12255257421306201, + "best1L": 0.08556925398962838, + "mean": 0.03422212341268148, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.597402600305421, + "best1L": 0.3688311715211187, + "mean": 0.3280719241925648, + "persist": 0.35974025992410524 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16517856983201845, + "mean": 0.10164835224194187, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.19001386953251703, + "best1L": 0.14008321639682567, + "mean": 0.04907713597640395, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.12291515565344266, + "best1L": 0.08629441793475832, + "mean": 0.034277904978288073, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.597402600305421, + "best1L": 0.3701298715812819, + "mean": 0.3274725303053856, + "persist": 0.3610389621130058 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16517856983201845, + "mean": 0.10222069599798747, + "persist": 0.0907738082376974 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.19417475589684077, + "best1L": 0.13730929073478496, + "mean": 0.04854368877464107, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11965191843254226, + "best1L": 0.0815808538879667, + "mean": 0.032855469666953595, + "persist": 0.02719361867223467 + }, + "order-ops": { + "min": 0.5961038959877831, + "best1L": 0.37012986998472897, + "mean": 0.3247752328004156, + "persist": 0.35714285873941015 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333155938555, + "best1L": 0.16071428518210137, + "mean": 0.09409340590770755, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.19278779413018907, + "best1L": 0.13730929073478496, + "mean": 0.04811693069392017, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.118201594799757, + "best1L": 0.08194343532834734, + "mean": 0.03319015985886965, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.5974025960479464, + "best1L": 0.36883117045675007, + "mean": 0.32537462402667316, + "persist": 0.35974025992410524 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232142644269125, + "best1L": 0.16071428198899543, + "mean": 0.0953525628096291, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.1914008323635374, + "best1L": 0.13730929073478496, + "mean": 0.04811693175828883, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11856417624013764, + "best1L": 0.08230601676872798, + "mean": 0.03313437829326306, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.5961038917303085, + "best1L": 0.36883117045675007, + "mean": 0.32527473143168856, + "persist": 0.35844155134899275 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095202616283, + "best1L": 0.16220238006540708, + "mean": 0.09569597124521222, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.1914008323635374, + "best1L": 0.13730929073478496, + "mean": 0.04833031106474144, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11856417624013764, + "best1L": 0.08230601676872798, + "mean": 0.03313437829326306, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.5961038959877831, + "best1L": 0.36883117045675007, + "mean": 0.32597402376788004, + "persist": 0.3610389621130058 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2723214253783226, + "best1L": 0.16220238006540708, + "mean": 0.09558150238756623, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.19278779413018907, + "best1L": 0.13730929073478496, + "mean": 0.04843699938750693, + "persist": 0.029126214129584178 + }, + "multilingual": { + "min": 0.11965192056127957, + "best1L": 0.08303118177822658, + "mean": 0.03338539553806186, + "persist": 0.02828136325946876 + }, + "order-ops": { + "min": 0.5974025960479464, + "best1L": 0.37012986998472897, + "mean": 0.3260739232812609, + "persist": 0.35974026205284254 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27380951919725965, + "best1L": 0.16220238006540708, + "mean": 0.09569597097912005, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + } + }, + "runtime_s": 1502.5 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Github_s2.json b/results/ladder410_cpu_rstrip/ladder_Github_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..bea07ec573dd69248fbec7512a2b04a32c2f6887 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Github_s2.json @@ -0,0 +1,661 @@ +{ + "corpus": "Github", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.18723994280610765, + "best1L": 0.13592232896813325, + "mean": 0.046196522390736, + "persist": 0.02219140050666673 + }, + "multilingual": { + "min": 0.11457577933158193, + "best1L": 0.07360406166740827, + "mean": 0.03067997996030109, + "persist": 0.024292965020452226 + }, + "order-ops": { + "min": 0.585714286991528, + "best1L": 0.3675324703965868, + "mean": 0.31758241408637594, + "persist": 0.35324675110833986 + }, + "poetry": { + "min": 0.020408163645437787, + "best1L": 0.013119533258889402, + "mean": 0.00504597436104502, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333155938555, + "best1L": 0.16369047654526575, + "mean": 0.08997252743159022, + "persist": 0.07589285700981106 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.20249653181859426, + "best1L": 0.14147018029221467, + "mean": 0.050784166769257615, + "persist": 0.02912621306521552 + }, + "multilingual": { + "min": 0.12146482989192009, + "best1L": 0.0815808538879667, + "mean": 0.03358063321294529, + "persist": 0.0286439446998494 + }, + "order-ops": { + "min": 0.5909090914896556, + "best1L": 0.3714285748345511, + "mean": 0.3261738286486694, + "persist": 0.3493506466703756 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.013119533658027649, + "mean": 0.004709576190050159, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27976190618106295, + "best1L": 0.16220238485506602, + "mean": 0.09809981659054756, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.19694868155888148, + "best1L": 0.13592232816985675, + "mean": 0.04907713704077261, + "persist": 0.02912621306521552 + }, + "multilingual": { + "min": 0.11675126584512847, + "best1L": 0.0790427838053022, + "mean": 0.032883360516279936, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.6103896072932652, + "best1L": 0.3675324703965868, + "mean": 0.3333666329937322, + "persist": 0.35714285873941015 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177886009216, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27083333900996615, + "best1L": 0.15773809727813518, + "mean": 0.09684065782598086, + "persist": 0.08630952358778034 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.19556171872786113, + "best1L": 0.13592232816985675, + "mean": 0.048650377829160006, + "persist": 0.02912621306521552 + }, + "multilingual": { + "min": 0.1160261050931045, + "best1L": 0.07759245751159531, + "mean": 0.03260445102517094, + "persist": 0.027556200112615312 + }, + "order-ops": { + "min": 0.605194804923875, + "best1L": 0.3675324613494532, + "mean": 0.3311688293303762, + "persist": 0.3545454501041344 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807474281107, + "mean": 0.004597443488559553, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529761833803995, + "best1L": 0.16071428890739167, + "mean": 0.0956959705799818, + "persist": 0.084821427640106 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.188626905637128, + "best1L": 0.13592232816985675, + "mean": 0.046836659112679105, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07868020209882941, + "mean": 0.03193506997610841, + "persist": 0.028281363658607006 + }, + "order-ops": { + "min": 0.605194804923875, + "best1L": 0.36883116140961647, + "mean": 0.32937063223549295, + "persist": 0.35974025992410524 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2723214307001659, + "best1L": 0.16071428305336408, + "mean": 0.09283425033624683, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000716674598931734, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.19417475589684077, + "best1L": 0.13730928993650846, + "mean": 0.04833031106474144, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11711385047861508, + "best1L": 0.08013052812644414, + "mean": 0.03263234234015856, + "persist": 0.029006526539368287 + }, + "order-ops": { + "min": 0.605194804923875, + "best1L": 0.3701298614697797, + "mean": 0.33086912759712767, + "persist": 0.3610389621130058 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428624647005, + "mean": 0.09512362802135092, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.19556171766349248, + "best1L": 0.13730928993650846, + "mean": 0.047903553782297034, + "persist": 0.02635228873363563 + }, + "multilingual": { + "min": 0.11675126903823443, + "best1L": 0.08013052812644414, + "mean": 0.03249288809352687, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.605194804923875, + "best1L": 0.3701298614697797, + "mean": 0.3304695316723415, + "persist": 0.3610389621130058 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004373177886009216, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2767857164144516, + "best1L": 0.1592261932258095, + "mean": 0.0943223430908152, + "persist": 0.08184523760740246 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.19001387059688568, + "best1L": 0.13869625250143663, + "mean": 0.04769017341147576, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11783901335937637, + "best1L": 0.08013052732816764, + "mean": 0.03210241533815861, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.5974025960479464, + "best1L": 0.3714285705770765, + "mean": 0.3268731268388884, + "persist": 0.3571428523531982 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380491154535, + "best1L": 0.16071428704474652, + "mean": 0.09500915730105979, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.1914008323635374, + "best1L": 0.13869625250143663, + "mean": 0.047476795435483964, + "persist": 0.02635228873363563 + }, + "multilingual": { + "min": 0.1163886875978538, + "best1L": 0.07831762012626443, + "mean": 0.03215819703681128, + "persist": 0.02719361853918859 + }, + "order-ops": { + "min": 0.5999999961682728, + "best1L": 0.37142857004489216, + "mean": 0.32887113253985134, + "persist": 0.3571428523531982 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976172736713, + "best1L": 0.16071428624647005, + "mean": 0.09478021931967565, + "persist": 0.08333333461944546 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.19278779413018907, + "best1L": 0.13869625250143663, + "mean": 0.0479035532501127, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11783901123063904, + "best1L": 0.08085569313594274, + "mean": 0.03246499724420054, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.601298696228436, + "best1L": 0.37142857004489216, + "mean": 0.32937062478491236, + "persist": 0.35974025247352465 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428624647005, + "mean": 0.09523809581462826, + "persist": 0.08333333461944546 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.19417475589684077, + "best1L": 0.13869625250143663, + "mean": 0.047796863330794234, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11820159267101969, + "best1L": 0.08121827244758606, + "mean": 0.03249288822657296, + "persist": 0.026831037098807946 + }, + "order-ops": { + "min": 0.5987012961081096, + "best1L": 0.37142857004489216, + "mean": 0.32867132499814034, + "persist": 0.3584415524133614 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428624647005, + "mean": 0.09535256414008993, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.18862690670149668, + "best1L": 0.13592232896813325, + "mean": 0.04705003735476306, + "persist": 0.02635228873363563 + }, + "multilingual": { + "min": 0.11675126903823443, + "best1L": 0.07940536524568285, + "mean": 0.0324371055965977, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.6000000046832221, + "best1L": 0.37012987051691326, + "mean": 0.3252747282385826, + "persist": 0.35454544903976576 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27232143389327185, + "best1L": 0.1592261932258095, + "mean": 0.09443681354501418, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.18585298316819326, + "best1L": 0.13730929073478496, + "mean": 0.04694334876590541, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11747642979025841, + "best1L": 0.08049310956682477, + "mean": 0.03252077880980713, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.6000000046832221, + "best1L": 0.37142857004489216, + "mean": 0.32637362820761545, + "persist": 0.35844155134899275 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095277122089, + "best1L": 0.1592261932258095, + "mean": 0.09420787609581437, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.18723994493484497, + "best1L": 0.13730929073478496, + "mean": 0.04694334849981325, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11638868546911649, + "best1L": 0.07940536524568285, + "mean": 0.03238132443012936, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.6025974048035485, + "best1L": 0.37142857004489216, + "mean": 0.32717282697558403, + "persist": 0.358441559863942 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2708333358168602, + "best1L": 0.16071428624647005, + "mean": 0.09409340723816838, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.18723994493484497, + "best1L": 0.13730929073478496, + "mean": 0.04705003788694739, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11711384834987777, + "best1L": 0.07976794668606349, + "mean": 0.0324928886257112, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.6012987047433853, + "best1L": 0.37142857004489216, + "mean": 0.3273726228092398, + "persist": 0.358441559863942 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2708333358168602, + "best1L": 0.16071428624647005, + "mean": 0.0942078758297222, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.18862690670149668, + "best1L": 0.13730929073478496, + "mean": 0.04694334849981325, + "persist": 0.027739251564655985 + }, + "multilingual": { + "min": 0.11711384834987777, + "best1L": 0.07976794668606349, + "mean": 0.03249288835961904, + "persist": 0.027556200245661393 + }, + "order-ops": { + "min": 0.6025974048035485, + "best1L": 0.37142857004489216, + "mean": 0.3273726249379771, + "persist": 0.358441559863942 + }, + "poetry": { + "min": 0.016034985626382486, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976215311459, + "best1L": 0.16071428198899543, + "mean": 0.09455128213656801, + "persist": 0.0833333335550768 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + } + }, + "runtime_s": 1297.8 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Pile-CC_s0.json b/results/ladder410_cpu_rstrip/ladder_Pile-CC_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..3563d8955d1b757ef8361ea2c7217e7376fb1b32 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Pile-CC_s0.json @@ -0,0 +1,465 @@ +{ + "corpus": "Pile-CC", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2552011111485107, + "best1L": 0.1733703182211944, + "mean": 0.07532273725207363, + "persist": 0.04576976464263031 + }, + "multilingual": { + "min": 0.1849166059068271, + "best1L": 0.15482233624373162, + "mean": 0.04507167956658772, + "persist": 0.03770848350333316 + }, + "order-ops": { + "min": 0.5090909025498799, + "best1L": 0.3922077917626926, + "mean": 0.27392607820885523, + "persist": 0.30779220296868254 + }, + "poetry": { + "min": 0.021865888631769588, + "best1L": 0.00728862958827189, + "mean": 0.003251850272395781, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3377976215311459, + "best1L": 0.2053571455180645, + "mean": 0.15178571401962213, + "persist": 0.16071428598037787 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180239915848, + "mean": 0.001791686484856265, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.26074896034385475, + "best1L": 0.17614424281886645, + "mean": 0.07585618165986878, + "persist": 0.04576976490872247 + }, + "multilingual": { + "min": 0.18527918734720775, + "best1L": 0.15300942904182843, + "mean": 0.04568527905004365, + "persist": 0.038796229154935906 + }, + "order-ops": { + "min": 0.49480519763060976, + "best1L": 0.3896103895136288, + "mean": 0.2717282734811306, + "persist": 0.30649350503725664 + }, + "poetry": { + "min": 0.021865889297, + "best1L": 0.008746355905064515, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3541666641831398, + "best1L": 0.2083333320915699, + "mean": 0.16220238033149922, + "persist": 0.16815476252564363 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180239915848, + "mean": 0.002030578109302691, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.2607489646013294, + "best1L": 0.170596394687891, + "mean": 0.07510935714734453, + "persist": 0.044382802077702115 + }, + "multilingual": { + "min": 0.1849166059068271, + "best1L": 0.1551849160875593, + "mean": 0.04565738833376339, + "persist": 0.03770848416856357 + }, + "order-ops": { + "min": 0.49610390194824766, + "best1L": 0.38701298726456507, + "mean": 0.27242756901042803, + "persist": 0.30519480630755424 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.005830903803663594, + "mean": 0.0037003811448812485, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3586309456399509, + "best1L": 0.2142857126891613, + "mean": 0.16723900900355407, + "persist": 0.1755952396030937 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.006211180239915848, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.25936199857720305, + "best1L": 0.1719833564545427, + "mean": 0.07585618259119135, + "persist": 0.04438280181160995 + }, + "multilingual": { + "min": 0.18527918521847045, + "best1L": 0.15591008056487357, + "mean": 0.045573716584060876, + "persist": 0.038433648246739595 + }, + "order-ops": { + "min": 0.49740260200841085, + "best1L": 0.38831168838909697, + "mean": 0.2725274738456522, + "persist": 0.30389609640198095 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.00728862958827189, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3541666641831398, + "best1L": 0.20833333102720125, + "mean": 0.16369047335215978, + "persist": 0.16964285634458065 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.006211180239915848, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.26629681459494997, + "best1L": 0.17198335725281919, + "mean": 0.07510935588340674, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.1831036987049239, + "best1L": 0.1526468460048948, + "mean": 0.04504378845116922, + "persist": 0.03734590286122901 + }, + "order-ops": { + "min": 0.4896103910037449, + "best1L": 0.38701298726456507, + "mean": 0.2704295739531517, + "persist": 0.29999999622149126 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.010204081822718893, + "mean": 0.004148912416504962, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.33928571322134565, + "best1L": 0.20684523933700152, + "mean": 0.1593406588903495, + "persist": 0.16666666657796927 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.26213592290878296, + "best1L": 0.17198335618845054, + "mean": 0.07542942431090134, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.18346628014530456, + "best1L": 0.15264684387615748, + "mean": 0.045378479840500016, + "persist": 0.037345901796860356 + }, + "order-ops": { + "min": 0.4909090910639082, + "best1L": 0.38701298726456507, + "mean": 0.2710289774196489, + "persist": 0.3012986962816545 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.008746355905064515, + "mean": 0.004036779648491314, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20684523720826423, + "mean": 0.16048534720071725, + "persist": 0.16666666657796927 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.26352288893290926, + "best1L": 0.17198335618845054, + "mean": 0.07574949380276459, + "persist": 0.04576976464263031 + }, + "multilingual": { + "min": 0.18527918947594507, + "best1L": 0.15518491715192795, + "mean": 0.04554582493645804, + "persist": 0.037708484301609654 + }, + "order-ops": { + "min": 0.49350649757044657, + "best1L": 0.38831168838909697, + "mean": 0.2720279725534575, + "persist": 0.3025973963418177 + }, + "poetry": { + "min": 0.024781340600124428, + "best1L": 0.008746355905064515, + "mean": 0.003812513979417937, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000319310595, + "best1L": 0.20386904744165285, + "mean": 0.16002747230231762, + "persist": 0.16517857063029492 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.2676837742328644, + "best1L": 0.1733703168907336, + "mean": 0.07489597843959928, + "persist": 0.042995840311050415 + }, + "multilingual": { + "min": 0.181290793631758, + "best1L": 0.14974619235311235, + "mean": 0.044681207676018984, + "persist": 0.03698332128780229 + }, + "order-ops": { + "min": 0.49350649969918386, + "best1L": 0.38701298726456507, + "mean": 0.27412587457469534, + "persist": 0.30519480497709345 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.008746355505926269, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738134264946, + "best1L": 0.2008928544819355, + "mean": 0.1587683167308569, + "persist": 0.16517857595213822 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.2662968124662127, + "best1L": 0.1747572807861226, + "mean": 0.07510935508513025, + "persist": 0.042995840311050415 + }, + "multilingual": { + "min": 0.18092821006264007, + "best1L": 0.14974619235311235, + "mean": 0.04476488049009016, + "persist": 0.03698332128780229 + }, + "order-ops": { + "min": 0.4948051997593471, + "best1L": 0.38701298726456507, + "mean": 0.27412587084940504, + "persist": 0.30519480497709345 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34226190724543165, + "best1L": 0.2038690485060215, + "mean": 0.15934066155127116, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.26352288467543467, + "best1L": 0.1719833551240819, + "mean": 0.0748959789717836, + "persist": 0.042995840311050415 + }, + "multilingual": { + "min": 0.18165337294340134, + "best1L": 0.150108773793493, + "mean": 0.04487644362130335, + "persist": 0.03698332128780229 + }, + "order-ops": { + "min": 0.4948051997593471, + "best1L": 0.38701298726456507, + "mean": 0.2737262706671442, + "persist": 0.3038961049169302 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.003588248509913683, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.2038690485060215, + "mean": 0.1600274746971471, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.26352288467543467, + "best1L": 0.17337031795510224, + "mean": 0.07532273492376719, + "persist": 0.045769765440906794 + }, + "multilingual": { + "min": 0.18274111726454326, + "best1L": 0.15083393773862294, + "mean": 0.044988006087286134, + "persist": 0.037708484301609654 + }, + "order-ops": { + "min": 0.49610390194824766, + "best1L": 0.38701298726456507, + "mean": 0.2739260771444866, + "persist": 0.30519480497709345 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.008746355505926269, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.2038690485060215, + "mean": 0.15991300184811866, + "persist": 0.16666666977107525 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 905.0 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Pile-CC_s1.json b/results/ladder410_cpu_rstrip/ladder_Pile-CC_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e146d8c7f90d392d108865512126aac80317ef88 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Pile-CC_s1.json @@ -0,0 +1,465 @@ +{ + "corpus": "Pile-CC", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.27461858625922886, + "best1L": 0.17337031955165522, + "mean": 0.07873679577772107, + "persist": 0.04993065313569137 + }, + "multilingual": { + "min": 0.186366933797087, + "best1L": 0.1544597510780607, + "mean": 0.04529480702642884, + "persist": 0.038071065076759884 + }, + "order-ops": { + "min": 0.503896102309227, + "best1L": 0.38571428826877047, + "mean": 0.2730269692838192, + "persist": 0.30259739980101585 + }, + "poetry": { + "min": 0.018950437062552998, + "best1L": 0.005830903803663594, + "mean": 0.003251850139349699, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3586309477686882, + "best1L": 0.21428571588226727, + "mean": 0.16483516645218646, + "persist": 0.17559523907090938 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.27045770308801104, + "best1L": 0.1733703190194709, + "mean": 0.07628294127061963, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.1849166037780898, + "best1L": 0.15482233464717865, + "mean": 0.04515535158238241, + "persist": 0.036620739182191234 + }, + "order-ops": { + "min": 0.49870129355362486, + "best1L": 0.38961039057799746, + "mean": 0.27452547049948145, + "persist": 0.3038961027881929 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.00728862958827189, + "mean": 0.0034761158084230764, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000319310595, + "best1L": 0.21130952079381263, + "mean": 0.16037087914134776, + "persist": 0.16964286006987095 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.26352288467543467, + "best1L": 0.1761442425527743, + "mean": 0.0748959789717836, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.18455402233770915, + "best1L": 0.15591007683958327, + "mean": 0.04509956988372973, + "persist": 0.03625815827399492 + }, + "order-ops": { + "min": 0.5064935024295535, + "best1L": 0.3909090885094234, + "mean": 0.27452547369258745, + "persist": 0.3051948020500796 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.005830903803663594, + "mean": 0.003588248576436724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34672619295971735, + "best1L": 0.20238095096179418, + "mean": 0.15922618843615055, + "persist": 0.16220238405678952 + }, + "association": { + "min": 0.009316770093781608, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.2676837742328644, + "best1L": 0.1733703190194709, + "mean": 0.07585618239162224, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.1849166059068271, + "best1L": 0.15699782222509384, + "mean": 0.045322696678340435, + "persist": 0.03734590272818293 + }, + "order-ops": { + "min": 0.5012987000601632, + "best1L": 0.38831168838909697, + "mean": 0.2743256741336414, + "persist": 0.30389610065945555 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.005830903803663594, + "mean": 0.00325185020587274, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023805975914, + "best1L": 0.20535714445369585, + "mean": 0.15956959421081202, + "persist": 0.16517857063029492 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001672240805679134, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.2732316205011947, + "best1L": 0.170596394687891, + "mean": 0.07745652366429567, + "persist": 0.048543689240302355 + }, + "multilingual": { + "min": 0.18709209454911097, + "best1L": 0.15663524291345052, + "mean": 0.045462152521525113, + "persist": 0.037708484035517485 + }, + "order-ops": { + "min": 0.5, + "best1L": 0.38831168838909697, + "mean": 0.27402597878660473, + "persist": 0.30389609640198095 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.005830903803663594, + "mean": 0.0033639829073633465, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047654526575, + "best1L": 0.19940476279173577, + "mean": 0.1603708786091634, + "persist": 0.1651785738234009 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.2704576969678913, + "best1L": 0.1719833548579897, + "mean": 0.07681638547884566, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.186729510979993, + "best1L": 0.1577229851058551, + "mean": 0.04551793448626995, + "persist": 0.03698332115475621 + }, + "order-ops": { + "min": 0.5000000021287373, + "best1L": 0.38831168838909697, + "mean": 0.27442557151828495, + "persist": 0.30519479646214415 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20089285661067283, + "mean": 0.1596840667937483, + "persist": 0.1651785738234009 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.2704576977661678, + "best1L": 0.1719833551240819, + "mean": 0.076923076795148, + "persist": 0.04715672667537417 + }, + "multilingual": { + "min": 0.18781725530113494, + "best1L": 0.1573604057942118, + "mean": 0.04562949748443706, + "persist": 0.03698332128780229 + }, + "order-ops": { + "min": 0.5000000021287373, + "best1L": 0.38831168838909697, + "mean": 0.2745254689029285, + "persist": 0.30389609640198095 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.005830903803663594, + "mean": 0.0033639829738863875, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3482142889073917, + "best1L": 0.2038690485060215, + "mean": 0.16117216327360698, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0015527950267174415, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.2732316205011947, + "best1L": 0.17198335751891136, + "mean": 0.07809666131756135, + "persist": 0.04854368977248669 + }, + "multilingual": { + "min": 0.18455402446644648, + "best1L": 0.15228426243577683, + "mean": 0.045322696811386516, + "persist": 0.03734590272818293 + }, + "order-ops": { + "min": 0.4961038998195103, + "best1L": 0.38571428826877047, + "mean": 0.2733266731458051, + "persist": 0.3012986962816545 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0038125141124640194, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34375000106436865, + "best1L": 0.20386904637728417, + "mean": 0.159455130142825, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.274618586525321, + "best1L": 0.17059639362352236, + "mean": 0.078203349906419, + "persist": 0.04715672694146633 + }, + "multilingual": { + "min": 0.1838288637144225, + "best1L": 0.1537345935191427, + "mean": 0.04537847890917744, + "persist": 0.037708484301609654 + }, + "order-ops": { + "min": 0.49740260200841085, + "best1L": 0.3870129883289337, + "mean": 0.2734265699982643, + "persist": 0.3012986962816545 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.15979853405484132, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.2732316205011947, + "best1L": 0.17337031875337874, + "mean": 0.07777659235788244, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.18491660803556442, + "best1L": 0.15482233784028462, + "mean": 0.04543426140610661, + "persist": 0.03734590272818293 + }, + "order-ops": { + "min": 0.49740260200841085, + "best1L": 0.3870129883289337, + "mean": 0.27342657691666056, + "persist": 0.3012986962816545 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.007288629721317973, + "mean": 0.0037003813444503714, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.15979853751403944, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.27323162129947115, + "best1L": 0.17198335725281919, + "mean": 0.07820335050512638, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.18455402659518377, + "best1L": 0.15445975427116668, + "mean": 0.04537847890917744, + "persist": 0.037708484301609654 + }, + "order-ops": { + "min": 0.4987013020685741, + "best1L": 0.3870129883289337, + "mean": 0.2738261728414467, + "persist": 0.3025973963418177 + }, + "poetry": { + "min": 0.021865889030907835, + "best1L": 0.005830903803663594, + "mean": 0.003588248576436724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.345238097012043, + "best1L": 0.20386904637728417, + "mean": 0.16025640815496445, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 908.8 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Pile-CC_s2.json b/results/ladder410_cpu_rstrip/ladder_Pile-CC_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..3a7a90ac77726b2f18940d1b6ed2440305952f55 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Pile-CC_s2.json @@ -0,0 +1,465 @@ +{ + "corpus": "Pile-CC", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2662968103374754, + "best1L": 0.17614424148840563, + "mean": 0.07574949460104108, + "persist": 0.04993065313569137 + }, + "multilingual": { + "min": 0.17150108782308443, + "best1L": 0.14140681390251433, + "mean": 0.04267306253314018, + "persist": 0.03662073904914515 + }, + "order-ops": { + "min": 0.49999999361378805, + "best1L": 0.387012990457671, + "mean": 0.26633366516658236, + "persist": 0.30129870266786646 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.007288629721317973, + "mean": 0.003924646747431585, + "persist": 0.0 + }, + "typo": { + "min": 0.34821428252117975, + "best1L": 0.20535714445369585, + "mean": 0.15464743679123266, + "persist": 0.16220238405678952 + }, + "association": { + "min": 0.009316769960735525, + "best1L": 0.004658385179936886, + "mean": 0.0013139034854248166, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.27184465740408215, + "best1L": 0.1747572807861226, + "mean": 0.07788328287590827, + "persist": 0.04854369030467102 + }, + "multilingual": { + "min": 0.18201595438378199, + "best1L": 0.15300942584872246, + "mean": 0.044932225186909945, + "persist": 0.0384336474484631 + }, + "order-ops": { + "min": 0.5000000042574746, + "best1L": 0.39090908957379206, + "mean": 0.27132867755634443, + "persist": 0.3025973987366472 + }, + "poetry": { + "min": 0.02332361534770046, + "best1L": 0.008746355505926269, + "mean": 0.003812513979417937, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.34672619083098005, + "best1L": 0.21279761780585563, + "mean": 0.15899725311568805, + "persist": 0.1666666687067066 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0017916865181177855, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.26490985069956097, + "best1L": 0.170596394687891, + "mean": 0.07638962932729296, + "persist": 0.04715672694146633 + }, + "multilingual": { + "min": 0.18600435022796905, + "best1L": 0.15591008216142654, + "mean": 0.04546215119106429, + "persist": 0.03843364758150918 + }, + "order-ops": { + "min": 0.49350649544170927, + "best1L": 0.38831168838909697, + "mean": 0.27132867010576384, + "persist": 0.3012986976121153 + }, + "poetry": { + "min": 0.023323614549423968, + "best1L": 0.008746355905064515, + "mean": 0.0037003812779273304, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023827263287, + "best1L": 0.21279761993459292, + "mean": 0.16311813199094363, + "persist": 0.1741071425910507 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.001911132363602519, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.2621359271662576, + "best1L": 0.1733703168907336, + "mean": 0.07638962852901646, + "persist": 0.04715672694146633 + }, + "multilingual": { + "min": 0.1856417709163257, + "best1L": 0.15337200675691878, + "mean": 0.045071679300495555, + "persist": 0.03734590206295252 + }, + "order-ops": { + "min": 0.4948051997593471, + "best1L": 0.3870129883289337, + "mean": 0.2705294681446893, + "persist": 0.2999999975519521 + }, + "poetry": { + "min": 0.026239067981285707, + "best1L": 0.00728862958827189, + "mean": 0.003924646880477667, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3526785724929401, + "best1L": 0.20982142697487557, + "mean": 0.16163003976855958, + "persist": 0.16964285794113362 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0020305780760411707, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.2662968124662127, + "best1L": 0.17198335698672704, + "mean": 0.07532273518985935, + "persist": 0.047156727739742825 + }, + "multilingual": { + "min": 0.18020304505314147, + "best1L": 0.15010877485786164, + "mean": 0.04426284347261701, + "persist": 0.03625815800790276 + }, + "order-ops": { + "min": 0.4909090931926455, + "best1L": 0.3883116894534656, + "mean": 0.27152846647160395, + "persist": 0.3012986962816545 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.33779762365988325, + "best1L": 0.20238094830087253, + "mean": 0.15670787702713693, + "persist": 0.16815476146127498 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.2676837721041271, + "best1L": 0.1733703182211944, + "mean": 0.07596287204484854, + "persist": 0.047156727739742825 + }, + "multilingual": { + "min": 0.1823785336954253, + "best1L": 0.1519216841885022, + "mean": 0.04473698950771775, + "persist": 0.03698332115475621 + }, + "order-ops": { + "min": 0.4948052018880844, + "best1L": 0.3883116894534656, + "mean": 0.27212787525994436, + "persist": 0.3012986962816545 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20535714019622123, + "mean": 0.15831043917153562, + "persist": 0.16666666764233792 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0016722407391560929, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.26352288680417196, + "best1L": 0.1747572807861226, + "mean": 0.07638962985947728, + "persist": 0.04854369030467102 + }, + "multilingual": { + "min": 0.181653370814664, + "best1L": 0.1508339388029916, + "mean": 0.04448597066636596, + "persist": 0.036620739847421646 + }, + "order-ops": { + "min": 0.4948052018880844, + "best1L": 0.38831168838909697, + "mean": 0.2722277710480349, + "persist": 0.3012986962816545 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.005830903803663594, + "mean": 0.0034761158749461174, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3407738112977573, + "best1L": 0.20535714019622123, + "mean": 0.15785256108003004, + "persist": 0.16666666551360063 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.2676837691771133, + "best1L": 0.17059639255915368, + "mean": 0.07713645630116973, + "persist": 0.04854368977248669 + }, + "multilingual": { + "min": 0.18201595651251928, + "best1L": 0.15119652024337224, + "mean": 0.044792771339416504, + "persist": 0.036620739581329484 + }, + "order-ops": { + "min": 0.49350650182792116, + "best1L": 0.38701298726456507, + "mean": 0.2757242726428168, + "persist": 0.30389609640198095 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.007288629721317973, + "mean": 0.00336398310693247, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3482142889073917, + "best1L": 0.20386904531291553, + "mean": 0.1620879146669592, + "persist": 0.16964285847331798 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.2690707359995161, + "best1L": 0.1747572807861226, + "mean": 0.07724314575482692, + "persist": 0.04715672694146633 + }, + "multilingual": { + "min": 0.18274111726454326, + "best1L": 0.15264684387615748, + "mean": 0.044904334736721854, + "persist": 0.036620739581329484 + }, + "order-ops": { + "min": 0.49350650182792116, + "best1L": 0.38701298726456507, + "mean": 0.2750249760491507, + "persist": 0.3025973963418177 + }, + "poetry": { + "min": 0.018950437461691245, + "best1L": 0.007288629721317973, + "mean": 0.00336398310693247, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.20535714126058988, + "mean": 0.16220237900103843, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.2690707359995161, + "best1L": 0.1719833551240819, + "mean": 0.07702976485182132, + "persist": 0.048543689506394525 + }, + "multilingual": { + "min": 0.18419144515480315, + "best1L": 0.15373458926166808, + "mean": 0.04501589826707329, + "persist": 0.03698332115475621 + }, + "order-ops": { + "min": 0.49350650182792116, + "best1L": 0.38701298726456507, + "mean": 0.27492507866450716, + "persist": 0.3025973963418177 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.007288629721317973, + "mean": 0.0034761156753769945, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.2068452350795269, + "mean": 0.1623168507856982, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0015527949601944005, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.2718446595328195, + "best1L": 0.1733703168907336, + "mean": 0.07713645450504762, + "persist": 0.04715672694146633 + }, + "multilingual": { + "min": 0.1838288637144225, + "best1L": 0.15337200782128743, + "mean": 0.044960115770144124, + "persist": 0.03698332115475621 + }, + "order-ops": { + "min": 0.49350650182792116, + "best1L": 0.38831168838909697, + "mean": 0.27452547901443075, + "persist": 0.3025973963418177 + }, + "poetry": { + "min": 0.020408163778483868, + "best1L": 0.007288629721317973, + "mean": 0.0034761156753769945, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3497023827263287, + "best1L": 0.20535714019622123, + "mean": 0.16151556745171547, + "persist": 0.171130952292255 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.004658385179936886, + "mean": 0.0014333492477557489, + "persist": 0.0 + } + } + }, + "runtime_s": 919.9 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_StackExchange_s0.json b/results/ladder410_cpu_rstrip/ladder_StackExchange_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..c75e7b122daf02087dacec164407d225f8a2e703 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_StackExchange_s0.json @@ -0,0 +1,661 @@ +{ + "corpus": "StackExchange", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2732316143810749, + "best1L": 0.17198335831718786, + "mean": 0.07756321398275239, + "persist": 0.047156725877097676 + }, + "multilingual": { + "min": 0.17512690648436546, + "best1L": 0.14321972216878617, + "mean": 0.04501589693661247, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.5064935045582908, + "best1L": 0.377922069813524, + "mean": 0.2962037936917373, + "persist": 0.32727272329585894 + }, + "poetry": { + "min": 0.013119533658027649, + "best1L": 0.008746355505926269, + "mean": 0.003588248576436724, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.37500000851494925, + "best1L": 0.22023809275456838, + "mean": 0.15819597031388963, + "persist": 0.16815475959862983 + }, + "association": { + "min": 0.015527950200651373, + "best1L": 0.006211180106869766, + "mean": 0.0014333492394403688, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.27461858466267586, + "best1L": 0.1775312072464398, + "mean": 0.07702976545052868, + "persist": 0.047156725877097676 + }, + "multilingual": { + "min": 0.1823785358241626, + "best1L": 0.14757070211427553, + "mean": 0.045796843245625496, + "persist": 0.0373459021959986 + }, + "order-ops": { + "min": 0.4870129791753633, + "best1L": 0.37662336975336075, + "mean": 0.28461538734180586, + "persist": 0.31038961719189373 + }, + "poetry": { + "min": 0.011661807873419352, + "best1L": 0.008746355505926269, + "mean": 0.0034761157419000354, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190469094685, + "best1L": 0.2217261865735054, + "mean": 0.15831043837325914, + "persist": 0.16964285873941012 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.006211180106869766, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.27045769936272074, + "best1L": 0.18723994493484497, + "mean": 0.07692307879083923, + "persist": 0.04715672800583499 + }, + "multilingual": { + "min": 0.18346627801656723, + "best1L": 0.15047135363732064, + "mean": 0.04649411501096828, + "persist": 0.03843364665018661 + }, + "order-ops": { + "min": 0.4883116888148444, + "best1L": 0.37922077838863644, + "mean": 0.2820179797708988, + "persist": 0.30519480417881695 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333262375425, + "best1L": 0.22470238219414437, + "mean": 0.16002747363277844, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.012422359947647368, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.27323161863854956, + "best1L": 0.18307905617569173, + "mean": 0.0774565227329731, + "persist": 0.047156725877097676 + }, + "multilingual": { + "min": 0.18020304611751012, + "best1L": 0.14684554136225156, + "mean": 0.046019970173282285, + "persist": 0.03843364651714053 + }, + "order-ops": { + "min": 0.4883116866861071, + "best1L": 0.37792207832847324, + "mean": 0.2846153825521469, + "persist": 0.3103896129344191 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.010204081822718893, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333262375425, + "best1L": 0.22172618923442705, + "mean": 0.1596840683903013, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.01397515514067241, + "best1L": 0.006211180106869766, + "mean": 0.0013139034687940563, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.27045769723398344, + "best1L": 0.1775312072464398, + "mean": 0.07745652406343392, + "persist": 0.05131761410406658 + }, + "multilingual": { + "min": 0.1831036987049239, + "best1L": 0.15192168578505516, + "mean": 0.04680091422051191, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.49350649544170927, + "best1L": 0.3779220687491553, + "mean": 0.28941059006111963, + "persist": 0.31818181968161036 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3749999978712627, + "best1L": 0.22767857196075575, + "mean": 0.16197344394666807, + "persist": 0.17559523986918585 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.26907073120985714, + "best1L": 0.18030513184411184, + "mean": 0.07756321371666022, + "persist": 0.05131761410406658 + }, + "multilingual": { + "min": 0.18274111513580596, + "best1L": 0.15010877645441464, + "mean": 0.046577787558947294, + "persist": 0.03879622795752117 + }, + "order-ops": { + "min": 0.4922077868665968, + "best1L": 0.3766233686889921, + "mean": 0.287912093102932, + "persist": 0.3155844195612839 + }, + "poetry": { + "min": 0.014577259442635946, + "best1L": 0.011661807873419352, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3749999978712627, + "best1L": 0.2291666679084301, + "mean": 0.16082875909549849, + "persist": 0.17113095255834715 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.27184465900063515, + "best1L": 0.18030513184411184, + "mean": 0.07798997020082814, + "persist": 0.04993065233741488 + }, + "multilingual": { + "min": 0.18092821006264007, + "best1L": 0.14865844696760178, + "mean": 0.046326770447194576, + "persist": 0.0395213919026511 + }, + "order-ops": { + "min": 0.49350648692675997, + "best1L": 0.3766233686889921, + "mean": 0.28861138969659805, + "persist": 0.31818181968161036 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0038125141124640194, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22619047920618737, + "mean": 0.1605998133974416, + "persist": 0.17261905063475882 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.27045769723398344, + "best1L": 0.1761442454797881, + "mean": 0.07809666151713048, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.18165337507213866, + "best1L": 0.15010877645441464, + "mean": 0.04643833264708519, + "persist": 0.039883973343031745 + }, + "order-ops": { + "min": 0.49350649757044657, + "best1L": 0.3779220687491553, + "mean": 0.29320680083973066, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246468139546254, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357110926083, + "best1L": 0.22767857302512443, + "mean": 0.15991300184811866, + "persist": 0.17559523986918585 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.2732316165098122, + "best1L": 0.1761442454797881, + "mean": 0.07798997133171984, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.18346628014530456, + "best1L": 0.1515591022159372, + "mean": 0.04666146143738713, + "persist": 0.04024655584778104 + }, + "order-ops": { + "min": 0.492207795381546, + "best1L": 0.3779220687491553, + "mean": 0.2918081980730806, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779648491314, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.37351190405232565, + "best1L": 0.22619047920618737, + "mean": 0.16025640975151742, + "persist": 0.17261905010257447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.2732316165098122, + "best1L": 0.1775312072464398, + "mean": 0.0777765903621912, + "persist": 0.049930651273046224 + }, + "multilingual": { + "min": 0.18201595438378199, + "best1L": 0.15083393827080727, + "mean": 0.04649411514401436, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.48961038674627033, + "best1L": 0.3779220687491553, + "mean": 0.29080918644155773, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22470238538725035, + "mean": 0.16014194062777928, + "persist": 0.17113095468708447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.2732316165098122, + "best1L": 0.1775312072464398, + "mean": 0.07766990097505706, + "persist": 0.049930651273046224 + }, + "multilingual": { + "min": 0.18092821006264007, + "best1L": 0.1493836103805474, + "mean": 0.04635466156261308, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.492207795381546, + "best1L": 0.3779220687491553, + "mean": 0.2913086877337524, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3705357164144516, + "best1L": 0.22470238538725035, + "mean": 0.15979853325656482, + "persist": 0.17113095468708447 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.27600554855806486, + "best1L": 0.1775312072464398, + "mean": 0.07916355432410326, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.18274111726454326, + "best1L": 0.1519216804632119, + "mean": 0.04641044206385102, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.4896103920681136, + "best1L": 0.3766233686889921, + "mean": 0.29240759196025984, + "persist": 0.32077921980193685 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246468139546254, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.15602106228470802, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.27600554855806486, + "best1L": 0.1775312072464398, + "mean": 0.0791635529271194, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.18201595438378199, + "best1L": 0.15119651864681924, + "mean": 0.04638255174670901, + "persist": 0.04024655584778104 + }, + "order-ops": { + "min": 0.4896103920681136, + "best1L": 0.3766233686889921, + "mean": 0.29210788916264263, + "persist": 0.32077921980193685 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1563644685915538, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.27600554855806486, + "best1L": 0.1775312072464398, + "mean": 0.07873679591076714, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.181653370814664, + "best1L": 0.1515591000871999, + "mean": 0.04649411527706044, + "persist": 0.04024655584778104 + }, + "order-ops": { + "min": 0.4896103920681136, + "best1L": 0.3766233686889921, + "mean": 0.291308695184333, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1567078743662153, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.27461858253393856, + "best1L": 0.1775312072464398, + "mean": 0.07884348609617778, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.18201595225504466, + "best1L": 0.1515591000871999, + "mean": 0.0465220059933407, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.492207795381546, + "best1L": 0.3766233686889921, + "mean": 0.29100898493613514, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904158762525, + "best1L": 0.22321428943957602, + "mean": 0.1569368147424289, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.27461858253393856, + "best1L": 0.1775312072464398, + "mean": 0.07852341713649887, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.1823785336954253, + "best1L": 0.1515591000871999, + "mean": 0.04646622456078019, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.492207795381546, + "best1L": 0.3766233686889921, + "mean": 0.29100898866142544, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713540656225, + "best1L": 0.22321428943957602, + "mean": 0.1568223461508751, + "persist": 0.16815475800207683 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0011944576898323639, + "persist": 0.0 + } + } + }, + "runtime_s": 1380.7 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_StackExchange_s1.json b/results/ladder410_cpu_rstrip/ladder_StackExchange_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..1e99ff7327d81c8f3a009c44a8dfc400c305003f --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_StackExchange_s1.json @@ -0,0 +1,661 @@ +{ + "corpus": "StackExchange", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2649098480386393, + "best1L": 0.17337032088211604, + "mean": 0.07660300850069948, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.17984046627368247, + "best1L": 0.14938361144491605, + "mean": 0.045768952396299155, + "persist": 0.0380710649437138 + }, + "order-ops": { + "min": 0.4935064890554973, + "best1L": 0.3766233686889921, + "mean": 0.292307692446879, + "persist": 0.32077922725251745 + }, + "poetry": { + "min": 0.014577259974820274, + "best1L": 0.011661807607327188, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523241860525, + "best1L": 0.22767857249294007, + "mean": 0.151785708963871, + "persist": 0.16369047601308143 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0009555661319089788, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.2773925145821912, + "best1L": 0.17475728264876775, + "mean": 0.07852341786825232, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.17911530126418387, + "best1L": 0.14720812227044786, + "mean": 0.045741060615650246, + "persist": 0.039883973343031745 + }, + "order-ops": { + "min": 0.49350649544170927, + "best1L": 0.37402596856866566, + "mean": 0.2929070969777448, + "persist": 0.3259740264288017 + }, + "poetry": { + "min": 0.01895043732864516, + "best1L": 0.013119533658027649, + "mean": 0.004036779648491314, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071364709309, + "best1L": 0.22619047920618737, + "mean": 0.15453297006232397, + "persist": 0.16666666844061442 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.2773925097925322, + "best1L": 0.1733703198177474, + "mean": 0.07916355565456408, + "persist": 0.05270457773336342 + }, + "multilingual": { + "min": 0.18274111726454326, + "best1L": 0.14974619182092802, + "mean": 0.04660567867436579, + "persist": 0.03952139097132853 + }, + "order-ops": { + "min": 0.4987013041973114, + "best1L": 0.3740259664399283, + "mean": 0.29250749360237804, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011905116694315, + "best1L": 0.22767856876764977, + "mean": 0.1565934100321361, + "persist": 0.1711309514939785 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.278779469962631, + "best1L": 0.17198335698672704, + "mean": 0.07895017767857228, + "persist": 0.048543689506394525 + }, + "multilingual": { + "min": 0.1809282068695341, + "best1L": 0.14865844643541745, + "mean": 0.046326770447194576, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.5000000085149493, + "best1L": 0.3740259664399283, + "mean": 0.294005992689303, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011905116694315, + "best1L": 0.22619047494871275, + "mean": 0.15579212669815337, + "persist": 0.16815476225955145 + }, + "association": { + "min": 0.009316770359873772, + "best1L": 0.006211180106869766, + "mean": 0.0008361203695780464, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.2746185889201505, + "best1L": 0.17475728264876775, + "mean": 0.07798997053344335, + "persist": 0.04715672667537417 + }, + "multilingual": { + "min": 0.1831037008336612, + "best1L": 0.1533720110143934, + "mean": 0.046884587965905666, + "persist": 0.04024655531559672 + }, + "order-ops": { + "min": 0.49220779112407137, + "best1L": 0.37532466862882885, + "mean": 0.2912087919456618, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714285182101385, + "best1L": 0.22767857302512443, + "mean": 0.15361721707241877, + "persist": 0.16815476279173577 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.2746185889201505, + "best1L": 0.1747572837131364, + "mean": 0.07788328034803271, + "persist": 0.04715672667537417 + }, + "multilingual": { + "min": 0.18201595438378199, + "best1L": 0.1508339403995446, + "mean": 0.046633569922830374, + "persist": 0.04097171926072666 + }, + "order-ops": { + "min": 0.49350649969918386, + "best1L": 0.37532466862882885, + "mean": 0.29160839691758156, + "persist": 0.32077921980193685 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904584509985, + "best1L": 0.22619047920618737, + "mean": 0.15418955896581923, + "persist": 0.1696428582072258 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.27045769936272074, + "best1L": 0.17614424654415675, + "mean": 0.07798996946907469, + "persist": 0.04715672667537417 + }, + "multilingual": { + "min": 0.17947788004364287, + "best1L": 0.14865844696760178, + "mean": 0.04638255174670901, + "persist": 0.04097171926072666 + }, + "order-ops": { + "min": 0.4948051997593471, + "best1L": 0.37532466862882885, + "mean": 0.2910089971763747, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004148912349981921, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35565476013081415, + "best1L": 0.22470238325851305, + "mean": 0.15327380704028265, + "persist": 0.16666666684406145 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.26907073333859444, + "best1L": 0.1789181690130915, + "mean": 0.07734983421063849, + "persist": 0.04715672667537417 + }, + "multilingual": { + "min": 0.17730239246572768, + "best1L": 0.14829586765595845, + "mean": 0.04585262494427817, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.4935064933129719, + "best1L": 0.3792207730667932, + "mean": 0.2926073915192059, + "persist": 0.32077921980193685 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22321428943957602, + "mean": 0.15350274635212763, + "persist": 0.16815476013081415 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.27045769936272074, + "best1L": 0.1789181690130915, + "mean": 0.07809666018666965, + "persist": 0.04715672667537417 + }, + "multilingual": { + "min": 0.1783901367868696, + "best1L": 0.14865844803197042, + "mean": 0.046019969907190116, + "persist": 0.039521392967019765 + }, + "order-ops": { + "min": 0.4935065007635525, + "best1L": 0.37792207300662994, + "mean": 0.2934065988021238, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22470238325851305, + "mean": 0.1539606223148959, + "persist": 0.16666666524750845 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.26907073546733173, + "best1L": 0.1775312072464398, + "mean": 0.07766990290422525, + "persist": 0.048543689506394525 + }, + "multilingual": { + "min": 0.1791153017963682, + "best1L": 0.1482958665915898, + "mean": 0.04607575213802712, + "persist": 0.03988397493958473 + }, + "order-ops": { + "min": 0.4935065007635525, + "best1L": 0.3779220687491553, + "mean": 0.29300699436238836, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22470238325851305, + "mean": 0.15361721866897174, + "persist": 0.16815476013081415 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.26907073546733173, + "best1L": 0.1775312072464398, + "mean": 0.07756321378318327, + "persist": 0.048543689506394525 + }, + "multilingual": { + "min": 0.17766497816358293, + "best1L": 0.14829586552722113, + "mean": 0.04593629775834935, + "persist": 0.039521392967019765 + }, + "order-ops": { + "min": 0.4935065007635525, + "best1L": 0.3766233686889921, + "mean": 0.2928071974643639, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3556547611951828, + "best1L": 0.22321428943957602, + "mean": 0.1537316877927099, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.27877947422010557, + "best1L": 0.1789181690130915, + "mean": 0.08012376040486353, + "persist": 0.052704576935086934 + }, + "multilingual": { + "min": 0.1791153039251055, + "best1L": 0.1493836087839944, + "mean": 0.0461594234885914, + "persist": 0.04024655584778104 + }, + "order-ops": { + "min": 0.49090908467769623, + "best1L": 0.3766233686889921, + "mean": 0.2928071900137833, + "persist": 0.31818181968161036 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.1546474370573248, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.0010750119108706713, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.2746185804052012, + "best1L": 0.1789181690130915, + "mean": 0.07969700019540531, + "persist": 0.052704576935086934 + }, + "multilingual": { + "min": 0.17693981102534703, + "best1L": 0.14793328515120915, + "mean": 0.046019969907190116, + "persist": 0.04024655584778104 + }, + "order-ops": { + "min": 0.4922077932528087, + "best1L": 0.3779220687491553, + "mean": 0.29370629521352903, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.1546474370573248, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.27739251245345387, + "best1L": 0.1789181690130915, + "mean": 0.07959031160654766, + "persist": 0.052704576935086934 + }, + "multilingual": { + "min": 0.1783901367868696, + "best1L": 0.14938361091273172, + "mean": 0.04618731447096382, + "persist": 0.04024655584778104 + }, + "order-ops": { + "min": 0.4922077932528087, + "best1L": 0.3779220687491553, + "mean": 0.29410589752452715, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15453296633703367, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.27739251245345387, + "best1L": 0.1775312072464398, + "mean": 0.0793769333644637, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.17947788110801152, + "best1L": 0.14938361091273172, + "mean": 0.046298879464822154, + "persist": 0.040609137288161685 + }, + "order-ops": { + "min": 0.4909090921282768, + "best1L": 0.3779220687491553, + "mean": 0.2934065982699394, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15453296580484935, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.27739251245345387, + "best1L": 0.1775312072464398, + "mean": 0.0793769333644637, + "persist": 0.052704576935086934 + }, + "multilingual": { + "min": 0.17911529966763087, + "best1L": 0.14938361091273172, + "mean": 0.04627098795026541, + "persist": 0.04024655584778104 + }, + "order-ops": { + "min": 0.4922077932528087, + "best1L": 0.3779220687491553, + "mean": 0.2933067056749548, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714284969227655, + "best1L": 0.22470238325851305, + "mean": 0.15441849987421716, + "persist": 0.1666666641831398 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + } + }, + "runtime_s": 1527.8 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_StackExchange_s2.json b/results/ladder410_cpu_rstrip/ladder_StackExchange_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..76aedf087056b8c4766d81b9f55de510da99d9c7 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_StackExchange_s2.json @@ -0,0 +1,661 @@ +{ + "corpus": "StackExchange", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.27461857934083256, + "best1L": 0.17614424441541945, + "mean": 0.07820335010598813, + "persist": 0.04854368977248669 + }, + "multilingual": { + "min": 0.17839013572250093, + "best1L": 0.14720812227044786, + "mean": 0.0456294978835753, + "persist": 0.03915880953094789 + }, + "order-ops": { + "min": 0.49480519763060976, + "best1L": 0.3792207730667932, + "mean": 0.29140859097242355, + "persist": 0.32077921980193685 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.004036779515445232, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071471146175, + "best1L": 0.2247023768723011, + "mean": 0.15144230664840766, + "persist": 0.16369047707745008 + }, + "association": { + "min": 0.01086956528680665, + "best1L": 0.004658385179936886, + "mean": 0.0009555661319089788, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.27461858466267586, + "best1L": 0.1789181690130915, + "mean": 0.0773498360067606, + "persist": 0.04854368844202587 + }, + "multilingual": { + "min": 0.17947788217238017, + "best1L": 0.1468455387013299, + "mean": 0.045741060482604165, + "persist": 0.03915881046227047 + }, + "order-ops": { + "min": 0.48701297491788864, + "best1L": 0.37792207300662994, + "mean": 0.2881118841469288, + "persist": 0.3207792229950428 + }, + "poetry": { + "min": 0.014577259974820274, + "best1L": 0.010204081822718893, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3690476215311459, + "best1L": 0.22321428518210137, + "mean": 0.15693681314587593, + "persist": 0.16815475959862983 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.27045769936272074, + "best1L": 0.18169209440904005, + "mean": 0.07681638853890556, + "persist": 0.04715672667537417 + }, + "multilingual": { + "min": 0.17621464814458573, + "best1L": 0.14684553816914558, + "mean": 0.04521113421235766, + "persist": 0.03843364758150918 + }, + "order-ops": { + "min": 0.4870129791753633, + "best1L": 0.3766233686889921, + "mean": 0.2856143851365362, + "persist": 0.3155844195612839 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3690476215311459, + "best1L": 0.22470238112977572, + "mean": 0.1575091587645667, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.27184465687189785, + "best1L": 0.18030513264238834, + "mean": 0.07713645430547851, + "persist": 0.047156725877097676 + }, + "multilingual": { + "min": 0.17657723277807236, + "best1L": 0.1472081212060792, + "mean": 0.04535058819289718, + "persist": 0.03843364758150918 + }, + "order-ops": { + "min": 0.48831167817115784, + "best1L": 0.3766233686889921, + "mean": 0.2878121856067862, + "persist": 0.31688311962144716 + }, + "poetry": { + "min": 0.01603498575942857, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3675595277122089, + "best1L": 0.22470238325851305, + "mean": 0.15636446912373816, + "persist": 0.16666666365095548 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.006211180106869766, + "mean": 0.000955566148539739, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.27045769510524614, + "best1L": 0.17753120804471628, + "mean": 0.07798997053344335, + "persist": 0.04854368977248669 + }, + "multilingual": { + "min": 0.17693981208971568, + "best1L": 0.14720812173826353, + "mean": 0.04535058845898935, + "persist": 0.03915881152663912 + }, + "order-ops": { + "min": 0.48441557586193085, + "best1L": 0.3779220687491553, + "mean": 0.2883116847702435, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22470238325851305, + "mean": 0.1548763715795108, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.27045769510524614, + "best1L": 0.17891817087573664, + "mean": 0.07820335063817245, + "persist": 0.04854368977248669 + }, + "multilingual": { + "min": 0.17730239565883363, + "best1L": 0.14648295885750226, + "mean": 0.04540637068982635, + "persist": 0.039521392967019765 + }, + "order-ops": { + "min": 0.4857142780508314, + "best1L": 0.3766233686889921, + "mean": 0.2886113907609667, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15556318578975542, + "persist": 0.16517856876764977 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.27045769510524614, + "best1L": 0.17753120804471628, + "mean": 0.07798997026735119, + "persist": 0.04854368977248669 + }, + "multilingual": { + "min": 0.17839014210871287, + "best1L": 0.14757070317864418, + "mean": 0.04557371525360005, + "persist": 0.0398839744074004 + }, + "order-ops": { + "min": 0.48441557586193085, + "best1L": 0.3779220687491553, + "mean": 0.28941059006111963, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.1556776538491249, + "persist": 0.16666666471532413 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.27323162502476145, + "best1L": 0.17475728264876775, + "mean": 0.0793769333644637, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.17875271716288157, + "best1L": 0.14757070104990685, + "mean": 0.046047860224332125, + "persist": 0.04060913742120777 + }, + "order-ops": { + "min": 0.4896103920681136, + "best1L": 0.3779220687491553, + "mean": 0.29270729262913975, + "persist": 0.32207792199083735 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15602106228470802, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.27323162502476145, + "best1L": 0.1761442454797881, + "mean": 0.07916355352582675, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.1787527192916189, + "best1L": 0.14720812173826353, + "mean": 0.045796842580395083, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.48831168349300114, + "best1L": 0.3766233686889921, + "mean": 0.29240759036370684, + "persist": 0.32337662630847525 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15544871719820158, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.27323162502476145, + "best1L": 0.1761442454797881, + "mean": 0.07927024371123739, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.1787527192916189, + "best1L": 0.14720812173826353, + "mean": 0.04582473382885967, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.48831168349300114, + "best1L": 0.3779220687491553, + "mean": 0.2915084904858044, + "persist": 0.32077922193067415 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15567765225257194, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.27323162502476145, + "best1L": 0.17475728264876775, + "mean": 0.07905686520306128, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.1780275564108576, + "best1L": 0.14757070211427553, + "mean": 0.04565738846680948, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.48831168349300114, + "best1L": 0.3779220687491553, + "mean": 0.2924075924924442, + "persist": 0.32207792199083735 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15556318578975542, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.27600554430059027, + "best1L": 0.17475728264876775, + "mean": 0.07991037923576576, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.1798404614840235, + "best1L": 0.1486584474997861, + "mean": 0.045992079323955944, + "persist": 0.04169688227453402 + }, + "order-ops": { + "min": 0.49350649544170927, + "best1L": 0.3779220687491553, + "mean": 0.2945054977067879, + "persist": 0.32077922405941145 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15670787755932128, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.2718446526144232, + "best1L": 0.17475728264876775, + "mean": 0.07959031267091632, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.17911529860326222, + "best1L": 0.14793328674776213, + "mean": 0.04599207879177162, + "persist": 0.04097171779721975 + }, + "order-ops": { + "min": 0.49090909532138277, + "best1L": 0.3779220687491553, + "mean": 0.29400599162493435, + "persist": 0.3220779241195747 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.22321428943957602, + "mean": 0.15636446912373816, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.27323161863854956, + "best1L": 0.17475728264876775, + "mean": 0.07948362461424299, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.17911530073199952, + "best1L": 0.14829586605940545, + "mean": 0.04585262441209385, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.4909090931926455, + "best1L": 0.3779220687491553, + "mean": 0.29330670088529587, + "persist": 0.32077921980193685 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15682234189340047, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.27461858253393856, + "best1L": 0.17475728264876775, + "mean": 0.07948362142113703, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.17802755215338298, + "best1L": 0.1475707053073815, + "mean": 0.04576895199716091, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.4909090931926455, + "best1L": 0.3779220687491553, + "mean": 0.2930069959589413, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.003812513979417937, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523774044855, + "best1L": 0.22321428943957602, + "mean": 0.15670787330184663, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.27461858253393856, + "best1L": 0.17475728264876775, + "mean": 0.07959031373528498, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.1787527214203562, + "best1L": 0.14865844962852343, + "mean": 0.04579684191516468, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.4909090931926455, + "best1L": 0.3779220687491553, + "mean": 0.2930069959589413, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.017492711011852537, + "best1L": 0.011661807873419352, + "mean": 0.0039246466809085435, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833336881229, + "best1L": 0.22321428943957602, + "mean": 0.15659340683903014, + "persist": 0.16815475853426115 + }, + "association": { + "min": 0.012422360213739532, + "best1L": 0.006211180106869766, + "mean": 0.0010750119275014316, + "persist": 0.0 + } + } + }, + "runtime_s": 1530.1 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s0.json b/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..6454e1860a85a225e0dfe161d284d8942ca1ef44 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s0.json @@ -0,0 +1,465 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2538141432617392, + "best1L": 0.1622746219592435, + "mean": 0.07094846991822124, + "persist": 0.051317615168435235 + }, + "multilingual": { + "min": 0.15047135204076767, + "best1L": 0.11203770738627229, + "mean": 0.040218665331069915, + "persist": 0.0387962299532124 + }, + "order-ops": { + "min": 0.518181820000921, + "best1L": 0.3714285705770765, + "mean": 0.28271727955767084, + "persist": 0.31688311429960386 + }, + "poetry": { + "min": 0.021865889563092163, + "best1L": 0.00874635563897235, + "mean": 0.004373177752963134, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3526785767504147, + "best1L": 0.1934523768723011, + "mean": 0.15934066155127116, + "persist": 0.17708333568381412 + }, + "association": { + "min": 0.017080744994538172, + "best1L": 0.006211180239915848, + "mean": 0.00203057804277965, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.25520111194678713, + "best1L": 0.16782246636492865, + "mean": 0.07265549818319934, + "persist": 0.05409154003219945 + }, + "multilingual": { + "min": 0.1515591000871999, + "best1L": 0.1116751278085368, + "mean": 0.040218665464115996, + "persist": 0.03770848337028708 + }, + "order-ops": { + "min": 0.5129870123096875, + "best1L": 0.3701298673238073, + "mean": 0.2833166883460113, + "persist": 0.3181818186172417 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004036779448922191, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3630952334829739, + "best1L": 0.20386904983648232, + "mean": 0.1652930397540331, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.007763975166848728, + "mean": 0.002030578100987311, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.2496532563652311, + "best1L": 0.16920942892985685, + "mean": 0.07212205204580512, + "persist": 0.058252427993076186 + }, + "multilingual": { + "min": 0.1504713578947953, + "best1L": 0.11457578066204276, + "mean": 0.040134991851768324, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.5012987000601632, + "best1L": 0.37272726957287106, + "mean": 0.2860139858509813, + "persist": 0.31818181968161036 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004373177886009216, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3601190426519939, + "best1L": 0.20684523401515825, + "mean": 0.16597984890852654, + "persist": 0.18303571242306912 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915346403207, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.2552011124789715, + "best1L": 0.16920942892985685, + "mean": 0.07308225706219673, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.14938361144491605, + "best1L": 0.11312545490052019, + "mean": 0.040051319769450595, + "persist": 0.03952139216874327 + }, + "order-ops": { + "min": 0.5051948002406529, + "best1L": 0.3740259696330343, + "mean": 0.28561438620090485, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071396640369, + "best1L": 0.20535714338932717, + "mean": 0.16540750674903393, + "persist": 0.17857142777315207 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915346403207, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.25381414751921383, + "best1L": 0.16643550539655344, + "mean": 0.0726554995801832, + "persist": 0.05963938896145139 + }, + "multilingual": { + "min": 0.15155910274812154, + "best1L": 0.11493836210242339, + "mean": 0.04044179245829582, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.5116883090564183, + "best1L": 0.37532467607940945, + "mean": 0.2877122860934053, + "persist": 0.32077921873756815 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004373177752963134, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3571428560784885, + "best1L": 0.19940476172736712, + "mean": 0.16185897588729858, + "persist": 0.17410714378846542 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.007763975166848728, + "mean": 0.0021500238134259625, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.2510402239859104, + "best1L": 0.1664355072591986, + "mean": 0.07286887915272798, + "persist": 0.058252426396523206 + }, + "multilingual": { + "min": 0.1515591000871999, + "best1L": 0.11566352498318468, + "mean": 0.04046968264239175, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.5116883111851556, + "best1L": 0.37272726744413376, + "mean": 0.28671329308833393, + "persist": 0.3168831185570785 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004261045051472527, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20386904637728417, + "mean": 0.16357600688934326, + "persist": 0.17708333461944545 + }, + "association": { + "min": 0.02173912977533681, + "best1L": 0.009316770226827689, + "mean": 0.0025083611253648996, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.25381414751921383, + "best1L": 0.1664355072591986, + "mean": 0.07318894704803824, + "persist": 0.05825242719479969 + }, + "multilingual": { + "min": 0.15083393933517592, + "best1L": 0.11493836210242339, + "mean": 0.04041390107678516, + "persist": 0.039883973609123914 + }, + "order-ops": { + "min": 0.5116883122495243, + "best1L": 0.37272726744413376, + "mean": 0.28651348395007, + "persist": 0.32077921873756815 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004148912216935839, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35565475800207685, + "best1L": 0.20089285767504148, + "mean": 0.16346154042652675, + "persist": 0.17857142950275115 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.007763975166848728, + "mean": 0.0025083612085187008, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.26074896060994696, + "best1L": 0.16920942813158035, + "mean": 0.07489597804046103, + "persist": 0.05825242746089186 + }, + "multilingual": { + "min": 0.1515591000871999, + "best1L": 0.11638868786394596, + "mean": 0.04044179192611149, + "persist": 0.04024655491645847 + }, + "order-ops": { + "min": 0.5168831231338638, + "best1L": 0.374025967504297, + "mean": 0.2866133930427687, + "persist": 0.3220779209264687 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833283662796, + "best1L": 0.20238095202616282, + "mean": 0.16597985343209334, + "persist": 0.18005952332168818 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.25797503707664354, + "best1L": 0.16920942813158035, + "mean": 0.07436253137088247, + "persist": 0.05825242719479969 + }, + "multilingual": { + "min": 0.1508339350777013, + "best1L": 0.11602610642356533, + "mean": 0.04035811951117856, + "persist": 0.040971717930265834 + }, + "order-ops": { + "min": 0.5129870101809502, + "best1L": 0.374025967504297, + "mean": 0.28721279012305395, + "persist": 0.3220779187977314 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523454734255, + "best1L": 0.20238095202616282, + "mean": 0.16506410363529408, + "persist": 0.17708333249070815 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.25797503707664354, + "best1L": 0.16920942813158035, + "mean": 0.07414915206442986, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.15010877219694002, + "best1L": 0.11602610642356533, + "mean": 0.0401628827676177, + "persist": 0.039883973476077826 + }, + "order-ops": { + "min": 0.511688310120787, + "best1L": 0.374025967504297, + "mean": 0.2863136902451515, + "persist": 0.32077921873756815 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3675595202616283, + "best1L": 0.2053571417927742, + "mean": 0.1655219742762191, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.25797503707664354, + "best1L": 0.16782246636492865, + "mean": 0.07414915259661418, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.15047135363732064, + "best1L": 0.11566352498318468, + "mean": 0.040274446830153465, + "persist": 0.04060913648988519 + }, + "order-ops": { + "min": 0.511688310120787, + "best1L": 0.374025967504297, + "mean": 0.28691308732543674, + "persist": 0.32337662098663195 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.20684523561171123, + "mean": 0.16609431936272553, + "persist": 0.18005952225731953 + }, + "association": { + "min": 0.023291925500546182, + "best1L": 0.009316770226827689, + "mean": 0.0026278069209573524, + "persist": 0.0 + } + } + }, + "runtime_s": 656.5 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s1.json b/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..befa17eba6f574e3a12e7e3d8cd24b2c20bcb70a --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s1.json @@ -0,0 +1,465 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.26768377157194273, + "best1L": 0.16643550912184374, + "mean": 0.07489597857264536, + "persist": 0.05547850179885115 + }, + "multilingual": { + "min": 0.1537345897938524, + "best1L": 0.1167512693043266, + "mean": 0.041334299237600396, + "persist": 0.04060913742120777 + }, + "order-ops": { + "min": 0.5012986979314259, + "best1L": 0.372727275959083, + "mean": 0.286013980529138, + "persist": 0.3220779187977314 + }, + "poetry": { + "min": 0.027696792967617512, + "best1L": 0.00874635563897235, + "mean": 0.004821708891540766, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3616071417927742, + "best1L": 0.19940476146127498, + "mean": 0.15945513040891715, + "persist": 0.16964285913854837 + }, + "association": { + "min": 0.02018633538058826, + "best1L": 0.01086956528680665, + "mean": 0.0020305780760411707, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.2565880742456232, + "best1L": 0.16643550912184374, + "mean": 0.07372239471546241, + "persist": 0.054091538967830796 + }, + "multilingual": { + "min": 0.15445975427116668, + "best1L": 0.11820159506584917, + "mean": 0.04139008153496044, + "persist": 0.041696881742349694 + }, + "order-ops": { + "min": 0.503896102309227, + "best1L": 0.3766233803970473, + "mean": 0.2827172763645649, + "persist": 0.31558441636817797 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004709576190050159, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36011904371636255, + "best1L": 0.20535714525197232, + "mean": 0.1656364481896162, + "persist": 0.18154761607625655 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.009316770226827689, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.25104022505027906, + "best1L": 0.16920942892985685, + "mean": 0.07190867380372115, + "persist": 0.05686546436377934 + }, + "multilingual": { + "min": 0.1580855670784201, + "best1L": 0.12037708424031734, + "mean": 0.04152953677943775, + "persist": 0.03952139283397368 + }, + "order-ops": { + "min": 0.5116883111851556, + "best1L": 0.37792208045721054, + "mean": 0.28531468553202494, + "persist": 0.32337661885789465 + }, + "poetry": { + "min": 0.023323615480746542, + "best1L": 0.00874635563897235, + "mean": 0.004036779648491314, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.3526785724929401, + "best1L": 0.19345238299242087, + "mean": 0.1564789390457528, + "persist": 0.16517857129552535 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.009316770226827689, + "mean": 0.002269469650595316, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.2538141507123198, + "best1L": 0.16643550592873776, + "mean": 0.07286888068275792, + "persist": 0.05686546462987151 + }, + "multilingual": { + "min": 0.1562726620052542, + "best1L": 0.11965192348829337, + "mean": 0.04150164579706533, + "persist": 0.03952139176960502 + }, + "order-ops": { + "min": 0.5077922024897167, + "best1L": 0.37662337826831, + "mean": 0.28591408101575716, + "persist": 0.32337661885789465 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004261044918426445, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35119047867400305, + "best1L": 0.19791666764233792, + "mean": 0.15716575139335223, + "persist": 0.16815476212650537 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.009316770226827689, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.24965326647673333, + "best1L": 0.1664355072591986, + "mean": 0.07201536325737834, + "persist": 0.056865465428148 + }, + "multilingual": { + "min": 0.15264684494052613, + "best1L": 0.11892675794661045, + "mean": 0.04105539054476789, + "persist": 0.03915881072836263 + }, + "order-ops": { + "min": 0.5155844198805946, + "best1L": 0.37662337188209805, + "mean": 0.2878121882677078, + "persist": 0.32337661779352594 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.35714285714285715, + "best1L": 0.19196428411773273, + "mean": 0.15739468644772256, + "persist": 0.17113095508622272 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.2538141517766884, + "best1L": 0.1664355072591986, + "mean": 0.07244212293465223, + "persist": 0.05963938922754356 + }, + "multilingual": { + "min": 0.15192168365631784, + "best1L": 0.11820159506584917, + "mean": 0.04088804538228682, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.5155844198805946, + "best1L": 0.37662337826831, + "mean": 0.28761239083749907, + "persist": 0.3246753178536892 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004373177686440093, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.35863095096179415, + "best1L": 0.19940475959862983, + "mean": 0.16037088047180856, + "persist": 0.1726190499695284 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.2538141517766884, + "best1L": 0.1664355072591986, + "mean": 0.07329563616908022, + "persist": 0.05963938922754356 + }, + "multilingual": { + "min": 0.1526468444083418, + "best1L": 0.11783901362546853, + "mean": 0.04102750029414892, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.5103896111249924, + "best1L": 0.37662337826831, + "mean": 0.2873126896364348, + "persist": 0.32207791773336275 + }, + "poetry": { + "min": 0.02478134073317051, + "best1L": 0.00874635563897235, + "mean": 0.004485310520976782, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3601190447807312, + "best1L": 0.20089285554630415, + "mean": 0.1607142865125622, + "persist": 0.1726190499695284 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.0021500238799490035, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.25520110928586554, + "best1L": 0.1650485454925469, + "mean": 0.07372239584635411, + "persist": 0.05825242746089186 + }, + "multilingual": { + "min": 0.1497461928852967, + "best1L": 0.11602610642356533, + "mean": 0.04030233781252589, + "persist": 0.04060913648988519 + }, + "order-ops": { + "min": 0.5155844177518573, + "best1L": 0.3753246696931975, + "mean": 0.28761238870876177, + "persist": 0.3207792176731995 + }, + "poetry": { + "min": 0.026239066384732723, + "best1L": 0.00874635563897235, + "mean": 0.004485310654022864, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.20089285767504148, + "mean": 0.1641483549028635, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.2524271857525621, + "best1L": 0.1664355072591986, + "mean": 0.07329563616908022, + "persist": 0.058252426396523206 + }, + "multilingual": { + "min": 0.15047135576605797, + "best1L": 0.11711385074470725, + "mean": 0.040413900810692995, + "persist": 0.04024655504950455 + }, + "order-ops": { + "min": 0.5155844177518573, + "best1L": 0.3753246696931975, + "mean": 0.28731268803988186, + "persist": 0.32207791773336275 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261044984949487, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160714285714285, + "best1L": 0.1979166679084301, + "mean": 0.1624313188450677, + "persist": 0.17857142630964518 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.2524271857525621, + "best1L": 0.1664355072591986, + "mean": 0.07361570512875915, + "persist": 0.05963938922754356 + }, + "multilingual": { + "min": 0.15083393933517592, + "best1L": 0.11711385074470725, + "mean": 0.04055335519037077, + "persist": 0.04060913648988519 + }, + "order-ops": { + "min": 0.5155844198805946, + "best1L": 0.3753246696931975, + "mean": 0.28731268271803856, + "persist": 0.32337661885789465 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36011904903820585, + "best1L": 0.19940476385610445, + "mean": 0.16254578450960772, + "persist": 0.1770833314263395 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.2524271857525621, + "best1L": 0.1664355072591986, + "mean": 0.07382908283865877, + "persist": 0.05963938922754356 + }, + "multilingual": { + "min": 0.15119651864681924, + "best1L": 0.11747643218508788, + "mean": 0.04066491858767612, + "persist": 0.04060913648988519 + }, + "order-ops": { + "min": 0.5155844198805946, + "best1L": 0.3753246696931975, + "mean": 0.28741259074636866, + "persist": 0.32337661885789465 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004373177819486175, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160714285714285, + "best1L": 0.19940476385610445, + "mean": 0.162660257092544, + "persist": 0.17857142737401382 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + } + }, + "runtime_s": 416.7 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s2.json b/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..5b0bd59977b1ad66370d9240724547784b3ea407 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s2.json @@ -0,0 +1,465 @@ +{ + "corpus": "USPTO_Backgrounds", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2593619977789266, + "best1L": 0.16920943318733148, + "mean": 0.07436253110479031, + "persist": 0.06102635179247175 + }, + "multilingual": { + "min": 0.15917331352829933, + "best1L": 0.12182741000183991, + "mean": 0.04219891796154635, + "persist": 0.0420594634488225 + }, + "order-ops": { + "min": 0.5207792179925101, + "best1L": 0.37402596856866566, + "mean": 0.2840159817465714, + "persist": 0.31298701411911417 + }, + "poetry": { + "min": 0.03061224440378802, + "best1L": 0.01020408155662673, + "mean": 0.005045974494091102, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713859966825, + "best1L": 0.20089285927159445, + "mean": 0.16357600609106676, + "persist": 0.17559523481343473 + }, + "association": { + "min": 0.024844719895294735, + "best1L": 0.009316770359873772, + "mean": 0.003225035799135055, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.25936199565018925, + "best1L": 0.1692094337195158, + "mean": 0.07446922069149357, + "persist": 0.05825242719479969 + }, + "multilingual": { + "min": 0.15989847906998225, + "best1L": 0.12146482882755143, + "mean": 0.0416689909595464, + "persist": 0.04133430030196905 + }, + "order-ops": { + "min": 0.515584412430014, + "best1L": 0.3740259696330343, + "mean": 0.28271727689674925, + "persist": 0.3181818164885044 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004709576123527118, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36309523241860525, + "best1L": 0.20238095388880797, + "mean": 0.16494963238281862, + "persist": 0.16964286020291702 + }, + "association": { + "min": 0.02173912977533681, + "best1L": 0.00776397529989481, + "mean": 0.002747252749811326, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.26490985229611397, + "best1L": 0.1705963984131813, + "mean": 0.07585618305685264, + "persist": 0.05686546356550285 + }, + "multilingual": { + "min": 0.1526468444083418, + "best1L": 0.11711385074470725, + "mean": 0.040218664931931665, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.5129870112453189, + "best1L": 0.3753246696931975, + "mean": 0.28531468553202494, + "persist": 0.3181818164885044 + }, + "poetry": { + "min": 0.02332361481551613, + "best1L": 0.00874635563897235, + "mean": 0.004261045117995569, + "persist": 0.002915451835308756 + }, + "typo": { + "min": 0.36607142644269125, + "best1L": 0.21428571215697698, + "mean": 0.16929945003773486, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.018633540453655378, + "best1L": 0.00776397529989481, + "mean": 0.00203057802614889, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.2621359223765986, + "best1L": 0.17059639522007533, + "mean": 0.07574949420190283, + "persist": 0.058252426396523206 + }, + "multilingual": { + "min": 0.15445975373898232, + "best1L": 0.11928933938699109, + "mean": 0.04091593636465924, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.5155844156231199, + "best1L": 0.3753246696931975, + "mean": 0.2864135870976107, + "persist": 0.3181818164885044 + }, + "poetry": { + "min": 0.024781341132308756, + "best1L": 0.00874635563897235, + "mean": 0.004485310654022864, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3675595223903656, + "best1L": 0.20982143070016587, + "mean": 0.16861264168151788, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.00776397529989481, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.2621359223765986, + "best1L": 0.16920942892985685, + "mean": 0.07318894758022257, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.15010877272912435, + "best1L": 0.1145757795976741, + "mean": 0.04055335572255509, + "persist": 0.038796228888843744 + }, + "order-ops": { + "min": 0.518181816807815, + "best1L": 0.374025967504297, + "mean": 0.2873126853789602, + "persist": 0.31948052293487955 + }, + "poetry": { + "min": 0.027696792302387103, + "best1L": 0.00874635563897235, + "mean": 0.0047095762565732, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36904762259551455, + "best1L": 0.20684523986918585, + "mean": 0.16540750887777125, + "persist": 0.17857142737401382 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.2621359223765986, + "best1L": 0.16920942892985685, + "mean": 0.07350901441116418, + "persist": 0.05686546569424016 + }, + "multilingual": { + "min": 0.15010877485786164, + "best1L": 0.11566352498318468, + "mean": 0.04063702720616545, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.5181818210652896, + "best1L": 0.374025967504297, + "mean": 0.2859140911272594, + "persist": 0.31818182074597906 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3645833304950169, + "best1L": 0.20982143176453455, + "mean": 0.16586538617100036, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.25797503707664354, + "best1L": 0.16782246636492865, + "mean": 0.07361570539485131, + "persist": 0.05686546569424016 + }, + "multilingual": { + "min": 0.1511965191790036, + "best1L": 0.11602610642356533, + "mean": 0.040720700685467036, + "persist": 0.03915881059531655 + }, + "order-ops": { + "min": 0.5168831210051265, + "best1L": 0.3753246696931975, + "mean": 0.2861138858965465, + "persist": 0.31948052293487955 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004597443422036511, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36607142857142855, + "best1L": 0.2083333358168602, + "mean": 0.16597985263381684, + "persist": 0.18154761926936253 + }, + "association": { + "min": 0.020186335114496096, + "best1L": 0.00776397529989481, + "mean": 0.002388915363033967, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.25658807530999184, + "best1L": 0.16504854735519206, + "mean": 0.07276218976559383, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.1515591000871999, + "best1L": 0.11566352498318468, + "mean": 0.040915937695120065, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.5155844166874886, + "best1L": 0.37532467607940945, + "mean": 0.2857142857142857, + "persist": 0.3181818143597671 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36160713859966825, + "best1L": 0.1994047649204731, + "mean": 0.16426282109958784, + "persist": 0.18005952225731953 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.25658807530999184, + "best1L": 0.16643550912184374, + "mean": 0.07350901520944067, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.15083393720643862, + "best1L": 0.11530094354280404, + "mean": 0.040637026873550246, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.5181818210652896, + "best1L": 0.374025967504297, + "mean": 0.28551448243004934, + "persist": 0.3181818186172417 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3630952366760799, + "best1L": 0.20238095255834715, + "mean": 0.16517857222684793, + "persist": 0.18005952438605682 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.25797503707664354, + "best1L": 0.16643550912184374, + "mean": 0.07361570486266698, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.15192168152758054, + "best1L": 0.11566352498318468, + "mean": 0.04074859053694776, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.5155844209449632, + "best1L": 0.374025967504297, + "mean": 0.2850149869918823, + "persist": 0.3181818186172417 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004485310587499823, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.3630952366760799, + "best1L": 0.20535714445369585, + "mean": 0.16472069839281694, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.02018633471535785, + "best1L": 0.006211180239915848, + "mean": 0.0023889154378723887, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.25658807530999184, + "best1L": 0.16643550912184374, + "mean": 0.07350901520944067, + "persist": 0.05547850286321981 + }, + "multilingual": { + "min": 0.15228426509669848, + "best1L": 0.11602610642356533, + "mean": 0.040832264481910636, + "persist": 0.03952139203569719 + }, + "order-ops": { + "min": 0.5168831210051265, + "best1L": 0.374025967504297, + "mean": 0.2851148865052632, + "persist": 0.31688311429960386 + }, + "poetry": { + "min": 0.026239067049963132, + "best1L": 0.00874635563897235, + "mean": 0.004597443422036511, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.36458333475249155, + "best1L": 0.20238095362271583, + "mean": 0.16483516432344913, + "persist": 0.18154761820499385 + }, + "association": { + "min": 0.018633540054517134, + "best1L": 0.006211180239915848, + "mean": 0.002269469592387655, + "persist": 0.0 + } + } + }, + "runtime_s": 396.9 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s0.json b/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..53ed03beac9fd57116ce5f9fb50e115fb7f12746 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s0.json @@ -0,0 +1,661 @@ +{ + "corpus": "Wikipedia_en", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2482662983238697, + "best1L": 0.1484049925846713, + "mean": 0.06636082294530102, + "persist": 0.048543689240302355 + }, + "multilingual": { + "min": 0.1490210292062589, + "best1L": 0.11675126957041877, + "mean": 0.03561666747555137, + "persist": 0.02864394509898765 + }, + "order-ops": { + "min": 0.48701298449720654, + "best1L": 0.39480519614049364, + "mean": 0.2752247713506222, + "persist": 0.32857142442039083 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.005158107195581708, + "persist": 0.0 + }, + "typo": { + "min": 0.29910714498588015, + "best1L": 0.17559523561171123, + "mean": 0.12557234375604562, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.010869565020714487, + "best1L": 0.004658385179936886, + "mean": 0.0020305781425642116, + "persist": 0.001552795059978962 + } + }, + "50": { + "multihop": { + "min": 0.24549237212964467, + "best1L": 0.151178918246712, + "mean": 0.06625413462253553, + "persist": 0.04854368844202587 + }, + "multilingual": { + "min": 0.1406816504895687, + "best1L": 0.10804931047771658, + "mean": 0.034752049549881904, + "persist": 0.027193618140050342 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.3896103873848915, + "mean": 0.27422577674899784, + "persist": 0.3194805176130363 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.005270239897072315, + "persist": 0.0 + }, + "typo": { + "min": 0.2991071439215115, + "best1L": 0.17857142697487557, + "mean": 0.1263736252273832, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.007763974900756564, + "best1L": 0.003105590119957924, + "mean": 0.001313903502055577, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.24826630364571298, + "best1L": 0.1567267682403326, + "mean": 0.06508055216233645, + "persist": 0.04438280154551778 + }, + "multilingual": { + "min": 0.13886874222329684, + "best1L": 0.1051486570920263, + "mean": 0.035003067792526314, + "persist": 0.028281363525560925 + }, + "order-ops": { + "min": 0.47922077987875256, + "best1L": 0.3909090885094234, + "mean": 0.2727272691471236, + "persist": 0.31558441397334847 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.00549450536657657, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3050595277122089, + "best1L": 0.17113095468708447, + "mean": 0.12328296580484935, + "persist": 0.12351190391927958 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.24549237319401332, + "best1L": 0.15117891611797468, + "mean": 0.06529392880786743, + "persist": 0.04438280154551778 + }, + "multilingual": { + "min": 0.14140681337032998, + "best1L": 0.10841189058763641, + "mean": 0.035365649831614325, + "persist": 0.028281363525560925 + }, + "order-ops": { + "min": 0.4805194820676531, + "best1L": 0.39090908744505476, + "mean": 0.2751248755625316, + "persist": 0.31818181755287306 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.01020408155662673, + "mean": 0.005158107328627791, + "persist": 0.0 + }, + "typo": { + "min": 0.30357143389327185, + "best1L": 0.17410714445369585, + "mean": 0.12465659342706203, + "persist": 0.1279761925605791 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.25104022505027906, + "best1L": 0.1553398043449436, + "mean": 0.06614744390494057, + "persist": 0.04438280260988644 + }, + "multilingual": { + "min": 0.13814358040690422, + "best1L": 0.10587382023887974, + "mean": 0.03508674107225878, + "persist": 0.028643944433757236 + }, + "order-ops": { + "min": 0.4766233797584261, + "best1L": 0.39090908744505476, + "mean": 0.27592407539486885, + "persist": 0.31948051867740496 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952381236212595, + "best1L": 0.17857143016798155, + "mean": 0.12854853511920997, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.25104022505027906, + "best1L": 0.1553398043449436, + "mean": 0.06582737654181463, + "persist": 0.04438280260988644 + }, + "multilingual": { + "min": 0.13778099790215492, + "best1L": 0.10551123826631478, + "mean": 0.035030958974467855, + "persist": 0.028643944433757236 + }, + "order-ops": { + "min": 0.47662338401590076, + "best1L": 0.3896103873848915, + "mean": 0.27562437525817324, + "persist": 0.31948051867740496 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004709576123527118, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3095238134264946, + "best1L": 0.17857143016798155, + "mean": 0.12774725072085857, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.24826629725950106, + "best1L": 0.15395284151392324, + "mean": 0.06604075504999075, + "persist": 0.04576976490872247 + }, + "multilingual": { + "min": 0.13959390723279544, + "best1L": 0.10696156429392951, + "mean": 0.03530986799991557, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.4805194841963904, + "best1L": 0.39220778750521795, + "mean": 0.27562437153288294, + "persist": 0.32077921873756815 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381236212595, + "best1L": 0.17708333422030723, + "mean": 0.12763278479022638, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.24965325902615274, + "best1L": 0.1525658786829029, + "mean": 0.06582737581006118, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.145757794380188, + "best1L": 0.11312545463442802, + "mean": 0.036230266959007294, + "persist": 0.028643944433757236 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.39220778750521795, + "mean": 0.27632367238402367, + "persist": 0.32337662098663195 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31398809381893705, + "best1L": 0.17857143016798155, + "mean": 0.12831959500908852, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.24965325902615274, + "best1L": 0.15395284151392324, + "mean": 0.06572068768686481, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14539521400417602, + "best1L": 0.1116751278085368, + "mean": 0.03611870435997844, + "persist": 0.02936910784670285 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.39220778750521795, + "mean": 0.27612387761473656, + "persist": 0.32337662098663195 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.01020408155662673, + "mean": 0.005158107328627791, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31398809275456835, + "best1L": 0.17857143016798155, + "mean": 0.12774725391396455, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0010750119358168117, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.24965325902615274, + "best1L": 0.15395284151392324, + "mean": 0.06550730731604355, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14394488824265345, + "best1L": 0.11094996279903821, + "mean": 0.03600713923307402, + "persist": 0.02936910784670285 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.39090908744505476, + "mean": 0.27582417854240965, + "persist": 0.3220779209264687 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.005270240030118397, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.17559523827263288, + "mean": 0.12751831619867257, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.24826629619513238, + "best1L": 0.1525658786829029, + "mean": 0.06572068742077265, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14358230467353547, + "best1L": 0.11094996492777552, + "mean": 0.03589557676709124, + "persist": 0.028643943901572908 + }, + "order-ops": { + "min": 0.4766233818871634, + "best1L": 0.39090908744505476, + "mean": 0.27532467458929333, + "persist": 0.3220779209264687 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.01020408155662673, + "mean": 0.005270240030118397, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.17559523827263288, + "mean": 0.1276327815971204, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.24965326434799603, + "best1L": 0.15395284151392324, + "mean": 0.0662541330925056, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14648295832531794, + "best1L": 0.1149383613041469, + "mean": 0.03639761285324182, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.39090908744505476, + "mean": 0.27622377233845846, + "persist": 0.32467532956174444 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.0 + }, + "typo": { + "min": 0.3139880895614624, + "best1L": 0.18005952505128725, + "mean": 0.12774725444614887, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.24826630151697568, + "best1L": 0.15395284151392324, + "mean": 0.06614744470321707, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.1439448855817318, + "best1L": 0.11385061645082065, + "mean": 0.036146594411028285, + "persist": 0.029369108378887177 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.39220778750521795, + "mean": 0.27612387548599926, + "persist": 0.32467532956174444 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.007288629721317973, + "mean": 0.004821708958063807, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.1785714291036129, + "mean": 0.1277472512530429, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.24826630151697568, + "best1L": 0.15395284151392324, + "mean": 0.06572068768686481, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14358230414135115, + "best1L": 0.11276287000094141, + "mean": 0.03609081284542169, + "persist": 0.03009427179183279 + }, + "order-ops": { + "min": 0.47792208194732666, + "best1L": 0.39090908744505476, + "mean": 0.27622377233845846, + "persist": 0.32467532956174444 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.12740384601056576, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.24826630151697568, + "best1L": 0.15395284151392324, + "mean": 0.06561399750145418, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14285714126058988, + "best1L": 0.11240029068929809, + "mean": 0.03603503141286118, + "persist": 0.03009427179183279 + }, + "order-ops": { + "min": 0.47792208194732666, + "best1L": 0.39090908744505476, + "mean": 0.2759240759270532, + "persist": 0.32337662098663195 + }, + "poetry": { + "min": 0.027696793233709677, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.1274038470749344, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.24826630151697568, + "best1L": 0.15395284151392324, + "mean": 0.06604075611435942, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14394488771046912, + "best1L": 0.11276287212967873, + "mean": 0.0361187031625637, + "persist": 0.03009427179183279 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.39090908744505476, + "mean": 0.2762237696775368, + "persist": 0.32337662098663195 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.31249999574252535, + "best1L": 0.17708333422030723, + "mean": 0.1274038470749344, + "persist": 0.12797618830310448 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + } + }, + "runtime_s": 1555.7 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s1.json b/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s1.json new file mode 100644 index 0000000000000000000000000000000000000000..38215c5d2aea9ee97f32c5f642c1d0ae864167fe --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s1.json @@ -0,0 +1,661 @@ +{ + "corpus": "Wikipedia_en", + "seed": 1, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.2510402261146477, + "best1L": 0.15533980514322007, + "mean": 0.06646751326375774, + "persist": 0.04576976331216948 + }, + "multilingual": { + "min": 0.15482233464717865, + "best1L": 0.11711385127689157, + "mean": 0.03756903065368533, + "persist": 0.029006526273276125 + }, + "order-ops": { + "min": 0.4740259732518877, + "best1L": 0.39090908744505476, + "mean": 0.2750249739204134, + "persist": 0.32597402217132704 + }, + "poetry": { + "min": 0.029154518486133645, + "best1L": 0.008746355505926269, + "mean": 0.005045974427568061, + "persist": 0.0 + }, + "typo": { + "min": 0.3035714307001659, + "best1L": 0.1770833320915699, + "mean": 0.12637362336473806, + "persist": 0.1309523812628218 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.004658385179936886, + "mean": 0.0014333492727018893, + "persist": 0.001552795059978962 + } + }, + "50": { + "multihop": { + "min": 0.24687933868595532, + "best1L": 0.1525658786829029, + "mean": 0.06582737640876855, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14684553976569856, + "best1L": 0.11276286946875709, + "mean": 0.03645339428580233, + "persist": 0.029006525608045713 + }, + "order-ops": { + "min": 0.47922077775001526, + "best1L": 0.3935064875653812, + "mean": 0.2772227791803224, + "persist": 0.32857143187097143 + }, + "poetry": { + "min": 0.030612245202064514, + "best1L": 0.007288629721317973, + "mean": 0.004485310587499823, + "persist": 0.0 + }, + "typo": { + "min": 0.30059524306229185, + "best1L": 0.17559524040137017, + "mean": 0.12568681447633676, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.004658385179936886, + "mean": 0.0013139034854248166, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.25104021972843577, + "best1L": 0.1525658786829029, + "mean": 0.06721433870760458, + "persist": 0.04438280181160995 + }, + "multilingual": { + "min": 0.14829586605940545, + "best1L": 0.1105873818908419, + "mean": 0.03645339448537145, + "persist": 0.028281362993376597 + }, + "order-ops": { + "min": 0.4844155865056174, + "best1L": 0.3935064875653812, + "mean": 0.2800199772630419, + "persist": 0.32077921873756815 + }, + "poetry": { + "min": 0.032069972184087546, + "best1L": 0.00728862958827189, + "mean": 0.004933841726077455, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.18005952771220887, + "mean": 0.12511446779327734, + "persist": 0.12351190538278647 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0008361203778934266, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.2496532622192587, + "best1L": 0.1525658786829029, + "mean": 0.06625413289293647, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.1501087716647557, + "best1L": 0.11312545250569071, + "mean": 0.03687175802354302, + "persist": 0.02864394403461899 + }, + "order-ops": { + "min": 0.48181818425655365, + "best1L": 0.3935064875653812, + "mean": 0.27922078168817926, + "persist": 0.3207792176731995 + }, + "poetry": { + "min": 0.03498542375330414, + "best1L": 0.008746355505926269, + "mean": 0.005270240030118397, + "persist": 0.0 + }, + "typo": { + "min": 0.3035714328289032, + "best1L": 0.17708333422030723, + "mean": 0.1254578780915056, + "persist": 0.12351190431841783 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.2413314820400306, + "best1L": 0.1525658786829029, + "mean": 0.06486717245674559, + "persist": 0.04438280260988644 + }, + "multilingual": { + "min": 0.14793328515120915, + "best1L": 0.11421319895556994, + "mean": 0.036732303710388284, + "persist": 0.029369107447564602 + }, + "order-ops": { + "min": 0.48571428656578064, + "best1L": 0.39090908744505476, + "mean": 0.2771228700876236, + "persist": 0.3194805250636169 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18005952398691857, + "mean": 0.12934981659054756, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0011944576565708434, + "persist": 0.001552795059978962 + } + }, + "175": { + "multihop": { + "min": 0.24410540983080864, + "best1L": 0.1525658786829029, + "mean": 0.0646537936824773, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14684554083006723, + "best1L": 0.11530094327671188, + "mean": 0.03648128593340516, + "persist": 0.029006525741091797 + }, + "order-ops": { + "min": 0.48571428656578064, + "best1L": 0.39090908744505476, + "mean": 0.2774225744817938, + "persist": 0.3194805197417736 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18005952398691857, + "mean": 0.12809065809207304, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.24271844806415693, + "best1L": 0.1525658786829029, + "mean": 0.06529392920700568, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14757070583956583, + "best1L": 0.11493836183633123, + "mean": 0.03670441173017025, + "persist": 0.029006525741091797 + }, + "order-ops": { + "min": 0.4844155865056174, + "best1L": 0.39090908744505476, + "mean": 0.27772228366562296, + "persist": 0.32337662843721254 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.311011902987957, + "best1L": 0.18154761993459292, + "mean": 0.12820513200547015, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0011944576565708434, + "persist": 0.001552795059978962 + } + }, + "300": { + "multihop": { + "min": 0.23717059514352254, + "best1L": 0.149791954351323, + "mean": 0.0629467620248241, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14104423352650233, + "best1L": 0.11094996386340686, + "mean": 0.03558877596099462, + "persist": 0.029731689420129572 + }, + "order-ops": { + "min": 0.47922077987875256, + "best1L": 0.39090908744505476, + "mean": 0.27422577142715454, + "persist": 0.3233766273728439 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.31101190405232565, + "best1L": 0.18005952558347157, + "mean": 0.12786172117505754, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.003105590119957924, + "mean": 0.0008361203862088067, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.24133148629750525, + "best1L": 0.1525658786829029, + "mean": 0.06369358893217784, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14213197784764425, + "best1L": 0.11058738242302622, + "mean": 0.0357561221213213, + "persist": 0.029731689420129572 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.39220778750521795, + "mean": 0.2747252769768238, + "persist": 0.322077926248312 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381023338865, + "best1L": 0.18005952558347157, + "mean": 0.12786172277161054, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.24271845232163156, + "best1L": 0.1525658786829029, + "mean": 0.06369358680344053, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14213197997638158, + "best1L": 0.11058738242302622, + "mean": 0.035644557526601214, + "persist": 0.02864394403461899 + }, + "order-ops": { + "min": 0.4792207820074899, + "best1L": 0.3935064875653812, + "mean": 0.27542457463485853, + "persist": 0.32467532956174444 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30952381449086325, + "best1L": 0.18005952558347157, + "mean": 0.12751831247338227, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.24271845232163156, + "best1L": 0.1525658786829029, + "mean": 0.06412034541634577, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14321972642626082, + "best1L": 0.11094996386340686, + "mean": 0.03583979553409985, + "persist": 0.02900652547499963 + }, + "order-ops": { + "min": 0.47792207981858936, + "best1L": 0.39220778750521795, + "mean": 0.2751248702406883, + "persist": 0.322077926248312 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.007288629721317973, + "mean": 0.004709575990481036, + "persist": 0.0 + }, + "typo": { + "min": 0.3080357164144516, + "best1L": 0.17857142963579722, + "mean": 0.12694597191044263, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.23855756329638617, + "best1L": 0.149791954351323, + "mean": 0.06337352037163717, + "persist": 0.047156725877097676 + }, + "multilingual": { + "min": 0.14176939640726363, + "best1L": 0.11058738242302622, + "mean": 0.03567244850897363, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.47792207981858936, + "best1L": 0.39220778750521795, + "mean": 0.275024979774441, + "persist": 0.32467532849737574 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30505952558347155, + "best1L": 0.17708333315593855, + "mean": 0.12637362868658134, + "persist": 0.1264880955485361 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.23855756329638617, + "best1L": 0.149791954351323, + "mean": 0.06358689861372113, + "persist": 0.047156725877097676 + }, + "multilingual": { + "min": 0.14213197997638158, + "best1L": 0.11058738135865756, + "mean": 0.03581190435215831, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.47792207981858936, + "best1L": 0.39220778750521795, + "mean": 0.275024981370994, + "persist": 0.3233766273728439 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952381023338865, + "best1L": 0.17559523827263288, + "mean": 0.12797619295971735, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.2427184528538159, + "best1L": 0.15117891611797468, + "mean": 0.06390696571075491, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.14321972536189215, + "best1L": 0.11094996279903821, + "mean": 0.03583979520148465, + "persist": 0.028643944965941564 + }, + "order-ops": { + "min": 0.47792207981858936, + "best1L": 0.39220778750521795, + "mean": 0.27492507600358557, + "persist": 0.3233766273728439 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004709575990481036, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.17559523827263288, + "mean": 0.1279761895005192, + "persist": 0.12797619043184177 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.245492372661829, + "best1L": 0.1525658786829029, + "mean": 0.06433372485584446, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.14321972536189215, + "best1L": 0.11094996279903821, + "mean": 0.0358676859177649, + "persist": 0.028643944965941564 + }, + "order-ops": { + "min": 0.47792207981858936, + "best1L": 0.39220778750521795, + "mean": 0.2754245751670429, + "persist": 0.3233766273728439 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.3065476215311459, + "best1L": 0.17708333422030723, + "mean": 0.12751831726304122, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.245492372661829, + "best1L": 0.1525658786829029, + "mean": 0.0641203444850232, + "persist": 0.045769764110445976 + }, + "multilingual": { + "min": 0.14213197997638158, + "best1L": 0.11058738295521055, + "mean": 0.035811903953020065, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.47792207981858936, + "best1L": 0.39220778750521795, + "mean": 0.2752247793333871, + "persist": 0.3233766273728439 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30505952345473425, + "best1L": 0.17708333422030723, + "mean": 0.12728937795119627, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180239915848, + "best1L": 0.003105590119957924, + "mean": 0.0010750119441321918, + "persist": 0.0 + } + } + }, + "runtime_s": 1546.9 +} \ No newline at end of file diff --git a/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s2.json b/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s2.json new file mode 100644 index 0000000000000000000000000000000000000000..9da70d9c17a05b0956cf9dff4654e16f3b6a1629 --- /dev/null +++ b/results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s2.json @@ -0,0 +1,465 @@ +{ + "corpus": "Wikipedia_en", + "seed": 2, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": true, + "readout_convention": "STRIPPED \u2014 the anchor rule: readout at the token immediately preceding `target`", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.24687933549284935, + "best1L": 0.15395284151392324, + "mean": 0.0657206878199109, + "persist": 0.04576976437653814 + }, + "multilingual": { + "min": 0.14829586632549763, + "best1L": 0.11675126797386579, + "mean": 0.03687175742483565, + "persist": 0.0319071796589664 + }, + "order-ops": { + "min": 0.4870129866259439, + "best1L": 0.3922077864408493, + "mean": 0.27852147604737965, + "persist": 0.32467532211116384 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30059523986918585, + "best1L": 0.1800595261156559, + "mean": 0.12339743519467967, + "persist": 0.1220238082376974 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.2496532622192587, + "best1L": 0.15395284151392324, + "mean": 0.06636082467490009, + "persist": 0.04715672694146633 + }, + "multilingual": { + "min": 0.1439448885087456, + "best1L": 0.11276287106531006, + "mean": 0.036146594277982204, + "persist": 0.0297316899523139 + }, + "order-ops": { + "min": 0.4818181757416044, + "best1L": 0.3909090853163174, + "mean": 0.27702298015356064, + "persist": 0.32467532211116384 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.008746355505926269, + "mean": 0.004821708825017724, + "persist": 0.0 + }, + "typo": { + "min": 0.3020833336881229, + "best1L": 0.17857142803924425, + "mean": 0.1250000007982765, + "persist": 0.12648809448416745 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "75": { + "multihop": { + "min": 0.24410540983080864, + "best1L": 0.1539528425782919, + "mean": 0.06508054990055305, + "persist": 0.04576976517481463 + }, + "multilingual": { + "min": 0.14140681682952813, + "best1L": 0.10877447415675436, + "mean": 0.035365648700722625, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.48181818425655365, + "best1L": 0.3935064875653812, + "mean": 0.27722277279411045, + "persist": 0.32337661885789465 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.01020408155662673, + "mean": 0.005158107195581708, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3020833304950169, + "best1L": 0.17261904890515975, + "mean": 0.12396978054727827, + "persist": 0.12202380943511214 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.245492372661829, + "best1L": 0.1567267682403326, + "mean": 0.06508055069882955, + "persist": 0.04438280234379428 + }, + "multilingual": { + "min": 0.1432197242975235, + "best1L": 0.11276287106531006, + "mean": 0.035839793871023824, + "persist": 0.030094271525740623 + }, + "order-ops": { + "min": 0.48181818425655365, + "best1L": 0.39090908744505476, + "mean": 0.27762238149132046, + "persist": 0.32467532849737574 + }, + "poetry": { + "min": 0.03206997111971889, + "best1L": 0.01020408155662673, + "mean": 0.005158107195581708, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.3005952388048172, + "best1L": 0.1755952350795269, + "mean": 0.12396977815244879, + "persist": 0.12202381089861904 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "150": { + "multihop": { + "min": 0.24271844487105096, + "best1L": 0.15117891691625118, + "mean": 0.06433372565412096, + "persist": 0.0416088772139379 + }, + "multilingual": { + "min": 0.14213197625109128, + "best1L": 0.11094996492777552, + "mean": 0.03556088590994477, + "persist": 0.028281362461192266 + }, + "order-ops": { + "min": 0.47792207981858936, + "best1L": 0.3935064896941185, + "mean": 0.2736263700893947, + "persist": 0.32077922725251745 + }, + "poetry": { + "min": 0.030612244802926267, + "best1L": 0.00728862958827189, + "mean": 0.00515810726210475, + "persist": 0.0 + }, + "typo": { + "min": 0.3124999978712627, + "best1L": 0.1770833320915699, + "mean": 0.12797619083098002, + "persist": 0.12351190471755606 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "175": { + "multihop": { + "min": 0.24549236946872302, + "best1L": 0.15395284151392324, + "mean": 0.0645471025657441, + "persist": 0.0416088772139379 + }, + "multilingual": { + "min": 0.14321972376533917, + "best1L": 0.11131254636815616, + "mean": 0.03553299545975668, + "persist": 0.027918781020811627 + }, + "order-ops": { + "min": 0.477922077689852, + "best1L": 0.3935064896941185, + "mean": 0.27392607608011793, + "persist": 0.32077922725251745 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004933841659554413, + "persist": 0.0 + }, + "typo": { + "min": 0.31101190405232565, + "best1L": 0.17559523933700152, + "mean": 0.12809065729379654, + "persist": 0.1250000006652304 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0013139034272171557, + "persist": 0.001552795059978962 + } + }, + "200": { + "multihop": { + "min": 0.24271844487105096, + "best1L": 0.15395284151392324, + "mean": 0.06476048253742712, + "persist": 0.0429958389805896 + }, + "multilingual": { + "min": 0.14176939693944796, + "best1L": 0.11131254743252482, + "mean": 0.03547721389415009, + "persist": 0.029369108511933258 + }, + "order-ops": { + "min": 0.4805194778101785, + "best1L": 0.39220778750521795, + "mean": 0.27512486970850397, + "persist": 0.32337662630847525 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004597443422036511, + "persist": 0.0 + }, + "typo": { + "min": 0.30803571215697695, + "best1L": 0.17261904637728417, + "mean": 0.12614468804427556, + "persist": 0.12202380664114441 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.004658385179936886, + "mean": 0.0011944577147785043, + "persist": 0.001552795059978962 + } + }, + "300": { + "multihop": { + "min": 0.24687933975032397, + "best1L": 0.15117891691625118, + "mean": 0.06582737687442984, + "persist": 0.04438280074724129 + }, + "multilingual": { + "min": 0.14394488877483777, + "best1L": 0.11240028962492943, + "mean": 0.035728230739810636, + "persist": 0.02973168981926782 + }, + "order-ops": { + "min": 0.47922077987875256, + "best1L": 0.3935064875653812, + "mean": 0.27632367770586697, + "persist": 0.3207792251237801 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.31249999680689405, + "best1L": 0.1785714291036129, + "mean": 0.1287774701735803, + "persist": 0.1294642842507788 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.24687933975032397, + "best1L": 0.1525658786829029, + "mean": 0.065827377140522, + "persist": 0.0429958389805896 + }, + "multilingual": { + "min": 0.1450326330959797, + "best1L": 0.11276287319404739, + "mean": 0.03586768565167274, + "persist": 0.0297316899523139 + }, + "order-ops": { + "min": 0.4779220787542207, + "best1L": 0.39220778750521795, + "mean": 0.2759240764592375, + "persist": 0.32077922725251745 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.1785714291036129, + "mean": 0.1279761895005192, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0008361203862088067, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.24549237691930362, + "best1L": 0.1525658786829029, + "mean": 0.06561399803363852, + "persist": 0.04438280074724129 + }, + "multilingual": { + "min": 0.14358230733445712, + "best1L": 0.1120377103132861, + "mean": 0.03578401270455548, + "persist": 0.02900652587413788 + }, + "order-ops": { + "min": 0.47922077987875256, + "best1L": 0.3935064875653812, + "mean": 0.2762237691453525, + "persist": 0.32207792731268065 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004821708825017724, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.17559523827263288, + "mean": 0.1279761895005192, + "persist": 0.1249999985364931 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.24549237691930362, + "best1L": 0.1525658786829029, + "mean": 0.06561399750145418, + "persist": 0.0429958389805896 + }, + "multilingual": { + "min": 0.14358230733445712, + "best1L": 0.11276287106531006, + "mean": 0.035700340289622545, + "persist": 0.029369107447564602 + }, + "order-ops": { + "min": 0.47922077987875256, + "best1L": 0.3935064875653812, + "mean": 0.2765234735395227, + "persist": 0.3233766273728439 + }, + "poetry": { + "min": 0.0291545195505023, + "best1L": 0.00728862958827189, + "mean": 0.004709575990481036, + "persist": 0.001457725917654378 + }, + "typo": { + "min": 0.30952380491154535, + "best1L": 0.17559523827263288, + "mean": 0.12763278585459506, + "persist": 0.12351190471755606 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.003105590119957924, + "mean": 0.0009555661651704993, + "persist": 0.0 + } + } + }, + "runtime_s": 1078.4 +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n100_s0.pt b/results/lenses/e28/e28_Github_410m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..cee2f0d8ae54d99595b6f388e0503ab6fbf8c290 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985f6d7aede6680ac4921866b6071fa89e4a72c9b15b494298f5cf10f3df909b +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n100_s0_provenance.json b/results/lenses/e28/e28_Github_410m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..295e79b1c63507becc9b5e64473a6b9332f12699 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n100_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n100_s0.pt", + "sha256": "985f6d7aede6680ac4921866b6071fa89e4a72c9b15b494298f5cf10f3df909b", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.238375, + "mean_fro": 8.237703, + "rms_per_prompt_fro": 18.85404, + "max_per_prompt_fro": 29.136555, + "heavy_tail_ratio": 1.5454, + "n_prompts": 100 + }, + "1": { + "dispersion": 3.767495, + "mean_fro": 7.736607, + "rms_per_prompt_fro": 16.892566, + "max_per_prompt_fro": 25.729753, + "heavy_tail_ratio": 1.5231, + "n_prompts": 100 + }, + "2": { + "dispersion": 3.060702, + "mean_fro": 8.341809, + "rms_per_prompt_fro": 16.809734, + "max_per_prompt_fro": 24.51263, + "heavy_tail_ratio": 1.4582, + "n_prompts": 100 + }, + "3": { + "dispersion": 2.503207, + "mean_fro": 7.967025, + "rms_per_prompt_fro": 14.911766, + "max_per_prompt_fro": 19.907942, + "heavy_tail_ratio": 1.335, + "n_prompts": 100 + }, + "4": { + "dispersion": 2.083485, + "mean_fro": 8.905653, + "rms_per_prompt_fro": 15.638198, + "max_per_prompt_fro": 20.365681, + "heavy_tail_ratio": 1.3023, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.792186, + "mean_fro": 9.172812, + "rms_per_prompt_fro": 15.327616, + "max_per_prompt_fro": 19.465076, + "heavy_tail_ratio": 1.2699, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.540362, + "mean_fro": 9.563564, + "rms_per_prompt_fro": 15.242899, + "max_per_prompt_fro": 18.996004, + "heavy_tail_ratio": 1.2462, + "n_prompts": 100 + }, + "7": { + "dispersion": 1.358054, + "mean_fro": 10.508536, + "rms_per_prompt_fro": 16.136862, + "max_per_prompt_fro": 19.43343, + "heavy_tail_ratio": 1.2043, + "n_prompts": 100 + }, + "8": { + "dispersion": 1.205992, + "mean_fro": 11.729362, + "rms_per_prompt_fro": 17.421132, + "max_per_prompt_fro": 20.644583, + "heavy_tail_ratio": 1.185, + "n_prompts": 100 + }, + "9": { + "dispersion": 1.100086, + "mean_fro": 12.958187, + "rms_per_prompt_fro": 18.778582, + "max_per_prompt_fro": 22.70639, + "heavy_tail_ratio": 1.2092, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.995326, + "mean_fro": 14.417933, + "rms_per_prompt_fro": 20.366197, + "max_per_prompt_fro": 24.566759, + "heavy_tail_ratio": 1.2063, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.886343, + "mean_fro": 16.39298, + "rms_per_prompt_fro": 22.514808, + "max_per_prompt_fro": 27.652733, + "heavy_tail_ratio": 1.2282, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.740101, + "mean_fro": 18.495439, + "rms_per_prompt_fro": 24.397871, + "max_per_prompt_fro": 29.19372, + "heavy_tail_ratio": 1.1966, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.643287, + "mean_fro": 19.93137, + "rms_per_prompt_fro": 25.550171, + "max_per_prompt_fro": 30.319773, + "heavy_tail_ratio": 1.1867, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.526934, + "mean_fro": 22.707644, + "rms_per_prompt_fro": 28.059652, + "max_per_prompt_fro": 33.417854, + "heavy_tail_ratio": 1.191, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.441062, + "mean_fro": 25.1247, + "rms_per_prompt_fro": 30.160759, + "max_per_prompt_fro": 35.608326, + "heavy_tail_ratio": 1.1806, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.319312, + "mean_fro": 28.510467, + "rms_per_prompt_fro": 32.747497, + "max_per_prompt_fro": 38.711796, + "heavy_tail_ratio": 1.1821, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.234019, + "mean_fro": 31.375315, + "rms_per_prompt_fro": 34.853711, + "max_per_prompt_fro": 39.857838, + "heavy_tail_ratio": 1.1436, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.167343, + "mean_fro": 33.100044, + "rms_per_prompt_fro": 35.762495, + "max_per_prompt_fro": 39.968056, + "heavy_tail_ratio": 1.1176, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.107684, + "mean_fro": 34.052238, + "rms_per_prompt_fro": 35.838812, + "max_per_prompt_fro": 38.831326, + "heavy_tail_ratio": 1.0835, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.051637, + "mean_fro": 34.721496, + "rms_per_prompt_fro": 35.606676, + "max_per_prompt_fro": 37.712627, + "heavy_tail_ratio": 1.0591, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.024058, + "mean_fro": 34.362915, + "rms_per_prompt_fro": 34.773804, + "max_per_prompt_fro": 35.857635, + "heavy_tail_ratio": 1.0312, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 311.9, + "fit_secs_per_prompt": 3.119, + "total_wall_seconds": 324.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.2376, + "top_sv": 1.9972, + "stable_rank": 17.012 + }, + { + "layer": 1, + "fro_norm": 7.7365, + "top_sv": 1.8287, + "stable_rank": 17.897 + }, + { + "layer": 2, + "fro_norm": 8.3417, + "top_sv": 1.9851, + "stable_rank": 17.659 + }, + { + "layer": 3, + "fro_norm": 7.9669, + "top_sv": 1.8975, + "stable_rank": 17.629 + }, + { + "layer": 4, + "fro_norm": 8.9055, + "top_sv": 2.1155, + "stable_rank": 17.721 + }, + { + "layer": 5, + "fro_norm": 9.1727, + "top_sv": 2.2715, + "stable_rank": 16.307 + }, + { + "layer": 6, + "fro_norm": 9.5634, + "top_sv": 2.3691, + "stable_rank": 16.295 + }, + { + "layer": 7, + "fro_norm": 10.5084, + "top_sv": 2.5202, + "stable_rank": 17.386 + }, + { + "layer": 8, + "fro_norm": 11.7292, + "top_sv": 2.8793, + "stable_rank": 16.594 + }, + { + "layer": 9, + "fro_norm": 12.958, + "top_sv": 3.1301, + "stable_rank": 17.138 + }, + { + "layer": 10, + "fro_norm": 14.4177, + "top_sv": 3.4622, + "stable_rank": 17.342 + }, + { + "layer": 11, + "fro_norm": 16.3927, + "top_sv": 3.8948, + "stable_rank": 17.715 + }, + { + "layer": 12, + "fro_norm": 18.4952, + "top_sv": 4.4708, + "stable_rank": 17.114 + }, + { + "layer": 13, + "fro_norm": 19.931, + "top_sv": 4.8734, + "stable_rank": 16.726 + }, + { + "layer": 14, + "fro_norm": 22.7073, + "top_sv": 5.5797, + "stable_rank": 16.562 + }, + { + "layer": 15, + "fro_norm": 25.1242, + "top_sv": 6.3335, + "stable_rank": 15.736 + }, + { + "layer": 16, + "fro_norm": 28.5099, + "top_sv": 7.3949, + "stable_rank": 14.864 + }, + { + "layer": 17, + "fro_norm": 31.3747, + "top_sv": 7.905, + "stable_rank": 15.753 + }, + { + "layer": 18, + "fro_norm": 33.0993, + "top_sv": 7.8843, + "stable_rank": 17.624 + }, + { + "layer": 19, + "fro_norm": 34.0513, + "top_sv": 7.6036, + "stable_rank": 20.055 + }, + { + "layer": 20, + "fro_norm": 34.7204, + "top_sv": 7.0079, + "stable_rank": 24.547 + }, + { + "layer": 21, + "fro_norm": 34.3615, + "top_sv": 5.3453, + "stable_rank": 41.324 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n100_s1.pt b/results/lenses/e28/e28_Github_410m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..7abdac4ec2d526cd68555fb2bd7ef7398c925982 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1733c165a94867d64c3bcc7fcb2388461b8aabfdf7700112823a2c3466b34e29 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n100_s1_provenance.json b/results/lenses/e28/e28_Github_410m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..74c549f60837c9ae862c87c9103c53dccd9c4e0a --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n100_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n100_s1.pt", + "sha256": "1733c165a94867d64c3bcc7fcb2388461b8aabfdf7700112823a2c3466b34e29", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.600407, + "mean_fro": 7.839603, + "rms_per_prompt_fro": 18.552562, + "max_per_prompt_fro": 26.577099, + "heavy_tail_ratio": 1.4325, + "n_prompts": 100 + }, + "1": { + "dispersion": 4.08687, + "mean_fro": 7.3748, + "rms_per_prompt_fro": 16.633191, + "max_per_prompt_fro": 23.843842, + "heavy_tail_ratio": 1.4335, + "n_prompts": 100 + }, + "2": { + "dispersion": 3.359652, + "mean_fro": 7.973579, + "rms_per_prompt_fro": 16.648656, + "max_per_prompt_fro": 24.031956, + "heavy_tail_ratio": 1.4435, + "n_prompts": 100 + }, + "3": { + "dispersion": 2.746337, + "mean_fro": 7.679114, + "rms_per_prompt_fro": 14.863277, + "max_per_prompt_fro": 20.982944, + "heavy_tail_ratio": 1.4117, + "n_prompts": 100 + }, + "4": { + "dispersion": 2.303098, + "mean_fro": 8.583681, + "rms_per_prompt_fro": 15.60035, + "max_per_prompt_fro": 21.62668, + "heavy_tail_ratio": 1.3863, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.99192, + "mean_fro": 8.839227, + "rms_per_prompt_fro": 15.289359, + "max_per_prompt_fro": 21.015985, + "heavy_tail_ratio": 1.3745, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.703366, + "mean_fro": 9.215621, + "rms_per_prompt_fro": 15.152246, + "max_per_prompt_fro": 20.854357, + "heavy_tail_ratio": 1.3763, + "n_prompts": 100 + }, + "7": { + "dispersion": 1.495027, + "mean_fro": 10.109595, + "rms_per_prompt_fro": 15.968768, + "max_per_prompt_fro": 22.127941, + "heavy_tail_ratio": 1.3857, + "n_prompts": 100 + }, + "8": { + "dispersion": 1.332485, + "mean_fro": 11.290414, + "rms_per_prompt_fro": 17.243256, + "max_per_prompt_fro": 23.444479, + "heavy_tail_ratio": 1.3596, + "n_prompts": 100 + }, + "9": { + "dispersion": 1.213432, + "mean_fro": 12.500295, + "rms_per_prompt_fro": 18.597449, + "max_per_prompt_fro": 24.862322, + "heavy_tail_ratio": 1.3369, + "n_prompts": 100 + }, + "10": { + "dispersion": 1.101433, + "mean_fro": 13.986209, + "rms_per_prompt_fro": 20.274859, + "max_per_prompt_fro": 26.610645, + "heavy_tail_ratio": 1.3125, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.987195, + "mean_fro": 15.960152, + "rms_per_prompt_fro": 22.498689, + "max_per_prompt_fro": 28.710007, + "heavy_tail_ratio": 1.2761, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.81707, + "mean_fro": 18.103376, + "rms_per_prompt_fro": 24.40312, + "max_per_prompt_fro": 29.876253, + "heavy_tail_ratio": 1.2243, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.710655, + "mean_fro": 19.621299, + "rms_per_prompt_fro": 25.663094, + "max_per_prompt_fro": 31.00931, + "heavy_tail_ratio": 1.2083, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.579265, + "mean_fro": 22.39794, + "rms_per_prompt_fro": 28.147225, + "max_per_prompt_fro": 33.104443, + "heavy_tail_ratio": 1.1761, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.485096, + "mean_fro": 24.835693, + "rms_per_prompt_fro": 30.265892, + "max_per_prompt_fro": 35.588196, + "heavy_tail_ratio": 1.1759, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.345568, + "mean_fro": 28.314726, + "rms_per_prompt_fro": 32.844686, + "max_per_prompt_fro": 37.617302, + "heavy_tail_ratio": 1.1453, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.252394, + "mean_fro": 31.233925, + "rms_per_prompt_fro": 34.954015, + "max_per_prompt_fro": 40.383511, + "heavy_tail_ratio": 1.1553, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.178443, + "mean_fro": 33.057456, + "rms_per_prompt_fro": 35.885897, + "max_per_prompt_fro": 40.659569, + "heavy_tail_ratio": 1.133, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.115206, + "mean_fro": 34.033366, + "rms_per_prompt_fro": 35.940361, + "max_per_prompt_fro": 40.076298, + "heavy_tail_ratio": 1.1151, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.054016, + "mean_fro": 34.710652, + "rms_per_prompt_fro": 35.63579, + "max_per_prompt_fro": 37.492172, + "heavy_tail_ratio": 1.0521, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.025327, + "mean_fro": 34.349735, + "rms_per_prompt_fro": 34.782004, + "max_per_prompt_fro": 36.109356, + "heavy_tail_ratio": 1.0382, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 304.8, + "fit_secs_per_prompt": 3.048, + "total_wall_seconds": 315.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.8395, + "top_sv": 1.8725, + "stable_rank": 17.528 + }, + { + "layer": 1, + "fro_norm": 7.3747, + "top_sv": 1.7502, + "stable_rank": 17.755 + }, + { + "layer": 2, + "fro_norm": 7.9735, + "top_sv": 1.9162, + "stable_rank": 17.315 + }, + { + "layer": 3, + "fro_norm": 7.679, + "top_sv": 1.8358, + "stable_rank": 17.497 + }, + { + "layer": 4, + "fro_norm": 8.5836, + "top_sv": 2.096, + "stable_rank": 16.771 + }, + { + "layer": 5, + "fro_norm": 8.8391, + "top_sv": 2.2027, + "stable_rank": 16.103 + }, + { + "layer": 6, + "fro_norm": 9.2155, + "top_sv": 2.2913, + "stable_rank": 16.175 + }, + { + "layer": 7, + "fro_norm": 10.1094, + "top_sv": 2.393, + "stable_rank": 17.847 + }, + { + "layer": 8, + "fro_norm": 11.2903, + "top_sv": 2.7226, + "stable_rank": 17.196 + }, + { + "layer": 9, + "fro_norm": 12.5001, + "top_sv": 3.0087, + "stable_rank": 17.262 + }, + { + "layer": 10, + "fro_norm": 13.986, + "top_sv": 3.3895, + "stable_rank": 17.026 + }, + { + "layer": 11, + "fro_norm": 15.9599, + "top_sv": 3.8538, + "stable_rank": 17.151 + }, + { + "layer": 12, + "fro_norm": 18.1031, + "top_sv": 4.396, + "stable_rank": 16.959 + }, + { + "layer": 13, + "fro_norm": 19.6209, + "top_sv": 4.8688, + "stable_rank": 16.24 + }, + { + "layer": 14, + "fro_norm": 22.3976, + "top_sv": 5.6025, + "stable_rank": 15.982 + }, + { + "layer": 15, + "fro_norm": 24.8352, + "top_sv": 6.4113, + "stable_rank": 15.005 + }, + { + "layer": 16, + "fro_norm": 28.3141, + "top_sv": 7.4719, + "stable_rank": 14.36 + }, + { + "layer": 17, + "fro_norm": 31.2333, + "top_sv": 7.9757, + "stable_rank": 15.336 + }, + { + "layer": 18, + "fro_norm": 33.0567, + "top_sv": 7.9777, + "stable_rank": 17.17 + }, + { + "layer": 19, + "fro_norm": 34.0324, + "top_sv": 7.6616, + "stable_rank": 19.731 + }, + { + "layer": 20, + "fro_norm": 34.7094, + "top_sv": 7.0988, + "stable_rank": 23.907 + }, + { + "layer": 21, + "fro_norm": 34.3483, + "top_sv": 5.4113, + "stable_rank": 40.29 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n100_s2.pt b/results/lenses/e28/e28_Github_410m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..d577465551b747d64fa63808e2d13a3c66706887 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac24e4cf9728827f63e99b145bd0291f72aa43ceaea363c90a78cdb967082536 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n100_s2_provenance.json b/results/lenses/e28/e28_Github_410m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7ef0d5733d7302fe24c3f2b4d418a42c80d6d7eb --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n100_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n100_s2.pt", + "sha256": "ac24e4cf9728827f63e99b145bd0291f72aa43ceaea363c90a78cdb967082536", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 5.336664, + "mean_fro": 7.99685, + "rms_per_prompt_fro": 20.130255, + "max_per_prompt_fro": 34.796436, + "heavy_tail_ratio": 1.7286, + "n_prompts": 100 + }, + "1": { + "dispersion": 4.723049, + "mean_fro": 7.506956, + "rms_per_prompt_fro": 17.958813, + "max_per_prompt_fro": 29.559164, + "heavy_tail_ratio": 1.6459, + "n_prompts": 100 + }, + "2": { + "dispersion": 3.803432, + "mean_fro": 8.133586, + "rms_per_prompt_fro": 17.826164, + "max_per_prompt_fro": 29.93277, + "heavy_tail_ratio": 1.6791, + "n_prompts": 100 + }, + "3": { + "dispersion": 3.076773, + "mean_fro": 7.783365, + "rms_per_prompt_fro": 15.715408, + "max_per_prompt_fro": 25.432657, + "heavy_tail_ratio": 1.6183, + "n_prompts": 100 + }, + "4": { + "dispersion": 2.543244, + "mean_fro": 8.709507, + "rms_per_prompt_fro": 16.394347, + "max_per_prompt_fro": 26.80678, + "heavy_tail_ratio": 1.6351, + "n_prompts": 100 + }, + "5": { + "dispersion": 2.190668, + "mean_fro": 8.979489, + "rms_per_prompt_fro": 16.039559, + "max_per_prompt_fro": 27.302526, + "heavy_tail_ratio": 1.7022, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.837321, + "mean_fro": 9.355662, + "rms_per_prompt_fro": 15.759006, + "max_per_prompt_fro": 24.270342, + "heavy_tail_ratio": 1.5401, + "n_prompts": 100 + }, + "7": { + "dispersion": 1.600749, + "mean_fro": 10.20199, + "rms_per_prompt_fro": 16.452584, + "max_per_prompt_fro": 24.17333, + "heavy_tail_ratio": 1.4693, + "n_prompts": 100 + }, + "8": { + "dispersion": 1.417503, + "mean_fro": 11.370419, + "rms_per_prompt_fro": 17.679094, + "max_per_prompt_fro": 26.130661, + "heavy_tail_ratio": 1.4781, + "n_prompts": 100 + }, + "9": { + "dispersion": 1.281595, + "mean_fro": 12.57713, + "rms_per_prompt_fro": 18.997691, + "max_per_prompt_fro": 27.802649, + "heavy_tail_ratio": 1.4635, + "n_prompts": 100 + }, + "10": { + "dispersion": 1.156077, + "mean_fro": 14.028139, + "rms_per_prompt_fro": 20.598338, + "max_per_prompt_fro": 29.789103, + "heavy_tail_ratio": 1.4462, + "n_prompts": 100 + }, + "11": { + "dispersion": 1.017982, + "mean_fro": 15.938703, + "rms_per_prompt_fro": 22.641836, + "max_per_prompt_fro": 27.419218, + "heavy_tail_ratio": 1.211, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.83942, + "mean_fro": 18.023148, + "rms_per_prompt_fro": 24.443932, + "max_per_prompt_fro": 28.603258, + "heavy_tail_ratio": 1.1702, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.72838, + "mean_fro": 19.517881, + "rms_per_prompt_fro": 25.659741, + "max_per_prompt_fro": 29.174259, + "heavy_tail_ratio": 1.137, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.590492, + "mean_fro": 22.208471, + "rms_per_prompt_fro": 28.008148, + "max_per_prompt_fro": 31.695009, + "heavy_tail_ratio": 1.1316, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.490281, + "mean_fro": 24.560476, + "rms_per_prompt_fro": 29.982704, + "max_per_prompt_fro": 34.53772, + "heavy_tail_ratio": 1.1519, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.354472, + "mean_fro": 28.027371, + "rms_per_prompt_fro": 32.618759, + "max_per_prompt_fro": 37.547672, + "heavy_tail_ratio": 1.1511, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.259816, + "mean_fro": 30.96706, + "rms_per_prompt_fro": 34.757905, + "max_per_prompt_fro": 38.719402, + "heavy_tail_ratio": 1.114, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.184341, + "mean_fro": 32.778474, + "rms_per_prompt_fro": 35.671973, + "max_per_prompt_fro": 39.30714, + "heavy_tail_ratio": 1.1019, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.118901, + "mean_fro": 33.803722, + "rms_per_prompt_fro": 35.756935, + "max_per_prompt_fro": 38.093178, + "heavy_tail_ratio": 1.0653, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.05694, + "mean_fro": 34.607643, + "rms_per_prompt_fro": 35.57928, + "max_per_prompt_fro": 37.237183, + "heavy_tail_ratio": 1.0466, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.026438, + "mean_fro": 34.321183, + "rms_per_prompt_fro": 34.771918, + "max_per_prompt_fro": 35.891567, + "heavy_tail_ratio": 1.0322, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 309.8, + "fit_secs_per_prompt": 3.098, + "total_wall_seconds": 317.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9968, + "top_sv": 2.0459, + "stable_rank": 15.277 + }, + { + "layer": 1, + "fro_norm": 7.5069, + "top_sv": 1.8589, + "stable_rank": 16.308 + }, + { + "layer": 2, + "fro_norm": 8.1335, + "top_sv": 2.1125, + "stable_rank": 14.824 + }, + { + "layer": 3, + "fro_norm": 7.7833, + "top_sv": 2.0172, + "stable_rank": 14.888 + }, + { + "layer": 4, + "fro_norm": 8.7094, + "top_sv": 2.2923, + "stable_rank": 14.436 + }, + { + "layer": 5, + "fro_norm": 8.9793, + "top_sv": 2.5007, + "stable_rank": 12.893 + }, + { + "layer": 6, + "fro_norm": 9.3555, + "top_sv": 2.6033, + "stable_rank": 12.915 + }, + { + "layer": 7, + "fro_norm": 10.2018, + "top_sv": 2.5304, + "stable_rank": 16.255 + }, + { + "layer": 8, + "fro_norm": 11.3703, + "top_sv": 2.8561, + "stable_rank": 15.849 + }, + { + "layer": 9, + "fro_norm": 12.5769, + "top_sv": 3.1179, + "stable_rank": 16.272 + }, + { + "layer": 10, + "fro_norm": 14.0279, + "top_sv": 3.4375, + "stable_rank": 16.654 + }, + { + "layer": 11, + "fro_norm": 15.9385, + "top_sv": 3.803, + "stable_rank": 17.564 + }, + { + "layer": 12, + "fro_norm": 18.0228, + "top_sv": 4.3196, + "stable_rank": 17.409 + }, + { + "layer": 13, + "fro_norm": 19.5176, + "top_sv": 4.7398, + "stable_rank": 16.956 + }, + { + "layer": 14, + "fro_norm": 22.2082, + "top_sv": 5.3502, + "stable_rank": 17.23 + }, + { + "layer": 15, + "fro_norm": 24.56, + "top_sv": 6.0439, + "stable_rank": 16.513 + }, + { + "layer": 16, + "fro_norm": 28.0268, + "top_sv": 7.0803, + "stable_rank": 15.669 + }, + { + "layer": 17, + "fro_norm": 30.9664, + "top_sv": 7.6026, + "stable_rank": 16.59 + }, + { + "layer": 18, + "fro_norm": 32.7778, + "top_sv": 7.7314, + "stable_rank": 17.974 + }, + { + "layer": 19, + "fro_norm": 33.8029, + "top_sv": 7.5032, + "stable_rank": 20.296 + }, + { + "layer": 20, + "fro_norm": 34.6065, + "top_sv": 6.9749, + "stable_rank": 24.617 + }, + { + "layer": 21, + "fro_norm": 34.3197, + "top_sv": 5.3121, + "stable_rank": 41.741 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n200_s0.pt b/results/lenses/e28/e28_Github_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..73dab69f2462c738cd5579152c43f5522846d837 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3895668dce09dae91c6300f6adbba2de8359c63bf45fd5676d554f6ed90e9dc8 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n200_s0_provenance.json b/results/lenses/e28/e28_Github_410m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..94fed3f034dcfe6db4d664c2cd62f448b5964f01 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n200_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n200_s0.pt", + "sha256": "3895668dce09dae91c6300f6adbba2de8359c63bf45fd5676d554f6ed90e9dc8", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.821137, + "mean_fro": 7.936097, + "rms_per_prompt_fro": 19.147447, + "max_per_prompt_fro": 29.136555, + "heavy_tail_ratio": 1.5217, + "n_prompts": 200 + }, + "1": { + "dispersion": 4.2568, + "mean_fro": 7.464025, + "rms_per_prompt_fro": 17.113301, + "max_per_prompt_fro": 25.729753, + "heavy_tail_ratio": 1.5035, + "n_prompts": 200 + }, + "2": { + "dispersion": 3.406551, + "mean_fro": 8.091766, + "rms_per_prompt_fro": 16.986064, + "max_per_prompt_fro": 24.51263, + "heavy_tail_ratio": 1.4431, + "n_prompts": 200 + }, + "3": { + "dispersion": 2.747913, + "mean_fro": 7.767516, + "rms_per_prompt_fro": 15.037543, + "max_per_prompt_fro": 19.907942, + "heavy_tail_ratio": 1.3239, + "n_prompts": 200 + }, + "4": { + "dispersion": 2.272438, + "mean_fro": 8.697895, + "rms_per_prompt_fro": 15.734388, + "max_per_prompt_fro": 20.7033, + "heavy_tail_ratio": 1.3158, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.939507, + "mean_fro": 8.960804, + "rms_per_prompt_fro": 15.36329, + "max_per_prompt_fro": 20.218361, + "heavy_tail_ratio": 1.316, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.650893, + "mean_fro": 9.347021, + "rms_per_prompt_fro": 15.218411, + "max_per_prompt_fro": 18.996004, + "heavy_tail_ratio": 1.2482, + "n_prompts": 200 + }, + "7": { + "dispersion": 1.450487, + "mean_fro": 10.247064, + "rms_per_prompt_fro": 16.040784, + "max_per_prompt_fro": 19.575705, + "heavy_tail_ratio": 1.2204, + "n_prompts": 200 + }, + "8": { + "dispersion": 1.288079, + "mean_fro": 11.417129, + "rms_per_prompt_fro": 17.270004, + "max_per_prompt_fro": 21.199211, + "heavy_tail_ratio": 1.2275, + "n_prompts": 200 + }, + "9": { + "dispersion": 1.173002, + "mean_fro": 12.62166, + "rms_per_prompt_fro": 18.605724, + "max_per_prompt_fro": 22.70639, + "heavy_tail_ratio": 1.2204, + "n_prompts": 200 + }, + "10": { + "dispersion": 1.061712, + "mean_fro": 14.079089, + "rms_per_prompt_fro": 20.215691, + "max_per_prompt_fro": 24.566759, + "heavy_tail_ratio": 1.2152, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.948139, + "mean_fro": 15.999458, + "rms_per_prompt_fro": 22.331363, + "max_per_prompt_fro": 27.652733, + "heavy_tail_ratio": 1.2383, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.788839, + "mean_fro": 18.099565, + "rms_per_prompt_fro": 24.207711, + "max_per_prompt_fro": 29.19372, + "heavy_tail_ratio": 1.206, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.68418, + "mean_fro": 19.566623, + "rms_per_prompt_fro": 25.392774, + "max_per_prompt_fro": 30.319773, + "heavy_tail_ratio": 1.194, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.556306, + "mean_fro": 22.294302, + "rms_per_prompt_fro": 27.812584, + "max_per_prompt_fro": 33.417854, + "heavy_tail_ratio": 1.2015, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.46308, + "mean_fro": 24.657129, + "rms_per_prompt_fro": 29.824732, + "max_per_prompt_fro": 35.608326, + "heavy_tail_ratio": 1.1939, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.336213, + "mean_fro": 28.061449, + "rms_per_prompt_fro": 32.437542, + "max_per_prompt_fro": 38.711796, + "heavy_tail_ratio": 1.1934, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.246915, + "mean_fro": 30.950378, + "rms_per_prompt_fro": 34.560849, + "max_per_prompt_fro": 39.857838, + "heavy_tail_ratio": 1.1533, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.175515, + "mean_fro": 32.74551, + "rms_per_prompt_fro": 35.503061, + "max_per_prompt_fro": 39.968056, + "heavy_tail_ratio": 1.1258, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.113427, + "mean_fro": 33.795434, + "rms_per_prompt_fro": 35.660613, + "max_per_prompt_fro": 38.831326, + "heavy_tail_ratio": 1.0889, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.053911, + "mean_fro": 34.57449, + "rms_per_prompt_fro": 35.494231, + "max_per_prompt_fro": 37.712627, + "heavy_tail_ratio": 1.0625, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.024929, + "mean_fro": 34.28349, + "rms_per_prompt_fro": 34.708188, + "max_per_prompt_fro": 36.229675, + "heavy_tail_ratio": 1.0438, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 624.4, + "fit_secs_per_prompt": 3.122, + "total_wall_seconds": 636.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.936, + "top_sv": 1.9239, + "stable_rank": 17.016 + }, + { + "layer": 1, + "fro_norm": 7.4639, + "top_sv": 1.7575, + "stable_rank": 18.037 + }, + { + "layer": 2, + "fro_norm": 8.0917, + "top_sv": 1.952, + "stable_rank": 17.184 + }, + { + "layer": 3, + "fro_norm": 7.7674, + "top_sv": 1.8825, + "stable_rank": 17.026 + }, + { + "layer": 4, + "fro_norm": 8.6978, + "top_sv": 2.115, + "stable_rank": 16.913 + }, + { + "layer": 5, + "fro_norm": 8.9607, + "top_sv": 2.2688, + "stable_rank": 15.599 + }, + { + "layer": 6, + "fro_norm": 9.3469, + "top_sv": 2.367, + "stable_rank": 15.593 + }, + { + "layer": 7, + "fro_norm": 10.2469, + "top_sv": 2.4496, + "stable_rank": 17.498 + }, + { + "layer": 8, + "fro_norm": 11.417, + "top_sv": 2.7858, + "stable_rank": 16.796 + }, + { + "layer": 9, + "fro_norm": 12.6215, + "top_sv": 3.044, + "stable_rank": 17.193 + }, + { + "layer": 10, + "fro_norm": 14.0789, + "top_sv": 3.3729, + "stable_rank": 17.423 + }, + { + "layer": 11, + "fro_norm": 15.9993, + "top_sv": 3.7652, + "stable_rank": 18.056 + }, + { + "layer": 12, + "fro_norm": 18.0993, + "top_sv": 4.3422, + "stable_rank": 17.374 + }, + { + "layer": 13, + "fro_norm": 19.5663, + "top_sv": 4.7366, + "stable_rank": 17.064 + }, + { + "layer": 14, + "fro_norm": 22.294, + "top_sv": 5.3986, + "stable_rank": 17.054 + }, + { + "layer": 15, + "fro_norm": 24.6567, + "top_sv": 6.0897, + "stable_rank": 16.393 + }, + { + "layer": 16, + "fro_norm": 28.0609, + "top_sv": 7.1134, + "stable_rank": 15.561 + }, + { + "layer": 17, + "fro_norm": 30.9498, + "top_sv": 7.6238, + "stable_rank": 16.481 + }, + { + "layer": 18, + "fro_norm": 32.7448, + "top_sv": 7.7362, + "stable_rank": 17.915 + }, + { + "layer": 19, + "fro_norm": 33.7945, + "top_sv": 7.5015, + "stable_rank": 20.295 + }, + { + "layer": 20, + "fro_norm": 34.5733, + "top_sv": 6.9447, + "stable_rank": 24.784 + }, + { + "layer": 21, + "fro_norm": 34.282, + "top_sv": 5.2603, + "stable_rank": 42.474 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n200_s1.pt b/results/lenses/e28/e28_Github_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..d035c3aca4e7f2753524cae7643a2d53d0f33296 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca08e40ea0973004e8c21790e59334ea3fa6839be08e4753570c8ccfa3784283 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n200_s1_provenance.json b/results/lenses/e28/e28_Github_410m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..89512ef38f7fc0dc539fdfcd0de1b063abb53c2e --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n200_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n200_s1.pt", + "sha256": "ca08e40ea0973004e8c21790e59334ea3fa6839be08e4753570c8ccfa3784283", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.694966, + "mean_fro": 7.984227, + "rms_per_prompt_fro": 19.053661, + "max_per_prompt_fro": 48.171146, + "heavy_tail_ratio": 2.5282, + "n_prompts": 200 + }, + "1": { + "dispersion": 4.16613, + "mean_fro": 7.507054, + "rms_per_prompt_fro": 17.062876, + "max_per_prompt_fro": 42.972279, + "heavy_tail_ratio": 2.5185, + "n_prompts": 200 + }, + "2": { + "dispersion": 3.380393, + "mean_fro": 8.108691, + "rms_per_prompt_fro": 16.970995, + "max_per_prompt_fro": 40.723667, + "heavy_tail_ratio": 2.3996, + "n_prompts": 200 + }, + "3": { + "dispersion": 2.781578, + "mean_fro": 7.809309, + "rms_per_prompt_fro": 15.186201, + "max_per_prompt_fro": 38.985329, + "heavy_tail_ratio": 2.5672, + "n_prompts": 200 + }, + "4": { + "dispersion": 2.312279, + "mean_fro": 8.729157, + "rms_per_prompt_fro": 15.886777, + "max_per_prompt_fro": 37.331726, + "heavy_tail_ratio": 2.3499, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.989524, + "mean_fro": 8.990346, + "rms_per_prompt_fro": 15.544524, + "max_per_prompt_fro": 34.034382, + "heavy_tail_ratio": 2.1895, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.700204, + "mean_fro": 9.375035, + "rms_per_prompt_fro": 15.405336, + "max_per_prompt_fro": 33.506432, + "heavy_tail_ratio": 2.175, + "n_prompts": 200 + }, + "7": { + "dispersion": 1.493574, + "mean_fro": 10.284642, + "rms_per_prompt_fro": 16.240534, + "max_per_prompt_fro": 34.340145, + "heavy_tail_ratio": 2.1145, + "n_prompts": 200 + }, + "8": { + "dispersion": 1.324641, + "mean_fro": 11.485946, + "rms_per_prompt_fro": 17.512361, + "max_per_prompt_fro": 34.944042, + "heavy_tail_ratio": 1.9954, + "n_prompts": 200 + }, + "9": { + "dispersion": 1.206187, + "mean_fro": 12.708508, + "rms_per_prompt_fro": 18.876248, + "max_per_prompt_fro": 37.255829, + "heavy_tail_ratio": 1.9737, + "n_prompts": 200 + }, + "10": { + "dispersion": 1.094904, + "mean_fro": 14.193278, + "rms_per_prompt_fro": 20.543044, + "max_per_prompt_fro": 39.984081, + "heavy_tail_ratio": 1.9464, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.979023, + "mean_fro": 16.177755, + "rms_per_prompt_fro": 22.758505, + "max_per_prompt_fro": 43.035824, + "heavy_tail_ratio": 1.891, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.812857, + "mean_fro": 18.323068, + "rms_per_prompt_fro": 24.670613, + "max_per_prompt_fro": 45.397015, + "heavy_tail_ratio": 1.8401, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.705282, + "mean_fro": 19.811655, + "rms_per_prompt_fro": 25.871338, + "max_per_prompt_fro": 44.361992, + "heavy_tail_ratio": 1.7147, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.572842, + "mean_fro": 22.598461, + "rms_per_prompt_fro": 28.341406, + "max_per_prompt_fro": 44.304882, + "heavy_tail_ratio": 1.5633, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.47668, + "mean_fro": 25.045508, + "rms_per_prompt_fro": 30.434979, + "max_per_prompt_fro": 43.403439, + "heavy_tail_ratio": 1.4261, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.335404, + "mean_fro": 28.516067, + "rms_per_prompt_fro": 32.953075, + "max_per_prompt_fro": 39.248611, + "heavy_tail_ratio": 1.191, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.243925, + "mean_fro": 31.390049, + "rms_per_prompt_fro": 35.009753, + "max_per_prompt_fro": 40.383511, + "heavy_tail_ratio": 1.1535, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.172249, + "mean_fro": 33.157645, + "rms_per_prompt_fro": 35.899927, + "max_per_prompt_fro": 40.659569, + "heavy_tail_ratio": 1.1326, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.111499, + "mean_fro": 34.08302, + "rms_per_prompt_fro": 35.932929, + "max_per_prompt_fro": 40.076298, + "heavy_tail_ratio": 1.1153, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.052996, + "mean_fro": 34.734176, + "rms_per_prompt_fro": 35.642676, + "max_per_prompt_fro": 37.898232, + "heavy_tail_ratio": 1.0633, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.024965, + "mean_fro": 34.366462, + "rms_per_prompt_fro": 34.792802, + "max_per_prompt_fro": 36.227936, + "heavy_tail_ratio": 1.0412, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 610.0, + "fit_secs_per_prompt": 3.05, + "total_wall_seconds": 620.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9841, + "top_sv": 1.8759, + "stable_rank": 18.114 + }, + { + "layer": 1, + "fro_norm": 7.507, + "top_sv": 1.7526, + "stable_rank": 18.348 + }, + { + "layer": 2, + "fro_norm": 8.1086, + "top_sv": 1.915, + "stable_rank": 17.929 + }, + { + "layer": 3, + "fro_norm": 7.8092, + "top_sv": 1.856, + "stable_rank": 17.704 + }, + { + "layer": 4, + "fro_norm": 8.729, + "top_sv": 2.0936, + "stable_rank": 17.383 + }, + { + "layer": 5, + "fro_norm": 8.9902, + "top_sv": 2.1994, + "stable_rank": 16.709 + }, + { + "layer": 6, + "fro_norm": 9.3749, + "top_sv": 2.2808, + "stable_rank": 16.895 + }, + { + "layer": 7, + "fro_norm": 10.2845, + "top_sv": 2.3955, + "stable_rank": 18.431 + }, + { + "layer": 8, + "fro_norm": 11.4858, + "top_sv": 2.7315, + "stable_rank": 17.682 + }, + { + "layer": 9, + "fro_norm": 12.7083, + "top_sv": 3.0312, + "stable_rank": 17.578 + }, + { + "layer": 10, + "fro_norm": 14.1931, + "top_sv": 3.4339, + "stable_rank": 17.084 + }, + { + "layer": 11, + "fro_norm": 16.1775, + "top_sv": 3.896, + "stable_rank": 17.242 + }, + { + "layer": 12, + "fro_norm": 18.3228, + "top_sv": 4.4333, + "stable_rank": 17.081 + }, + { + "layer": 13, + "fro_norm": 19.8113, + "top_sv": 4.8886, + "stable_rank": 16.423 + }, + { + "layer": 14, + "fro_norm": 22.5981, + "top_sv": 5.6214, + "stable_rank": 16.16 + }, + { + "layer": 15, + "fro_norm": 25.0451, + "top_sv": 6.4174, + "stable_rank": 15.231 + }, + { + "layer": 16, + "fro_norm": 28.5155, + "top_sv": 7.5091, + "stable_rank": 14.421 + }, + { + "layer": 17, + "fro_norm": 31.3895, + "top_sv": 8.0161, + "stable_rank": 15.334 + }, + { + "layer": 18, + "fro_norm": 33.1568, + "top_sv": 7.9611, + "stable_rank": 17.346 + }, + { + "layer": 19, + "fro_norm": 34.0821, + "top_sv": 7.646, + "stable_rank": 19.869 + }, + { + "layer": 20, + "fro_norm": 34.733, + "top_sv": 7.0929, + "stable_rank": 23.979 + }, + { + "layer": 21, + "fro_norm": 34.3649, + "top_sv": 5.4173, + "stable_rank": 40.24 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n200_s2.pt b/results/lenses/e28/e28_Github_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3c2e5e2126f0c546d57ca70abc05642b8b4c43e --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fabb7cc8e515078c224375f23949ba336c33a4f86f35bf3e96df6cfbdba97bfc +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n200_s2_provenance.json b/results/lenses/e28/e28_Github_410m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9c66c179762b888a623091a7705d59ad83f0b49f --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n200_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n200_s2.pt", + "sha256": "fabb7cc8e515078c224375f23949ba336c33a4f86f35bf3e96df6cfbdba97bfc", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 5.086925, + "mean_fro": 7.95514, + "rms_per_prompt_fro": 19.626679, + "max_per_prompt_fro": 34.796436, + "heavy_tail_ratio": 1.7729, + "n_prompts": 200 + }, + "1": { + "dispersion": 4.491357, + "mean_fro": 7.470737, + "rms_per_prompt_fro": 17.50666, + "max_per_prompt_fro": 29.559164, + "heavy_tail_ratio": 1.6885, + "n_prompts": 200 + }, + "2": { + "dispersion": 3.62262, + "mean_fro": 8.096563, + "rms_per_prompt_fro": 17.407838, + "max_per_prompt_fro": 29.93277, + "heavy_tail_ratio": 1.7195, + "n_prompts": 200 + }, + "3": { + "dispersion": 2.935255, + "mean_fro": 7.777223, + "rms_per_prompt_fro": 15.428048, + "max_per_prompt_fro": 25.432657, + "heavy_tail_ratio": 1.6485, + "n_prompts": 200 + }, + "4": { + "dispersion": 2.43814, + "mean_fro": 8.703015, + "rms_per_prompt_fro": 16.137323, + "max_per_prompt_fro": 26.80678, + "heavy_tail_ratio": 1.6612, + "n_prompts": 200 + }, + "5": { + "dispersion": 2.103337, + "mean_fro": 8.953816, + "rms_per_prompt_fro": 15.773303, + "max_per_prompt_fro": 27.302526, + "heavy_tail_ratio": 1.7309, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.781155, + "mean_fro": 9.335913, + "rms_per_prompt_fro": 15.56931, + "max_per_prompt_fro": 24.270342, + "heavy_tail_ratio": 1.5589, + "n_prompts": 200 + }, + "7": { + "dispersion": 1.561259, + "mean_fro": 10.205282, + "rms_per_prompt_fro": 16.332464, + "max_per_prompt_fro": 24.17333, + "heavy_tail_ratio": 1.4801, + "n_prompts": 200 + }, + "8": { + "dispersion": 1.388417, + "mean_fro": 11.364063, + "rms_per_prompt_fro": 17.562598, + "max_per_prompt_fro": 26.130661, + "heavy_tail_ratio": 1.4879, + "n_prompts": 200 + }, + "9": { + "dispersion": 1.25944, + "mean_fro": 12.577224, + "rms_per_prompt_fro": 18.90537, + "max_per_prompt_fro": 27.802649, + "heavy_tail_ratio": 1.4706, + "n_prompts": 200 + }, + "10": { + "dispersion": 1.138363, + "mean_fro": 14.044445, + "rms_per_prompt_fro": 20.537392, + "max_per_prompt_fro": 29.789103, + "heavy_tail_ratio": 1.4505, + "n_prompts": 200 + }, + "11": { + "dispersion": 1.009864, + "mean_fro": 15.958629, + "rms_per_prompt_fro": 22.624493, + "max_per_prompt_fro": 28.692446, + "heavy_tail_ratio": 1.2682, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.836028, + "mean_fro": 18.069786, + "rms_per_prompt_fro": 24.48458, + "max_per_prompt_fro": 30.583603, + "heavy_tail_ratio": 1.2491, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.725415, + "mean_fro": 19.58297, + "rms_per_prompt_fro": 25.723217, + "max_per_prompt_fro": 31.018682, + "heavy_tail_ratio": 1.2059, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.588856, + "mean_fro": 22.30224, + "rms_per_prompt_fro": 28.111938, + "max_per_prompt_fro": 34.177765, + "heavy_tail_ratio": 1.2158, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.491775, + "mean_fro": 24.711716, + "rms_per_prompt_fro": 30.182458, + "max_per_prompt_fro": 36.843662, + "heavy_tail_ratio": 1.2207, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.34989, + "mean_fro": 28.165209, + "rms_per_prompt_fro": 32.723684, + "max_per_prompt_fro": 39.856323, + "heavy_tail_ratio": 1.218, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.254494, + "mean_fro": 31.104332, + "rms_per_prompt_fro": 34.838151, + "max_per_prompt_fro": 40.543797, + "heavy_tail_ratio": 1.1638, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.179382, + "mean_fro": 32.914115, + "rms_per_prompt_fro": 35.744522, + "max_per_prompt_fro": 40.60183, + "heavy_tail_ratio": 1.1359, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.11511, + "mean_fro": 33.905053, + "rms_per_prompt_fro": 35.803323, + "max_per_prompt_fro": 39.329338, + "heavy_tail_ratio": 1.0985, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.054826, + "mean_fro": 34.656799, + "rms_per_prompt_fro": 35.594175, + "max_per_prompt_fro": 38.151516, + "heavy_tail_ratio": 1.0718, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.025452, + "mean_fro": 34.329842, + "rms_per_prompt_fro": 34.763974, + "max_per_prompt_fro": 36.411808, + "heavy_tail_ratio": 1.0474, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 619.6, + "fit_secs_per_prompt": 3.098, + "total_wall_seconds": 627.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9551, + "top_sv": 1.9407, + "stable_rank": 16.802 + }, + { + "layer": 1, + "fro_norm": 7.4706, + "top_sv": 1.7752, + "stable_rank": 17.71 + }, + { + "layer": 2, + "fro_norm": 8.0965, + "top_sv": 1.9913, + "stable_rank": 16.531 + }, + { + "layer": 3, + "fro_norm": 7.7771, + "top_sv": 1.9164, + "stable_rank": 16.469 + }, + { + "layer": 4, + "fro_norm": 8.7029, + "top_sv": 2.1691, + "stable_rank": 16.097 + }, + { + "layer": 5, + "fro_norm": 8.9537, + "top_sv": 2.3344, + "stable_rank": 14.711 + }, + { + "layer": 6, + "fro_norm": 9.3358, + "top_sv": 2.4366, + "stable_rank": 14.68 + }, + { + "layer": 7, + "fro_norm": 10.2052, + "top_sv": 2.4553, + "stable_rank": 17.275 + }, + { + "layer": 8, + "fro_norm": 11.3639, + "top_sv": 2.7749, + "stable_rank": 16.771 + }, + { + "layer": 9, + "fro_norm": 12.577, + "top_sv": 3.0516, + "stable_rank": 16.986 + }, + { + "layer": 10, + "fro_norm": 14.0442, + "top_sv": 3.3918, + "stable_rank": 17.144 + }, + { + "layer": 11, + "fro_norm": 15.9584, + "top_sv": 3.7765, + "stable_rank": 17.857 + }, + { + "layer": 12, + "fro_norm": 18.0695, + "top_sv": 4.3151, + "stable_rank": 17.535 + }, + { + "layer": 13, + "fro_norm": 19.5826, + "top_sv": 4.8151, + "stable_rank": 16.54 + }, + { + "layer": 14, + "fro_norm": 22.3019, + "top_sv": 5.5274, + "stable_rank": 16.28 + }, + { + "layer": 15, + "fro_norm": 24.7112, + "top_sv": 6.3358, + "stable_rank": 15.212 + }, + { + "layer": 16, + "fro_norm": 28.1646, + "top_sv": 7.3671, + "stable_rank": 14.616 + }, + { + "layer": 17, + "fro_norm": 31.1037, + "top_sv": 7.8893, + "stable_rank": 15.543 + }, + { + "layer": 18, + "fro_norm": 32.9134, + "top_sv": 7.9338, + "stable_rank": 17.21 + }, + { + "layer": 19, + "fro_norm": 33.9041, + "top_sv": 7.5933, + "stable_rank": 19.936 + }, + { + "layer": 20, + "fro_norm": 34.6557, + "top_sv": 7.0427, + "stable_rank": 24.214 + }, + { + "layer": 21, + "fro_norm": 34.3284, + "top_sv": 5.3526, + "stable_rank": 41.132 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n25_s0.pt b/results/lenses/e28/e28_Github_410m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4867eeab7b044e96cc1e021fe13ff0d68c768a76 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b229a4f060278dd2c95c9ce8a9db97396385907c0742f4f0bc14ea782e62ba2 +size 46144773 diff --git a/results/lenses/e28/e28_Github_410m_n25_s0_provenance.json b/results/lenses/e28/e28_Github_410m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..00baf2149a52de01ab0a53abd263893411c0330c --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n25_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n25_s0.pt", + "sha256": "5b229a4f060278dd2c95c9ce8a9db97396385907c0742f4f0bc14ea782e62ba2", + "bytes": 46144773, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.107046, + "mean_fro": 8.604546, + "rms_per_prompt_fro": 19.445219, + "max_per_prompt_fro": 29.136555, + "heavy_tail_ratio": 1.4984, + "n_prompts": 25 + }, + "1": { + "dispersion": 3.722414, + "mean_fro": 8.027612, + "rms_per_prompt_fro": 17.444896, + "max_per_prompt_fro": 25.729753, + "heavy_tail_ratio": 1.4749, + "n_prompts": 25 + }, + "2": { + "dispersion": 3.055227, + "mean_fro": 8.571342, + "rms_per_prompt_fro": 17.260622, + "max_per_prompt_fro": 24.51263, + "heavy_tail_ratio": 1.4201, + "n_prompts": 25 + }, + "3": { + "dispersion": 2.509291, + "mean_fro": 8.099916, + "rms_per_prompt_fro": 15.173655, + "max_per_prompt_fro": 19.907942, + "heavy_tail_ratio": 1.312, + "n_prompts": 25 + }, + "4": { + "dispersion": 2.089657, + "mean_fro": 8.997174, + "rms_per_prompt_fro": 15.814712, + "max_per_prompt_fro": 20.365681, + "heavy_tail_ratio": 1.2878, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.780717, + "mean_fro": 9.232176, + "rms_per_prompt_fro": 15.395099, + "max_per_prompt_fro": 19.429564, + "heavy_tail_ratio": 1.2621, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.523345, + "mean_fro": 9.581198, + "rms_per_prompt_fro": 15.219771, + "max_per_prompt_fro": 18.996004, + "heavy_tail_ratio": 1.2481, + "n_prompts": 25 + }, + "7": { + "dispersion": 1.34173, + "mean_fro": 10.446308, + "rms_per_prompt_fro": 15.985685, + "max_per_prompt_fro": 19.020069, + "heavy_tail_ratio": 1.1898, + "n_prompts": 25 + }, + "8": { + "dispersion": 1.188101, + "mean_fro": 11.600814, + "rms_per_prompt_fro": 17.160192, + "max_per_prompt_fro": 19.865673, + "heavy_tail_ratio": 1.1577, + "n_prompts": 25 + }, + "9": { + "dispersion": 1.076929, + "mean_fro": 12.781285, + "rms_per_prompt_fro": 18.419818, + "max_per_prompt_fro": 20.911674, + "heavy_tail_ratio": 1.1353, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.973783, + "mean_fro": 14.182546, + "rms_per_prompt_fro": 19.925255, + "max_per_prompt_fro": 22.553274, + "heavy_tail_ratio": 1.1319, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.871579, + "mean_fro": 15.997174, + "rms_per_prompt_fro": 21.885041, + "max_per_prompt_fro": 24.606277, + "heavy_tail_ratio": 1.1243, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.726012, + "mean_fro": 18.041336, + "rms_per_prompt_fro": 23.702308, + "max_per_prompt_fro": 26.225662, + "heavy_tail_ratio": 1.1065, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.62831, + "mean_fro": 19.492437, + "rms_per_prompt_fro": 24.873372, + "max_per_prompt_fro": 27.642759, + "heavy_tail_ratio": 1.1113, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.516351, + "mean_fro": 22.168077, + "rms_per_prompt_fro": 27.297815, + "max_per_prompt_fro": 30.010412, + "heavy_tail_ratio": 1.0994, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.431939, + "mean_fro": 24.554627, + "rms_per_prompt_fro": 29.382965, + "max_per_prompt_fro": 32.179237, + "heavy_tail_ratio": 1.0952, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.31255, + "mean_fro": 27.867519, + "rms_per_prompt_fro": 31.926863, + "max_per_prompt_fro": 34.125912, + "heavy_tail_ratio": 1.0689, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.227904, + "mean_fro": 30.795515, + "rms_per_prompt_fro": 34.124766, + "max_per_prompt_fro": 36.33057, + "heavy_tail_ratio": 1.0646, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.161395, + "mean_fro": 32.600845, + "rms_per_prompt_fro": 35.133284, + "max_per_prompt_fro": 37.322865, + "heavy_tail_ratio": 1.0623, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.103898, + "mean_fro": 33.675308, + "rms_per_prompt_fro": 35.38148, + "max_per_prompt_fro": 37.082958, + "heavy_tail_ratio": 1.0481, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.049628, + "mean_fro": 34.451167, + "rms_per_prompt_fro": 35.295693, + "max_per_prompt_fro": 36.369907, + "heavy_tail_ratio": 1.0304, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.02316, + "mean_fro": 34.209424, + "rms_per_prompt_fro": 34.603301, + "max_per_prompt_fro": 35.23341, + "heavy_tail_ratio": 1.0182, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 78.1, + "fit_secs_per_prompt": 3.124, + "total_wall_seconds": 89.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.6044, + "top_sv": 2.6164, + "stable_rank": 10.815 + }, + { + "layer": 1, + "fro_norm": 8.0275, + "top_sv": 2.315, + "stable_rank": 12.025 + }, + { + "layer": 2, + "fro_norm": 8.5712, + "top_sv": 2.4462, + "stable_rank": 12.278 + }, + { + "layer": 3, + "fro_norm": 8.0998, + "top_sv": 2.2325, + "stable_rank": 13.163 + }, + { + "layer": 4, + "fro_norm": 8.997, + "top_sv": 2.468, + "stable_rank": 13.29 + }, + { + "layer": 5, + "fro_norm": 9.232, + "top_sv": 2.6391, + "stable_rank": 12.238 + }, + { + "layer": 6, + "fro_norm": 9.5811, + "top_sv": 2.7234, + "stable_rank": 12.377 + }, + { + "layer": 7, + "fro_norm": 10.4462, + "top_sv": 2.759, + "stable_rank": 14.335 + }, + { + "layer": 8, + "fro_norm": 11.6007, + "top_sv": 3.0873, + "stable_rank": 14.119 + }, + { + "layer": 9, + "fro_norm": 12.7811, + "top_sv": 3.337, + "stable_rank": 14.67 + }, + { + "layer": 10, + "fro_norm": 14.1823, + "top_sv": 3.6458, + "stable_rank": 15.132 + }, + { + "layer": 11, + "fro_norm": 15.997, + "top_sv": 3.9615, + "stable_rank": 16.307 + }, + { + "layer": 12, + "fro_norm": 18.0411, + "top_sv": 4.4212, + "stable_rank": 16.651 + }, + { + "layer": 13, + "fro_norm": 19.4921, + "top_sv": 4.7277, + "stable_rank": 16.999 + }, + { + "layer": 14, + "fro_norm": 22.1678, + "top_sv": 5.3653, + "stable_rank": 17.071 + }, + { + "layer": 15, + "fro_norm": 24.5542, + "top_sv": 6.0462, + "stable_rank": 16.493 + }, + { + "layer": 16, + "fro_norm": 27.867, + "top_sv": 7.0453, + "stable_rank": 15.645 + }, + { + "layer": 17, + "fro_norm": 30.7949, + "top_sv": 7.6119, + "stable_rank": 16.367 + }, + { + "layer": 18, + "fro_norm": 32.6002, + "top_sv": 7.7365, + "stable_rank": 17.756 + }, + { + "layer": 19, + "fro_norm": 33.6744, + "top_sv": 7.5471, + "stable_rank": 19.908 + }, + { + "layer": 20, + "fro_norm": 34.45, + "top_sv": 6.9118, + "stable_rank": 24.842 + }, + { + "layer": 21, + "fro_norm": 34.2079, + "top_sv": 5.2554, + "stable_rank": 42.368 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n25_s1.pt b/results/lenses/e28/e28_Github_410m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..66c8e9a9910cdff4495ff90d8d9a235aa1ed1fd9 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7436c9a8282f2c2ff743271fb84efafdb1c78de5449110ed77020b1c849096d0 +size 46144773 diff --git a/results/lenses/e28/e28_Github_410m_n25_s1_provenance.json b/results/lenses/e28/e28_Github_410m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f70548a9e091808457e3ad93dce6432bed42c1df --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n25_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n25_s1.pt", + "sha256": "7436c9a8282f2c2ff743271fb84efafdb1c78de5449110ed77020b1c849096d0", + "bytes": 46144773, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 3.715841, + "mean_fro": 8.373424, + "rms_per_prompt_fro": 18.183717, + "max_per_prompt_fro": 23.731688, + "heavy_tail_ratio": 1.3051, + "n_prompts": 25 + }, + "1": { + "dispersion": 3.325021, + "mean_fro": 7.833279, + "rms_per_prompt_fro": 16.290622, + "max_per_prompt_fro": 20.814186, + "heavy_tail_ratio": 1.2777, + "n_prompts": 25 + }, + "2": { + "dispersion": 2.800677, + "mean_fro": 8.370706, + "rms_per_prompt_fro": 16.318965, + "max_per_prompt_fro": 20.783924, + "heavy_tail_ratio": 1.2736, + "n_prompts": 25 + }, + "3": { + "dispersion": 2.315834, + "mean_fro": 7.993171, + "rms_per_prompt_fro": 14.55511, + "max_per_prompt_fro": 17.847971, + "heavy_tail_ratio": 1.2262, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.973277, + "mean_fro": 8.876344, + "rms_per_prompt_fro": 15.30565, + "max_per_prompt_fro": 18.516159, + "heavy_tail_ratio": 1.2098, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.731194, + "mean_fro": 9.136035, + "rms_per_prompt_fro": 15.098507, + "max_per_prompt_fro": 18.304998, + "heavy_tail_ratio": 1.2124, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.493209, + "mean_fro": 9.489906, + "rms_per_prompt_fro": 14.984467, + "max_per_prompt_fro": 17.768753, + "heavy_tail_ratio": 1.1858, + "n_prompts": 25 + }, + "7": { + "dispersion": 1.312615, + "mean_fro": 10.386933, + "rms_per_prompt_fro": 15.795704, + "max_per_prompt_fro": 19.002712, + "heavy_tail_ratio": 1.203, + "n_prompts": 25 + }, + "8": { + "dispersion": 1.163811, + "mean_fro": 11.607539, + "rms_per_prompt_fro": 17.074572, + "max_per_prompt_fro": 20.459017, + "heavy_tail_ratio": 1.1982, + "n_prompts": 25 + }, + "9": { + "dispersion": 1.063739, + "mean_fro": 12.844036, + "rms_per_prompt_fro": 18.451384, + "max_per_prompt_fro": 21.9069, + "heavy_tail_ratio": 1.1873, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.966222, + "mean_fro": 14.341062, + "rms_per_prompt_fro": 20.109326, + "max_per_prompt_fro": 23.540939, + "heavy_tail_ratio": 1.1706, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.865293, + "mean_fro": 16.381113, + "rms_per_prompt_fro": 22.372625, + "max_per_prompt_fro": 26.063936, + "heavy_tail_ratio": 1.165, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.721842, + "mean_fro": 18.538583, + "rms_per_prompt_fro": 24.326142, + "max_per_prompt_fro": 27.788324, + "heavy_tail_ratio": 1.1423, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.625352, + "mean_fro": 19.992445, + "rms_per_prompt_fro": 25.488223, + "max_per_prompt_fro": 29.076735, + "heavy_tail_ratio": 1.1408, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.511744, + "mean_fro": 22.752952, + "rms_per_prompt_fro": 27.975439, + "max_per_prompt_fro": 31.73115, + "heavy_tail_ratio": 1.1343, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.428709, + "mean_fro": 25.173759, + "rms_per_prompt_fro": 30.089847, + "max_per_prompt_fro": 33.857239, + "heavy_tail_ratio": 1.1252, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.310957, + "mean_fro": 28.584868, + "rms_per_prompt_fro": 32.728827, + "max_per_prompt_fro": 36.559162, + "heavy_tail_ratio": 1.117, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.229419, + "mean_fro": 31.410629, + "rms_per_prompt_fro": 34.827841, + "max_per_prompt_fro": 38.012127, + "heavy_tail_ratio": 1.0914, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.164161, + "mean_fro": 33.116062, + "rms_per_prompt_fro": 35.730998, + "max_per_prompt_fro": 38.426544, + "heavy_tail_ratio": 1.0754, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.106756, + "mean_fro": 34.05687, + "rms_per_prompt_fro": 35.828662, + "max_per_prompt_fro": 37.938892, + "heavy_tail_ratio": 1.0589, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.050224, + "mean_fro": 34.691543, + "rms_per_prompt_fro": 35.552042, + "max_per_prompt_fro": 37.117928, + "heavy_tail_ratio": 1.044, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.023589, + "mean_fro": 34.316276, + "rms_per_prompt_fro": 34.71866, + "max_per_prompt_fro": 35.871658, + "heavy_tail_ratio": 1.0332, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 76.2, + "fit_secs_per_prompt": 3.05, + "total_wall_seconds": 86.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.3733, + "top_sv": 2.1725, + "stable_rank": 14.855 + }, + { + "layer": 1, + "fro_norm": 7.8332, + "top_sv": 2.0215, + "stable_rank": 15.015 + }, + { + "layer": 2, + "fro_norm": 8.3706, + "top_sv": 2.112, + "stable_rank": 15.708 + }, + { + "layer": 3, + "fro_norm": 7.9931, + "top_sv": 2.0097, + "stable_rank": 15.819 + }, + { + "layer": 4, + "fro_norm": 8.8762, + "top_sv": 2.1987, + "stable_rank": 16.297 + }, + { + "layer": 5, + "fro_norm": 9.1359, + "top_sv": 2.3037, + "stable_rank": 15.727 + }, + { + "layer": 6, + "fro_norm": 9.4898, + "top_sv": 2.376, + "stable_rank": 15.953 + }, + { + "layer": 7, + "fro_norm": 10.3868, + "top_sv": 2.4565, + "stable_rank": 17.879 + }, + { + "layer": 8, + "fro_norm": 11.6074, + "top_sv": 2.7887, + "stable_rank": 17.325 + }, + { + "layer": 9, + "fro_norm": 12.8438, + "top_sv": 3.0525, + "stable_rank": 17.705 + }, + { + "layer": 10, + "fro_norm": 14.3408, + "top_sv": 3.412, + "stable_rank": 17.666 + }, + { + "layer": 11, + "fro_norm": 16.3809, + "top_sv": 3.896, + "stable_rank": 17.678 + }, + { + "layer": 12, + "fro_norm": 18.5383, + "top_sv": 4.5057, + "stable_rank": 16.929 + }, + { + "layer": 13, + "fro_norm": 19.9921, + "top_sv": 4.9144, + "stable_rank": 16.549 + }, + { + "layer": 14, + "fro_norm": 22.7526, + "top_sv": 5.5507, + "stable_rank": 16.802 + }, + { + "layer": 15, + "fro_norm": 25.1733, + "top_sv": 6.2735, + "stable_rank": 16.101 + }, + { + "layer": 16, + "fro_norm": 28.5843, + "top_sv": 7.3552, + "stable_rank": 15.103 + }, + { + "layer": 17, + "fro_norm": 31.41, + "top_sv": 7.8529, + "stable_rank": 15.998 + }, + { + "layer": 18, + "fro_norm": 33.1153, + "top_sv": 7.8929, + "stable_rank": 17.603 + }, + { + "layer": 19, + "fro_norm": 34.056, + "top_sv": 7.7182, + "stable_rank": 19.469 + }, + { + "layer": 20, + "fro_norm": 34.6904, + "top_sv": 7.0736, + "stable_rank": 24.051 + }, + { + "layer": 21, + "fro_norm": 34.3148, + "top_sv": 5.3997, + "stable_rank": 40.385 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n25_s2.pt b/results/lenses/e28/e28_Github_410m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bee83f54e6e593b70f6dcd47775f0bb802e37cb --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed2d4e34a916474f0db2c3e5eaebbcf8b7c1d19901e73846533ffc6d7feb9005 +size 46144773 diff --git a/results/lenses/e28/e28_Github_410m_n25_s2_provenance.json b/results/lenses/e28/e28_Github_410m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..05b19ed70a483a7edb64b7d25bf32c7f716bacfb --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n25_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n25_s2.pt", + "sha256": "ed2d4e34a916474f0db2c3e5eaebbcf8b7c1d19901e73846533ffc6d7feb9005", + "bytes": 46144773, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.715521, + "mean_fro": 8.717437, + "rms_per_prompt_fro": 20.840912, + "max_per_prompt_fro": 29.450354, + "heavy_tail_ratio": 1.4131, + "n_prompts": 25 + }, + "1": { + "dispersion": 4.248028, + "mean_fro": 8.142386, + "rms_per_prompt_fro": 18.653046, + "max_per_prompt_fro": 27.164385, + "heavy_tail_ratio": 1.4563, + "n_prompts": 25 + }, + "2": { + "dispersion": 3.468486, + "mean_fro": 8.722924, + "rms_per_prompt_fro": 18.43921, + "max_per_prompt_fro": 24.959475, + "heavy_tail_ratio": 1.3536, + "n_prompts": 25 + }, + "3": { + "dispersion": 2.848194, + "mean_fro": 8.296365, + "rms_per_prompt_fro": 16.274825, + "max_per_prompt_fro": 22.403288, + "heavy_tail_ratio": 1.3766, + "n_prompts": 25 + }, + "4": { + "dispersion": 2.385069, + "mean_fro": 9.209169, + "rms_per_prompt_fro": 16.943544, + "max_per_prompt_fro": 22.562243, + "heavy_tail_ratio": 1.3316, + "n_prompts": 25 + }, + "5": { + "dispersion": 2.067821, + "mean_fro": 9.462673, + "rms_per_prompt_fro": 16.574058, + "max_per_prompt_fro": 23.27467, + "heavy_tail_ratio": 1.4043, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.746821, + "mean_fro": 9.814678, + "rms_per_prompt_fro": 16.266392, + "max_per_prompt_fro": 22.96879, + "heavy_tail_ratio": 1.412, + "n_prompts": 25 + }, + "7": { + "dispersion": 1.53254, + "mean_fro": 10.639992, + "rms_per_prompt_fro": 16.932437, + "max_per_prompt_fro": 24.17333, + "heavy_tail_ratio": 1.4276, + "n_prompts": 25 + }, + "8": { + "dispersion": 1.369504, + "mean_fro": 11.781492, + "rms_per_prompt_fro": 18.135477, + "max_per_prompt_fro": 26.130661, + "heavy_tail_ratio": 1.4409, + "n_prompts": 25 + }, + "9": { + "dispersion": 1.238637, + "mean_fro": 12.997072, + "rms_per_prompt_fro": 19.446318, + "max_per_prompt_fro": 27.802649, + "heavy_tail_ratio": 1.4297, + "n_prompts": 25 + }, + "10": { + "dispersion": 1.120991, + "mean_fro": 14.480065, + "rms_per_prompt_fro": 21.08822, + "max_per_prompt_fro": 29.789103, + "heavy_tail_ratio": 1.4126, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.96823, + "mean_fro": 16.353802, + "rms_per_prompt_fro": 22.943341, + "max_per_prompt_fro": 27.419218, + "heavy_tail_ratio": 1.1951, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.802775, + "mean_fro": 18.415079, + "rms_per_prompt_fro": 24.725457, + "max_per_prompt_fro": 28.338881, + "heavy_tail_ratio": 1.1461, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.698777, + "mean_fro": 19.956186, + "rms_per_prompt_fro": 26.010325, + "max_per_prompt_fro": 29.052856, + "heavy_tail_ratio": 1.117, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.5627, + "mean_fro": 22.580702, + "rms_per_prompt_fro": 28.227685, + "max_per_prompt_fro": 31.660454, + "heavy_tail_ratio": 1.1216, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.468008, + "mean_fro": 24.894722, + "rms_per_prompt_fro": 30.162785, + "max_per_prompt_fro": 33.299244, + "heavy_tail_ratio": 1.104, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.336723, + "mean_fro": 28.329224, + "rms_per_prompt_fro": 32.753324, + "max_per_prompt_fro": 36.125347, + "heavy_tail_ratio": 1.103, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.246898, + "mean_fro": 31.289978, + "rms_per_prompt_fro": 34.93983, + "max_per_prompt_fro": 37.866962, + "heavy_tail_ratio": 1.0838, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.174871, + "mean_fro": 33.071666, + "rms_per_prompt_fro": 35.846858, + "max_per_prompt_fro": 38.430702, + "heavy_tail_ratio": 1.0721, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.112258, + "mean_fro": 34.047992, + "rms_per_prompt_fro": 35.908246, + "max_per_prompt_fro": 37.773045, + "heavy_tail_ratio": 1.0519, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.052404, + "mean_fro": 34.732688, + "rms_per_prompt_fro": 35.631141, + "max_per_prompt_fro": 37.091408, + "heavy_tail_ratio": 1.041, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.024246, + "mean_fro": 34.37778, + "rms_per_prompt_fro": 34.792042, + "max_per_prompt_fro": 35.690605, + "heavy_tail_ratio": 1.0258, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 77.4, + "fit_secs_per_prompt": 3.095, + "total_wall_seconds": 84.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.7173, + "top_sv": 2.46, + "stable_rank": 12.557 + }, + { + "layer": 1, + "fro_norm": 8.1423, + "top_sv": 2.2471, + "stable_rank": 13.13 + }, + { + "layer": 2, + "fro_norm": 8.7228, + "top_sv": 2.4656, + "stable_rank": 12.516 + }, + { + "layer": 3, + "fro_norm": 8.2962, + "top_sv": 2.3233, + "stable_rank": 12.751 + }, + { + "layer": 4, + "fro_norm": 9.209, + "top_sv": 2.6176, + "stable_rank": 12.378 + }, + { + "layer": 5, + "fro_norm": 9.4625, + "top_sv": 2.8524, + "stable_rank": 11.005 + }, + { + "layer": 6, + "fro_norm": 9.8145, + "top_sv": 2.9549, + "stable_rank": 11.032 + }, + { + "layer": 7, + "fro_norm": 10.6399, + "top_sv": 2.8432, + "stable_rank": 14.004 + }, + { + "layer": 8, + "fro_norm": 11.7813, + "top_sv": 3.1339, + "stable_rank": 14.132 + }, + { + "layer": 9, + "fro_norm": 12.9969, + "top_sv": 3.3984, + "stable_rank": 14.626 + }, + { + "layer": 10, + "fro_norm": 14.4799, + "top_sv": 3.752, + "stable_rank": 14.894 + }, + { + "layer": 11, + "fro_norm": 16.3536, + "top_sv": 4.126, + "stable_rank": 15.71 + }, + { + "layer": 12, + "fro_norm": 18.4148, + "top_sv": 4.5723, + "stable_rank": 16.221 + }, + { + "layer": 13, + "fro_norm": 19.9559, + "top_sv": 5.0049, + "stable_rank": 15.898 + }, + { + "layer": 14, + "fro_norm": 22.5804, + "top_sv": 5.6327, + "stable_rank": 16.07 + }, + { + "layer": 15, + "fro_norm": 24.8943, + "top_sv": 6.3714, + "stable_rank": 15.266 + }, + { + "layer": 16, + "fro_norm": 28.3287, + "top_sv": 7.4009, + "stable_rank": 14.651 + }, + { + "layer": 17, + "fro_norm": 31.2894, + "top_sv": 7.9116, + "stable_rank": 15.641 + }, + { + "layer": 18, + "fro_norm": 33.0709, + "top_sv": 8.1588, + "stable_rank": 16.43 + }, + { + "layer": 19, + "fro_norm": 34.0471, + "top_sv": 7.9263, + "stable_rank": 18.451 + }, + { + "layer": 20, + "fro_norm": 34.7316, + "top_sv": 7.0925, + "stable_rank": 23.98 + }, + { + "layer": 21, + "fro_norm": 34.3763, + "top_sv": 5.3156, + "stable_rank": 41.824 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n400_s0.pt b/results/lenses/e28/e28_Github_410m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bc8454373c4678e4215c916cd9a52f61e9a358f --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c87ed9a5f778ef0280a17ff0df5e623e8732fb7b706936fcc7caf624979511a +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n400_s0_provenance.json b/results/lenses/e28/e28_Github_410m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..565f70f23b5c58781567daba7636eae1b1dac032 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n400_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n400_s0.pt", + "sha256": "4c87ed9a5f778ef0280a17ff0df5e623e8732fb7b706936fcc7caf624979511a", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.829229, + "mean_fro": 7.923328, + "rms_per_prompt_fro": 19.129921, + "max_per_prompt_fro": 30.175514, + "heavy_tail_ratio": 1.5774, + "n_prompts": 400 + }, + "1": { + "dispersion": 4.271572, + "mean_fro": 7.456116, + "rms_per_prompt_fro": 17.119171, + "max_per_prompt_fro": 29.6033, + "heavy_tail_ratio": 1.7292, + "n_prompts": 400 + }, + "2": { + "dispersion": 3.423528, + "mean_fro": 8.081048, + "rms_per_prompt_fro": 16.99621, + "max_per_prompt_fro": 28.624714, + "heavy_tail_ratio": 1.6842, + "n_prompts": 400 + }, + "3": { + "dispersion": 2.775732, + "mean_fro": 7.765443, + "rms_per_prompt_fro": 15.089221, + "max_per_prompt_fro": 27.685934, + "heavy_tail_ratio": 1.8348, + "n_prompts": 400 + }, + "4": { + "dispersion": 2.293017, + "mean_fro": 8.702435, + "rms_per_prompt_fro": 15.792023, + "max_per_prompt_fro": 26.827032, + "heavy_tail_ratio": 1.6988, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.964391, + "mean_fro": 8.97168, + "rms_per_prompt_fro": 15.446907, + "max_per_prompt_fro": 26.705669, + "heavy_tail_ratio": 1.7289, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.670075, + "mean_fro": 9.361331, + "rms_per_prompt_fro": 15.296755, + "max_per_prompt_fro": 22.787937, + "heavy_tail_ratio": 1.4897, + "n_prompts": 400 + }, + "7": { + "dispersion": 1.46617, + "mean_fro": 10.273088, + "rms_per_prompt_fro": 16.132902, + "max_per_prompt_fro": 19.958813, + "heavy_tail_ratio": 1.2371, + "n_prompts": 400 + }, + "8": { + "dispersion": 1.302672, + "mean_fro": 11.463929, + "rms_per_prompt_fro": 17.396006, + "max_per_prompt_fro": 21.227322, + "heavy_tail_ratio": 1.2202, + "n_prompts": 400 + }, + "9": { + "dispersion": 1.187323, + "mean_fro": 12.678997, + "rms_per_prompt_fro": 18.751729, + "max_per_prompt_fro": 23.041128, + "heavy_tail_ratio": 1.2287, + "n_prompts": 400 + }, + "10": { + "dispersion": 1.076126, + "mean_fro": 14.155739, + "rms_per_prompt_fro": 20.396677, + "max_per_prompt_fro": 24.674379, + "heavy_tail_ratio": 1.2097, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.961066, + "mean_fro": 16.116413, + "rms_per_prompt_fro": 22.569114, + "max_per_prompt_fro": 27.652733, + "heavy_tail_ratio": 1.2252, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.798688, + "mean_fro": 18.239088, + "rms_per_prompt_fro": 24.461383, + "max_per_prompt_fro": 29.19372, + "heavy_tail_ratio": 1.1935, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.693547, + "mean_fro": 19.71694, + "rms_per_prompt_fro": 25.658907, + "max_per_prompt_fro": 30.319773, + "heavy_tail_ratio": 1.1816, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.564815, + "mean_fro": 22.479078, + "rms_per_prompt_fro": 28.119657, + "max_per_prompt_fro": 33.417854, + "heavy_tail_ratio": 1.1884, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.47075, + "mean_fro": 24.874882, + "rms_per_prompt_fro": 30.166887, + "max_per_prompt_fro": 35.608326, + "heavy_tail_ratio": 1.1804, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.339242, + "mean_fro": 28.315901, + "rms_per_prompt_fro": 32.768748, + "max_per_prompt_fro": 38.711796, + "heavy_tail_ratio": 1.1814, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.248509, + "mean_fro": 31.21234, + "rms_per_prompt_fro": 34.875634, + "max_per_prompt_fro": 39.857838, + "heavy_tail_ratio": 1.1429, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.176372, + "mean_fro": 32.995949, + "rms_per_prompt_fro": 35.787637, + "max_per_prompt_fro": 39.968056, + "heavy_tail_ratio": 1.1168, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.113889, + "mean_fro": 33.98993, + "rms_per_prompt_fro": 35.87329, + "max_per_prompt_fro": 39.266659, + "heavy_tail_ratio": 1.0946, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.054225, + "mean_fro": 34.709817, + "rms_per_prompt_fro": 35.638463, + "max_per_prompt_fro": 38.154331, + "heavy_tail_ratio": 1.0706, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.025279, + "mean_fro": 34.36985, + "rms_per_prompt_fro": 34.801553, + "max_per_prompt_fro": 36.432076, + "heavy_tail_ratio": 1.0469, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1247.2, + "fit_secs_per_prompt": 3.118, + "total_wall_seconds": 1260.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9232, + "top_sv": 1.8434, + "stable_rank": 18.474 + }, + { + "layer": 1, + "fro_norm": 7.456, + "top_sv": 1.7121, + "stable_rank": 18.964 + }, + { + "layer": 2, + "fro_norm": 8.081, + "top_sv": 1.8898, + "stable_rank": 18.284 + }, + { + "layer": 3, + "fro_norm": 7.7653, + "top_sv": 1.832, + "stable_rank": 17.966 + }, + { + "layer": 4, + "fro_norm": 8.7023, + "top_sv": 2.0673, + "stable_rank": 17.72 + }, + { + "layer": 5, + "fro_norm": 8.9716, + "top_sv": 2.2189, + "stable_rank": 16.348 + }, + { + "layer": 6, + "fro_norm": 9.3612, + "top_sv": 2.3138, + "stable_rank": 16.368 + }, + { + "layer": 7, + "fro_norm": 10.2729, + "top_sv": 2.4068, + "stable_rank": 18.218 + }, + { + "layer": 8, + "fro_norm": 11.4638, + "top_sv": 2.7522, + "stable_rank": 17.35 + }, + { + "layer": 9, + "fro_norm": 12.6788, + "top_sv": 3.0072, + "stable_rank": 17.776 + }, + { + "layer": 10, + "fro_norm": 14.1555, + "top_sv": 3.3412, + "stable_rank": 17.949 + }, + { + "layer": 11, + "fro_norm": 16.1162, + "top_sv": 3.7739, + "stable_rank": 18.236 + }, + { + "layer": 12, + "fro_norm": 18.2388, + "top_sv": 4.3577, + "stable_rank": 17.517 + }, + { + "layer": 13, + "fro_norm": 19.7166, + "top_sv": 4.7918, + "stable_rank": 16.931 + }, + { + "layer": 14, + "fro_norm": 22.4787, + "top_sv": 5.493, + "stable_rank": 16.746 + }, + { + "layer": 15, + "fro_norm": 24.8744, + "top_sv": 6.2392, + "stable_rank": 15.895 + }, + { + "layer": 16, + "fro_norm": 28.3153, + "top_sv": 7.2957, + "stable_rank": 15.063 + }, + { + "layer": 17, + "fro_norm": 31.2117, + "top_sv": 7.8066, + "stable_rank": 15.985 + }, + { + "layer": 18, + "fro_norm": 32.9952, + "top_sv": 7.8744, + "stable_rank": 17.558 + }, + { + "layer": 19, + "fro_norm": 33.989, + "top_sv": 7.6082, + "stable_rank": 19.958 + }, + { + "layer": 20, + "fro_norm": 34.7086, + "top_sv": 7.0149, + "stable_rank": 24.481 + }, + { + "layer": 21, + "fro_norm": 34.3684, + "top_sv": 5.331, + "stable_rank": 41.562 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n400_s1.pt b/results/lenses/e28/e28_Github_410m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2a0f93ae8610a893a73546715eb86c31fd52e16 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5c4f4c5a773a337be04a0269f0d75746b3286a64ff1a6908abe8067626f6f4b +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n400_s1_provenance.json b/results/lenses/e28/e28_Github_410m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f5459f1713b73fd79643938edc418a48eea6e237 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n400_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n400_s1.pt", + "sha256": "a5c4f4c5a773a337be04a0269f0d75746b3286a64ff1a6908abe8067626f6f4b", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.767306, + "mean_fro": 8.000013, + "rms_per_prompt_fro": 19.212205, + "max_per_prompt_fro": 48.171146, + "heavy_tail_ratio": 2.5073, + "n_prompts": 400 + }, + "1": { + "dispersion": 4.219813, + "mean_fro": 7.524622, + "rms_per_prompt_fro": 17.191437, + "max_per_prompt_fro": 42.972279, + "heavy_tail_ratio": 2.4996, + "n_prompts": 400 + }, + "2": { + "dispersion": 3.400724, + "mean_fro": 8.139547, + "rms_per_prompt_fro": 17.075062, + "max_per_prompt_fro": 40.723667, + "heavy_tail_ratio": 2.385, + "n_prompts": 400 + }, + "3": { + "dispersion": 2.77769, + "mean_fro": 7.831055, + "rms_per_prompt_fro": 15.220657, + "max_per_prompt_fro": 38.985329, + "heavy_tail_ratio": 2.5613, + "n_prompts": 400 + }, + "4": { + "dispersion": 2.308221, + "mean_fro": 8.757376, + "rms_per_prompt_fro": 15.928367, + "max_per_prompt_fro": 37.331726, + "heavy_tail_ratio": 2.3437, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.986994, + "mean_fro": 9.015461, + "rms_per_prompt_fro": 15.581352, + "max_per_prompt_fro": 34.034382, + "heavy_tail_ratio": 2.1843, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.6953, + "mean_fro": 9.4002, + "rms_per_prompt_fro": 15.432654, + "max_per_prompt_fro": 33.506432, + "heavy_tail_ratio": 2.1711, + "n_prompts": 400 + }, + "7": { + "dispersion": 1.490956, + "mean_fro": 10.299476, + "rms_per_prompt_fro": 16.25542, + "max_per_prompt_fro": 34.340145, + "heavy_tail_ratio": 2.1125, + "n_prompts": 400 + }, + "8": { + "dispersion": 1.324193, + "mean_fro": 11.492829, + "rms_per_prompt_fro": 17.521169, + "max_per_prompt_fro": 34.944042, + "heavy_tail_ratio": 1.9944, + "n_prompts": 400 + }, + "9": { + "dispersion": 1.206038, + "mean_fro": 12.708601, + "rms_per_prompt_fro": 18.875752, + "max_per_prompt_fro": 37.255829, + "heavy_tail_ratio": 1.9737, + "n_prompts": 400 + }, + "10": { + "dispersion": 1.092995, + "mean_fro": 14.188343, + "rms_per_prompt_fro": 20.52654, + "max_per_prompt_fro": 39.984081, + "heavy_tail_ratio": 1.9479, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.976967, + "mean_fro": 16.155182, + "rms_per_prompt_fro": 22.71494, + "max_per_prompt_fro": 43.035824, + "heavy_tail_ratio": 1.8946, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.810107, + "mean_fro": 18.294557, + "rms_per_prompt_fro": 24.613538, + "max_per_prompt_fro": 45.397015, + "heavy_tail_ratio": 1.8444, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.702926, + "mean_fro": 19.789822, + "rms_per_prompt_fro": 25.824965, + "max_per_prompt_fro": 44.361992, + "heavy_tail_ratio": 1.7178, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.571208, + "mean_fro": 22.557335, + "rms_per_prompt_fro": 28.27513, + "max_per_prompt_fro": 44.304882, + "heavy_tail_ratio": 1.5669, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.476056, + "mean_fro": 24.985992, + "rms_per_prompt_fro": 30.356239, + "max_per_prompt_fro": 43.403439, + "heavy_tail_ratio": 1.4298, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.337814, + "mean_fro": 28.437093, + "rms_per_prompt_fro": 32.891453, + "max_per_prompt_fro": 39.248611, + "heavy_tail_ratio": 1.1933, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.246294, + "mean_fro": 31.307186, + "rms_per_prompt_fro": 34.950572, + "max_per_prompt_fro": 40.383511, + "heavy_tail_ratio": 1.1554, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.173844, + "mean_fro": 33.074715, + "rms_per_prompt_fro": 35.834499, + "max_per_prompt_fro": 40.659569, + "heavy_tail_ratio": 1.1346, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.112059, + "mean_fro": 34.022927, + "rms_per_prompt_fro": 35.87861, + "max_per_prompt_fro": 40.076298, + "heavy_tail_ratio": 1.117, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.053297, + "mean_fro": 34.718913, + "rms_per_prompt_fro": 35.632118, + "max_per_prompt_fro": 37.898232, + "heavy_tail_ratio": 1.0636, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.024909, + "mean_fro": 34.358796, + "rms_per_prompt_fro": 34.784081, + "max_per_prompt_fro": 36.227936, + "heavy_tail_ratio": 1.0415, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1219.8, + "fit_secs_per_prompt": 3.049, + "total_wall_seconds": 1230.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9999, + "top_sv": 1.8601, + "stable_rank": 18.497 + }, + { + "layer": 1, + "fro_norm": 7.5245, + "top_sv": 1.7411, + "stable_rank": 18.678 + }, + { + "layer": 2, + "fro_norm": 8.1395, + "top_sv": 1.9141, + "stable_rank": 18.083 + }, + { + "layer": 3, + "fro_norm": 7.831, + "top_sv": 1.8514, + "stable_rank": 17.89 + }, + { + "layer": 4, + "fro_norm": 8.7573, + "top_sv": 2.0895, + "stable_rank": 17.565 + }, + { + "layer": 5, + "fro_norm": 9.0153, + "top_sv": 2.2172, + "stable_rank": 16.533 + }, + { + "layer": 6, + "fro_norm": 9.4001, + "top_sv": 2.309, + "stable_rank": 16.573 + }, + { + "layer": 7, + "fro_norm": 10.2993, + "top_sv": 2.3984, + "stable_rank": 18.44 + }, + { + "layer": 8, + "fro_norm": 11.4927, + "top_sv": 2.7377, + "stable_rank": 17.622 + }, + { + "layer": 9, + "fro_norm": 12.7084, + "top_sv": 3.0012, + "stable_rank": 17.93 + }, + { + "layer": 10, + "fro_norm": 14.1881, + "top_sv": 3.3609, + "stable_rank": 17.821 + }, + { + "layer": 11, + "fro_norm": 16.1549, + "top_sv": 3.8233, + "stable_rank": 17.854 + }, + { + "layer": 12, + "fro_norm": 18.2942, + "top_sv": 4.3748, + "stable_rank": 17.486 + }, + { + "layer": 13, + "fro_norm": 19.7895, + "top_sv": 4.843, + "stable_rank": 16.697 + }, + { + "layer": 14, + "fro_norm": 22.557, + "top_sv": 5.5609, + "stable_rank": 16.454 + }, + { + "layer": 15, + "fro_norm": 24.9855, + "top_sv": 6.3365, + "stable_rank": 15.548 + }, + { + "layer": 16, + "fro_norm": 28.4366, + "top_sv": 7.4058, + "stable_rank": 14.744 + }, + { + "layer": 17, + "fro_norm": 31.3066, + "top_sv": 7.9081, + "stable_rank": 15.672 + }, + { + "layer": 18, + "fro_norm": 33.074, + "top_sv": 7.9182, + "stable_rank": 17.447 + }, + { + "layer": 19, + "fro_norm": 34.0219, + "top_sv": 7.6207, + "stable_rank": 19.931 + }, + { + "layer": 20, + "fro_norm": 34.7178, + "top_sv": 7.0692, + "stable_rank": 24.119 + }, + { + "layer": 21, + "fro_norm": 34.3573, + "top_sv": 5.3936, + "stable_rank": 40.577 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n400_s2.pt b/results/lenses/e28/e28_Github_410m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ec210548cf7af6c58306fa6cf65db65401f7c8e --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3bff4c7693cbc54d171edb7fc4affd07abc8ca7a75982d25773a79d153d9140 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n400_s2_provenance.json b/results/lenses/e28/e28_Github_410m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..349a51ad5d1c176d162326e71fc8fccf3b6e6618 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n400_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n400_s2.pt", + "sha256": "d3bff4c7693cbc54d171edb7fc4affd07abc8ca7a75982d25773a79d153d9140", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 5.266542, + "mean_fro": 7.920797, + "rms_per_prompt_fro": 19.828181, + "max_per_prompt_fro": 48.171146, + "heavy_tail_ratio": 2.4294, + "n_prompts": 400 + }, + "1": { + "dispersion": 4.650404, + "mean_fro": 7.450091, + "rms_per_prompt_fro": 17.709297, + "max_per_prompt_fro": 42.972279, + "heavy_tail_ratio": 2.4265, + "n_prompts": 400 + }, + "2": { + "dispersion": 3.728196, + "mean_fro": 8.081403, + "rms_per_prompt_fro": 17.572538, + "max_per_prompt_fro": 40.723667, + "heavy_tail_ratio": 2.3175, + "n_prompts": 400 + }, + "3": { + "dispersion": 3.044278, + "mean_fro": 7.780714, + "rms_per_prompt_fro": 15.647319, + "max_per_prompt_fro": 38.985329, + "heavy_tail_ratio": 2.4915, + "n_prompts": 400 + }, + "4": { + "dispersion": 2.511102, + "mean_fro": 8.712531, + "rms_per_prompt_fro": 16.325485, + "max_per_prompt_fro": 37.331726, + "heavy_tail_ratio": 2.2867, + "n_prompts": 400 + }, + "5": { + "dispersion": 2.15232, + "mean_fro": 8.970257, + "rms_per_prompt_fro": 15.926489, + "max_per_prompt_fro": 34.034382, + "heavy_tail_ratio": 2.137, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.823311, + "mean_fro": 9.352829, + "rms_per_prompt_fro": 15.715287, + "max_per_prompt_fro": 33.506432, + "heavy_tail_ratio": 2.1321, + "n_prompts": 400 + }, + "7": { + "dispersion": 1.594304, + "mean_fro": 10.225265, + "rms_per_prompt_fro": 16.469676, + "max_per_prompt_fro": 34.340145, + "heavy_tail_ratio": 2.0851, + "n_prompts": 400 + }, + "8": { + "dispersion": 1.413247, + "mean_fro": 11.393586, + "rms_per_prompt_fro": 17.699515, + "max_per_prompt_fro": 34.944042, + "heavy_tail_ratio": 1.9743, + "n_prompts": 400 + }, + "9": { + "dispersion": 1.283332, + "mean_fro": 12.608759, + "rms_per_prompt_fro": 19.052716, + "max_per_prompt_fro": 37.255829, + "heavy_tail_ratio": 1.9554, + "n_prompts": 400 + }, + "10": { + "dispersion": 1.159379, + "mean_fro": 14.082212, + "rms_per_prompt_fro": 20.693565, + "max_per_prompt_fro": 39.984081, + "heavy_tail_ratio": 1.9322, + "n_prompts": 400 + }, + "11": { + "dispersion": 1.031518, + "mean_fro": 16.00548, + "rms_per_prompt_fro": 22.812825, + "max_per_prompt_fro": 43.035824, + "heavy_tail_ratio": 1.8865, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.853123, + "mean_fro": 18.133358, + "rms_per_prompt_fro": 24.68484, + "max_per_prompt_fro": 45.397015, + "heavy_tail_ratio": 1.8391, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.737058, + "mean_fro": 19.652322, + "rms_per_prompt_fro": 25.901271, + "max_per_prompt_fro": 44.361992, + "heavy_tail_ratio": 1.7127, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.595739, + "mean_fro": 22.38994, + "rms_per_prompt_fro": 28.283548, + "max_per_prompt_fro": 44.304882, + "heavy_tail_ratio": 1.5665, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.493956, + "mean_fro": 24.796095, + "rms_per_prompt_fro": 30.30764, + "max_per_prompt_fro": 43.403439, + "heavy_tail_ratio": 1.4321, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.347647, + "mean_fro": 28.257166, + "rms_per_prompt_fro": 32.803233, + "max_per_prompt_fro": 39.856323, + "heavy_tail_ratio": 1.215, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.252123, + "mean_fro": 31.174824, + "rms_per_prompt_fro": 34.884093, + "max_per_prompt_fro": 40.543797, + "heavy_tail_ratio": 1.1622, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.177464, + "mean_fro": 32.982987, + "rms_per_prompt_fro": 35.790172, + "max_per_prompt_fro": 40.60183, + "heavy_tail_ratio": 1.1344, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.113927, + "mean_fro": 33.967449, + "rms_per_prompt_fro": 35.850174, + "max_per_prompt_fro": 39.329338, + "heavy_tail_ratio": 1.097, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.054032, + "mean_fro": 34.706802, + "rms_per_prompt_fro": 35.632102, + "max_per_prompt_fro": 38.151516, + "heavy_tail_ratio": 1.0707, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.025086, + "mean_fro": 34.362, + "rms_per_prompt_fro": 34.790328, + "max_per_prompt_fro": 36.419445, + "heavy_tail_ratio": 1.0468, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1240.0, + "fit_secs_per_prompt": 3.1, + "total_wall_seconds": 1247.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9207, + "top_sv": 1.9268, + "stable_rank": 16.898 + }, + { + "layer": 1, + "fro_norm": 7.45, + "top_sv": 1.7715, + "stable_rank": 17.687 + }, + { + "layer": 2, + "fro_norm": 8.0813, + "top_sv": 1.9831, + "stable_rank": 16.607 + }, + { + "layer": 3, + "fro_norm": 7.7806, + "top_sv": 1.9194, + "stable_rank": 16.433 + }, + { + "layer": 4, + "fro_norm": 8.7124, + "top_sv": 2.1685, + "stable_rank": 16.141 + }, + { + "layer": 5, + "fro_norm": 8.9701, + "top_sv": 2.339, + "stable_rank": 14.708 + }, + { + "layer": 6, + "fro_norm": 9.3527, + "top_sv": 2.4322, + "stable_rank": 14.787 + }, + { + "layer": 7, + "fro_norm": 10.2251, + "top_sv": 2.4559, + "stable_rank": 17.335 + }, + { + "layer": 8, + "fro_norm": 11.3934, + "top_sv": 2.7791, + "stable_rank": 16.807 + }, + { + "layer": 9, + "fro_norm": 12.6086, + "top_sv": 3.0498, + "stable_rank": 17.092 + }, + { + "layer": 10, + "fro_norm": 14.082, + "top_sv": 3.3873, + "stable_rank": 17.283 + }, + { + "layer": 11, + "fro_norm": 16.0053, + "top_sv": 3.7691, + "stable_rank": 18.032 + }, + { + "layer": 12, + "fro_norm": 18.1331, + "top_sv": 4.3291, + "stable_rank": 17.545 + }, + { + "layer": 13, + "fro_norm": 19.652, + "top_sv": 4.8182, + "stable_rank": 16.636 + }, + { + "layer": 14, + "fro_norm": 22.3896, + "top_sv": 5.5174, + "stable_rank": 16.467 + }, + { + "layer": 15, + "fro_norm": 24.7956, + "top_sv": 6.3066, + "stable_rank": 15.458 + }, + { + "layer": 16, + "fro_norm": 28.2566, + "top_sv": 7.3408, + "stable_rank": 14.817 + }, + { + "layer": 17, + "fro_norm": 31.1742, + "top_sv": 7.8506, + "stable_rank": 15.768 + }, + { + "layer": 18, + "fro_norm": 32.9823, + "top_sv": 7.9058, + "stable_rank": 17.405 + }, + { + "layer": 19, + "fro_norm": 33.9665, + "top_sv": 7.5882, + "stable_rank": 20.037 + }, + { + "layer": 20, + "fro_norm": 34.7056, + "top_sv": 7.0402, + "stable_rank": 24.301 + }, + { + "layer": 21, + "fro_norm": 34.3605, + "top_sv": 5.3509, + "stable_rank": 41.235 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n50_s0.pt b/results/lenses/e28/e28_Github_410m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d910dbd286ad390db3619126fb7293773c6b2f1 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31b20e81de8508703f25faf1111fe4db0c1820f0189dcf1b7be0d8c1a9430280 +size 46144773 diff --git a/results/lenses/e28/e28_Github_410m_n50_s0_provenance.json b/results/lenses/e28/e28_Github_410m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..68c9607c95f7f4580931dffeecc87d46cd69da21 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n50_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n50_s0.pt", + "sha256": "31b20e81de8508703f25faf1111fe4db0c1820f0189dcf1b7be0d8c1a9430280", + "bytes": 46144773, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.202762, + "mean_fro": 8.549232, + "rms_per_prompt_fro": 19.500426, + "max_per_prompt_fro": 29.136555, + "heavy_tail_ratio": 1.4941, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.749666, + "mean_fro": 8.009236, + "rms_per_prompt_fro": 17.455113, + "max_per_prompt_fro": 25.729753, + "heavy_tail_ratio": 1.4741, + "n_prompts": 50 + }, + "2": { + "dispersion": 3.053861, + "mean_fro": 8.608604, + "rms_per_prompt_fro": 17.332738, + "max_per_prompt_fro": 24.51263, + "heavy_tail_ratio": 1.4142, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.493921, + "mean_fro": 8.17181, + "rms_per_prompt_fro": 15.274774, + "max_per_prompt_fro": 19.907942, + "heavy_tail_ratio": 1.3033, + "n_prompts": 50 + }, + "4": { + "dispersion": 2.071845, + "mean_fro": 9.114976, + "rms_per_prompt_fro": 15.975526, + "max_per_prompt_fro": 20.365681, + "heavy_tail_ratio": 1.2748, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.774895, + "mean_fro": 9.375838, + "rms_per_prompt_fro": 15.618285, + "max_per_prompt_fro": 19.465076, + "heavy_tail_ratio": 1.2463, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.519752, + "mean_fro": 9.756063, + "rms_per_prompt_fro": 15.486507, + "max_per_prompt_fro": 18.996004, + "heavy_tail_ratio": 1.2266, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.340019, + "mean_fro": 10.678515, + "rms_per_prompt_fro": 16.335053, + "max_per_prompt_fro": 19.43343, + "heavy_tail_ratio": 1.1897, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.187795, + "mean_fro": 11.88381, + "rms_per_prompt_fro": 17.577577, + "max_per_prompt_fro": 20.495922, + "heavy_tail_ratio": 1.166, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.08011, + "mean_fro": 13.090305, + "rms_per_prompt_fro": 18.879606, + "max_per_prompt_fro": 22.202566, + "heavy_tail_ratio": 1.176, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.975542, + "mean_fro": 14.517827, + "rms_per_prompt_fro": 20.405381, + "max_per_prompt_fro": 23.720745, + "heavy_tail_ratio": 1.1625, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.869301, + "mean_fro": 16.406012, + "rms_per_prompt_fro": 22.430689, + "max_per_prompt_fro": 25.624063, + "heavy_tail_ratio": 1.1424, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.724646, + "mean_fro": 18.478067, + "rms_per_prompt_fro": 24.266463, + "max_per_prompt_fro": 27.266958, + "heavy_tail_ratio": 1.1236, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.62976, + "mean_fro": 19.888199, + "rms_per_prompt_fro": 25.389682, + "max_per_prompt_fro": 27.780455, + "heavy_tail_ratio": 1.0942, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.516572, + "mean_fro": 22.590067, + "rms_per_prompt_fro": 27.81948, + "max_per_prompt_fro": 30.815104, + "heavy_tail_ratio": 1.1077, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.432427, + "mean_fro": 24.984344, + "rms_per_prompt_fro": 29.902278, + "max_per_prompt_fro": 33.102242, + "heavy_tail_ratio": 1.107, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.313166, + "mean_fro": 28.294241, + "rms_per_prompt_fro": 32.423347, + "max_per_prompt_fro": 36.214497, + "heavy_tail_ratio": 1.1169, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.230061, + "mean_fro": 31.128226, + "rms_per_prompt_fro": 34.523725, + "max_per_prompt_fro": 38.127647, + "heavy_tail_ratio": 1.1044, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.164173, + "mean_fro": 32.868308, + "rms_per_prompt_fro": 35.463869, + "max_per_prompt_fro": 38.36879, + "heavy_tail_ratio": 1.0819, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.106023, + "mean_fro": 33.862778, + "rms_per_prompt_fro": 35.612685, + "max_per_prompt_fro": 37.90659, + "heavy_tail_ratio": 1.0644, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.05056, + "mean_fro": 34.582908, + "rms_per_prompt_fro": 35.44638, + "max_per_prompt_fro": 37.184475, + "heavy_tail_ratio": 1.049, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.023508, + "mean_fro": 34.2877, + "rms_per_prompt_fro": 34.688381, + "max_per_prompt_fro": 35.642113, + "heavy_tail_ratio": 1.0275, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 156.0, + "fit_secs_per_prompt": 3.12, + "total_wall_seconds": 168.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.5491, + "top_sv": 2.2447, + "stable_rank": 14.505 + }, + { + "layer": 1, + "fro_norm": 8.0091, + "top_sv": 2.0244, + "stable_rank": 15.652 + }, + { + "layer": 2, + "fro_norm": 8.6085, + "top_sv": 2.1552, + "stable_rank": 15.955 + }, + { + "layer": 3, + "fro_norm": 8.1717, + "top_sv": 2.0345, + "stable_rank": 16.132 + }, + { + "layer": 4, + "fro_norm": 9.1148, + "top_sv": 2.2611, + "stable_rank": 16.25 + }, + { + "layer": 5, + "fro_norm": 9.3757, + "top_sv": 2.4406, + "stable_rank": 14.758 + }, + { + "layer": 6, + "fro_norm": 9.7559, + "top_sv": 2.5434, + "stable_rank": 14.713 + }, + { + "layer": 7, + "fro_norm": 10.6784, + "top_sv": 2.6602, + "stable_rank": 16.113 + }, + { + "layer": 8, + "fro_norm": 11.8837, + "top_sv": 3.0121, + "stable_rank": 15.565 + }, + { + "layer": 9, + "fro_norm": 13.0901, + "top_sv": 3.2499, + "stable_rank": 16.224 + }, + { + "layer": 10, + "fro_norm": 14.5176, + "top_sv": 3.5709, + "stable_rank": 16.529 + }, + { + "layer": 11, + "fro_norm": 16.4058, + "top_sv": 3.8958, + "stable_rank": 17.733 + }, + { + "layer": 12, + "fro_norm": 18.4777, + "top_sv": 4.4566, + "stable_rank": 17.191 + }, + { + "layer": 13, + "fro_norm": 19.8879, + "top_sv": 4.7996, + "stable_rank": 17.17 + }, + { + "layer": 14, + "fro_norm": 22.5897, + "top_sv": 5.472, + "stable_rank": 17.042 + }, + { + "layer": 15, + "fro_norm": 24.9839, + "top_sv": 6.1531, + "stable_rank": 16.487 + }, + { + "layer": 16, + "fro_norm": 28.2936, + "top_sv": 7.1795, + "stable_rank": 15.531 + }, + { + "layer": 17, + "fro_norm": 31.1276, + "top_sv": 7.6884, + "stable_rank": 16.392 + }, + { + "layer": 18, + "fro_norm": 32.8676, + "top_sv": 7.7536, + "stable_rank": 17.969 + }, + { + "layer": 19, + "fro_norm": 33.8619, + "top_sv": 7.5592, + "stable_rank": 20.067 + }, + { + "layer": 20, + "fro_norm": 34.5817, + "top_sv": 6.9122, + "stable_rank": 25.03 + }, + { + "layer": 21, + "fro_norm": 34.2862, + "top_sv": 5.2922, + "stable_rank": 41.973 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n50_s1.pt b/results/lenses/e28/e28_Github_410m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..b731db2eebdea0216c4ac6c741f5fe3b3057b3f4 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9b85e54f1e56ab1f30f28f9aaa0a6424b180e9f5756ce0bc903d915e27be209 +size 46144773 diff --git a/results/lenses/e28/e28_Github_410m_n50_s1_provenance.json b/results/lenses/e28/e28_Github_410m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..32b3ae218bb3e2e1c6144d561777c73ba717ae49 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n50_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n50_s1.pt", + "sha256": "b9b85e54f1e56ab1f30f28f9aaa0a6424b180e9f5756ce0bc903d915e27be209", + "bytes": 46144773, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.300414, + "mean_fro": 7.958564, + "rms_per_prompt_fro": 18.322706, + "max_per_prompt_fro": 25.659891, + "heavy_tail_ratio": 1.4004, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.8418, + "mean_fro": 7.468062, + "rms_per_prompt_fro": 16.432791, + "max_per_prompt_fro": 23.562773, + "heavy_tail_ratio": 1.4339, + "n_prompts": 50 + }, + "2": { + "dispersion": 3.210923, + "mean_fro": 8.021205, + "rms_per_prompt_fro": 16.459941, + "max_per_prompt_fro": 22.937246, + "heavy_tail_ratio": 1.3935, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.645976, + "mean_fro": 7.721067, + "rms_per_prompt_fro": 14.742945, + "max_per_prompt_fro": 20.982944, + "heavy_tail_ratio": 1.4233, + "n_prompts": 50 + }, + "4": { + "dispersion": 2.246017, + "mean_fro": 8.604046, + "rms_per_prompt_fro": 15.501655, + "max_per_prompt_fro": 21.62668, + "heavy_tail_ratio": 1.3951, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.955443, + "mean_fro": 8.857028, + "rms_per_prompt_fro": 15.226474, + "max_per_prompt_fro": 21.015985, + "heavy_tail_ratio": 1.3802, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.690737, + "mean_fro": 9.228494, + "rms_per_prompt_fro": 15.137928, + "max_per_prompt_fro": 20.854357, + "heavy_tail_ratio": 1.3776, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.493553, + "mean_fro": 10.140206, + "rms_per_prompt_fro": 16.012385, + "max_per_prompt_fro": 22.127941, + "heavy_tail_ratio": 1.3819, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.32912, + "mean_fro": 11.329706, + "rms_per_prompt_fro": 17.290779, + "max_per_prompt_fro": 23.444479, + "heavy_tail_ratio": 1.3559, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.2111, + "mean_fro": 12.560007, + "rms_per_prompt_fro": 18.67644, + "max_per_prompt_fro": 24.862322, + "heavy_tail_ratio": 1.3312, + "n_prompts": 50 + }, + "10": { + "dispersion": 1.100759, + "mean_fro": 14.041858, + "rms_per_prompt_fro": 20.352263, + "max_per_prompt_fro": 26.610645, + "heavy_tail_ratio": 1.3075, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.983501, + "mean_fro": 16.026514, + "rms_per_prompt_fro": 22.571232, + "max_per_prompt_fro": 28.710007, + "heavy_tail_ratio": 1.272, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.818758, + "mean_fro": 18.163324, + "rms_per_prompt_fro": 24.495302, + "max_per_prompt_fro": 29.876253, + "heavy_tail_ratio": 1.2197, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.711317, + "mean_fro": 19.63553, + "rms_per_prompt_fro": 25.686672, + "max_per_prompt_fro": 31.00931, + "heavy_tail_ratio": 1.2072, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.582455, + "mean_fro": 22.40204, + "rms_per_prompt_fro": 28.180793, + "max_per_prompt_fro": 33.104443, + "heavy_tail_ratio": 1.1747, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.488728, + "mean_fro": 24.835361, + "rms_per_prompt_fro": 30.302483, + "max_per_prompt_fro": 34.397076, + "heavy_tail_ratio": 1.1351, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.348942, + "mean_fro": 28.308037, + "rms_per_prompt_fro": 32.878074, + "max_per_prompt_fro": 37.135166, + "heavy_tail_ratio": 1.1295, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.255039, + "mean_fro": 31.218734, + "rms_per_prompt_fro": 34.973894, + "max_per_prompt_fro": 38.627228, + "heavy_tail_ratio": 1.1045, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.180995, + "mean_fro": 33.021801, + "rms_per_prompt_fro": 35.885973, + "max_per_prompt_fro": 38.426544, + "heavy_tail_ratio": 1.0708, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.116941, + "mean_fro": 33.980152, + "rms_per_prompt_fro": 35.912066, + "max_per_prompt_fro": 37.938892, + "heavy_tail_ratio": 1.0564, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.055067, + "mean_fro": 34.660107, + "rms_per_prompt_fro": 35.601625, + "max_per_prompt_fro": 37.117928, + "heavy_tail_ratio": 1.0426, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.026138, + "mean_fro": 34.314072, + "rms_per_prompt_fro": 34.759627, + "max_per_prompt_fro": 35.871658, + "heavy_tail_ratio": 1.032, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 152.3, + "fit_secs_per_prompt": 3.047, + "total_wall_seconds": 162.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9585, + "top_sv": 1.935, + "stable_rank": 16.915 + }, + { + "layer": 1, + "fro_norm": 7.468, + "top_sv": 1.8026, + "stable_rank": 17.163 + }, + { + "layer": 2, + "fro_norm": 8.0211, + "top_sv": 1.9084, + "stable_rank": 17.666 + }, + { + "layer": 3, + "fro_norm": 7.721, + "top_sv": 1.8438, + "stable_rank": 17.535 + }, + { + "layer": 4, + "fro_norm": 8.6039, + "top_sv": 2.0774, + "stable_rank": 17.154 + }, + { + "layer": 5, + "fro_norm": 8.8569, + "top_sv": 2.169, + "stable_rank": 16.675 + }, + { + "layer": 6, + "fro_norm": 9.2283, + "top_sv": 2.2527, + "stable_rank": 16.781 + }, + { + "layer": 7, + "fro_norm": 10.1401, + "top_sv": 2.3883, + "stable_rank": 18.026 + }, + { + "layer": 8, + "fro_norm": 11.3296, + "top_sv": 2.7179, + "stable_rank": 17.376 + }, + { + "layer": 9, + "fro_norm": 12.5598, + "top_sv": 3.0863, + "stable_rank": 16.561 + }, + { + "layer": 10, + "fro_norm": 14.0416, + "top_sv": 3.4787, + "stable_rank": 16.293 + }, + { + "layer": 11, + "fro_norm": 16.0263, + "top_sv": 3.9661, + "stable_rank": 16.328 + }, + { + "layer": 12, + "fro_norm": 18.163, + "top_sv": 4.5075, + "stable_rank": 16.237 + }, + { + "layer": 13, + "fro_norm": 19.6352, + "top_sv": 4.9625, + "stable_rank": 15.655 + }, + { + "layer": 14, + "fro_norm": 22.4017, + "top_sv": 5.6888, + "stable_rank": 15.507 + }, + { + "layer": 15, + "fro_norm": 24.8349, + "top_sv": 6.5005, + "stable_rank": 14.596 + }, + { + "layer": 16, + "fro_norm": 28.3075, + "top_sv": 7.5888, + "stable_rank": 13.914 + }, + { + "layer": 17, + "fro_norm": 31.2181, + "top_sv": 8.0992, + "stable_rank": 14.857 + }, + { + "layer": 18, + "fro_norm": 33.021, + "top_sv": 8.0433, + "stable_rank": 16.854 + }, + { + "layer": 19, + "fro_norm": 33.9792, + "top_sv": 7.6921, + "stable_rank": 19.514 + }, + { + "layer": 20, + "fro_norm": 34.659, + "top_sv": 7.0912, + "stable_rank": 23.889 + }, + { + "layer": 21, + "fro_norm": 34.3125, + "top_sv": 5.4192, + "stable_rank": 40.09 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n50_s2.pt b/results/lenses/e28/e28_Github_410m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..993a29b32df958d44c02d18ee70745ba943250fb --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:551df46a3acdc70eb26d222db228bbea280b15cdfb5e9e3534d97e294342469e +size 46144773 diff --git a/results/lenses/e28/e28_Github_410m_n50_s2_provenance.json b/results/lenses/e28/e28_Github_410m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ef411015f29d6821d63677df8c80a9d759bebf4a --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n50_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n50_s2.pt", + "sha256": "551df46a3acdc70eb26d222db228bbea280b15cdfb5e9e3534d97e294342469e", + "bytes": 46144773, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 5.172782, + "mean_fro": 8.172922, + "rms_per_prompt_fro": 20.305694, + "max_per_prompt_fro": 34.796436, + "heavy_tail_ratio": 1.7136, + "n_prompts": 50 + }, + "1": { + "dispersion": 4.596408, + "mean_fro": 7.648867, + "rms_per_prompt_fro": 18.094717, + "max_per_prompt_fro": 29.559164, + "heavy_tail_ratio": 1.6336, + "n_prompts": 50 + }, + "2": { + "dispersion": 3.737582, + "mean_fro": 8.247642, + "rms_per_prompt_fro": 17.951808, + "max_per_prompt_fro": 29.93277, + "heavy_tail_ratio": 1.6674, + "n_prompts": 50 + }, + "3": { + "dispersion": 3.050494, + "mean_fro": 7.863117, + "rms_per_prompt_fro": 15.825183, + "max_per_prompt_fro": 25.432657, + "heavy_tail_ratio": 1.6071, + "n_prompts": 50 + }, + "4": { + "dispersion": 2.533791, + "mean_fro": 8.795523, + "rms_per_prompt_fro": 16.534159, + "max_per_prompt_fro": 26.80678, + "heavy_tail_ratio": 1.6213, + "n_prompts": 50 + }, + "5": { + "dispersion": 2.200196, + "mean_fro": 9.06531, + "rms_per_prompt_fro": 16.217017, + "max_per_prompt_fro": 27.302526, + "heavy_tail_ratio": 1.6836, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.836424, + "mean_fro": 9.437178, + "rms_per_prompt_fro": 15.8938, + "max_per_prompt_fro": 24.270342, + "heavy_tail_ratio": 1.527, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.583473, + "mean_fro": 10.265362, + "rms_per_prompt_fro": 16.499706, + "max_per_prompt_fro": 24.17333, + "heavy_tail_ratio": 1.4651, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.397752, + "mean_fro": 11.443294, + "rms_per_prompt_fro": 17.719569, + "max_per_prompt_fro": 26.130661, + "heavy_tail_ratio": 1.4747, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.259222, + "mean_fro": 12.66415, + "rms_per_prompt_fro": 19.035114, + "max_per_prompt_fro": 27.802649, + "heavy_tail_ratio": 1.4606, + "n_prompts": 50 + }, + "10": { + "dispersion": 1.13109, + "mean_fro": 14.137788, + "rms_per_prompt_fro": 20.638702, + "max_per_prompt_fro": 29.789103, + "heavy_tail_ratio": 1.4434, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.987306, + "mean_fro": 16.092873, + "rms_per_prompt_fro": 22.686421, + "max_per_prompt_fro": 27.419218, + "heavy_tail_ratio": 1.2086, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.807951, + "mean_fro": 18.185078, + "rms_per_prompt_fro": 24.451669, + "max_per_prompt_fro": 28.603258, + "heavy_tail_ratio": 1.1698, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.701627, + "mean_fro": 19.701626, + "rms_per_prompt_fro": 25.700069, + "max_per_prompt_fro": 29.174259, + "heavy_tail_ratio": 1.1352, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.565487, + "mean_fro": 22.413132, + "rms_per_prompt_fro": 28.043183, + "max_per_prompt_fro": 31.695009, + "heavy_tail_ratio": 1.1302, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.468172, + "mean_fro": 24.773625, + "rms_per_prompt_fro": 30.017742, + "max_per_prompt_fro": 34.53772, + "heavy_tail_ratio": 1.1506, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.338812, + "mean_fro": 28.3027, + "rms_per_prompt_fro": 32.748218, + "max_per_prompt_fro": 37.547672, + "heavy_tail_ratio": 1.1466, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.247764, + "mean_fro": 31.249558, + "rms_per_prompt_fro": 34.906803, + "max_per_prompt_fro": 38.719402, + "heavy_tail_ratio": 1.1092, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.175973, + "mean_fro": 33.052818, + "rms_per_prompt_fro": 35.843231, + "max_per_prompt_fro": 39.30714, + "heavy_tail_ratio": 1.0966, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.113757, + "mean_fro": 34.043015, + "rms_per_prompt_fro": 35.927194, + "max_per_prompt_fro": 38.093178, + "heavy_tail_ratio": 1.0603, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.054714, + "mean_fro": 34.773449, + "rms_per_prompt_fro": 35.712078, + "max_per_prompt_fro": 37.219135, + "heavy_tail_ratio": 1.0422, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.025347, + "mean_fro": 34.435348, + "rms_per_prompt_fro": 34.869029, + "max_per_prompt_fro": 35.718357, + "heavy_tail_ratio": 1.0244, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 154.8, + "fit_secs_per_prompt": 3.097, + "total_wall_seconds": 162.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.1728, + "top_sv": 2.1334, + "stable_rank": 14.675 + }, + { + "layer": 1, + "fro_norm": 7.6488, + "top_sv": 1.9375, + "stable_rank": 15.585 + }, + { + "layer": 2, + "fro_norm": 8.2475, + "top_sv": 2.175, + "stable_rank": 14.379 + }, + { + "layer": 3, + "fro_norm": 7.863, + "top_sv": 2.09, + "stable_rank": 14.154 + }, + { + "layer": 4, + "fro_norm": 8.7954, + "top_sv": 2.3802, + "stable_rank": 13.655 + }, + { + "layer": 5, + "fro_norm": 9.0652, + "top_sv": 2.6285, + "stable_rank": 11.894 + }, + { + "layer": 6, + "fro_norm": 9.437, + "top_sv": 2.7418, + "stable_rank": 11.847 + }, + { + "layer": 7, + "fro_norm": 10.2652, + "top_sv": 2.596, + "stable_rank": 15.636 + }, + { + "layer": 8, + "fro_norm": 11.4432, + "top_sv": 2.934, + "stable_rank": 15.211 + }, + { + "layer": 9, + "fro_norm": 12.6639, + "top_sv": 3.1915, + "stable_rank": 15.746 + }, + { + "layer": 10, + "fro_norm": 14.1376, + "top_sv": 3.5093, + "stable_rank": 16.229 + }, + { + "layer": 11, + "fro_norm": 16.0927, + "top_sv": 3.8927, + "stable_rank": 17.09 + }, + { + "layer": 12, + "fro_norm": 18.1848, + "top_sv": 4.3846, + "stable_rank": 17.201 + }, + { + "layer": 13, + "fro_norm": 19.7013, + "top_sv": 4.8024, + "stable_rank": 16.83 + }, + { + "layer": 14, + "fro_norm": 22.4128, + "top_sv": 5.3923, + "stable_rank": 17.276 + }, + { + "layer": 15, + "fro_norm": 24.7732, + "top_sv": 6.0587, + "stable_rank": 16.719 + }, + { + "layer": 16, + "fro_norm": 28.3022, + "top_sv": 7.1236, + "stable_rank": 15.785 + }, + { + "layer": 17, + "fro_norm": 31.249, + "top_sv": 7.6399, + "stable_rank": 16.73 + }, + { + "layer": 18, + "fro_norm": 33.0521, + "top_sv": 7.8137, + "stable_rank": 17.893 + }, + { + "layer": 19, + "fro_norm": 34.042, + "top_sv": 7.6161, + "stable_rank": 19.978 + }, + { + "layer": 20, + "fro_norm": 34.7723, + "top_sv": 7.062, + "stable_rank": 24.245 + }, + { + "layer": 21, + "fro_norm": 34.4338, + "top_sv": 5.3644, + "stable_rank": 41.203 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n800_s0.pt b/results/lenses/e28/e28_Github_410m_n800_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..29e50b5f070debf4fa6a683001c195f6ac8943a3 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n800_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1667fe86a60a0cecdf86f7204c3f9734fd1e94f92b15df0159bb3194930ccc2 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n800_s0_provenance.json b/results/lenses/e28/e28_Github_410m_n800_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9fe26794dac707caacc163addafa97b3b2749ed3 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n800_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n800_s0.pt", + "sha256": "e1667fe86a60a0cecdf86f7204c3f9734fd1e94f92b15df0159bb3194930ccc2", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.87004, + "mean_fro": 7.85677, + "rms_per_prompt_fro": 19.035513, + "max_per_prompt_fro": 30.175514, + "heavy_tail_ratio": 1.5852, + "n_prompts": 800 + }, + "1": { + "dispersion": 4.300576, + "mean_fro": 7.39771, + "rms_per_prompt_fro": 17.031733, + "max_per_prompt_fro": 29.6033, + "heavy_tail_ratio": 1.7381, + "n_prompts": 800 + }, + "2": { + "dispersion": 3.457025, + "mean_fro": 8.021754, + "rms_per_prompt_fro": 16.93526, + "max_per_prompt_fro": 28.624714, + "heavy_tail_ratio": 1.6902, + "n_prompts": 800 + }, + "3": { + "dispersion": 2.795654, + "mean_fro": 7.715559, + "rms_per_prompt_fro": 15.03179, + "max_per_prompt_fro": 27.685934, + "heavy_tail_ratio": 1.8418, + "n_prompts": 800 + }, + "4": { + "dispersion": 2.313291, + "mean_fro": 8.648088, + "rms_per_prompt_fro": 15.741636, + "max_per_prompt_fro": 26.827032, + "heavy_tail_ratio": 1.7042, + "n_prompts": 800 + }, + "5": { + "dispersion": 1.982941, + "mean_fro": 8.915126, + "rms_per_prompt_fro": 15.397486, + "max_per_prompt_fro": 26.705669, + "heavy_tail_ratio": 1.7344, + "n_prompts": 800 + }, + "6": { + "dispersion": 1.68991, + "mean_fro": 9.303899, + "rms_per_prompt_fro": 15.259274, + "max_per_prompt_fro": 24.696411, + "heavy_tail_ratio": 1.6185, + "n_prompts": 800 + }, + "7": { + "dispersion": 1.486242, + "mean_fro": 10.209386, + "rms_per_prompt_fro": 16.097978, + "max_per_prompt_fro": 26.140972, + "heavy_tail_ratio": 1.6239, + "n_prompts": 800 + }, + "8": { + "dispersion": 1.322067, + "mean_fro": 11.394146, + "rms_per_prompt_fro": 17.362775, + "max_per_prompt_fro": 28.608419, + "heavy_tail_ratio": 1.6477, + "n_prompts": 800 + }, + "9": { + "dispersion": 1.203738, + "mean_fro": 12.610738, + "rms_per_prompt_fro": 18.72063, + "max_per_prompt_fro": 29.910036, + "heavy_tail_ratio": 1.5977, + "n_prompts": 800 + }, + "10": { + "dispersion": 1.089496, + "mean_fro": 14.088331, + "rms_per_prompt_fro": 20.364808, + "max_per_prompt_fro": 31.275078, + "heavy_tail_ratio": 1.5357, + "n_prompts": 800 + }, + "11": { + "dispersion": 0.972358, + "mean_fro": 16.042309, + "rms_per_prompt_fro": 22.529927, + "max_per_prompt_fro": 32.343155, + "heavy_tail_ratio": 1.4356, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.806434, + "mean_fro": 18.164074, + "rms_per_prompt_fro": 24.41318, + "max_per_prompt_fro": 33.483078, + "heavy_tail_ratio": 1.3715, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.700069, + "mean_fro": 19.652124, + "rms_per_prompt_fro": 25.623753, + "max_per_prompt_fro": 33.654221, + "heavy_tail_ratio": 1.3134, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.569292, + "mean_fro": 22.408438, + "rms_per_prompt_fro": 28.071363, + "max_per_prompt_fro": 36.006378, + "heavy_tail_ratio": 1.2827, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.474228, + "mean_fro": 24.811038, + "rms_per_prompt_fro": 30.125011, + "max_per_prompt_fro": 38.474888, + "heavy_tail_ratio": 1.2772, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.341, + "mean_fro": 28.268051, + "rms_per_prompt_fro": 32.734846, + "max_per_prompt_fro": 40.500286, + "heavy_tail_ratio": 1.2372, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.249265, + "mean_fro": 31.176923, + "rms_per_prompt_fro": 34.846615, + "max_per_prompt_fro": 41.440643, + "heavy_tail_ratio": 1.1892, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.176672, + "mean_fro": 32.976212, + "rms_per_prompt_fro": 35.770792, + "max_per_prompt_fro": 40.749237, + "heavy_tail_ratio": 1.1392, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.113946, + "mean_fro": 33.968852, + "rms_per_prompt_fro": 35.851971, + "max_per_prompt_fro": 39.937675, + "heavy_tail_ratio": 1.114, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.05412, + "mean_fro": 34.684479, + "rms_per_prompt_fro": 35.610672, + "max_per_prompt_fro": 38.154331, + "heavy_tail_ratio": 1.0714, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.02526, + "mean_fro": 34.347695, + "rms_per_prompt_fro": 34.778801, + "max_per_prompt_fro": 36.432076, + "heavy_tail_ratio": 1.0475, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2495.7, + "fit_secs_per_prompt": 3.12, + "total_wall_seconds": 2508.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.8567, + "top_sv": 1.8101, + "stable_rank": 18.84 + }, + { + "layer": 1, + "fro_norm": 7.3976, + "top_sv": 1.6887, + "stable_rank": 19.19 + }, + { + "layer": 2, + "fro_norm": 8.0217, + "top_sv": 1.8755, + "stable_rank": 18.293 + }, + { + "layer": 3, + "fro_norm": 7.7155, + "top_sv": 1.8153, + "stable_rank": 18.065 + }, + { + "layer": 4, + "fro_norm": 8.648, + "top_sv": 2.0535, + "stable_rank": 17.736 + }, + { + "layer": 5, + "fro_norm": 8.915, + "top_sv": 2.1992, + "stable_rank": 16.433 + }, + { + "layer": 6, + "fro_norm": 9.3038, + "top_sv": 2.2904, + "stable_rank": 16.5 + }, + { + "layer": 7, + "fro_norm": 10.2092, + "top_sv": 2.3803, + "stable_rank": 18.395 + }, + { + "layer": 8, + "fro_norm": 11.394, + "top_sv": 2.7217, + "stable_rank": 17.526 + }, + { + "layer": 9, + "fro_norm": 12.6105, + "top_sv": 2.9839, + "stable_rank": 17.861 + }, + { + "layer": 10, + "fro_norm": 14.0881, + "top_sv": 3.3199, + "stable_rank": 18.007 + }, + { + "layer": 11, + "fro_norm": 16.0421, + "top_sv": 3.7655, + "stable_rank": 18.15 + }, + { + "layer": 12, + "fro_norm": 18.1638, + "top_sv": 4.3425, + "stable_rank": 17.496 + }, + { + "layer": 13, + "fro_norm": 19.6518, + "top_sv": 4.7837, + "stable_rank": 16.876 + }, + { + "layer": 14, + "fro_norm": 22.4081, + "top_sv": 5.4847, + "stable_rank": 16.692 + }, + { + "layer": 15, + "fro_norm": 24.8106, + "top_sv": 6.2374, + "stable_rank": 15.822 + }, + { + "layer": 16, + "fro_norm": 28.2675, + "top_sv": 7.2966, + "stable_rank": 15.008 + }, + { + "layer": 17, + "fro_norm": 31.1764, + "top_sv": 7.8161, + "stable_rank": 15.91 + }, + { + "layer": 18, + "fro_norm": 32.9755, + "top_sv": 7.8874, + "stable_rank": 17.479 + }, + { + "layer": 19, + "fro_norm": 33.9679, + "top_sv": 7.6067, + "stable_rank": 19.941 + }, + { + "layer": 20, + "fro_norm": 34.6833, + "top_sv": 7.0269, + "stable_rank": 24.362 + }, + { + "layer": 21, + "fro_norm": 34.3462, + "top_sv": 5.3368, + "stable_rank": 41.419 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n800_s1.pt b/results/lenses/e28/e28_Github_410m_n800_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..736287a9a2f722aa35d28a9b44fc376e3657156a --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n800_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da8d4fbac1f4c3251cbe2e24251e8dd12bd6cc50db1c1a640c344c6a9d4abd76 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n800_s1_provenance.json b/results/lenses/e28/e28_Github_410m_n800_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ecb71583c87a7855c7602851e96364a76bece685 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n800_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n800_s1.pt", + "sha256": "da8d4fbac1f4c3251cbe2e24251e8dd12bd6cc50db1c1a640c344c6a9d4abd76", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 4.809027, + "mean_fro": 7.940568, + "rms_per_prompt_fro": 19.138297, + "max_per_prompt_fro": 48.171146, + "heavy_tail_ratio": 2.517, + "n_prompts": 800 + }, + "1": { + "dispersion": 4.257743, + "mean_fro": 7.472869, + "rms_per_prompt_fro": 17.135115, + "max_per_prompt_fro": 42.972279, + "heavy_tail_ratio": 2.5078, + "n_prompts": 800 + }, + "2": { + "dispersion": 3.429588, + "mean_fro": 8.092362, + "rms_per_prompt_fro": 17.031661, + "max_per_prompt_fro": 40.723667, + "heavy_tail_ratio": 2.3911, + "n_prompts": 800 + }, + "3": { + "dispersion": 2.782543, + "mean_fro": 7.792934, + "rms_per_prompt_fro": 15.156291, + "max_per_prompt_fro": 38.985329, + "heavy_tail_ratio": 2.5722, + "n_prompts": 800 + }, + "4": { + "dispersion": 2.316555, + "mean_fro": 8.712758, + "rms_per_prompt_fro": 15.867162, + "max_per_prompt_fro": 37.331726, + "heavy_tail_ratio": 2.3528, + "n_prompts": 800 + }, + "5": { + "dispersion": 1.990998, + "mean_fro": 8.968841, + "rms_per_prompt_fro": 15.511164, + "max_per_prompt_fro": 34.034382, + "heavy_tail_ratio": 2.1942, + "n_prompts": 800 + }, + "6": { + "dispersion": 1.699045, + "mean_fro": 9.352987, + "rms_per_prompt_fro": 15.365809, + "max_per_prompt_fro": 33.506432, + "heavy_tail_ratio": 2.1806, + "n_prompts": 800 + }, + "7": { + "dispersion": 1.497647, + "mean_fro": 10.242533, + "rms_per_prompt_fro": 16.187243, + "max_per_prompt_fro": 34.340145, + "heavy_tail_ratio": 2.1214, + "n_prompts": 800 + }, + "8": { + "dispersion": 1.332699, + "mean_fro": 11.417873, + "rms_per_prompt_fro": 17.438717, + "max_per_prompt_fro": 34.944042, + "heavy_tail_ratio": 2.0038, + "n_prompts": 800 + }, + "9": { + "dispersion": 1.213649, + "mean_fro": 12.628009, + "rms_per_prompt_fro": 18.788376, + "max_per_prompt_fro": 37.255829, + "heavy_tail_ratio": 1.9829, + "n_prompts": 800 + }, + "10": { + "dispersion": 1.099426, + "mean_fro": 14.103224, + "rms_per_prompt_fro": 20.434719, + "max_per_prompt_fro": 39.984081, + "heavy_tail_ratio": 1.9567, + "n_prompts": 800 + }, + "11": { + "dispersion": 0.982984, + "mean_fro": 16.043213, + "rms_per_prompt_fro": 22.591807, + "max_per_prompt_fro": 43.035824, + "heavy_tail_ratio": 1.9049, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.815207, + "mean_fro": 18.175756, + "rms_per_prompt_fro": 24.488129, + "max_per_prompt_fro": 45.397015, + "heavy_tail_ratio": 1.8538, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.706576, + "mean_fro": 19.68203, + "rms_per_prompt_fro": 25.711817, + "max_per_prompt_fro": 44.361992, + "heavy_tail_ratio": 1.7254, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.573762, + "mean_fro": 22.428335, + "rms_per_prompt_fro": 28.136276, + "max_per_prompt_fro": 44.304882, + "heavy_tail_ratio": 1.5747, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.478603, + "mean_fro": 24.837578, + "rms_per_prompt_fro": 30.20195, + "max_per_prompt_fro": 43.403439, + "heavy_tail_ratio": 1.4371, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.340722, + "mean_fro": 28.290195, + "rms_per_prompt_fro": 32.757095, + "max_per_prompt_fro": 39.248611, + "heavy_tail_ratio": 1.1982, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.248028, + "mean_fro": 31.185976, + "rms_per_prompt_fro": 34.839468, + "max_per_prompt_fro": 40.383511, + "heavy_tail_ratio": 1.1591, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.174742, + "mean_fro": 32.979614, + "rms_per_prompt_fro": 35.745124, + "max_per_prompt_fro": 40.659569, + "heavy_tail_ratio": 1.1375, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.112373, + "mean_fro": 33.960464, + "rms_per_prompt_fro": 35.817795, + "max_per_prompt_fro": 40.076298, + "heavy_tail_ratio": 1.1189, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.053119, + "mean_fro": 34.685622, + "rms_per_prompt_fro": 35.594934, + "max_per_prompt_fro": 37.898232, + "heavy_tail_ratio": 1.0647, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.024679, + "mean_fro": 34.337608, + "rms_per_prompt_fro": 34.758732, + "max_per_prompt_fro": 36.227936, + "heavy_tail_ratio": 1.0423, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2442.1, + "fit_secs_per_prompt": 3.053, + "total_wall_seconds": 2452.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9405, + "top_sv": 1.8637, + "stable_rank": 18.154 + }, + { + "layer": 1, + "fro_norm": 7.4728, + "top_sv": 1.7344, + "stable_rank": 18.563 + }, + { + "layer": 2, + "fro_norm": 8.0923, + "top_sv": 1.9132, + "stable_rank": 17.891 + }, + { + "layer": 3, + "fro_norm": 7.7928, + "top_sv": 1.8536, + "stable_rank": 17.676 + }, + { + "layer": 4, + "fro_norm": 8.7126, + "top_sv": 2.0921, + "stable_rank": 17.343 + }, + { + "layer": 5, + "fro_norm": 8.9687, + "top_sv": 2.2251, + "stable_rank": 16.247 + }, + { + "layer": 6, + "fro_norm": 9.3528, + "top_sv": 2.3181, + "stable_rank": 16.279 + }, + { + "layer": 7, + "fro_norm": 10.2424, + "top_sv": 2.397, + "stable_rank": 18.259 + }, + { + "layer": 8, + "fro_norm": 11.4177, + "top_sv": 2.734, + "stable_rank": 17.44 + }, + { + "layer": 9, + "fro_norm": 12.6278, + "top_sv": 3.0025, + "stable_rank": 17.688 + }, + { + "layer": 10, + "fro_norm": 14.103, + "top_sv": 3.3391, + "stable_rank": 17.838 + }, + { + "layer": 11, + "fro_norm": 16.043, + "top_sv": 3.8092, + "stable_rank": 17.738 + }, + { + "layer": 12, + "fro_norm": 18.1754, + "top_sv": 4.3601, + "stable_rank": 17.377 + }, + { + "layer": 13, + "fro_norm": 19.6817, + "top_sv": 4.8429, + "stable_rank": 16.516 + }, + { + "layer": 14, + "fro_norm": 22.428, + "top_sv": 5.5552, + "stable_rank": 16.3 + }, + { + "layer": 15, + "fro_norm": 24.8371, + "top_sv": 6.3395, + "stable_rank": 15.349 + }, + { + "layer": 16, + "fro_norm": 28.2896, + "top_sv": 7.3847, + "stable_rank": 14.675 + }, + { + "layer": 17, + "fro_norm": 31.1854, + "top_sv": 7.8975, + "stable_rank": 15.593 + }, + { + "layer": 18, + "fro_norm": 32.9788, + "top_sv": 7.9388, + "stable_rank": 17.257 + }, + { + "layer": 19, + "fro_norm": 33.9595, + "top_sv": 7.6341, + "stable_rank": 19.789 + }, + { + "layer": 20, + "fro_norm": 34.6844, + "top_sv": 7.0637, + "stable_rank": 24.111 + }, + { + "layer": 21, + "fro_norm": 34.3361, + "top_sv": 5.3736, + "stable_rank": 40.829 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_410m_n800_s2.pt b/results/lenses/e28/e28_Github_410m_n800_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..48ee9b138882be77d3af032fa7aae04c52b8b3e1 --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n800_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:448959b64da304821a20fb59e5ee485bf6185efb9394e1d2f1ffd75ee825a505 +size 46144865 diff --git a/results/lenses/e28/e28_Github_410m_n800_s2_provenance.json b/results/lenses/e28/e28_Github_410m_n800_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c6f8f67d917059c9016ccbc8d18c9e82b58eb82f --- /dev/null +++ b/results/lenses/e28/e28_Github_410m_n800_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Github_410m_n800_s2.pt", + "sha256": "448959b64da304821a20fb59e5ee485bf6185efb9394e1d2f1ffd75ee825a505", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Github", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 5.104112, + "mean_fro": 7.892968, + "rms_per_prompt_fro": 19.500763, + "max_per_prompt_fro": 48.171146, + "heavy_tail_ratio": 2.4702, + "n_prompts": 800 + }, + "1": { + "dispersion": 4.508725, + "mean_fro": 7.42996, + "rms_per_prompt_fro": 17.438616, + "max_per_prompt_fro": 42.972279, + "heavy_tail_ratio": 2.4642, + "n_prompts": 800 + }, + "2": { + "dispersion": 3.621842, + "mean_fro": 8.05822, + "rms_per_prompt_fro": 17.323939, + "max_per_prompt_fro": 40.723667, + "heavy_tail_ratio": 2.3507, + "n_prompts": 800 + }, + "3": { + "dispersion": 2.942882, + "mean_fro": 7.757555, + "rms_per_prompt_fro": 15.403937, + "max_per_prompt_fro": 38.985329, + "heavy_tail_ratio": 2.5309, + "n_prompts": 800 + }, + "4": { + "dispersion": 2.434775, + "mean_fro": 8.685299, + "rms_per_prompt_fro": 16.096592, + "max_per_prompt_fro": 37.331726, + "heavy_tail_ratio": 2.3192, + "n_prompts": 800 + }, + "5": { + "dispersion": 2.088621, + "mean_fro": 8.947359, + "rms_per_prompt_fro": 15.724512, + "max_per_prompt_fro": 34.034382, + "heavy_tail_ratio": 2.1644, + "n_prompts": 800 + }, + "6": { + "dispersion": 1.776515, + "mean_fro": 9.333132, + "rms_per_prompt_fro": 15.551682, + "max_per_prompt_fro": 33.506432, + "heavy_tail_ratio": 2.1545, + "n_prompts": 800 + }, + "7": { + "dispersion": 1.559439, + "mean_fro": 10.215868, + "rms_per_prompt_fro": 16.343599, + "max_per_prompt_fro": 34.340145, + "heavy_tail_ratio": 2.1011, + "n_prompts": 800 + }, + "8": { + "dispersion": 1.386583, + "mean_fro": 11.386799, + "rms_per_prompt_fro": 17.590974, + "max_per_prompt_fro": 34.944042, + "heavy_tail_ratio": 1.9865, + "n_prompts": 800 + }, + "9": { + "dispersion": 1.261097, + "mean_fro": 12.59927, + "rms_per_prompt_fro": 18.945452, + "max_per_prompt_fro": 37.255829, + "heavy_tail_ratio": 1.9665, + "n_prompts": 800 + }, + "10": { + "dispersion": 1.139963, + "mean_fro": 14.07068, + "rms_per_prompt_fro": 20.583451, + "max_per_prompt_fro": 39.984081, + "heavy_tail_ratio": 1.9425, + "n_prompts": 800 + }, + "11": { + "dispersion": 1.015619, + "mean_fro": 15.994929, + "rms_per_prompt_fro": 22.7084, + "max_per_prompt_fro": 43.035824, + "heavy_tail_ratio": 1.8951, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.840514, + "mean_fro": 18.128898, + "rms_per_prompt_fro": 24.594667, + "max_per_prompt_fro": 45.397015, + "heavy_tail_ratio": 1.8458, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.726406, + "mean_fro": 19.627714, + "rms_per_prompt_fro": 25.789396, + "max_per_prompt_fro": 44.361992, + "heavy_tail_ratio": 1.7202, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.588609, + "mean_fro": 22.371956, + "rms_per_prompt_fro": 28.197624, + "max_per_prompt_fro": 44.304882, + "heavy_tail_ratio": 1.5712, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.488984, + "mean_fro": 24.775534, + "rms_per_prompt_fro": 30.232079, + "max_per_prompt_fro": 43.403439, + "heavy_tail_ratio": 1.4357, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.347815, + "mean_fro": 28.24378, + "rms_per_prompt_fro": 32.789742, + "max_per_prompt_fro": 39.856323, + "heavy_tail_ratio": 1.2155, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.252838, + "mean_fro": 31.163729, + "rms_per_prompt_fro": 34.881637, + "max_per_prompt_fro": 40.543797, + "heavy_tail_ratio": 1.1623, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.178336, + "mean_fro": 32.963062, + "rms_per_prompt_fro": 35.781799, + "max_per_prompt_fro": 40.60183, + "heavy_tail_ratio": 1.1347, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.11472, + "mean_fro": 33.949578, + "rms_per_prompt_fro": 35.844064, + "max_per_prompt_fro": 39.687809, + "heavy_tail_ratio": 1.1072, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.054369, + "mean_fro": 34.689022, + "rms_per_prompt_fro": 35.619538, + "max_per_prompt_fro": 38.625275, + "heavy_tail_ratio": 1.0844, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.025334, + "mean_fro": 34.35202, + "rms_per_prompt_fro": 34.784434, + "max_per_prompt_fro": 36.567379, + "heavy_tail_ratio": 1.0513, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2465.4, + "fit_secs_per_prompt": 3.082, + "total_wall_seconds": 2473.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.8929, + "top_sv": 1.8987, + "stable_rank": 17.281 + }, + { + "layer": 1, + "fro_norm": 7.4299, + "top_sv": 1.7574, + "stable_rank": 17.873 + }, + { + "layer": 2, + "fro_norm": 8.0581, + "top_sv": 1.9565, + "stable_rank": 16.963 + }, + { + "layer": 3, + "fro_norm": 7.7575, + "top_sv": 1.8923, + "stable_rank": 16.806 + }, + { + "layer": 4, + "fro_norm": 8.6852, + "top_sv": 2.1309, + "stable_rank": 16.612 + }, + { + "layer": 5, + "fro_norm": 8.9472, + "top_sv": 2.293, + "stable_rank": 15.225 + }, + { + "layer": 6, + "fro_norm": 9.333, + "top_sv": 2.3859, + "stable_rank": 15.302 + }, + { + "layer": 7, + "fro_norm": 10.2157, + "top_sv": 2.4347, + "stable_rank": 17.606 + }, + { + "layer": 8, + "fro_norm": 11.3867, + "top_sv": 2.7639, + "stable_rank": 16.973 + }, + { + "layer": 9, + "fro_norm": 12.5991, + "top_sv": 3.0311, + "stable_rank": 17.277 + }, + { + "layer": 10, + "fro_norm": 14.0705, + "top_sv": 3.3729, + "stable_rank": 17.402 + }, + { + "layer": 11, + "fro_norm": 15.9947, + "top_sv": 3.7616, + "stable_rank": 18.08 + }, + { + "layer": 12, + "fro_norm": 18.1286, + "top_sv": 4.3383, + "stable_rank": 17.462 + }, + { + "layer": 13, + "fro_norm": 19.6274, + "top_sv": 4.8051, + "stable_rank": 16.685 + }, + { + "layer": 14, + "fro_norm": 22.3716, + "top_sv": 5.5024, + "stable_rank": 16.53 + }, + { + "layer": 15, + "fro_norm": 24.775, + "top_sv": 6.2767, + "stable_rank": 15.58 + }, + { + "layer": 16, + "fro_norm": 28.2432, + "top_sv": 7.3332, + "stable_rank": 14.833 + }, + { + "layer": 17, + "fro_norm": 31.1631, + "top_sv": 7.849, + "stable_rank": 15.764 + }, + { + "layer": 18, + "fro_norm": 32.9623, + "top_sv": 7.8956, + "stable_rank": 17.429 + }, + { + "layer": 19, + "fro_norm": 33.9486, + "top_sv": 7.586, + "stable_rank": 20.027 + }, + { + "layer": 20, + "fro_norm": 34.6878, + "top_sv": 7.0316, + "stable_rank": 24.336 + }, + { + "layer": 21, + "fro_norm": 34.3505, + "top_sv": 5.3482, + "stable_rank": 41.252 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n100_s0.pt b/results/lenses/e28/e28_Github_70m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a12b219435e6cfe6c452e81e5ff02b713e863080 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cbae5ce45c68678d6484f81907307fd79f16cb3914548dbb3dcf18224a66ae6 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n100_s0_provenance.json b/results/lenses/e28/e28_Github_70m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b0e81ceab4d3d5249da5ac1676f214e4737a7473 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n100_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n100_s0.pt", + "sha256": "7cbae5ce45c68678d6484f81907307fd79f16cb3914548dbb3dcf18224a66ae6", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.327171, + "mean_fro": 32.564206, + "rms_per_prompt_fro": 37.514917, + "max_per_prompt_fro": 45.55867, + "heavy_tail_ratio": 1.2144, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.265089, + "mean_fro": 25.540941, + "rms_per_prompt_fro": 28.727477, + "max_per_prompt_fro": 34.294407, + "heavy_tail_ratio": 1.1938, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.198287, + "mean_fro": 24.255951, + "rms_per_prompt_fro": 26.552094, + "max_per_prompt_fro": 31.167931, + "heavy_tail_ratio": 1.1738, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.107549, + "mean_fro": 23.061306, + "rms_per_prompt_fro": 24.269751, + "max_per_prompt_fro": 26.52655, + "heavy_tail_ratio": 1.093, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.2, + "fit_secs_per_prompt": 0.112, + "total_wall_seconds": 19.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.5642, + "top_sv": 4.8364, + "stable_rank": 45.334 + }, + { + "layer": 1, + "fro_norm": 25.5409, + "top_sv": 4.4818, + "stable_rank": 32.476 + }, + { + "layer": 2, + "fro_norm": 24.2559, + "top_sv": 3.8961, + "stable_rank": 38.76 + }, + { + "layer": 3, + "fro_norm": 23.0613, + "top_sv": 4.7207, + "stable_rank": 23.865 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n100_s1.pt b/results/lenses/e28/e28_Github_70m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c3791d3f1c487885728bf6fddd4a1c69ecf7634 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019c76267d59a4c58a993ba26f69bdf4a707d5874be4f7eb219e439cfd9d344a +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n100_s1_provenance.json b/results/lenses/e28/e28_Github_70m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7ff40bc495e16141e86a800495e7831944dcfd3f --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n100_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n100_s1.pt", + "sha256": "019c76267d59a4c58a993ba26f69bdf4a707d5874be4f7eb219e439cfd9d344a", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.368997, + "mean_fro": 31.797211, + "rms_per_prompt_fro": 37.204054, + "max_per_prompt_fro": 44.725929, + "heavy_tail_ratio": 1.2022, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.299839, + "mean_fro": 24.998882, + "rms_per_prompt_fro": 28.50135, + "max_per_prompt_fro": 36.304962, + "heavy_tail_ratio": 1.2738, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.22289, + "mean_fro": 23.772023, + "rms_per_prompt_fro": 26.288142, + "max_per_prompt_fro": 32.164768, + "heavy_tail_ratio": 1.2235, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.122358, + "mean_fro": 22.813432, + "rms_per_prompt_fro": 24.168868, + "max_per_prompt_fro": 27.334352, + "heavy_tail_ratio": 1.131, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 12.5, + "fit_secs_per_prompt": 0.125, + "total_wall_seconds": 21.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.7972, + "top_sv": 4.7219, + "stable_rank": 45.347 + }, + { + "layer": 1, + "fro_norm": 24.9988, + "top_sv": 4.3932, + "stable_rank": 32.38 + }, + { + "layer": 2, + "fro_norm": 23.772, + "top_sv": 3.8915, + "stable_rank": 37.315 + }, + { + "layer": 3, + "fro_norm": 22.8134, + "top_sv": 4.6537, + "stable_rank": 24.032 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n100_s2.pt b/results/lenses/e28/e28_Github_70m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..2eca085fad0364bfb9a56fb55d0be92251b7745d --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4ef9329e51dcd511a33f909aae382071abda3821f9b4e01d1af4732f9724197 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n100_s2_provenance.json b/results/lenses/e28/e28_Github_70m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a55d6389ffa046a21f5c2bf7a47088ba3017c003 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n100_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n100_s2.pt", + "sha256": "b4ef9329e51dcd511a33f909aae382071abda3821f9b4e01d1af4732f9724197", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.358088, + "mean_fro": 32.261658, + "rms_per_prompt_fro": 37.596774, + "max_per_prompt_fro": 44.985344, + "heavy_tail_ratio": 1.1965, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.2905, + "mean_fro": 25.308, + "rms_per_prompt_fro": 28.74993, + "max_per_prompt_fro": 33.807781, + "heavy_tail_ratio": 1.1759, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.21887, + "mean_fro": 23.939398, + "rms_per_prompt_fro": 26.429686, + "max_per_prompt_fro": 30.770025, + "heavy_tail_ratio": 1.1642, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.11703, + "mean_fro": 22.846368, + "rms_per_prompt_fro": 24.146242, + "max_per_prompt_fro": 27.119856, + "heavy_tail_ratio": 1.1232, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 12.5, + "fit_secs_per_prompt": 0.125, + "total_wall_seconds": 21.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.2616, + "top_sv": 4.7962, + "stable_rank": 45.246 + }, + { + "layer": 1, + "fro_norm": 25.3079, + "top_sv": 4.4598, + "stable_rank": 32.203 + }, + { + "layer": 2, + "fro_norm": 23.9394, + "top_sv": 3.8939, + "stable_rank": 37.797 + }, + { + "layer": 3, + "fro_norm": 22.8463, + "top_sv": 4.6698, + "stable_rank": 23.935 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n200_s0.pt b/results/lenses/e28/e28_Github_70m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a9385bb3d8a8bd8f7b673bbc9c6efa7ffe7f842 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39342f80a279888a8a04cc096969cd05e387afe2a5b7ae9c90b6a03df0d906b6 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n200_s0_provenance.json b/results/lenses/e28/e28_Github_70m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..bde9c3a2cad951f721d7a8329d66a5a746249a62 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n200_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n200_s0.pt", + "sha256": "39342f80a279888a8a04cc096969cd05e387afe2a5b7ae9c90b6a03df0d906b6", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.3334, + "mean_fro": 32.125621, + "rms_per_prompt_fro": 37.096396, + "max_per_prompt_fro": 45.55867, + "heavy_tail_ratio": 1.2281, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.26835, + "mean_fro": 25.276489, + "rms_per_prompt_fro": 28.466646, + "max_per_prompt_fro": 34.294407, + "heavy_tail_ratio": 1.2047, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.199926, + "mean_fro": 24.026628, + "rms_per_prompt_fro": 26.319043, + "max_per_prompt_fro": 31.167931, + "heavy_tail_ratio": 1.1842, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.108289, + "mean_fro": 22.966512, + "rms_per_prompt_fro": 24.178068, + "max_per_prompt_fro": 26.52655, + "heavy_tail_ratio": 1.0971, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 22.6, + "fit_secs_per_prompt": 0.113, + "total_wall_seconds": 30.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.1256, + "top_sv": 4.7803, + "stable_rank": 45.163 + }, + { + "layer": 1, + "fro_norm": 25.2765, + "top_sv": 4.4374, + "stable_rank": 32.447 + }, + { + "layer": 2, + "fro_norm": 24.0266, + "top_sv": 3.9235, + "stable_rank": 37.5 + }, + { + "layer": 3, + "fro_norm": 22.9665, + "top_sv": 4.7107, + "stable_rank": 23.77 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n200_s1.pt b/results/lenses/e28/e28_Github_70m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a00c5ed63f8e89032add39b6aa6eba1c6444aa4 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d483e10ac99485d770ed5c7ec3b3704664fc6d8fbae582afb2968819f87f9417 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n200_s1_provenance.json b/results/lenses/e28/e28_Github_70m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f86a2c65b8e9b0c7c4bb41806eadfb4fe92bd045 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n200_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n200_s1.pt", + "sha256": "d483e10ac99485d770ed5c7ec3b3704664fc6d8fbae582afb2968819f87f9417", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.353646, + "mean_fro": 31.953391, + "rms_per_prompt_fro": 37.176582, + "max_per_prompt_fro": 49.233932, + "heavy_tail_ratio": 1.3243, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.286721, + "mean_fro": 25.1085, + "rms_per_prompt_fro": 28.481506, + "max_per_prompt_fro": 37.362392, + "heavy_tail_ratio": 1.3118, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.213388, + "mean_fro": 23.898515, + "rms_per_prompt_fro": 26.32514, + "max_per_prompt_fro": 32.164768, + "heavy_tail_ratio": 1.2218, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.117352, + "mean_fro": 22.895727, + "rms_per_prompt_fro": 24.201903, + "max_per_prompt_fro": 27.334352, + "heavy_tail_ratio": 1.1294, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 23.9, + "fit_secs_per_prompt": 0.12, + "total_wall_seconds": 32.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.9533, + "top_sv": 4.7407, + "stable_rank": 45.43 + }, + { + "layer": 1, + "fro_norm": 25.1085, + "top_sv": 4.3953, + "stable_rank": 32.633 + }, + { + "layer": 2, + "fro_norm": 23.8985, + "top_sv": 3.8979, + "stable_rank": 37.592 + }, + { + "layer": 3, + "fro_norm": 22.8957, + "top_sv": 4.6827, + "stable_rank": 23.906 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n200_s2.pt b/results/lenses/e28/e28_Github_70m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..b26c2b335d5f81d7c61d81a18490acaf57a419ec --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6404ec1c9bdd298426724313253d2300a5950026aeaa7ff5dddd5e090c2d3dac +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n200_s2_provenance.json b/results/lenses/e28/e28_Github_70m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..3eec356fdaf8586940822c25bb9e5aad954e5d76 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n200_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n200_s2.pt", + "sha256": "6404ec1c9bdd298426724313253d2300a5950026aeaa7ff5dddd5e090c2d3dac", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.362176, + "mean_fro": 31.961943, + "rms_per_prompt_fro": 37.303511, + "max_per_prompt_fro": 46.130817, + "heavy_tail_ratio": 1.2366, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.294416, + "mean_fro": 25.116077, + "rms_per_prompt_fro": 28.57516, + "max_per_prompt_fro": 33.979523, + "heavy_tail_ratio": 1.1891, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.221449, + "mean_fro": 23.833469, + "rms_per_prompt_fro": 26.340552, + "max_per_prompt_fro": 30.770025, + "heavy_tail_ratio": 1.1682, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.118398, + "mean_fro": 22.829507, + "rms_per_prompt_fro": 24.143196, + "max_per_prompt_fro": 27.119856, + "heavy_tail_ratio": 1.1233, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 24.7, + "fit_secs_per_prompt": 0.123, + "total_wall_seconds": 33.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.9619, + "top_sv": 4.7387, + "stable_rank": 45.493 + }, + { + "layer": 1, + "fro_norm": 25.116, + "top_sv": 4.4098, + "stable_rank": 32.44 + }, + { + "layer": 2, + "fro_norm": 23.8334, + "top_sv": 3.9121, + "stable_rank": 37.115 + }, + { + "layer": 3, + "fro_norm": 22.8294, + "top_sv": 4.683, + "stable_rank": 23.765 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n25_s0.pt b/results/lenses/e28/e28_Github_70m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..805c837c922e783de1866d9a8e13c855e0ae3e2e --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cd9589c619ef6be3d7bce67bc371185f732b8972dc59a9d05a5318bb0981cb5 +size 2099753 diff --git a/results/lenses/e28/e28_Github_70m_n25_s0_provenance.json b/results/lenses/e28/e28_Github_70m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ae514aa1d0b4cb975566b07d660a48bf38f612cb --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n25_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n25_s0.pt", + "sha256": "0cd9589c619ef6be3d7bce67bc371185f732b8972dc59a9d05a5318bb0981cb5", + "bytes": 2099753, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.327985, + "mean_fro": 32.248344, + "rms_per_prompt_fro": 37.162424, + "max_per_prompt_fro": 42.198063, + "heavy_tail_ratio": 1.1355, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.264825, + "mean_fro": 25.346392, + "rms_per_prompt_fro": 28.505674, + "max_per_prompt_fro": 31.429424, + "heavy_tail_ratio": 1.1026, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.198135, + "mean_fro": 24.126537, + "rms_per_prompt_fro": 26.408755, + "max_per_prompt_fro": 28.708977, + "heavy_tail_ratio": 1.0871, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.106601, + "mean_fro": 22.955664, + "rms_per_prompt_fro": 24.148239, + "max_per_prompt_fro": 26.468372, + "heavy_tail_ratio": 1.0961, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.0, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 10.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.2483, + "top_sv": 4.7181, + "stable_rank": 46.717 + }, + { + "layer": 1, + "fro_norm": 25.3463, + "top_sv": 4.3627, + "stable_rank": 33.754 + }, + { + "layer": 2, + "fro_norm": 24.1265, + "top_sv": 3.8513, + "stable_rank": 39.245 + }, + { + "layer": 3, + "fro_norm": 22.9556, + "top_sv": 4.699, + "stable_rank": 23.865 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n25_s1.pt b/results/lenses/e28/e28_Github_70m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3e59ff74f6dbadf0c6636ce190afdf33e8f603a --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977fee0bb6a6d13a53037b14dab0c366d8e3243416d38940dbac346226a22847 +size 2099753 diff --git a/results/lenses/e28/e28_Github_70m_n25_s1_provenance.json b/results/lenses/e28/e28_Github_70m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..995b5354e52680ffae0fddc0588d319c33eb3553 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n25_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n25_s1.pt", + "sha256": "977fee0bb6a6d13a53037b14dab0c366d8e3243416d38940dbac346226a22847", + "bytes": 2099753, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.333234, + "mean_fro": 32.429128, + "rms_per_prompt_fro": 37.444542, + "max_per_prompt_fro": 44.064457, + "heavy_tail_ratio": 1.1768, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.265806, + "mean_fro": 25.402172, + "rms_per_prompt_fro": 28.579484, + "max_per_prompt_fro": 33.146976, + "heavy_tail_ratio": 1.1598, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.194499, + "mean_fro": 24.175496, + "rms_per_prompt_fro": 26.422158, + "max_per_prompt_fro": 29.527843, + "heavy_tail_ratio": 1.1175, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.106067, + "mean_fro": 23.036653, + "rms_per_prompt_fro": 24.227579, + "max_per_prompt_fro": 25.607801, + "heavy_tail_ratio": 1.057, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.8, + "fit_secs_per_prompt": 0.233, + "total_wall_seconds": 15.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.4291, + "top_sv": 4.858, + "stable_rank": 44.56 + }, + { + "layer": 1, + "fro_norm": 25.4021, + "top_sv": 4.5065, + "stable_rank": 31.773 + }, + { + "layer": 2, + "fro_norm": 24.1755, + "top_sv": 3.9119, + "stable_rank": 38.191 + }, + { + "layer": 3, + "fro_norm": 23.0366, + "top_sv": 4.6765, + "stable_rank": 24.266 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n25_s2.pt b/results/lenses/e28/e28_Github_70m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..464b8417692cc757aad2c9dadab8baf52472b796 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2131a7b899d2562c83fbf87bbbe633f48322b71ddc8c5ece93ad3d5785e83965 +size 2099753 diff --git a/results/lenses/e28/e28_Github_70m_n25_s2_provenance.json b/results/lenses/e28/e28_Github_70m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..02029b32fdb1620747f1ffc24750a7f49daed09e --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n25_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n25_s2.pt", + "sha256": "2131a7b899d2562c83fbf87bbbe633f48322b71ddc8c5ece93ad3d5785e83965", + "bytes": 2099753, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.357736, + "mean_fro": 32.245902, + "rms_per_prompt_fro": 37.573551, + "max_per_prompt_fro": 44.985344, + "heavy_tail_ratio": 1.1973, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.281501, + "mean_fro": 25.323391, + "rms_per_prompt_fro": 28.666939, + "max_per_prompt_fro": 33.242966, + "heavy_tail_ratio": 1.1596, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.207622, + "mean_fro": 23.988657, + "rms_per_prompt_fro": 26.361576, + "max_per_prompt_fro": 29.50971, + "heavy_tail_ratio": 1.1194, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.112651, + "mean_fro": 22.935917, + "rms_per_prompt_fro": 24.193329, + "max_per_prompt_fro": 26.163595, + "heavy_tail_ratio": 1.0814, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.5, + "fit_secs_per_prompt": 0.14, + "total_wall_seconds": 12.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.2459, + "top_sv": 4.8501, + "stable_rank": 44.203 + }, + { + "layer": 1, + "fro_norm": 25.3233, + "top_sv": 4.4471, + "stable_rank": 32.425 + }, + { + "layer": 2, + "fro_norm": 23.9886, + "top_sv": 3.9542, + "stable_rank": 36.805 + }, + { + "layer": 3, + "fro_norm": 22.9359, + "top_sv": 4.6836, + "stable_rank": 23.981 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n400_s0.pt b/results/lenses/e28/e28_Github_70m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d1ad9f7671d1f41476308e73ec057d16f9395e3 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed6ae13e4867a77af882f2eca7913b78e4dc00b030716cb0aae4dc64c5908b7a +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n400_s0_provenance.json b/results/lenses/e28/e28_Github_70m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..60a7edfe89415bfd3c2ce788e4677492dc080187 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n400_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n400_s0.pt", + "sha256": "ed6ae13e4867a77af882f2eca7913b78e4dc00b030716cb0aae4dc64c5908b7a", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.340993, + "mean_fro": 32.088649, + "rms_per_prompt_fro": 37.159063, + "max_per_prompt_fro": 47.552181, + "heavy_tail_ratio": 1.2797, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.276433, + "mean_fro": 25.207349, + "rms_per_prompt_fro": 28.479101, + "max_per_prompt_fro": 35.018219, + "heavy_tail_ratio": 1.2296, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.206497, + "mean_fro": 23.96093, + "rms_per_prompt_fro": 26.318845, + "max_per_prompt_fro": 31.167931, + "heavy_tail_ratio": 1.1842, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.112214, + "mean_fro": 22.907524, + "rms_per_prompt_fro": 24.158635, + "max_per_prompt_fro": 26.52655, + "heavy_tail_ratio": 1.098, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 44.9, + "fit_secs_per_prompt": 0.112, + "total_wall_seconds": 52.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.0886, + "top_sv": 4.7892, + "stable_rank": 44.892 + }, + { + "layer": 1, + "fro_norm": 25.2073, + "top_sv": 4.4442, + "stable_rank": 32.171 + }, + { + "layer": 2, + "fro_norm": 23.9609, + "top_sv": 3.8986, + "stable_rank": 37.773 + }, + { + "layer": 3, + "fro_norm": 22.9075, + "top_sv": 4.6968, + "stable_rank": 23.787 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n400_s1.pt b/results/lenses/e28/e28_Github_70m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..76c9cf2030ec6b302c4c19ac23b829e4ef436071 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cfc88ad4ae8063c5cc821f24b4acf9c2f0dd42c03e50f6c2d2086b1d6c3ad82 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n400_s1_provenance.json b/results/lenses/e28/e28_Github_70m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..17e334d46c8dcb153ca335accfc70c48c972ece9 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n400_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n400_s1.pt", + "sha256": "4cfc88ad4ae8063c5cc821f24b4acf9c2f0dd42c03e50f6c2d2086b1d6c3ad82", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.350236, + "mean_fro": 32.026067, + "rms_per_prompt_fro": 37.214177, + "max_per_prompt_fro": 49.233932, + "heavy_tail_ratio": 1.323, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.283535, + "mean_fro": 25.172195, + "rms_per_prompt_fro": 28.518386, + "max_per_prompt_fro": 37.362392, + "heavy_tail_ratio": 1.3101, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.211265, + "mean_fro": 23.922939, + "rms_per_prompt_fro": 26.32898, + "max_per_prompt_fro": 32.164768, + "heavy_tail_ratio": 1.2216, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.115027, + "mean_fro": 22.889237, + "rms_per_prompt_fro": 24.169851, + "max_per_prompt_fro": 27.334352, + "heavy_tail_ratio": 1.1309, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 49.8, + "fit_secs_per_prompt": 0.125, + "total_wall_seconds": 58.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.026, + "top_sv": 4.7502, + "stable_rank": 45.454 + }, + { + "layer": 1, + "fro_norm": 25.1722, + "top_sv": 4.4078, + "stable_rank": 32.614 + }, + { + "layer": 2, + "fro_norm": 23.9229, + "top_sv": 3.9004, + "stable_rank": 37.618 + }, + { + "layer": 3, + "fro_norm": 22.8892, + "top_sv": 4.6714, + "stable_rank": 24.009 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n400_s2.pt b/results/lenses/e28/e28_Github_70m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f13b1e38e0809b2bfdba11bb974c8365509a94a --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d15b0f8320df9a459cfb88bf43df3c45cba27dcc471810c599b488d0da9d6f +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n400_s2_provenance.json b/results/lenses/e28/e28_Github_70m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e980e198d23be7e65b9c4d3a84f77e45c26f65f4 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n400_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n400_s2.pt", + "sha256": "c4d15b0f8320df9a459cfb88bf43df3c45cba27dcc471810c599b488d0da9d6f", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.358425, + "mean_fro": 32.038349, + "rms_per_prompt_fro": 37.341175, + "max_per_prompt_fro": 46.130817, + "heavy_tail_ratio": 1.2354, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.290734, + "mean_fro": 25.179296, + "rms_per_prompt_fro": 28.606323, + "max_per_prompt_fro": 35.109913, + "heavy_tail_ratio": 1.2273, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.218367, + "mean_fro": 23.891748, + "rms_per_prompt_fro": 26.371631, + "max_per_prompt_fro": 31.680578, + "heavy_tail_ratio": 1.2013, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.116383, + "mean_fro": 22.850461, + "rms_per_prompt_fro": 24.143571, + "max_per_prompt_fro": 27.119856, + "heavy_tail_ratio": 1.1233, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 47.5, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 56.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.0383, + "top_sv": 4.7559, + "stable_rank": 45.381 + }, + { + "layer": 1, + "fro_norm": 25.1793, + "top_sv": 4.4211, + "stable_rank": 32.436 + }, + { + "layer": 2, + "fro_norm": 23.8917, + "top_sv": 3.9277, + "stable_rank": 37.001 + }, + { + "layer": 3, + "fro_norm": 22.8504, + "top_sv": 4.6719, + "stable_rank": 23.922 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n50_s0.pt b/results/lenses/e28/e28_Github_70m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a63a2680f2e8d88e7b6e21c6f1d0d33b5bebc93 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:120b35b89571b816d3c1e199a9bd9da978a8cf569c19042656bd28eb1e996b10 +size 2099753 diff --git a/results/lenses/e28/e28_Github_70m_n50_s0_provenance.json b/results/lenses/e28/e28_Github_70m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..5addff0bb9b109ccb0ddd1e8e1323fff8716ba6a --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n50_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n50_s0.pt", + "sha256": "120b35b89571b816d3c1e199a9bd9da978a8cf569c19042656bd28eb1e996b10", + "bytes": 2099753, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.320175, + "mean_fro": 32.802876, + "rms_per_prompt_fro": 37.690133, + "max_per_prompt_fro": 44.663265, + "heavy_tail_ratio": 1.185, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.258502, + "mean_fro": 25.702496, + "rms_per_prompt_fro": 28.833825, + "max_per_prompt_fro": 34.294407, + "heavy_tail_ratio": 1.1894, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.193771, + "mean_fro": 24.389649, + "rms_per_prompt_fro": 26.648087, + "max_per_prompt_fro": 31.074966, + "heavy_tail_ratio": 1.1661, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.104826, + "mean_fro": 23.093664, + "rms_per_prompt_fro": 24.273909, + "max_per_prompt_fro": 26.468372, + "heavy_tail_ratio": 1.0904, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.7, + "fit_secs_per_prompt": 0.115, + "total_wall_seconds": 18.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.8028, + "top_sv": 4.8343, + "stable_rank": 46.041 + }, + { + "layer": 1, + "fro_norm": 25.7025, + "top_sv": 4.4663, + "stable_rank": 33.118 + }, + { + "layer": 2, + "fro_norm": 24.3896, + "top_sv": 3.881, + "stable_rank": 39.494 + }, + { + "layer": 3, + "fro_norm": 23.0936, + "top_sv": 4.7207, + "stable_rank": 23.932 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n50_s1.pt b/results/lenses/e28/e28_Github_70m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..77435b7abe5c8a94fd56fa1f48d96888a32a909e --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b831f1444b7cd8a4f9afc49b4b2ac70effadcc31d24baab983b66fcd66110f4 +size 2099753 diff --git a/results/lenses/e28/e28_Github_70m_n50_s1_provenance.json b/results/lenses/e28/e28_Github_70m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..38a6e1d2f103b431369318f74fab82fcf9b7c8f6 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n50_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n50_s1.pt", + "sha256": "5b831f1444b7cd8a4f9afc49b4b2ac70effadcc31d24baab983b66fcd66110f4", + "bytes": 2099753, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.374132, + "mean_fro": 32.1231, + "rms_per_prompt_fro": 37.655784, + "max_per_prompt_fro": 44.725929, + "heavy_tail_ratio": 1.1878, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.304745, + "mean_fro": 25.172739, + "rms_per_prompt_fro": 28.753675, + "max_per_prompt_fro": 36.304962, + "heavy_tail_ratio": 1.2626, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.22509, + "mean_fro": 23.964706, + "rms_per_prompt_fro": 26.525038, + "max_per_prompt_fro": 32.164768, + "heavy_tail_ratio": 1.2126, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.123704, + "mean_fro": 22.909346, + "rms_per_prompt_fro": 24.285035, + "max_per_prompt_fro": 27.334352, + "heavy_tail_ratio": 1.1256, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 6.6, + "fit_secs_per_prompt": 0.132, + "total_wall_seconds": 16.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.1231, + "top_sv": 4.7597, + "stable_rank": 45.549 + }, + { + "layer": 1, + "fro_norm": 25.1727, + "top_sv": 4.4198, + "stable_rank": 32.438 + }, + { + "layer": 2, + "fro_norm": 23.9647, + "top_sv": 3.9277, + "stable_rank": 37.228 + }, + { + "layer": 3, + "fro_norm": 22.9093, + "top_sv": 4.6771, + "stable_rank": 23.992 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n50_s2.pt b/results/lenses/e28/e28_Github_70m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..141b272e311c51467bb4e0c8370c25a2267a3693 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e3192762cfb069f17a261abadbc9d1e73b2088ab5fe8ce80e4ae87947f78dbd +size 2099753 diff --git a/results/lenses/e28/e28_Github_70m_n50_s2_provenance.json b/results/lenses/e28/e28_Github_70m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9b13d10d4bde47aa6619f36a92beabe81cb96074 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n50_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n50_s2.pt", + "sha256": "6e3192762cfb069f17a261abadbc9d1e73b2088ab5fe8ce80e4ae87947f78dbd", + "bytes": 2099753, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.348832, + "mean_fro": 32.306281, + "rms_per_prompt_fro": 37.52026, + "max_per_prompt_fro": 44.985344, + "heavy_tail_ratio": 1.199, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.279352, + "mean_fro": 25.329843, + "rms_per_prompt_fro": 28.650191, + "max_per_prompt_fro": 33.242966, + "heavy_tail_ratio": 1.1603, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.20897, + "mean_fro": 23.939088, + "rms_per_prompt_fro": 26.321783, + "max_per_prompt_fro": 29.50971, + "heavy_tail_ratio": 1.1211, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.111675, + "mean_fro": 22.81836, + "rms_per_prompt_fro": 24.05877, + "max_per_prompt_fro": 26.163595, + "heavy_tail_ratio": 1.0875, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 6.5, + "fit_secs_per_prompt": 0.13, + "total_wall_seconds": 15.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.3062, + "top_sv": 4.8571, + "stable_rank": 44.241 + }, + { + "layer": 1, + "fro_norm": 25.3298, + "top_sv": 4.5118, + "stable_rank": 31.519 + }, + { + "layer": 2, + "fro_norm": 23.939, + "top_sv": 3.9093, + "stable_rank": 37.498 + }, + { + "layer": 3, + "fro_norm": 22.8183, + "top_sv": 4.6646, + "stable_rank": 23.93 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n800_s0.pt b/results/lenses/e28/e28_Github_70m_n800_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..2780e500587b00efa8cdc3537f8ce5752ab80b7a --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n800_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a1ccb5f2f437f158e0376944f74aa5e6a270535c4667af655e9783e145ce89 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n800_s0_provenance.json b/results/lenses/e28/e28_Github_70m_n800_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..428d5a663c6a3a4faa55c62a6a0892a24e0d5c10 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n800_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n800_s0.pt", + "sha256": "56a1ccb5f2f437f158e0376944f74aa5e6a270535c4667af655e9783e145ce89", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.346657, + "mean_fro": 31.947918, + "rms_per_prompt_fro": 37.074136, + "max_per_prompt_fro": 47.552181, + "heavy_tail_ratio": 1.2826, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.280408, + "mean_fro": 25.12513, + "rms_per_prompt_fro": 28.430374, + "max_per_prompt_fro": 35.704212, + "heavy_tail_ratio": 1.2558, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.209314, + "mean_fro": 23.901537, + "rms_per_prompt_fro": 26.284235, + "max_per_prompt_fro": 31.287842, + "heavy_tail_ratio": 1.1904, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.1135, + "mean_fro": 22.877952, + "rms_per_prompt_fro": 24.141387, + "max_per_prompt_fro": 27.428726, + "heavy_tail_ratio": 1.1362, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 89.6, + "fit_secs_per_prompt": 0.112, + "total_wall_seconds": 97.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.9479, + "top_sv": 4.7634, + "stable_rank": 44.983 + }, + { + "layer": 1, + "fro_norm": 25.1251, + "top_sv": 4.4259, + "stable_rank": 32.226 + }, + { + "layer": 2, + "fro_norm": 23.9015, + "top_sv": 3.9105, + "stable_rank": 37.358 + }, + { + "layer": 3, + "fro_norm": 22.8779, + "top_sv": 4.6913, + "stable_rank": 23.782 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n800_s1.pt b/results/lenses/e28/e28_Github_70m_n800_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..82170e1fbeea974651b2767095b588a6aa627de6 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n800_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a69469ee71f893c26b08795aacadae6a4b48c3757b0296736d7e13c683c5a33 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n800_s1_provenance.json b/results/lenses/e28/e28_Github_70m_n800_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9da819ba9f74442f9c2a68403084de091a85b9a0 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n800_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n800_s1.pt", + "sha256": "5a69469ee71f893c26b08795aacadae6a4b48c3757b0296736d7e13c683c5a33", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.349282, + "mean_fro": 31.913689, + "rms_per_prompt_fro": 37.070494, + "max_per_prompt_fro": 49.233932, + "heavy_tail_ratio": 1.3281, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.283348, + "mean_fro": 25.087875, + "rms_per_prompt_fro": 28.420783, + "max_per_prompt_fro": 37.362392, + "heavy_tail_ratio": 1.3146, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.212452, + "mean_fro": 23.849404, + "rms_per_prompt_fro": 26.260912, + "max_per_prompt_fro": 32.425072, + "heavy_tail_ratio": 1.2347, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.115184, + "mean_fro": 22.879992, + "rms_per_prompt_fro": 24.161789, + "max_per_prompt_fro": 27.334352, + "heavy_tail_ratio": 1.1313, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 99.2, + "fit_secs_per_prompt": 0.124, + "total_wall_seconds": 108.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.9136, + "top_sv": 4.7156, + "stable_rank": 45.802 + }, + { + "layer": 1, + "fro_norm": 25.0878, + "top_sv": 4.3777, + "stable_rank": 32.843 + }, + { + "layer": 2, + "fro_norm": 23.8493, + "top_sv": 3.9139, + "stable_rank": 37.131 + }, + { + "layer": 3, + "fro_norm": 22.8799, + "top_sv": 4.6802, + "stable_rank": 23.899 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Github_70m_n800_s2.pt b/results/lenses/e28/e28_Github_70m_n800_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..4481663b4bbf45e1a231ff465657d95d404afd3e --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n800_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:793f38bfd6b5ce319189af994612d9bf6b141603ef4746a114a677c05c443dd7 +size 2099827 diff --git a/results/lenses/e28/e28_Github_70m_n800_s2_provenance.json b/results/lenses/e28/e28_Github_70m_n800_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a3e88d4a2f21f02be0e2646c112a721007eb0390 --- /dev/null +++ b/results/lenses/e28/e28_Github_70m_n800_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Github_70m_n800_s2.pt", + "sha256": "793f38bfd6b5ce319189af994612d9bf6b141603ef4746a114a677c05c443dd7", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Github", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Github.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.357242, + "mean_fro": 32.114307, + "rms_per_prompt_fro": 37.413405, + "max_per_prompt_fro": 50.974575, + "heavy_tail_ratio": 1.3625, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.289341, + "mean_fro": 25.215485, + "rms_per_prompt_fro": 28.631966, + "max_per_prompt_fro": 37.426861, + "heavy_tail_ratio": 1.3072, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.217313, + "mean_fro": 23.929194, + "rms_per_prompt_fro": 26.401534, + "max_per_prompt_fro": 34.060989, + "heavy_tail_ratio": 1.2901, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.116421, + "mean_fro": 22.888561, + "rms_per_prompt_fro": 24.184245, + "max_per_prompt_fro": 27.119856, + "heavy_tail_ratio": 1.1214, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 96.8, + "fit_secs_per_prompt": 0.121, + "total_wall_seconds": 105.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.1143, + "top_sv": 4.7686, + "stable_rank": 45.354 + }, + { + "layer": 1, + "fro_norm": 25.2154, + "top_sv": 4.4266, + "stable_rank": 32.449 + }, + { + "layer": 2, + "fro_norm": 23.9292, + "top_sv": 3.9428, + "stable_rank": 36.834 + }, + { + "layer": 3, + "fro_norm": 22.8885, + "top_sv": 4.6927, + "stable_rank": 23.79 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n100_s0.pt b/results/lenses/e28/e28_Pile-CC_410m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa2f01bfadfd396fe1c4f35b17ecca375144dd26 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7592ccff0fe57aac7d2fa9b694d0ed6df64d590c681ce7a19a9b05e38d2fc02b +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n100_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..6c3c7e80147fbbea8434e5e47c5b1e69e65b1147 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n100_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n100_s0.pt", + "sha256": "7592ccff0fe57aac7d2fa9b694d0ed6df64d590c681ce7a19a9b05e38d2fc02b", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.972192, + "mean_fro": 7.180087, + "rms_per_prompt_fro": 14.31017, + "max_per_prompt_fro": 32.681721, + "heavy_tail_ratio": 2.2838, + "n_prompts": 100 + }, + "1": { + "dispersion": 2.62867, + "mean_fro": 6.744402, + "rms_per_prompt_fro": 12.847457, + "max_per_prompt_fro": 27.193319, + "heavy_tail_ratio": 2.1166, + "n_prompts": 100 + }, + "2": { + "dispersion": 2.078689, + "mean_fro": 7.497194, + "rms_per_prompt_fro": 13.154722, + "max_per_prompt_fro": 26.869106, + "heavy_tail_ratio": 2.0425, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.775269, + "mean_fro": 7.196253, + "rms_per_prompt_fro": 11.988337, + "max_per_prompt_fro": 23.182907, + "heavy_tail_ratio": 1.9338, + "n_prompts": 100 + }, + "4": { + "dispersion": 1.426601, + "mean_fro": 8.396161, + "rms_per_prompt_fro": 13.079163, + "max_per_prompt_fro": 25.410982, + "heavy_tail_ratio": 1.9429, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.280928, + "mean_fro": 8.612876, + "rms_per_prompt_fro": 13.007803, + "max_per_prompt_fro": 22.737774, + "heavy_tail_ratio": 1.748, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.100459, + "mean_fro": 8.986485, + "rms_per_prompt_fro": 13.024078, + "max_per_prompt_fro": 18.687145, + "heavy_tail_ratio": 1.4348, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.936203, + "mean_fro": 10.010284, + "rms_per_prompt_fro": 13.929061, + "max_per_prompt_fro": 19.361792, + "heavy_tail_ratio": 1.39, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.837171, + "mean_fro": 11.603151, + "rms_per_prompt_fro": 15.727174, + "max_per_prompt_fro": 21.465254, + "heavy_tail_ratio": 1.3649, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.756447, + "mean_fro": 13.095919, + "rms_per_prompt_fro": 17.356151, + "max_per_prompt_fro": 23.202282, + "heavy_tail_ratio": 1.3368, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.678553, + "mean_fro": 14.963225, + "rms_per_prompt_fro": 19.386203, + "max_per_prompt_fro": 25.345011, + "heavy_tail_ratio": 1.3074, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.60796, + "mean_fro": 17.696364, + "rms_per_prompt_fro": 22.439936, + "max_per_prompt_fro": 28.636795, + "heavy_tail_ratio": 1.2762, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.472585, + "mean_fro": 19.450712, + "rms_per_prompt_fro": 23.603458, + "max_per_prompt_fro": 29.116224, + "heavy_tail_ratio": 1.2336, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.401077, + "mean_fro": 21.325329, + "rms_per_prompt_fro": 25.242176, + "max_per_prompt_fro": 30.482628, + "heavy_tail_ratio": 1.2076, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.336003, + "mean_fro": 24.650009, + "rms_per_prompt_fro": 28.491861, + "max_per_prompt_fro": 33.269478, + "heavy_tail_ratio": 1.1677, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.281968, + "mean_fro": 27.61426, + "rms_per_prompt_fro": 31.265975, + "max_per_prompt_fro": 36.050182, + "heavy_tail_ratio": 1.153, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.20047, + "mean_fro": 30.820045, + "rms_per_prompt_fro": 33.768281, + "max_per_prompt_fro": 37.680023, + "heavy_tail_ratio": 1.1158, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.147277, + "mean_fro": 33.223564, + "rms_per_prompt_fro": 35.586098, + "max_per_prompt_fro": 38.701962, + "heavy_tail_ratio": 1.0876, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.106935, + "mean_fro": 34.75977, + "rms_per_prompt_fro": 36.571098, + "max_per_prompt_fro": 39.161575, + "heavy_tail_ratio": 1.0708, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.07238, + "mean_fro": 34.875485, + "rms_per_prompt_fro": 36.115586, + "max_per_prompt_fro": 37.874847, + "heavy_tail_ratio": 1.0487, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.042928, + "mean_fro": 34.973209, + "rms_per_prompt_fro": 35.715988, + "max_per_prompt_fro": 36.998234, + "heavy_tail_ratio": 1.0359, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.020188, + "mean_fro": 34.55188, + "rms_per_prompt_fro": 34.898912, + "max_per_prompt_fro": 35.682949, + "heavy_tail_ratio": 1.0225, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 307.7, + "fit_secs_per_prompt": 3.077, + "total_wall_seconds": 315.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.18, + "top_sv": 1.4269, + "stable_rank": 25.32 + }, + { + "layer": 1, + "fro_norm": 6.7443, + "top_sv": 1.3595, + "stable_rank": 24.61 + }, + { + "layer": 2, + "fro_norm": 7.4971, + "top_sv": 1.6439, + "stable_rank": 20.799 + }, + { + "layer": 3, + "fro_norm": 7.1962, + "top_sv": 1.5271, + "stable_rank": 22.207 + }, + { + "layer": 4, + "fro_norm": 8.3961, + "top_sv": 1.8689, + "stable_rank": 20.182 + }, + { + "layer": 5, + "fro_norm": 8.6128, + "top_sv": 1.9678, + "stable_rank": 19.156 + }, + { + "layer": 6, + "fro_norm": 8.9864, + "top_sv": 1.9873, + "stable_rank": 20.448 + }, + { + "layer": 7, + "fro_norm": 10.0101, + "top_sv": 2.228, + "stable_rank": 20.186 + }, + { + "layer": 8, + "fro_norm": 11.603, + "top_sv": 2.8114, + "stable_rank": 17.034 + }, + { + "layer": 9, + "fro_norm": 13.0957, + "top_sv": 3.1198, + "stable_rank": 17.621 + }, + { + "layer": 10, + "fro_norm": 14.963, + "top_sv": 3.4989, + "stable_rank": 18.289 + }, + { + "layer": 11, + "fro_norm": 17.6961, + "top_sv": 4.0847, + "stable_rank": 18.768 + }, + { + "layer": 12, + "fro_norm": 19.4504, + "top_sv": 4.2476, + "stable_rank": 20.968 + }, + { + "layer": 13, + "fro_norm": 21.325, + "top_sv": 4.6434, + "stable_rank": 21.092 + }, + { + "layer": 14, + "fro_norm": 24.6496, + "top_sv": 5.2719, + "stable_rank": 21.862 + }, + { + "layer": 15, + "fro_norm": 27.6137, + "top_sv": 6.1736, + "stable_rank": 20.007 + }, + { + "layer": 16, + "fro_norm": 30.8195, + "top_sv": 6.6744, + "stable_rank": 21.322 + }, + { + "layer": 17, + "fro_norm": 33.2228, + "top_sv": 7.2284, + "stable_rank": 21.124 + }, + { + "layer": 18, + "fro_norm": 34.7588, + "top_sv": 7.7075, + "stable_rank": 20.338 + }, + { + "layer": 19, + "fro_norm": 34.8743, + "top_sv": 7.657, + "stable_rank": 20.744 + }, + { + "layer": 20, + "fro_norm": 34.9719, + "top_sv": 7.13, + "stable_rank": 24.058 + }, + { + "layer": 21, + "fro_norm": 34.5502, + "top_sv": 5.6373, + "stable_rank": 37.562 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n100_s1.pt b/results/lenses/e28/e28_Pile-CC_410m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a33e24b766f644c568081ede9737c99564f0682 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd295aa65cbce257e7fd5670cf0e40ee0d10902af3ab74937fc70edc37654f2 +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n100_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e73198ec8349a59a66b3807eac09a7824b6dce78 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n100_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n100_s1.pt", + "sha256": "cbd295aa65cbce257e7fd5670cf0e40ee0d10902af3ab74937fc70edc37654f2", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.928151, + "mean_fro": 7.186759, + "rms_per_prompt_fro": 14.243842, + "max_per_prompt_fro": 23.687071, + "heavy_tail_ratio": 1.663, + "n_prompts": 100 + }, + "1": { + "dispersion": 2.610462, + "mean_fro": 6.759484, + "rms_per_prompt_fro": 12.843842, + "max_per_prompt_fro": 20.697523, + "heavy_tail_ratio": 1.6115, + "n_prompts": 100 + }, + "2": { + "dispersion": 2.03913, + "mean_fro": 7.530202, + "rms_per_prompt_fro": 13.127479, + "max_per_prompt_fro": 22.714045, + "heavy_tail_ratio": 1.7303, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.753774, + "mean_fro": 7.215669, + "rms_per_prompt_fro": 11.97404, + "max_per_prompt_fro": 21.115162, + "heavy_tail_ratio": 1.7634, + "n_prompts": 100 + }, + "4": { + "dispersion": 1.411668, + "mean_fro": 8.421621, + "rms_per_prompt_fro": 13.078395, + "max_per_prompt_fro": 22.915529, + "heavy_tail_ratio": 1.7522, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.276738, + "mean_fro": 8.62588, + "rms_per_prompt_fro": 13.015474, + "max_per_prompt_fro": 22.637657, + "heavy_tail_ratio": 1.7393, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.089955, + "mean_fro": 8.999806, + "rms_per_prompt_fro": 13.010728, + "max_per_prompt_fro": 17.876284, + "heavy_tail_ratio": 1.374, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.928981, + "mean_fro": 10.017249, + "rms_per_prompt_fro": 13.912734, + "max_per_prompt_fro": 18.760618, + "heavy_tail_ratio": 1.3484, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.825542, + "mean_fro": 11.58859, + "rms_per_prompt_fro": 15.657645, + "max_per_prompt_fro": 20.646214, + "heavy_tail_ratio": 1.3186, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.745343, + "mean_fro": 13.052509, + "rms_per_prompt_fro": 17.243858, + "max_per_prompt_fro": 21.89467, + "heavy_tail_ratio": 1.2697, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.66814, + "mean_fro": 14.917901, + "rms_per_prompt_fro": 19.26744, + "max_per_prompt_fro": 23.614281, + "heavy_tail_ratio": 1.2256, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.59559, + "mean_fro": 17.627729, + "rms_per_prompt_fro": 22.266761, + "max_per_prompt_fro": 27.168297, + "heavy_tail_ratio": 1.2201, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.467896, + "mean_fro": 19.378916, + "rms_per_prompt_fro": 23.47887, + "max_per_prompt_fro": 28.181065, + "heavy_tail_ratio": 1.2003, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.396034, + "mean_fro": 21.237143, + "rms_per_prompt_fro": 25.092511, + "max_per_prompt_fro": 29.310579, + "heavy_tail_ratio": 1.1681, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.329251, + "mean_fro": 24.555543, + "rms_per_prompt_fro": 28.310861, + "max_per_prompt_fro": 32.12635, + "heavy_tail_ratio": 1.1348, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.276261, + "mean_fro": 27.50174, + "rms_per_prompt_fro": 31.069185, + "max_per_prompt_fro": 35.279549, + "heavy_tail_ratio": 1.1355, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.196657, + "mean_fro": 30.722147, + "rms_per_prompt_fro": 33.60751, + "max_per_prompt_fro": 36.715282, + "heavy_tail_ratio": 1.0925, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.144297, + "mean_fro": 33.139111, + "rms_per_prompt_fro": 35.44951, + "max_per_prompt_fro": 37.917965, + "heavy_tail_ratio": 1.0696, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.105672, + "mean_fro": 34.677905, + "rms_per_prompt_fro": 36.464145, + "max_per_prompt_fro": 38.668629, + "heavy_tail_ratio": 1.0605, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.071198, + "mean_fro": 34.812328, + "rms_per_prompt_fro": 36.030305, + "max_per_prompt_fro": 37.653812, + "heavy_tail_ratio": 1.0451, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.04149, + "mean_fro": 34.937813, + "rms_per_prompt_fro": 35.655226, + "max_per_prompt_fro": 36.732857, + "heavy_tail_ratio": 1.0302, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.019411, + "mean_fro": 34.532005, + "rms_per_prompt_fro": 34.865537, + "max_per_prompt_fro": 35.864395, + "heavy_tail_ratio": 1.0286, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 311.1, + "fit_secs_per_prompt": 3.111, + "total_wall_seconds": 321.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.1867, + "top_sv": 1.4345, + "stable_rank": 25.1 + }, + { + "layer": 1, + "fro_norm": 6.7594, + "top_sv": 1.3767, + "stable_rank": 24.108 + }, + { + "layer": 2, + "fro_norm": 7.5301, + "top_sv": 1.6758, + "stable_rank": 20.191 + }, + { + "layer": 3, + "fro_norm": 7.2156, + "top_sv": 1.5408, + "stable_rank": 21.929 + }, + { + "layer": 4, + "fro_norm": 8.4215, + "top_sv": 1.8852, + "stable_rank": 19.957 + }, + { + "layer": 5, + "fro_norm": 8.6258, + "top_sv": 1.9891, + "stable_rank": 18.805 + }, + { + "layer": 6, + "fro_norm": 8.9997, + "top_sv": 2.0086, + "stable_rank": 20.076 + }, + { + "layer": 7, + "fro_norm": 10.0171, + "top_sv": 2.2311, + "stable_rank": 20.159 + }, + { + "layer": 8, + "fro_norm": 11.5884, + "top_sv": 2.8022, + "stable_rank": 17.103 + }, + { + "layer": 9, + "fro_norm": 13.0523, + "top_sv": 3.0833, + "stable_rank": 17.92 + }, + { + "layer": 10, + "fro_norm": 14.9177, + "top_sv": 3.4598, + "stable_rank": 18.591 + }, + { + "layer": 11, + "fro_norm": 17.6275, + "top_sv": 4.0037, + "stable_rank": 19.385 + }, + { + "layer": 12, + "fro_norm": 19.3786, + "top_sv": 4.1806, + "stable_rank": 21.486 + }, + { + "layer": 13, + "fro_norm": 21.2368, + "top_sv": 4.5762, + "stable_rank": 21.537 + }, + { + "layer": 14, + "fro_norm": 24.5551, + "top_sv": 5.2077, + "stable_rank": 22.233 + }, + { + "layer": 15, + "fro_norm": 27.5012, + "top_sv": 6.1108, + "stable_rank": 20.254 + }, + { + "layer": 16, + "fro_norm": 30.7216, + "top_sv": 6.5907, + "stable_rank": 21.729 + }, + { + "layer": 17, + "fro_norm": 33.1383, + "top_sv": 7.1491, + "stable_rank": 21.486 + }, + { + "layer": 18, + "fro_norm": 34.677, + "top_sv": 7.6665, + "stable_rank": 20.459 + }, + { + "layer": 19, + "fro_norm": 34.8112, + "top_sv": 7.5949, + "stable_rank": 21.008 + }, + { + "layer": 20, + "fro_norm": 34.9365, + "top_sv": 7.0904, + "stable_rank": 24.279 + }, + { + "layer": 21, + "fro_norm": 34.5303, + "top_sv": 5.5986, + "stable_rank": 38.04 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n100_s2.pt b/results/lenses/e28/e28_Pile-CC_410m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..aab52fb0e87e681197744424a09406cfe71aca68 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:474269538b22f5b1bc84fe7ec0cbcccefa81efb2913a6537226846864e23aa27 +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n100_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..49af678641f0dd09feacc4fabc2d93e050a14671 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n100_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n100_s2.pt", + "sha256": "474269538b22f5b1bc84fe7ec0cbcccefa81efb2913a6537226846864e23aa27", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.88477, + "mean_fro": 7.203023, + "rms_per_prompt_fro": 14.197028, + "max_per_prompt_fro": 22.358212, + "heavy_tail_ratio": 1.5749, + "n_prompts": 100 + }, + "1": { + "dispersion": 2.582452, + "mean_fro": 6.785412, + "rms_per_prompt_fro": 12.842998, + "max_per_prompt_fro": 20.413586, + "heavy_tail_ratio": 1.5895, + "n_prompts": 100 + }, + "2": { + "dispersion": 2.025045, + "mean_fro": 7.534794, + "rms_per_prompt_fro": 13.105009, + "max_per_prompt_fro": 20.335821, + "heavy_tail_ratio": 1.5518, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.721043, + "mean_fro": 7.235653, + "rms_per_prompt_fro": 11.935633, + "max_per_prompt_fro": 17.48867, + "heavy_tail_ratio": 1.4652, + "n_prompts": 100 + }, + "4": { + "dispersion": 1.379722, + "mean_fro": 8.430609, + "rms_per_prompt_fro": 13.005351, + "max_per_prompt_fro": 18.102705, + "heavy_tail_ratio": 1.3919, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.253895, + "mean_fro": 8.656929, + "rms_per_prompt_fro": 12.996628, + "max_per_prompt_fro": 17.888645, + "heavy_tail_ratio": 1.3764, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.084946, + "mean_fro": 9.025168, + "rms_per_prompt_fro": 13.031749, + "max_per_prompt_fro": 17.365171, + "heavy_tail_ratio": 1.3325, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.929743, + "mean_fro": 10.034892, + "rms_per_prompt_fro": 13.93999, + "max_per_prompt_fro": 18.48444, + "heavy_tail_ratio": 1.326, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.831163, + "mean_fro": 11.622604, + "rms_per_prompt_fro": 15.72776, + "max_per_prompt_fro": 20.566521, + "heavy_tail_ratio": 1.3077, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.75181, + "mean_fro": 13.106467, + "rms_per_prompt_fro": 17.347191, + "max_per_prompt_fro": 22.233456, + "heavy_tail_ratio": 1.2817, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.674745, + "mean_fro": 14.988751, + "rms_per_prompt_fro": 19.397233, + "max_per_prompt_fro": 24.590446, + "heavy_tail_ratio": 1.2677, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.607422, + "mean_fro": 17.739012, + "rms_per_prompt_fro": 22.490253, + "max_per_prompt_fro": 28.712776, + "heavy_tail_ratio": 1.2767, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.467659, + "mean_fro": 19.466765, + "rms_per_prompt_fro": 23.5834, + "max_per_prompt_fro": 27.595303, + "heavy_tail_ratio": 1.1701, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.396418, + "mean_fro": 21.311512, + "rms_per_prompt_fro": 25.183845, + "max_per_prompt_fro": 29.178133, + "heavy_tail_ratio": 1.1586, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.330672, + "mean_fro": 24.673301, + "rms_per_prompt_fro": 28.461829, + "max_per_prompt_fro": 32.315968, + "heavy_tail_ratio": 1.1354, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.275315, + "mean_fro": 27.654129, + "rms_per_prompt_fro": 31.229768, + "max_per_prompt_fro": 35.050976, + "heavy_tail_ratio": 1.1224, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.198573, + "mean_fro": 30.961451, + "rms_per_prompt_fro": 33.896392, + "max_per_prompt_fro": 36.951889, + "heavy_tail_ratio": 1.0901, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.143601, + "mean_fro": 33.311905, + "rms_per_prompt_fro": 35.623515, + "max_per_prompt_fro": 38.267223, + "heavy_tail_ratio": 1.0742, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.104463, + "mean_fro": 34.857197, + "rms_per_prompt_fro": 36.632629, + "max_per_prompt_fro": 39.066708, + "heavy_tail_ratio": 1.0664, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.070066, + "mean_fro": 34.961696, + "rms_per_prompt_fro": 36.165767, + "max_per_prompt_fro": 37.570126, + "heavy_tail_ratio": 1.0388, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.041328, + "mean_fro": 35.003498, + "rms_per_prompt_fro": 35.719491, + "max_per_prompt_fro": 36.963787, + "heavy_tail_ratio": 1.0348, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.019388, + "mean_fro": 34.554298, + "rms_per_prompt_fro": 34.887659, + "max_per_prompt_fro": 35.649303, + "heavy_tail_ratio": 1.0218, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 304.2, + "fit_secs_per_prompt": 3.042, + "total_wall_seconds": 313.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.2029, + "top_sv": 1.4135, + "stable_rank": 25.969 + }, + { + "layer": 1, + "fro_norm": 6.7853, + "top_sv": 1.3602, + "stable_rank": 24.886 + }, + { + "layer": 2, + "fro_norm": 7.5347, + "top_sv": 1.6259, + "stable_rank": 21.475 + }, + { + "layer": 3, + "fro_norm": 7.2356, + "top_sv": 1.5215, + "stable_rank": 22.614 + }, + { + "layer": 4, + "fro_norm": 8.4305, + "top_sv": 1.8699, + "stable_rank": 20.326 + }, + { + "layer": 5, + "fro_norm": 8.6568, + "top_sv": 1.9757, + "stable_rank": 19.198 + }, + { + "layer": 6, + "fro_norm": 9.025, + "top_sv": 1.9983, + "stable_rank": 20.398 + }, + { + "layer": 7, + "fro_norm": 10.0348, + "top_sv": 2.2302, + "stable_rank": 20.246 + }, + { + "layer": 8, + "fro_norm": 11.6225, + "top_sv": 2.7882, + "stable_rank": 17.376 + }, + { + "layer": 9, + "fro_norm": 13.1063, + "top_sv": 3.0837, + "stable_rank": 18.064 + }, + { + "layer": 10, + "fro_norm": 14.9886, + "top_sv": 3.4643, + "stable_rank": 18.719 + }, + { + "layer": 11, + "fro_norm": 17.7387, + "top_sv": 4.0593, + "stable_rank": 19.096 + }, + { + "layer": 12, + "fro_norm": 19.4665, + "top_sv": 4.2006, + "stable_rank": 21.476 + }, + { + "layer": 13, + "fro_norm": 21.3112, + "top_sv": 4.5769, + "stable_rank": 21.68 + }, + { + "layer": 14, + "fro_norm": 24.6729, + "top_sv": 5.2077, + "stable_rank": 22.447 + }, + { + "layer": 15, + "fro_norm": 27.6536, + "top_sv": 6.1078, + "stable_rank": 20.499 + }, + { + "layer": 16, + "fro_norm": 30.9609, + "top_sv": 6.6675, + "stable_rank": 21.563 + }, + { + "layer": 17, + "fro_norm": 33.3111, + "top_sv": 7.2267, + "stable_rank": 21.247 + }, + { + "layer": 18, + "fro_norm": 34.8562, + "top_sv": 7.7792, + "stable_rank": 20.077 + }, + { + "layer": 19, + "fro_norm": 34.9606, + "top_sv": 7.7486, + "stable_rank": 20.357 + }, + { + "layer": 20, + "fro_norm": 35.0021, + "top_sv": 7.1175, + "stable_rank": 24.184 + }, + { + "layer": 21, + "fro_norm": 34.5526, + "top_sv": 5.6241, + "stable_rank": 37.745 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n200_s0.pt b/results/lenses/e28/e28_Pile-CC_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8fc2ad4c7d4d6a40e34bdc233ed2df00d3c2e05 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666e119f8cdcc6b218c020862e3de834c6920dcbed894121a108defa90877325 +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n200_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..208264b363fd8581e5e6377ecefb0b721b1c9f46 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n200_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n200_s0.pt", + "sha256": "666e119f8cdcc6b218c020862e3de834c6920dcbed894121a108defa90877325", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 3.036369, + "mean_fro": 6.956831, + "rms_per_prompt_fro": 13.976772, + "max_per_prompt_fro": 32.681721, + "heavy_tail_ratio": 2.3383, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.694082, + "mean_fro": 6.545879, + "rms_per_prompt_fro": 12.581175, + "max_per_prompt_fro": 27.193319, + "heavy_tail_ratio": 2.1614, + "n_prompts": 200 + }, + "2": { + "dispersion": 2.125136, + "mean_fro": 7.26742, + "rms_per_prompt_fro": 12.847384, + "max_per_prompt_fro": 26.869106, + "heavy_tail_ratio": 2.0914, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.815448, + "mean_fro": 7.001361, + "rms_per_prompt_fro": 11.747793, + "max_per_prompt_fro": 23.182907, + "heavy_tail_ratio": 1.9734, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.461345, + "mean_fro": 8.160456, + "rms_per_prompt_fro": 12.802673, + "max_per_prompt_fro": 25.410982, + "heavy_tail_ratio": 1.9848, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.315985, + "mean_fro": 8.386017, + "rms_per_prompt_fro": 12.762144, + "max_per_prompt_fro": 22.737774, + "heavy_tail_ratio": 1.7817, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.131339, + "mean_fro": 8.762808, + "rms_per_prompt_fro": 12.792917, + "max_per_prompt_fro": 18.687145, + "heavy_tail_ratio": 1.4607, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.966026, + "mean_fro": 9.771487, + "rms_per_prompt_fro": 13.701095, + "max_per_prompt_fro": 19.361792, + "heavy_tail_ratio": 1.4132, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.861795, + "mean_fro": 11.330014, + "rms_per_prompt_fro": 15.459533, + "max_per_prompt_fro": 21.465254, + "heavy_tail_ratio": 1.3885, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.778725, + "mean_fro": 12.802855, + "rms_per_prompt_fro": 17.075021, + "max_per_prompt_fro": 23.202282, + "heavy_tail_ratio": 1.3588, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.697414, + "mean_fro": 14.656428, + "rms_per_prompt_fro": 19.095102, + "max_per_prompt_fro": 25.345011, + "heavy_tail_ratio": 1.3273, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.625186, + "mean_fro": 17.360378, + "rms_per_prompt_fro": 22.131495, + "max_per_prompt_fro": 28.636795, + "heavy_tail_ratio": 1.2939, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.490879, + "mean_fro": 19.152956, + "rms_per_prompt_fro": 23.386056, + "max_per_prompt_fro": 29.116224, + "heavy_tail_ratio": 1.245, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.416966, + "mean_fro": 21.015762, + "rms_per_prompt_fro": 25.016401, + "max_per_prompt_fro": 30.482628, + "heavy_tail_ratio": 1.2185, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.34913, + "mean_fro": 24.363184, + "rms_per_prompt_fro": 28.298344, + "max_per_prompt_fro": 33.269478, + "heavy_tail_ratio": 1.1757, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.291626, + "mean_fro": 27.333891, + "rms_per_prompt_fro": 31.064898, + "max_per_prompt_fro": 36.050182, + "heavy_tail_ratio": 1.1605, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.209382, + "mean_fro": 30.596893, + "rms_per_prompt_fro": 33.64799, + "max_per_prompt_fro": 37.680023, + "heavy_tail_ratio": 1.1198, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.153627, + "mean_fro": 33.038341, + "rms_per_prompt_fro": 35.485508, + "max_per_prompt_fro": 38.701962, + "heavy_tail_ratio": 1.0906, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.111669, + "mean_fro": 34.62125, + "rms_per_prompt_fro": 36.50316, + "max_per_prompt_fro": 39.161575, + "heavy_tail_ratio": 1.0728, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.075481, + "mean_fro": 34.803887, + "rms_per_prompt_fro": 36.093511, + "max_per_prompt_fro": 37.874847, + "heavy_tail_ratio": 1.0494, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.044254, + "mean_fro": 34.942269, + "rms_per_prompt_fro": 35.707065, + "max_per_prompt_fro": 37.17144, + "heavy_tail_ratio": 1.041, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.020729, + "mean_fro": 34.542908, + "rms_per_prompt_fro": 34.899093, + "max_per_prompt_fro": 36.107208, + "heavy_tail_ratio": 1.0346, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 617.6, + "fit_secs_per_prompt": 3.088, + "total_wall_seconds": 625.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9567, + "top_sv": 1.3351, + "stable_rank": 27.15 + }, + { + "layer": 1, + "fro_norm": 6.5458, + "top_sv": 1.282, + "stable_rank": 26.07 + }, + { + "layer": 2, + "fro_norm": 7.2673, + "top_sv": 1.5411, + "stable_rank": 22.238 + }, + { + "layer": 3, + "fro_norm": 7.0013, + "top_sv": 1.4418, + "stable_rank": 23.58 + }, + { + "layer": 4, + "fro_norm": 8.1603, + "top_sv": 1.7638, + "stable_rank": 21.406 + }, + { + "layer": 5, + "fro_norm": 8.3859, + "top_sv": 1.8685, + "stable_rank": 20.144 + }, + { + "layer": 6, + "fro_norm": 8.7627, + "top_sv": 1.892, + "stable_rank": 21.451 + }, + { + "layer": 7, + "fro_norm": 9.7714, + "top_sv": 2.1299, + "stable_rank": 21.048 + }, + { + "layer": 8, + "fro_norm": 11.3299, + "top_sv": 2.6985, + "stable_rank": 17.628 + }, + { + "layer": 9, + "fro_norm": 12.8027, + "top_sv": 3.0043, + "stable_rank": 18.16 + }, + { + "layer": 10, + "fro_norm": 14.6562, + "top_sv": 3.3834, + "stable_rank": 18.765 + }, + { + "layer": 11, + "fro_norm": 17.3601, + "top_sv": 3.9556, + "stable_rank": 19.261 + }, + { + "layer": 12, + "fro_norm": 19.1527, + "top_sv": 4.1583, + "stable_rank": 21.214 + }, + { + "layer": 13, + "fro_norm": 21.0155, + "top_sv": 4.5472, + "stable_rank": 21.359 + }, + { + "layer": 14, + "fro_norm": 24.3627, + "top_sv": 5.178, + "stable_rank": 22.137 + }, + { + "layer": 15, + "fro_norm": 27.3334, + "top_sv": 6.0859, + "stable_rank": 20.171 + }, + { + "layer": 16, + "fro_norm": 30.5963, + "top_sv": 6.6112, + "stable_rank": 21.418 + }, + { + "layer": 17, + "fro_norm": 33.0375, + "top_sv": 7.1618, + "stable_rank": 21.28 + }, + { + "layer": 18, + "fro_norm": 34.6203, + "top_sv": 7.6594, + "stable_rank": 20.43 + }, + { + "layer": 19, + "fro_norm": 34.8028, + "top_sv": 7.6223, + "stable_rank": 20.848 + }, + { + "layer": 20, + "fro_norm": 34.9409, + "top_sv": 7.1028, + "stable_rank": 24.2 + }, + { + "layer": 21, + "fro_norm": 34.5412, + "top_sv": 5.6166, + "stable_rank": 37.82 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n200_s1.pt b/results/lenses/e28/e28_Pile-CC_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b8677b87a5aaa890d3665977d413029c391aa94 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c8e7e8900cdc01e54107cd6d4d786d12d199a7c50110f29606ea3ddbe64e0bc +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n200_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..61f142fe4e25773cf9c46240df168f0a8e689bd2 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n200_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n200_s1.pt", + "sha256": "6c8e7e8900cdc01e54107cd6d4d786d12d199a7c50110f29606ea3ddbe64e0bc", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 3.009703, + "mean_fro": 7.083348, + "rms_per_prompt_fro": 14.183868, + "max_per_prompt_fro": 29.8395, + "heavy_tail_ratio": 2.1038, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.684606, + "mean_fro": 6.674137, + "rms_per_prompt_fro": 12.811224, + "max_per_prompt_fro": 27.580046, + "heavy_tail_ratio": 2.1528, + "n_prompts": 200 + }, + "2": { + "dispersion": 2.104245, + "mean_fro": 7.433255, + "rms_per_prompt_fro": 13.096554, + "max_per_prompt_fro": 26.860525, + "heavy_tail_ratio": 2.051, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.813421, + "mean_fro": 7.129845, + "rms_per_prompt_fro": 11.959072, + "max_per_prompt_fro": 23.617346, + "heavy_tail_ratio": 1.9748, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.458849, + "mean_fro": 8.323232, + "rms_per_prompt_fro": 13.051425, + "max_per_prompt_fro": 24.629473, + "heavy_tail_ratio": 1.8871, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.318642, + "mean_fro": 8.52969, + "rms_per_prompt_fro": 12.988232, + "max_per_prompt_fro": 24.065071, + "heavy_tail_ratio": 1.8528, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.137426, + "mean_fro": 8.90294, + "rms_per_prompt_fro": 13.016042, + "max_per_prompt_fro": 24.010962, + "heavy_tail_ratio": 1.8447, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.974571, + "mean_fro": 9.916944, + "rms_per_prompt_fro": 13.935231, + "max_per_prompt_fro": 24.991608, + "heavy_tail_ratio": 1.7934, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.864777, + "mean_fro": 11.481923, + "rms_per_prompt_fro": 15.679351, + "max_per_prompt_fro": 27.02479, + "heavy_tail_ratio": 1.7236, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.779669, + "mean_fro": 12.944084, + "rms_per_prompt_fro": 17.267954, + "max_per_prompt_fro": 28.387159, + "heavy_tail_ratio": 1.6439, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.697805, + "mean_fro": 14.808922, + "rms_per_prompt_fro": 19.296003, + "max_per_prompt_fro": 29.992128, + "heavy_tail_ratio": 1.5543, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.621603, + "mean_fro": 17.525823, + "rms_per_prompt_fro": 22.317765, + "max_per_prompt_fro": 32.060299, + "heavy_tail_ratio": 1.4365, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.489945, + "mean_fro": 19.298785, + "rms_per_prompt_fro": 23.556738, + "max_per_prompt_fro": 32.468254, + "heavy_tail_ratio": 1.3783, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.415342, + "mean_fro": 21.155535, + "rms_per_prompt_fro": 25.168343, + "max_per_prompt_fro": 33.436691, + "heavy_tail_ratio": 1.3285, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.345444, + "mean_fro": 24.50314, + "rms_per_prompt_fro": 28.421994, + "max_per_prompt_fro": 36.206196, + "heavy_tail_ratio": 1.2739, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.288199, + "mean_fro": 27.453876, + "rms_per_prompt_fro": 31.15984, + "max_per_prompt_fro": 39.191433, + "heavy_tail_ratio": 1.2578, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.206223, + "mean_fro": 30.68403, + "rms_per_prompt_fro": 33.699718, + "max_per_prompt_fro": 39.902481, + "heavy_tail_ratio": 1.1841, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.15132, + "mean_fro": 33.134845, + "rms_per_prompt_fro": 35.553555, + "max_per_prompt_fro": 40.019886, + "heavy_tail_ratio": 1.1256, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.110233, + "mean_fro": 34.683266, + "rms_per_prompt_fro": 36.544916, + "max_per_prompt_fro": 39.960537, + "heavy_tail_ratio": 1.0935, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.074216, + "mean_fro": 34.836257, + "rms_per_prompt_fro": 36.105833, + "max_per_prompt_fro": 38.099815, + "heavy_tail_ratio": 1.0552, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.043075, + "mean_fro": 34.965645, + "rms_per_prompt_fro": 35.71077, + "max_per_prompt_fro": 36.913273, + "heavy_tail_ratio": 1.0337, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.020113, + "mean_fro": 34.552601, + "rms_per_prompt_fro": 34.898355, + "max_per_prompt_fro": 35.864395, + "heavy_tail_ratio": 1.0277, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 622.0, + "fit_secs_per_prompt": 3.11, + "total_wall_seconds": 631.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.0832, + "top_sv": 1.4074, + "stable_rank": 25.329 + }, + { + "layer": 1, + "fro_norm": 6.674, + "top_sv": 1.3577, + "stable_rank": 24.163 + }, + { + "layer": 2, + "fro_norm": 7.4332, + "top_sv": 1.6498, + "stable_rank": 20.3 + }, + { + "layer": 3, + "fro_norm": 7.1298, + "top_sv": 1.5224, + "stable_rank": 21.932 + }, + { + "layer": 4, + "fro_norm": 8.3231, + "top_sv": 1.8642, + "stable_rank": 19.933 + }, + { + "layer": 5, + "fro_norm": 8.5296, + "top_sv": 1.966, + "stable_rank": 18.822 + }, + { + "layer": 6, + "fro_norm": 8.9028, + "top_sv": 1.9849, + "stable_rank": 20.117 + }, + { + "layer": 7, + "fro_norm": 9.9168, + "top_sv": 2.216, + "stable_rank": 20.027 + }, + { + "layer": 8, + "fro_norm": 11.4818, + "top_sv": 2.7937, + "stable_rank": 16.891 + }, + { + "layer": 9, + "fro_norm": 12.9439, + "top_sv": 3.0789, + "stable_rank": 17.674 + }, + { + "layer": 10, + "fro_norm": 14.8087, + "top_sv": 3.4645, + "stable_rank": 18.271 + }, + { + "layer": 11, + "fro_norm": 17.5256, + "top_sv": 4.0256, + "stable_rank": 18.953 + }, + { + "layer": 12, + "fro_norm": 19.2985, + "top_sv": 4.2202, + "stable_rank": 20.911 + }, + { + "layer": 13, + "fro_norm": 21.1552, + "top_sv": 4.609, + "stable_rank": 21.068 + }, + { + "layer": 14, + "fro_norm": 24.5026, + "top_sv": 5.2544, + "stable_rank": 21.746 + }, + { + "layer": 15, + "fro_norm": 27.4534, + "top_sv": 6.1679, + "stable_rank": 19.812 + }, + { + "layer": 16, + "fro_norm": 30.6834, + "top_sv": 6.6425, + "stable_rank": 21.338 + }, + { + "layer": 17, + "fro_norm": 33.134, + "top_sv": 7.2085, + "stable_rank": 21.128 + }, + { + "layer": 18, + "fro_norm": 34.6823, + "top_sv": 7.7056, + "stable_rank": 20.258 + }, + { + "layer": 19, + "fro_norm": 34.8351, + "top_sv": 7.6315, + "stable_rank": 20.836 + }, + { + "layer": 20, + "fro_norm": 34.9642, + "top_sv": 7.1239, + "stable_rank": 24.089 + }, + { + "layer": 21, + "fro_norm": 34.5509, + "top_sv": 5.6208, + "stable_rank": 37.785 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n200_s2.pt b/results/lenses/e28/e28_Pile-CC_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a0f283262269b944ce5006411f0ab76fc71dd6b --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb4fc80ba2990f27e9343260491cfafbdb1d3bcaf9abc79cfc66b51a5fd38037 +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n200_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..879abe8bcf830819f2ff7f059f7f07c226ad481b --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n200_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n200_s2.pt", + "sha256": "fb4fc80ba2990f27e9343260491cfafbdb1d3bcaf9abc79cfc66b51a5fd38037", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 3.066588, + "mean_fro": 7.002972, + "rms_per_prompt_fro": 14.122042, + "max_per_prompt_fro": 22.358212, + "heavy_tail_ratio": 1.5832, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.730623, + "mean_fro": 6.600887, + "rms_per_prompt_fro": 12.749495, + "max_per_prompt_fro": 20.413586, + "heavy_tail_ratio": 1.6011, + "n_prompts": 200 + }, + "2": { + "dispersion": 2.148243, + "mean_fro": 7.330992, + "rms_per_prompt_fro": 13.007592, + "max_per_prompt_fro": 20.335821, + "heavy_tail_ratio": 1.5634, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.835915, + "mean_fro": 7.050425, + "rms_per_prompt_fro": 11.87304, + "max_per_prompt_fro": 17.48867, + "heavy_tail_ratio": 1.473, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.475527, + "mean_fro": 8.202711, + "rms_per_prompt_fro": 12.905987, + "max_per_prompt_fro": 18.102705, + "heavy_tail_ratio": 1.4027, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.338207, + "mean_fro": 8.429496, + "rms_per_prompt_fro": 12.889708, + "max_per_prompt_fro": 17.888645, + "heavy_tail_ratio": 1.3878, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.161914, + "mean_fro": 8.794281, + "rms_per_prompt_fro": 12.930625, + "max_per_prompt_fro": 17.365171, + "heavy_tail_ratio": 1.3429, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.998302, + "mean_fro": 9.787388, + "rms_per_prompt_fro": 13.835579, + "max_per_prompt_fro": 18.48444, + "heavy_tail_ratio": 1.336, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.892248, + "mean_fro": 11.336003, + "rms_per_prompt_fro": 15.593694, + "max_per_prompt_fro": 20.566521, + "heavy_tail_ratio": 1.3189, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.806978, + "mean_fro": 12.792356, + "rms_per_prompt_fro": 17.195982, + "max_per_prompt_fro": 22.233456, + "heavy_tail_ratio": 1.2929, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.723614, + "mean_fro": 14.641533, + "rms_per_prompt_fro": 19.222353, + "max_per_prompt_fro": 24.590446, + "heavy_tail_ratio": 1.2793, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.650782, + "mean_fro": 17.32804, + "rms_per_prompt_fro": 22.263541, + "max_per_prompt_fro": 28.712776, + "heavy_tail_ratio": 1.2897, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.506858, + "mean_fro": 19.128269, + "rms_per_prompt_fro": 23.480747, + "max_per_prompt_fro": 27.595303, + "heavy_tail_ratio": 1.1752, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.432497, + "mean_fro": 20.967239, + "rms_per_prompt_fro": 25.095053, + "max_per_prompt_fro": 29.178133, + "heavy_tail_ratio": 1.1627, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.362125, + "mean_fro": 24.332775, + "rms_per_prompt_fro": 28.398806, + "max_per_prompt_fro": 33.074104, + "heavy_tail_ratio": 1.1646, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.30152, + "mean_fro": 27.288885, + "rms_per_prompt_fro": 31.132303, + "max_per_prompt_fro": 35.90984, + "heavy_tail_ratio": 1.1535, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.218696, + "mean_fro": 30.574838, + "rms_per_prompt_fro": 33.752963, + "max_per_prompt_fro": 37.529732, + "heavy_tail_ratio": 1.1119, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.160258, + "mean_fro": 32.970157, + "rms_per_prompt_fro": 35.513887, + "max_per_prompt_fro": 38.829151, + "heavy_tail_ratio": 1.0934, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.117673, + "mean_fro": 34.551701, + "rms_per_prompt_fro": 36.52808, + "max_per_prompt_fro": 39.466694, + "heavy_tail_ratio": 1.0804, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.07947, + "mean_fro": 34.74973, + "rms_per_prompt_fro": 36.104123, + "max_per_prompt_fro": 37.570126, + "heavy_tail_ratio": 1.0406, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.045964, + "mean_fro": 34.901656, + "rms_per_prompt_fro": 35.694748, + "max_per_prompt_fro": 36.963787, + "heavy_tail_ratio": 1.0356, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.02151, + "mean_fro": 34.518978, + "rms_per_prompt_fro": 34.888257, + "max_per_prompt_fro": 35.681271, + "heavy_tail_ratio": 1.0227, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 608.7, + "fit_secs_per_prompt": 3.043, + "total_wall_seconds": 618.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.0029, + "top_sv": 1.3677, + "stable_rank": 26.215 + }, + { + "layer": 1, + "fro_norm": 6.6008, + "top_sv": 1.3235, + "stable_rank": 24.873 + }, + { + "layer": 2, + "fro_norm": 7.3309, + "top_sv": 1.597, + "stable_rank": 21.072 + }, + { + "layer": 3, + "fro_norm": 7.0503, + "top_sv": 1.4815, + "stable_rank": 22.647 + }, + { + "layer": 4, + "fro_norm": 8.2026, + "top_sv": 1.81, + "stable_rank": 20.536 + }, + { + "layer": 5, + "fro_norm": 8.4294, + "top_sv": 1.9166, + "stable_rank": 19.343 + }, + { + "layer": 6, + "fro_norm": 8.7942, + "top_sv": 1.9265, + "stable_rank": 20.837 + }, + { + "layer": 7, + "fro_norm": 9.7872, + "top_sv": 2.1494, + "stable_rank": 20.735 + }, + { + "layer": 8, + "fro_norm": 11.3359, + "top_sv": 2.7129, + "stable_rank": 17.46 + }, + { + "layer": 9, + "fro_norm": 12.7922, + "top_sv": 3.0036, + "stable_rank": 18.139 + }, + { + "layer": 10, + "fro_norm": 14.6413, + "top_sv": 3.3806, + "stable_rank": 18.757 + }, + { + "layer": 11, + "fro_norm": 17.3277, + "top_sv": 3.9487, + "stable_rank": 19.257 + }, + { + "layer": 12, + "fro_norm": 19.128, + "top_sv": 4.1279, + "stable_rank": 21.472 + }, + { + "layer": 13, + "fro_norm": 20.9669, + "top_sv": 4.4944, + "stable_rank": 21.764 + }, + { + "layer": 14, + "fro_norm": 24.3324, + "top_sv": 5.1302, + "stable_rank": 22.496 + }, + { + "layer": 15, + "fro_norm": 27.2884, + "top_sv": 6.0172, + "stable_rank": 20.567 + }, + { + "layer": 16, + "fro_norm": 30.5743, + "top_sv": 6.5266, + "stable_rank": 21.945 + }, + { + "layer": 17, + "fro_norm": 32.9694, + "top_sv": 7.0787, + "stable_rank": 21.693 + }, + { + "layer": 18, + "fro_norm": 34.5507, + "top_sv": 7.6014, + "stable_rank": 20.66 + }, + { + "layer": 19, + "fro_norm": 34.7486, + "top_sv": 7.5822, + "stable_rank": 21.003 + }, + { + "layer": 20, + "fro_norm": 34.9003, + "top_sv": 7.0562, + "stable_rank": 24.463 + }, + { + "layer": 21, + "fro_norm": 34.5173, + "top_sv": 5.5677, + "stable_rank": 38.434 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n25_s0.pt b/results/lenses/e28/e28_Pile-CC_410m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4007a75185c4d1a0c3348d1c3db1949bcd02c98e --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1745452027ba4e8bf117b5c261080000e145240126e432dcffd7278ad522c9f9 +size 46144865 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n25_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9514bb5cde34b00ab4da5ca09dc25832d449b3dd --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n25_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n25_s0.pt", + "sha256": "1745452027ba4e8bf117b5c261080000e145240126e432dcffd7278ad522c9f9", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.42313, + "mean_fro": 7.047497, + "rms_per_prompt_fro": 13.039071, + "max_per_prompt_fro": 17.665543, + "heavy_tail_ratio": 1.3548, + "n_prompts": 25 + }, + "1": { + "dispersion": 2.174292, + "mean_fro": 6.602843, + "rms_per_prompt_fro": 11.763983, + "max_per_prompt_fro": 15.904056, + "heavy_tail_ratio": 1.3519, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.762661, + "mean_fro": 7.291238, + "rms_per_prompt_fro": 12.118951, + "max_per_prompt_fro": 16.468616, + "heavy_tail_ratio": 1.3589, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.538494, + "mean_fro": 6.997995, + "rms_per_prompt_fro": 11.149661, + "max_per_prompt_fro": 15.067781, + "heavy_tail_ratio": 1.3514, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.263226, + "mean_fro": 8.172483, + "rms_per_prompt_fro": 12.294703, + "max_per_prompt_fro": 16.663425, + "heavy_tail_ratio": 1.3553, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.151865, + "mean_fro": 8.41049, + "rms_per_prompt_fro": 12.337547, + "max_per_prompt_fro": 16.346407, + "heavy_tail_ratio": 1.3249, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.023499, + "mean_fro": 8.783868, + "rms_per_prompt_fro": 12.495029, + "max_per_prompt_fro": 16.568392, + "heavy_tail_ratio": 1.326, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.893962, + "mean_fro": 9.783892, + "rms_per_prompt_fro": 13.464719, + "max_per_prompt_fro": 17.899071, + "heavy_tail_ratio": 1.3293, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.804074, + "mean_fro": 11.378238, + "rms_per_prompt_fro": 15.282773, + "max_per_prompt_fro": 19.758896, + "heavy_tail_ratio": 1.2929, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.728368, + "mean_fro": 12.893047, + "rms_per_prompt_fro": 16.950152, + "max_per_prompt_fro": 21.443424, + "heavy_tail_ratio": 1.2651, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.654094, + "mean_fro": 14.80251, + "rms_per_prompt_fro": 19.037745, + "max_per_prompt_fro": 23.286634, + "heavy_tail_ratio": 1.2232, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.583645, + "mean_fro": 17.578789, + "rms_per_prompt_fro": 22.12167, + "max_per_prompt_fro": 26.242798, + "heavy_tail_ratio": 1.1863, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.457996, + "mean_fro": 19.296801, + "rms_per_prompt_fro": 23.300408, + "max_per_prompt_fro": 27.498411, + "heavy_tail_ratio": 1.1802, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.389275, + "mean_fro": 21.17629, + "rms_per_prompt_fro": 24.959966, + "max_per_prompt_fro": 28.693182, + "heavy_tail_ratio": 1.1496, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.327766, + "mean_fro": 24.451534, + "rms_per_prompt_fro": 28.175189, + "max_per_prompt_fro": 32.119045, + "heavy_tail_ratio": 1.14, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.273834, + "mean_fro": 27.369965, + "rms_per_prompt_fro": 30.890911, + "max_per_prompt_fro": 34.809654, + "heavy_tail_ratio": 1.1269, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.198082, + "mean_fro": 30.63578, + "rms_per_prompt_fro": 33.532991, + "max_per_prompt_fro": 37.032356, + "heavy_tail_ratio": 1.1044, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.146076, + "mean_fro": 33.079537, + "rms_per_prompt_fro": 35.413274, + "max_per_prompt_fro": 38.701962, + "heavy_tail_ratio": 1.0929, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.105081, + "mean_fro": 34.616406, + "rms_per_prompt_fro": 36.389751, + "max_per_prompt_fro": 39.161575, + "heavy_tail_ratio": 1.0762, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.071159, + "mean_fro": 34.79594, + "rms_per_prompt_fro": 36.012681, + "max_per_prompt_fro": 37.874847, + "heavy_tail_ratio": 1.0517, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.0422, + "mean_fro": 34.929309, + "rms_per_prompt_fro": 35.6587, + "max_per_prompt_fro": 36.998234, + "heavy_tail_ratio": 1.0376, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.019883, + "mean_fro": 34.527077, + "rms_per_prompt_fro": 34.868641, + "max_per_prompt_fro": 35.682949, + "heavy_tail_ratio": 1.0234, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 76.0, + "fit_secs_per_prompt": 3.04, + "total_wall_seconds": 98.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.0474, + "top_sv": 1.4142, + "stable_rank": 24.835 + }, + { + "layer": 1, + "fro_norm": 6.6028, + "top_sv": 1.3507, + "stable_rank": 23.898 + }, + { + "layer": 2, + "fro_norm": 7.2912, + "top_sv": 1.6544, + "stable_rank": 19.422 + }, + { + "layer": 3, + "fro_norm": 6.9979, + "top_sv": 1.4909, + "stable_rank": 22.032 + }, + { + "layer": 4, + "fro_norm": 8.1724, + "top_sv": 1.8737, + "stable_rank": 19.024 + }, + { + "layer": 5, + "fro_norm": 8.4104, + "top_sv": 1.9759, + "stable_rank": 18.119 + }, + { + "layer": 6, + "fro_norm": 8.7838, + "top_sv": 1.9877, + "stable_rank": 19.527 + }, + { + "layer": 7, + "fro_norm": 9.7837, + "top_sv": 2.177, + "stable_rank": 20.198 + }, + { + "layer": 8, + "fro_norm": 11.3781, + "top_sv": 2.7781, + "stable_rank": 16.774 + }, + { + "layer": 9, + "fro_norm": 12.8928, + "top_sv": 3.0718, + "stable_rank": 17.616 + }, + { + "layer": 10, + "fro_norm": 14.8023, + "top_sv": 3.4659, + "stable_rank": 18.24 + }, + { + "layer": 11, + "fro_norm": 17.5785, + "top_sv": 4.0504, + "stable_rank": 18.835 + }, + { + "layer": 12, + "fro_norm": 19.2965, + "top_sv": 4.1741, + "stable_rank": 21.371 + }, + { + "layer": 13, + "fro_norm": 21.176, + "top_sv": 4.5636, + "stable_rank": 21.531 + }, + { + "layer": 14, + "fro_norm": 24.4511, + "top_sv": 5.1537, + "stable_rank": 22.509 + }, + { + "layer": 15, + "fro_norm": 27.3695, + "top_sv": 6.0503, + "stable_rank": 20.463 + }, + { + "layer": 16, + "fro_norm": 30.6352, + "top_sv": 6.6217, + "stable_rank": 21.405 + }, + { + "layer": 17, + "fro_norm": 33.0787, + "top_sv": 7.144, + "stable_rank": 21.44 + }, + { + "layer": 18, + "fro_norm": 34.6154, + "top_sv": 7.5827, + "stable_rank": 20.84 + }, + { + "layer": 19, + "fro_norm": 34.7948, + "top_sv": 7.5583, + "stable_rank": 21.193 + }, + { + "layer": 20, + "fro_norm": 34.9279, + "top_sv": 7.1282, + "stable_rank": 24.01 + }, + { + "layer": 21, + "fro_norm": 34.5254, + "top_sv": 5.6408, + "stable_rank": 37.463 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n25_s1.pt b/results/lenses/e28/e28_Pile-CC_410m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ba620af34dc5656a6921ecd37cd276f718e125d --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2017712b8cd2247cd421bb25e95cd0a6a248973ce4f07ff440e7679fe0b08b9 +size 46144865 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n25_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b7f4a5b7f8287538519587c72d4705b5ebfcf266 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n25_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n25_s1.pt", + "sha256": "a2017712b8cd2247cd421bb25e95cd0a6a248973ce4f07ff440e7679fe0b08b9", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.667275, + "mean_fro": 7.610957, + "rms_per_prompt_fro": 14.575081, + "max_per_prompt_fro": 20.557617, + "heavy_tail_ratio": 1.4105, + "n_prompts": 25 + }, + "1": { + "dispersion": 2.414941, + "mean_fro": 7.131857, + "rms_per_prompt_fro": 13.179358, + "max_per_prompt_fro": 18.014589, + "heavy_tail_ratio": 1.3669, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.906341, + "mean_fro": 7.886795, + "rms_per_prompt_fro": 13.445403, + "max_per_prompt_fro": 18.787886, + "heavy_tail_ratio": 1.3973, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.655733, + "mean_fro": 7.534519, + "rms_per_prompt_fro": 12.278569, + "max_per_prompt_fro": 17.171862, + "heavy_tail_ratio": 1.3985, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.343404, + "mean_fro": 8.741087, + "rms_per_prompt_fro": 13.381013, + "max_per_prompt_fro": 19.045858, + "heavy_tail_ratio": 1.4233, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.225774, + "mean_fro": 8.907432, + "rms_per_prompt_fro": 13.289023, + "max_per_prompt_fro": 18.511869, + "heavy_tail_ratio": 1.393, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.070212, + "mean_fro": 9.255034, + "rms_per_prompt_fro": 13.316358, + "max_per_prompt_fro": 17.876284, + "heavy_tail_ratio": 1.3424, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.916704, + "mean_fro": 10.28163, + "rms_per_prompt_fro": 14.23441, + "max_per_prompt_fro": 18.760618, + "heavy_tail_ratio": 1.318, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.818147, + "mean_fro": 11.873193, + "rms_per_prompt_fro": 16.009657, + "max_per_prompt_fro": 20.646214, + "heavy_tail_ratio": 1.2896, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.736895, + "mean_fro": 13.336007, + "rms_per_prompt_fro": 17.5757, + "max_per_prompt_fro": 21.89467, + "heavy_tail_ratio": 1.2457, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.650928, + "mean_fro": 15.172649, + "rms_per_prompt_fro": 19.495099, + "max_per_prompt_fro": 23.248056, + "heavy_tail_ratio": 1.1925, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.578415, + "mean_fro": 17.891239, + "rms_per_prompt_fro": 22.477657, + "max_per_prompt_fro": 26.493469, + "heavy_tail_ratio": 1.1787, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.44968, + "mean_fro": 19.54953, + "rms_per_prompt_fro": 23.538153, + "max_per_prompt_fro": 26.665606, + "heavy_tail_ratio": 1.1329, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.378511, + "mean_fro": 21.402331, + "rms_per_prompt_fro": 25.128479, + "max_per_prompt_fro": 28.148933, + "heavy_tail_ratio": 1.1202, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.316503, + "mean_fro": 24.720731, + "rms_per_prompt_fro": 28.364308, + "max_per_prompt_fro": 32.12635, + "heavy_tail_ratio": 1.1326, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.267913, + "mean_fro": 27.651568, + "rms_per_prompt_fro": 31.136125, + "max_per_prompt_fro": 35.279549, + "heavy_tail_ratio": 1.1331, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.186326, + "mean_fro": 30.907997, + "rms_per_prompt_fro": 33.664552, + "max_per_prompt_fro": 36.173946, + "heavy_tail_ratio": 1.0745, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.137023, + "mean_fro": 33.323918, + "rms_per_prompt_fro": 35.533724, + "max_per_prompt_fro": 37.810223, + "heavy_tail_ratio": 1.0641, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.09968, + "mean_fro": 34.816178, + "rms_per_prompt_fro": 36.510198, + "max_per_prompt_fro": 38.158878, + "heavy_tail_ratio": 1.0452, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.066713, + "mean_fro": 34.878586, + "rms_per_prompt_fro": 36.023234, + "max_per_prompt_fro": 36.920586, + "heavy_tail_ratio": 1.0249, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.038957, + "mean_fro": 34.972118, + "rms_per_prompt_fro": 35.64682, + "max_per_prompt_fro": 36.314686, + "heavy_tail_ratio": 1.0187, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.018514, + "mean_fro": 34.506512, + "rms_per_prompt_fro": 34.824466, + "max_per_prompt_fro": 35.260006, + "heavy_tail_ratio": 1.0125, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 76.8, + "fit_secs_per_prompt": 3.072, + "total_wall_seconds": 105.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.6109, + "top_sv": 1.7092, + "stable_rank": 19.828 + }, + { + "layer": 1, + "fro_norm": 7.1318, + "top_sv": 1.6316, + "stable_rank": 19.106 + }, + { + "layer": 2, + "fro_norm": 7.8867, + "top_sv": 1.947, + "stable_rank": 16.409 + }, + { + "layer": 3, + "fro_norm": 7.5344, + "top_sv": 1.7699, + "stable_rank": 18.121 + }, + { + "layer": 4, + "fro_norm": 8.741, + "top_sv": 2.146, + "stable_rank": 16.59 + }, + { + "layer": 5, + "fro_norm": 8.9073, + "top_sv": 2.2064, + "stable_rank": 16.297 + }, + { + "layer": 6, + "fro_norm": 9.2549, + "top_sv": 2.1981, + "stable_rank": 17.727 + }, + { + "layer": 7, + "fro_norm": 10.2815, + "top_sv": 2.447, + "stable_rank": 17.654 + }, + { + "layer": 8, + "fro_norm": 11.873, + "top_sv": 3.0362, + "stable_rank": 15.292 + }, + { + "layer": 9, + "fro_norm": 13.3358, + "top_sv": 3.3033, + "stable_rank": 16.298 + }, + { + "layer": 10, + "fro_norm": 15.1724, + "top_sv": 3.6341, + "stable_rank": 17.431 + }, + { + "layer": 11, + "fro_norm": 17.891, + "top_sv": 4.171, + "stable_rank": 18.398 + }, + { + "layer": 12, + "fro_norm": 19.5492, + "top_sv": 4.2866, + "stable_rank": 20.799 + }, + { + "layer": 13, + "fro_norm": 21.402, + "top_sv": 4.696, + "stable_rank": 20.771 + }, + { + "layer": 14, + "fro_norm": 24.7203, + "top_sv": 5.3246, + "stable_rank": 21.555 + }, + { + "layer": 15, + "fro_norm": 27.6511, + "top_sv": 6.1469, + "stable_rank": 20.235 + }, + { + "layer": 16, + "fro_norm": 30.9074, + "top_sv": 6.6308, + "stable_rank": 21.726 + }, + { + "layer": 17, + "fro_norm": 33.3232, + "top_sv": 7.201, + "stable_rank": 21.415 + }, + { + "layer": 18, + "fro_norm": 34.8153, + "top_sv": 7.647, + "stable_rank": 20.728 + }, + { + "layer": 19, + "fro_norm": 34.8775, + "top_sv": 7.5584, + "stable_rank": 21.293 + }, + { + "layer": 20, + "fro_norm": 34.9708, + "top_sv": 7.152, + "stable_rank": 23.909 + }, + { + "layer": 21, + "fro_norm": 34.5049, + "top_sv": 5.6437, + "stable_rank": 37.38 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n25_s2.pt b/results/lenses/e28/e28_Pile-CC_410m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5cd606e938d6cc09dcb1ae6b0d37c7a045924346 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e6ab007579ede452eff34d350c419b76e39a0c456db3f4137a041a1f36dd090 +size 46144865 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n25_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..75ba235f920eca142d49e1f00441c21ae8bacf46 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n25_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n25_s2.pt", + "sha256": "2e6ab007579ede452eff34d350c419b76e39a0c456db3f4137a041a1f36dd090", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.501145, + "mean_fro": 7.533795, + "rms_per_prompt_fro": 14.096746, + "max_per_prompt_fro": 18.16094, + "heavy_tail_ratio": 1.2883, + "n_prompts": 25 + }, + "1": { + "dispersion": 2.253783, + "mean_fro": 7.066357, + "rms_per_prompt_fro": 12.746468, + "max_per_prompt_fro": 16.435219, + "heavy_tail_ratio": 1.2894, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.850081, + "mean_fro": 7.747249, + "rms_per_prompt_fro": 13.079049, + "max_per_prompt_fro": 16.61902, + "heavy_tail_ratio": 1.2707, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.606401, + "mean_fro": 7.42687, + "rms_per_prompt_fro": 11.990201, + "max_per_prompt_fro": 15.148478, + "heavy_tail_ratio": 1.2634, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.325828, + "mean_fro": 8.606028, + "rms_per_prompt_fro": 13.124764, + "max_per_prompt_fro": 16.41441, + "heavy_tail_ratio": 1.2506, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.214563, + "mean_fro": 8.834551, + "rms_per_prompt_fro": 13.147056, + "max_per_prompt_fro": 16.989782, + "heavy_tail_ratio": 1.2923, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.055727, + "mean_fro": 9.17985, + "rms_per_prompt_fro": 13.161891, + "max_per_prompt_fro": 17.235607, + "heavy_tail_ratio": 1.3095, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.904746, + "mean_fro": 10.178905, + "rms_per_prompt_fro": 14.048167, + "max_per_prompt_fro": 17.64271, + "heavy_tail_ratio": 1.2559, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.816326, + "mean_fro": 11.787092, + "rms_per_prompt_fro": 15.885601, + "max_per_prompt_fro": 20.109953, + "heavy_tail_ratio": 1.2659, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.739783, + "mean_fro": 13.280719, + "rms_per_prompt_fro": 17.517379, + "max_per_prompt_fro": 22.150478, + "heavy_tail_ratio": 1.2645, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.668199, + "mean_fro": 15.18846, + "rms_per_prompt_fro": 19.617229, + "max_per_prompt_fro": 24.590446, + "heavy_tail_ratio": 1.2535, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.60679, + "mean_fro": 17.982157, + "rms_per_prompt_fro": 22.794045, + "max_per_prompt_fro": 28.712776, + "heavy_tail_ratio": 1.2597, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.457267, + "mean_fro": 19.640173, + "rms_per_prompt_fro": 23.709091, + "max_per_prompt_fro": 26.464548, + "heavy_tail_ratio": 1.1162, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.38936, + "mean_fro": 21.467352, + "rms_per_prompt_fro": 25.303804, + "max_per_prompt_fro": 28.109467, + "heavy_tail_ratio": 1.1109, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.326846, + "mean_fro": 24.86814, + "rms_per_prompt_fro": 28.645313, + "max_per_prompt_fro": 31.434555, + "heavy_tail_ratio": 1.0974, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.272715, + "mean_fro": 27.836111, + "rms_per_prompt_fro": 31.403216, + "max_per_prompt_fro": 34.146469, + "heavy_tail_ratio": 1.0874, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.198772, + "mean_fro": 31.117842, + "rms_per_prompt_fro": 34.070442, + "max_per_prompt_fro": 36.94334, + "heavy_tail_ratio": 1.0843, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.144863, + "mean_fro": 33.399955, + "rms_per_prompt_fro": 35.737381, + "max_per_prompt_fro": 38.267223, + "heavy_tail_ratio": 1.0708, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.105787, + "mean_fro": 34.919762, + "rms_per_prompt_fro": 36.720372, + "max_per_prompt_fro": 38.573795, + "heavy_tail_ratio": 1.0505, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.07096, + "mean_fro": 35.008434, + "rms_per_prompt_fro": 36.229243, + "max_per_prompt_fro": 37.383549, + "heavy_tail_ratio": 1.0319, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.042105, + "mean_fro": 35.023128, + "rms_per_prompt_fro": 35.752852, + "max_per_prompt_fro": 36.469814, + "heavy_tail_ratio": 1.0201, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.019441, + "mean_fro": 34.570409, + "rms_per_prompt_fro": 34.904827, + "max_per_prompt_fro": 35.246788, + "heavy_tail_ratio": 1.0098, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 75.6, + "fit_secs_per_prompt": 3.022, + "total_wall_seconds": 102.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.5337, + "top_sv": 1.5704, + "stable_rank": 23.013 + }, + { + "layer": 1, + "fro_norm": 7.0663, + "top_sv": 1.4792, + "stable_rank": 22.821 + }, + { + "layer": 2, + "fro_norm": 7.7472, + "top_sv": 1.6996, + "stable_rank": 20.777 + }, + { + "layer": 3, + "fro_norm": 7.4268, + "top_sv": 1.6072, + "stable_rank": 21.353 + }, + { + "layer": 4, + "fro_norm": 8.6059, + "top_sv": 1.9565, + "stable_rank": 19.348 + }, + { + "layer": 5, + "fro_norm": 8.8344, + "top_sv": 2.0874, + "stable_rank": 17.913 + }, + { + "layer": 6, + "fro_norm": 9.1797, + "top_sv": 2.0984, + "stable_rank": 19.137 + }, + { + "layer": 7, + "fro_norm": 10.1788, + "top_sv": 2.3367, + "stable_rank": 18.976 + }, + { + "layer": 8, + "fro_norm": 11.7869, + "top_sv": 2.8995, + "stable_rank": 16.525 + }, + { + "layer": 9, + "fro_norm": 13.2805, + "top_sv": 3.2205, + "stable_rank": 17.006 + }, + { + "layer": 10, + "fro_norm": 15.1883, + "top_sv": 3.6167, + "stable_rank": 17.636 + }, + { + "layer": 11, + "fro_norm": 17.9819, + "top_sv": 4.2351, + "stable_rank": 18.028 + }, + { + "layer": 12, + "fro_norm": 19.6399, + "top_sv": 4.3244, + "stable_rank": 20.627 + }, + { + "layer": 13, + "fro_norm": 21.467, + "top_sv": 4.6597, + "stable_rank": 21.224 + }, + { + "layer": 14, + "fro_norm": 24.8677, + "top_sv": 5.2768, + "stable_rank": 22.209 + }, + { + "layer": 15, + "fro_norm": 27.8356, + "top_sv": 6.1805, + "stable_rank": 20.284 + }, + { + "layer": 16, + "fro_norm": 31.1173, + "top_sv": 6.6964, + "stable_rank": 21.593 + }, + { + "layer": 17, + "fro_norm": 33.3991, + "top_sv": 7.275, + "stable_rank": 21.077 + }, + { + "layer": 18, + "fro_norm": 34.9188, + "top_sv": 7.8334, + "stable_rank": 19.871 + }, + { + "layer": 19, + "fro_norm": 35.0073, + "top_sv": 7.788, + "stable_rank": 20.205 + }, + { + "layer": 20, + "fro_norm": 35.0218, + "top_sv": 7.1531, + "stable_rank": 23.971 + }, + { + "layer": 21, + "fro_norm": 34.5688, + "top_sv": 5.6304, + "stable_rank": 37.696 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n400_s0.pt b/results/lenses/e28/e28_Pile-CC_410m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..23c7b3aedf00ef8ef3ddd1b14f877098272226c4 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4754d23b9e6eb16ac8159661cc393328374d8cc8d286bbb9c971ee95166fca62 +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n400_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ee12bc585fd619cc42e23008eed1ed2c0b414e2b --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n400_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n400_s0.pt", + "sha256": "4754d23b9e6eb16ac8159661cc393328374d8cc8d286bbb9c971ee95166fca62", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 3.177652, + "mean_fro": 6.966621, + "rms_per_prompt_fro": 14.239288, + "max_per_prompt_fro": 32.681721, + "heavy_tail_ratio": 2.2952, + "n_prompts": 400 + }, + "1": { + "dispersion": 2.812605, + "mean_fro": 6.558509, + "rms_per_prompt_fro": 12.806075, + "max_per_prompt_fro": 27.193319, + "heavy_tail_ratio": 2.1235, + "n_prompts": 400 + }, + "2": { + "dispersion": 2.221855, + "mean_fro": 7.282961, + "rms_per_prompt_fro": 13.072571, + "max_per_prompt_fro": 26.869106, + "heavy_tail_ratio": 2.0554, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.897533, + "mean_fro": 7.00468, + "rms_per_prompt_fro": 11.923465, + "max_per_prompt_fro": 23.182907, + "heavy_tail_ratio": 1.9443, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.528682, + "mean_fro": 8.158419, + "rms_per_prompt_fro": 12.97338, + "max_per_prompt_fro": 25.410982, + "heavy_tail_ratio": 1.9587, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.37875, + "mean_fro": 8.374163, + "rms_per_prompt_fro": 12.915635, + "max_per_prompt_fro": 23.339247, + "heavy_tail_ratio": 1.8071, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.184871, + "mean_fro": 8.752378, + "rms_per_prompt_fro": 12.937162, + "max_per_prompt_fro": 22.96962, + "heavy_tail_ratio": 1.7755, + "n_prompts": 400 + }, + "7": { + "dispersion": 1.00983, + "mean_fro": 9.754026, + "rms_per_prompt_fro": 13.828133, + "max_per_prompt_fro": 23.381296, + "heavy_tail_ratio": 1.6908, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.900536, + "mean_fro": 11.296676, + "rms_per_prompt_fro": 15.573589, + "max_per_prompt_fro": 26.290297, + "heavy_tail_ratio": 1.6881, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.812847, + "mean_fro": 12.755312, + "rms_per_prompt_fro": 17.17401, + "max_per_prompt_fro": 27.34864, + "heavy_tail_ratio": 1.5924, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.728189, + "mean_fro": 14.591442, + "rms_per_prompt_fro": 19.181997, + "max_per_prompt_fro": 29.051102, + "heavy_tail_ratio": 1.5145, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.653769, + "mean_fro": 17.268312, + "rms_per_prompt_fro": 22.206868, + "max_per_prompt_fro": 32.765968, + "heavy_tail_ratio": 1.4755, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.511253, + "mean_fro": 19.070321, + "rms_per_prompt_fro": 23.443723, + "max_per_prompt_fro": 30.238277, + "heavy_tail_ratio": 1.2898, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.435529, + "mean_fro": 20.919666, + "rms_per_prompt_fro": 25.0646, + "max_per_prompt_fro": 31.074577, + "heavy_tail_ratio": 1.2398, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.363134, + "mean_fro": 24.255104, + "rms_per_prompt_fro": 28.318644, + "max_per_prompt_fro": 33.278355, + "heavy_tail_ratio": 1.1751, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.303205, + "mean_fro": 27.212483, + "rms_per_prompt_fro": 31.065222, + "max_per_prompt_fro": 36.133087, + "heavy_tail_ratio": 1.1631, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.218333, + "mean_fro": 30.471796, + "rms_per_prompt_fro": 33.634194, + "max_per_prompt_fro": 38.343491, + "heavy_tail_ratio": 1.14, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.160218, + "mean_fro": 32.912415, + "rms_per_prompt_fro": 35.451079, + "max_per_prompt_fro": 38.843258, + "heavy_tail_ratio": 1.0957, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.116693, + "mean_fro": 34.509197, + "rms_per_prompt_fro": 36.467144, + "max_per_prompt_fro": 39.431427, + "heavy_tail_ratio": 1.0813, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.078765, + "mean_fro": 34.717999, + "rms_per_prompt_fro": 36.059368, + "max_per_prompt_fro": 38.199406, + "heavy_tail_ratio": 1.0593, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.045969, + "mean_fro": 34.90258, + "rms_per_prompt_fro": 35.695789, + "max_per_prompt_fro": 37.17144, + "heavy_tail_ratio": 1.0413, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.021468, + "mean_fro": 34.517609, + "rms_per_prompt_fro": 34.886149, + "max_per_prompt_fro": 36.107208, + "heavy_tail_ratio": 1.035, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1237.3, + "fit_secs_per_prompt": 3.093, + "total_wall_seconds": 1245.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9665, + "top_sv": 1.3256, + "stable_rank": 27.617 + }, + { + "layer": 1, + "fro_norm": 6.5584, + "top_sv": 1.2756, + "stable_rank": 26.436 + }, + { + "layer": 2, + "fro_norm": 7.2829, + "top_sv": 1.5314, + "stable_rank": 22.616 + }, + { + "layer": 3, + "fro_norm": 7.0046, + "top_sv": 1.4248, + "stable_rank": 24.168 + }, + { + "layer": 4, + "fro_norm": 8.1583, + "top_sv": 1.7366, + "stable_rank": 22.069 + }, + { + "layer": 5, + "fro_norm": 8.3741, + "top_sv": 1.8397, + "stable_rank": 20.72 + }, + { + "layer": 6, + "fro_norm": 8.7523, + "top_sv": 1.8689, + "stable_rank": 21.931 + }, + { + "layer": 7, + "fro_norm": 9.7539, + "top_sv": 2.1048, + "stable_rank": 21.475 + }, + { + "layer": 8, + "fro_norm": 11.2965, + "top_sv": 2.6655, + "stable_rank": 17.961 + }, + { + "layer": 9, + "fro_norm": 12.7551, + "top_sv": 2.9653, + "stable_rank": 18.503 + }, + { + "layer": 10, + "fro_norm": 14.5912, + "top_sv": 3.3439, + "stable_rank": 19.04 + }, + { + "layer": 11, + "fro_norm": 17.2681, + "top_sv": 3.9035, + "stable_rank": 19.569 + }, + { + "layer": 12, + "fro_norm": 19.07, + "top_sv": 4.1168, + "stable_rank": 21.457 + }, + { + "layer": 13, + "fro_norm": 20.9193, + "top_sv": 4.4892, + "stable_rank": 21.715 + }, + { + "layer": 14, + "fro_norm": 24.2547, + "top_sv": 5.1141, + "stable_rank": 22.493 + }, + { + "layer": 15, + "fro_norm": 27.212, + "top_sv": 6.0205, + "stable_rank": 20.429 + }, + { + "layer": 16, + "fro_norm": 30.4712, + "top_sv": 6.5384, + "stable_rank": 21.719 + }, + { + "layer": 17, + "fro_norm": 32.9116, + "top_sv": 7.0948, + "stable_rank": 21.519 + }, + { + "layer": 18, + "fro_norm": 34.5082, + "top_sv": 7.5971, + "stable_rank": 20.632 + }, + { + "layer": 19, + "fro_norm": 34.7168, + "top_sv": 7.5601, + "stable_rank": 21.088 + }, + { + "layer": 20, + "fro_norm": 34.9012, + "top_sv": 7.088, + "stable_rank": 24.246 + }, + { + "layer": 21, + "fro_norm": 34.5159, + "top_sv": 5.5928, + "stable_rank": 38.088 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n400_s1.pt b/results/lenses/e28/e28_Pile-CC_410m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f176c77cbaac925a13e650fb92a9933aaad932b --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e9a962ee48623d7cf0ed646f4a59119c2a9c6eca02e4e4c809b5ef7659a4dd +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n400_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..5e4189102ec14dde5fae9e760bb7bcef705f377c --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n400_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n400_s1.pt", + "sha256": "31e9a962ee48623d7cf0ed646f4a59119c2a9c6eca02e4e4c809b5ef7659a4dd", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 3.036196, + "mean_fro": 7.071123, + "rms_per_prompt_fro": 14.206089, + "max_per_prompt_fro": 29.8395, + "heavy_tail_ratio": 2.1005, + "n_prompts": 400 + }, + "1": { + "dispersion": 2.701819, + "mean_fro": 6.66526, + "rms_per_prompt_fro": 12.824034, + "max_per_prompt_fro": 27.580046, + "heavy_tail_ratio": 2.1507, + "n_prompts": 400 + }, + "2": { + "dispersion": 2.108312, + "mean_fro": 7.419964, + "rms_per_prompt_fro": 13.081698, + "max_per_prompt_fro": 26.860525, + "heavy_tail_ratio": 2.0533, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.806186, + "mean_fro": 7.127373, + "rms_per_prompt_fro": 11.939543, + "max_per_prompt_fro": 23.617346, + "heavy_tail_ratio": 1.9781, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.44351, + "mean_fro": 8.317573, + "rms_per_prompt_fro": 13.001805, + "max_per_prompt_fro": 24.629473, + "heavy_tail_ratio": 1.8943, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.302624, + "mean_fro": 8.536059, + "rms_per_prompt_fro": 12.952957, + "max_per_prompt_fro": 24.065071, + "heavy_tail_ratio": 1.8579, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.128119, + "mean_fro": 8.911734, + "rms_per_prompt_fro": 13.000504, + "max_per_prompt_fro": 24.010962, + "heavy_tail_ratio": 1.8469, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.969708, + "mean_fro": 9.931164, + "rms_per_prompt_fro": 13.938019, + "max_per_prompt_fro": 24.991608, + "heavy_tail_ratio": 1.7931, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.862224, + "mean_fro": 11.501196, + "rms_per_prompt_fro": 15.694916, + "max_per_prompt_fro": 27.02479, + "heavy_tail_ratio": 1.7219, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.779548, + "mean_fro": 12.97298, + "rms_per_prompt_fro": 17.305916, + "max_per_prompt_fro": 28.387159, + "heavy_tail_ratio": 1.6403, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.69983, + "mean_fro": 14.838013, + "rms_per_prompt_fro": 19.345433, + "max_per_prompt_fro": 29.992128, + "heavy_tail_ratio": 1.5503, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.625795, + "mean_fro": 17.559275, + "rms_per_prompt_fro": 22.389243, + "max_per_prompt_fro": 32.060299, + "heavy_tail_ratio": 1.432, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.494142, + "mean_fro": 19.325299, + "rms_per_prompt_fro": 23.622298, + "max_per_prompt_fro": 32.468254, + "heavy_tail_ratio": 1.3745, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.420501, + "mean_fro": 21.168661, + "rms_per_prompt_fro": 25.229822, + "max_per_prompt_fro": 33.436691, + "heavy_tail_ratio": 1.3253, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.35096, + "mean_fro": 24.515664, + "rms_per_prompt_fro": 28.494756, + "max_per_prompt_fro": 36.206196, + "heavy_tail_ratio": 1.2706, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.292997, + "mean_fro": 27.464228, + "rms_per_prompt_fro": 31.229579, + "max_per_prompt_fro": 39.191433, + "heavy_tail_ratio": 1.2549, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.211272, + "mean_fro": 30.677214, + "rms_per_prompt_fro": 33.762669, + "max_per_prompt_fro": 39.902481, + "heavy_tail_ratio": 1.1819, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.155593, + "mean_fro": 33.100536, + "rms_per_prompt_fro": 35.582588, + "max_per_prompt_fro": 40.019886, + "heavy_tail_ratio": 1.1247, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.113419, + "mean_fro": 34.639973, + "rms_per_prompt_fro": 36.551646, + "max_per_prompt_fro": 39.960537, + "heavy_tail_ratio": 1.0933, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.076712, + "mean_fro": 34.801407, + "rms_per_prompt_fro": 36.11158, + "max_per_prompt_fro": 38.099815, + "heavy_tail_ratio": 1.0551, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.044625, + "mean_fro": 34.944466, + "rms_per_prompt_fro": 35.715657, + "max_per_prompt_fro": 37.157249, + "heavy_tail_ratio": 1.0404, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.020789, + "mean_fro": 34.539002, + "rms_per_prompt_fro": 34.896173, + "max_per_prompt_fro": 35.864395, + "heavy_tail_ratio": 1.0277, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1243.4, + "fit_secs_per_prompt": 3.108, + "total_wall_seconds": 1254.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.071, + "top_sv": 1.3847, + "stable_rank": 26.077 + }, + { + "layer": 1, + "fro_norm": 6.6652, + "top_sv": 1.3372, + "stable_rank": 24.845 + }, + { + "layer": 2, + "fro_norm": 7.4199, + "top_sv": 1.6191, + "stable_rank": 21.002 + }, + { + "layer": 3, + "fro_norm": 7.1273, + "top_sv": 1.498, + "stable_rank": 22.638 + }, + { + "layer": 4, + "fro_norm": 8.3175, + "top_sv": 1.8337, + "stable_rank": 20.574 + }, + { + "layer": 5, + "fro_norm": 8.5359, + "top_sv": 1.9392, + "stable_rank": 19.375 + }, + { + "layer": 6, + "fro_norm": 8.9116, + "top_sv": 1.9591, + "stable_rank": 20.693 + }, + { + "layer": 7, + "fro_norm": 9.931, + "top_sv": 2.1894, + "stable_rank": 20.575 + }, + { + "layer": 8, + "fro_norm": 11.5011, + "top_sv": 2.7686, + "stable_rank": 17.256 + }, + { + "layer": 9, + "fro_norm": 12.9728, + "top_sv": 3.0615, + "stable_rank": 17.956 + }, + { + "layer": 10, + "fro_norm": 14.8378, + "top_sv": 3.4512, + "stable_rank": 18.484 + }, + { + "layer": 11, + "fro_norm": 17.559, + "top_sv": 4.0164, + "stable_rank": 19.113 + }, + { + "layer": 12, + "fro_norm": 19.325, + "top_sv": 4.2168, + "stable_rank": 21.003 + }, + { + "layer": 13, + "fro_norm": 21.1684, + "top_sv": 4.594, + "stable_rank": 21.232 + }, + { + "layer": 14, + "fro_norm": 24.5152, + "top_sv": 5.2302, + "stable_rank": 21.97 + }, + { + "layer": 15, + "fro_norm": 27.4637, + "top_sv": 6.1427, + "stable_rank": 19.989 + }, + { + "layer": 16, + "fro_norm": 30.6766, + "top_sv": 6.6197, + "stable_rank": 21.475 + }, + { + "layer": 17, + "fro_norm": 33.0997, + "top_sv": 7.1808, + "stable_rank": 21.247 + }, + { + "layer": 18, + "fro_norm": 34.639, + "top_sv": 7.6666, + "stable_rank": 20.414 + }, + { + "layer": 19, + "fro_norm": 34.8003, + "top_sv": 7.601, + "stable_rank": 20.962 + }, + { + "layer": 20, + "fro_norm": 34.9431, + "top_sv": 7.1154, + "stable_rank": 24.117 + }, + { + "layer": 21, + "fro_norm": 34.5374, + "top_sv": 5.6078, + "stable_rank": 37.931 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n400_s2.pt b/results/lenses/e28/e28_Pile-CC_410m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..aff6e42f2adf71d0141e141d4f3046fffa7d7f8a --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a53750f0a8695fa3ddcd996a2be068f1dcabf986a9ae76ec8949c3fcb5097197 +size 46144893 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n400_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c7a3f04cede5edf7b7df0a9c4449e9ca3e33f7e2 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n400_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n400_s2.pt", + "sha256": "a53750f0a8695fa3ddcd996a2be068f1dcabf986a9ae76ec8949c3fcb5097197", + "bytes": 46144893, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 3.123278, + "mean_fro": 7.047083, + "rms_per_prompt_fro": 14.309706, + "max_per_prompt_fro": 42.869869, + "heavy_tail_ratio": 2.9959, + "n_prompts": 400 + }, + "1": { + "dispersion": 2.781346, + "mean_fro": 6.644301, + "rms_per_prompt_fro": 12.920297, + "max_per_prompt_fro": 37.738842, + "heavy_tail_ratio": 2.9209, + "n_prompts": 400 + }, + "2": { + "dispersion": 2.175331, + "mean_fro": 7.382938, + "rms_per_prompt_fro": 13.155996, + "max_per_prompt_fro": 36.94894, + "heavy_tail_ratio": 2.8085, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.863223, + "mean_fro": 7.099915, + "rms_per_prompt_fro": 12.013809, + "max_per_prompt_fro": 34.755753, + "heavy_tail_ratio": 2.893, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.488941, + "mean_fro": 8.269826, + "rms_per_prompt_fro": 13.046791, + "max_per_prompt_fro": 35.437588, + "heavy_tail_ratio": 2.7162, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.354206, + "mean_fro": 8.492033, + "rms_per_prompt_fro": 13.029683, + "max_per_prompt_fro": 36.492474, + "heavy_tail_ratio": 2.8007, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.170977, + "mean_fro": 8.862399, + "rms_per_prompt_fro": 13.058066, + "max_per_prompt_fro": 32.523952, + "heavy_tail_ratio": 2.4907, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.993255, + "mean_fro": 9.863107, + "rms_per_prompt_fro": 13.925, + "max_per_prompt_fro": 24.222897, + "heavy_tail_ratio": 1.7395, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.883819, + "mean_fro": 11.414226, + "rms_per_prompt_fro": 15.666288, + "max_per_prompt_fro": 24.296886, + "heavy_tail_ratio": 1.5509, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.798276, + "mean_fro": 12.871034, + "rms_per_prompt_fro": 17.260031, + "max_per_prompt_fro": 23.998451, + "heavy_tail_ratio": 1.3904, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.715342, + "mean_fro": 14.716165, + "rms_per_prompt_fro": 19.273917, + "max_per_prompt_fro": 24.590446, + "heavy_tail_ratio": 1.2758, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.641908, + "mean_fro": 17.400209, + "rms_per_prompt_fro": 22.296098, + "max_per_prompt_fro": 28.712776, + "heavy_tail_ratio": 1.2878, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.501811, + "mean_fro": 19.189928, + "rms_per_prompt_fro": 23.516948, + "max_per_prompt_fro": 28.855286, + "heavy_tail_ratio": 1.227, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.427474, + "mean_fro": 21.028179, + "rms_per_prompt_fro": 25.123822, + "max_per_prompt_fro": 29.987915, + "heavy_tail_ratio": 1.1936, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.357854, + "mean_fro": 24.380711, + "rms_per_prompt_fro": 28.410109, + "max_per_prompt_fro": 33.074104, + "heavy_tail_ratio": 1.1642, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.298553, + "mean_fro": 27.34305, + "rms_per_prompt_fro": 31.158516, + "max_per_prompt_fro": 35.90984, + "heavy_tail_ratio": 1.1525, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.215495, + "mean_fro": 30.598885, + "rms_per_prompt_fro": 33.735108, + "max_per_prompt_fro": 38.290558, + "heavy_tail_ratio": 1.135, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.158114, + "mean_fro": 32.993526, + "rms_per_prompt_fro": 35.506213, + "max_per_prompt_fro": 38.9352, + "heavy_tail_ratio": 1.0966, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.115762, + "mean_fro": 34.566341, + "rms_per_prompt_fro": 36.5123, + "max_per_prompt_fro": 39.541519, + "heavy_tail_ratio": 1.083, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.078266, + "mean_fro": 34.755172, + "rms_per_prompt_fro": 36.089632, + "max_per_prompt_fro": 38.293179, + "heavy_tail_ratio": 1.0611, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.04537, + "mean_fro": 34.910885, + "rms_per_prompt_fro": 35.694051, + "max_per_prompt_fro": 37.352703, + "heavy_tail_ratio": 1.0465, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.021268, + "mean_fro": 34.517857, + "rms_per_prompt_fro": 34.88299, + "max_per_prompt_fro": 35.766071, + "heavy_tail_ratio": 1.0253, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1217.1, + "fit_secs_per_prompt": 3.043, + "total_wall_seconds": 1226.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.047, + "top_sv": 1.3589, + "stable_rank": 26.893 + }, + { + "layer": 1, + "fro_norm": 6.6442, + "top_sv": 1.3146, + "stable_rank": 25.543 + }, + { + "layer": 2, + "fro_norm": 7.3828, + "top_sv": 1.5895, + "stable_rank": 21.573 + }, + { + "layer": 3, + "fro_norm": 7.0998, + "top_sv": 1.4755, + "stable_rank": 23.154 + }, + { + "layer": 4, + "fro_norm": 8.2697, + "top_sv": 1.8032, + "stable_rank": 21.033 + }, + { + "layer": 5, + "fro_norm": 8.4919, + "top_sv": 1.911, + "stable_rank": 19.747 + }, + { + "layer": 6, + "fro_norm": 8.8623, + "top_sv": 1.928, + "stable_rank": 21.128 + }, + { + "layer": 7, + "fro_norm": 9.863, + "top_sv": 2.1561, + "stable_rank": 20.926 + }, + { + "layer": 8, + "fro_norm": 11.4141, + "top_sv": 2.7183, + "stable_rank": 17.632 + }, + { + "layer": 9, + "fro_norm": 12.8709, + "top_sv": 3.0086, + "stable_rank": 18.301 + }, + { + "layer": 10, + "fro_norm": 14.7159, + "top_sv": 3.3832, + "stable_rank": 18.92 + }, + { + "layer": 11, + "fro_norm": 17.4, + "top_sv": 3.9452, + "stable_rank": 19.452 + }, + { + "layer": 12, + "fro_norm": 19.1896, + "top_sv": 4.1382, + "stable_rank": 21.503 + }, + { + "layer": 13, + "fro_norm": 21.0279, + "top_sv": 4.5048, + "stable_rank": 21.789 + }, + { + "layer": 14, + "fro_norm": 24.3803, + "top_sv": 5.136, + "stable_rank": 22.534 + }, + { + "layer": 15, + "fro_norm": 27.3426, + "top_sv": 6.0351, + "stable_rank": 20.526 + }, + { + "layer": 16, + "fro_norm": 30.5983, + "top_sv": 6.5336, + "stable_rank": 21.932 + }, + { + "layer": 17, + "fro_norm": 32.9928, + "top_sv": 7.0868, + "stable_rank": 21.674 + }, + { + "layer": 18, + "fro_norm": 34.5654, + "top_sv": 7.5951, + "stable_rank": 20.712 + }, + { + "layer": 19, + "fro_norm": 34.7541, + "top_sv": 7.5689, + "stable_rank": 21.084 + }, + { + "layer": 20, + "fro_norm": 34.9095, + "top_sv": 7.0842, + "stable_rank": 24.284 + }, + { + "layer": 21, + "fro_norm": 34.5163, + "top_sv": 5.5933, + "stable_rank": 38.081 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n50_s0.pt b/results/lenses/e28/e28_Pile-CC_410m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..dff1ef9ce90de68a0af1d44e40cba61afae04603 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c76f2b9fb699bcf929b34ebe1279d413d651d33f6fb689584a1dadd64b5ca03c +size 46144865 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n50_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..768977c6e728a4abf61e1c911503d4b52198d84d --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n50_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n50_s0.pt", + "sha256": "c76f2b9fb699bcf929b34ebe1279d413d651d33f6fb689584a1dadd64b5ca03c", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.668064, + "mean_fro": 7.217607, + "rms_per_prompt_fro": 13.823297, + "max_per_prompt_fro": 20.072716, + "heavy_tail_ratio": 1.4521, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.378874, + "mean_fro": 6.777027, + "rms_per_prompt_fro": 12.457337, + "max_per_prompt_fro": 17.730816, + "heavy_tail_ratio": 1.4233, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.901635, + "mean_fro": 7.527195, + "rms_per_prompt_fro": 12.821965, + "max_per_prompt_fro": 17.277742, + "heavy_tail_ratio": 1.3475, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.644374, + "mean_fro": 7.209287, + "rms_per_prompt_fro": 11.723404, + "max_per_prompt_fro": 15.108992, + "heavy_tail_ratio": 1.2888, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.340058, + "mean_fro": 8.428416, + "rms_per_prompt_fro": 12.893156, + "max_per_prompt_fro": 16.663425, + "heavy_tail_ratio": 1.2924, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.216569, + "mean_fro": 8.657208, + "rms_per_prompt_fro": 12.888979, + "max_per_prompt_fro": 16.346407, + "heavy_tail_ratio": 1.2682, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.071548, + "mean_fro": 9.029159, + "rms_per_prompt_fro": 12.995555, + "max_per_prompt_fro": 16.568392, + "heavy_tail_ratio": 1.2749, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.930443, + "mean_fro": 10.051927, + "rms_per_prompt_fro": 13.966185, + "max_per_prompt_fro": 17.899071, + "heavy_tail_ratio": 1.2816, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.833758, + "mean_fro": 11.683747, + "rms_per_prompt_fro": 15.821699, + "max_per_prompt_fro": 19.758896, + "heavy_tail_ratio": 1.2488, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.754599, + "mean_fro": 13.193501, + "rms_per_prompt_fro": 17.476281, + "max_per_prompt_fro": 21.443424, + "heavy_tail_ratio": 1.227, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.679093, + "mean_fro": 15.090686, + "rms_per_prompt_fro": 19.554486, + "max_per_prompt_fro": 23.286634, + "heavy_tail_ratio": 1.1909, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.607964, + "mean_fro": 17.882916, + "rms_per_prompt_fro": 22.676523, + "max_per_prompt_fro": 26.577879, + "heavy_tail_ratio": 1.172, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.473736, + "mean_fro": 19.540075, + "rms_per_prompt_fro": 23.721171, + "max_per_prompt_fro": 27.498411, + "heavy_tail_ratio": 1.1592, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.402077, + "mean_fro": 21.417637, + "rms_per_prompt_fro": 25.360478, + "max_per_prompt_fro": 28.693182, + "heavy_tail_ratio": 1.1314, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.339368, + "mean_fro": 24.733163, + "rms_per_prompt_fro": 28.623952, + "max_per_prompt_fro": 32.119045, + "heavy_tail_ratio": 1.1221, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.283671, + "mean_fro": 27.694989, + "rms_per_prompt_fro": 31.378201, + "max_per_prompt_fro": 34.93475, + "heavy_tail_ratio": 1.1133, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.203675, + "mean_fro": 30.902782, + "rms_per_prompt_fro": 33.904103, + "max_per_prompt_fro": 37.032356, + "heavy_tail_ratio": 1.0923, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.150717, + "mean_fro": 33.315814, + "rms_per_prompt_fro": 35.738371, + "max_per_prompt_fro": 38.701962, + "heavy_tail_ratio": 1.0829, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.109333, + "mean_fro": 34.84742, + "rms_per_prompt_fro": 36.703008, + "max_per_prompt_fro": 39.161575, + "heavy_tail_ratio": 1.067, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.074209, + "mean_fro": 34.950066, + "rms_per_prompt_fro": 36.223671, + "max_per_prompt_fro": 37.874847, + "heavy_tail_ratio": 1.0456, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.044131, + "mean_fro": 35.021309, + "rms_per_prompt_fro": 35.785737, + "max_per_prompt_fro": 36.998234, + "heavy_tail_ratio": 1.0339, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.020749, + "mean_fro": 34.579327, + "rms_per_prompt_fro": 34.936221, + "max_per_prompt_fro": 35.682949, + "heavy_tail_ratio": 1.0214, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 153.5, + "fit_secs_per_prompt": 3.071, + "total_wall_seconds": 161.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.2175, + "top_sv": 1.4555, + "stable_rank": 24.589 + }, + { + "layer": 1, + "fro_norm": 6.7769, + "top_sv": 1.4039, + "stable_rank": 23.303 + }, + { + "layer": 2, + "fro_norm": 7.5271, + "top_sv": 1.7248, + "stable_rank": 19.045 + }, + { + "layer": 3, + "fro_norm": 7.2092, + "top_sv": 1.5659, + "stable_rank": 21.196 + }, + { + "layer": 4, + "fro_norm": 8.4283, + "top_sv": 1.9461, + "stable_rank": 18.756 + }, + { + "layer": 5, + "fro_norm": 8.6571, + "top_sv": 2.0517, + "stable_rank": 17.804 + }, + { + "layer": 6, + "fro_norm": 9.029, + "top_sv": 2.0553, + "stable_rank": 19.299 + }, + { + "layer": 7, + "fro_norm": 10.0518, + "top_sv": 2.2639, + "stable_rank": 19.714 + }, + { + "layer": 8, + "fro_norm": 11.6836, + "top_sv": 2.8841, + "stable_rank": 16.411 + }, + { + "layer": 9, + "fro_norm": 13.1933, + "top_sv": 3.181, + "stable_rank": 17.202 + }, + { + "layer": 10, + "fro_norm": 15.0905, + "top_sv": 3.5782, + "stable_rank": 17.786 + }, + { + "layer": 11, + "fro_norm": 17.8826, + "top_sv": 4.1757, + "stable_rank": 18.34 + }, + { + "layer": 12, + "fro_norm": 19.5397, + "top_sv": 4.2821, + "stable_rank": 20.822 + }, + { + "layer": 13, + "fro_norm": 21.4173, + "top_sv": 4.6684, + "stable_rank": 21.047 + }, + { + "layer": 14, + "fro_norm": 24.7327, + "top_sv": 5.2903, + "stable_rank": 21.857 + }, + { + "layer": 15, + "fro_norm": 27.6945, + "top_sv": 6.1942, + "stable_rank": 19.99 + }, + { + "layer": 16, + "fro_norm": 30.9022, + "top_sv": 6.7337, + "stable_rank": 21.06 + }, + { + "layer": 17, + "fro_norm": 33.315, + "top_sv": 7.2683, + "stable_rank": 21.01 + }, + { + "layer": 18, + "fro_norm": 34.8465, + "top_sv": 7.717, + "stable_rank": 20.39 + }, + { + "layer": 19, + "fro_norm": 34.9489, + "top_sv": 7.6783, + "stable_rank": 20.717 + }, + { + "layer": 20, + "fro_norm": 35.02, + "top_sv": 7.1422, + "stable_rank": 24.042 + }, + { + "layer": 21, + "fro_norm": 34.5777, + "top_sv": 5.6408, + "stable_rank": 37.576 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n50_s1.pt b/results/lenses/e28/e28_Pile-CC_410m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..93980eff33af6c5663f3ffffe1f28f5ed7df0e3a --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b9ca06df6d95d1b3fbb858527b178a25bfd74f5126808d87cf9a024523f9bda +size 46144865 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n50_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..1829a78d1f1ea376643ff376ea9d5a246b1f5adf --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n50_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n50_s1.pt", + "sha256": "4b9ca06df6d95d1b3fbb858527b178a25bfd74f5126808d87cf9a024523f9bda", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.961879, + "mean_fro": 7.317984, + "rms_per_prompt_fro": 14.566059, + "max_per_prompt_fro": 23.687071, + "heavy_tail_ratio": 1.6262, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.655853, + "mean_fro": 6.861464, + "rms_per_prompt_fro": 13.119313, + "max_per_prompt_fro": 20.697523, + "heavy_tail_ratio": 1.5776, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.084905, + "mean_fro": 7.604895, + "rms_per_prompt_fro": 13.357159, + "max_per_prompt_fro": 22.714045, + "heavy_tail_ratio": 1.7005, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.79802, + "mean_fro": 7.287024, + "rms_per_prompt_fro": 12.189211, + "max_per_prompt_fro": 21.115162, + "heavy_tail_ratio": 1.7323, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.455571, + "mean_fro": 8.44567, + "rms_per_prompt_fro": 13.234588, + "max_per_prompt_fro": 22.915529, + "heavy_tail_ratio": 1.7315, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.319382, + "mean_fro": 8.64061, + "rms_per_prompt_fro": 13.159231, + "max_per_prompt_fro": 22.637657, + "heavy_tail_ratio": 1.7203, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.103217, + "mean_fro": 8.996456, + "rms_per_prompt_fro": 13.047086, + "max_per_prompt_fro": 17.876284, + "heavy_tail_ratio": 1.3701, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.927155, + "mean_fro": 9.992232, + "rms_per_prompt_fro": 13.871419, + "max_per_prompt_fro": 18.760618, + "heavy_tail_ratio": 1.3525, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.820507, + "mean_fro": 11.52595, + "rms_per_prompt_fro": 15.551523, + "max_per_prompt_fro": 20.646214, + "heavy_tail_ratio": 1.3276, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.739857, + "mean_fro": 12.964554, + "rms_per_prompt_fro": 17.100719, + "max_per_prompt_fro": 21.89467, + "heavy_tail_ratio": 1.2803, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.656736, + "mean_fro": 14.784197, + "rms_per_prompt_fro": 19.029369, + "max_per_prompt_fro": 23.248056, + "heavy_tail_ratio": 1.2217, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.581881, + "mean_fro": 17.437283, + "rms_per_prompt_fro": 21.931365, + "max_per_prompt_fro": 26.493469, + "heavy_tail_ratio": 1.208, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.456321, + "mean_fro": 19.199108, + "rms_per_prompt_fro": 23.169124, + "max_per_prompt_fro": 26.665606, + "heavy_tail_ratio": 1.1509, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.386081, + "mean_fro": 21.053776, + "rms_per_prompt_fro": 24.787015, + "max_per_prompt_fro": 28.148933, + "heavy_tail_ratio": 1.1356, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.322034, + "mean_fro": 24.394991, + "rms_per_prompt_fro": 28.049293, + "max_per_prompt_fro": 32.12635, + "heavy_tail_ratio": 1.1454, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.272115, + "mean_fro": 27.328912, + "rms_per_prompt_fro": 30.823753, + "max_per_prompt_fro": 35.279549, + "heavy_tail_ratio": 1.1446, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.192496, + "mean_fro": 30.613344, + "rms_per_prompt_fro": 33.430219, + "max_per_prompt_fro": 36.173946, + "heavy_tail_ratio": 1.0821, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.141806, + "mean_fro": 33.040791, + "rms_per_prompt_fro": 35.305839, + "max_per_prompt_fro": 37.810223, + "heavy_tail_ratio": 1.0709, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.10429, + "mean_fro": 34.572861, + "rms_per_prompt_fro": 36.33096, + "max_per_prompt_fro": 38.158878, + "heavy_tail_ratio": 1.0503, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.070312, + "mean_fro": 34.710854, + "rms_per_prompt_fro": 35.910423, + "max_per_prompt_fro": 37.161911, + "heavy_tail_ratio": 1.0349, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.041023, + "mean_fro": 34.874589, + "rms_per_prompt_fro": 35.582721, + "max_per_prompt_fro": 36.678207, + "heavy_tail_ratio": 1.0308, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.019109, + "mean_fro": 34.493275, + "rms_per_prompt_fro": 34.821274, + "max_per_prompt_fro": 35.864395, + "heavy_tail_ratio": 1.03, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 155.0, + "fit_secs_per_prompt": 3.101, + "total_wall_seconds": 165.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.3179, + "top_sv": 1.4922, + "stable_rank": 24.052 + }, + { + "layer": 1, + "fro_norm": 6.8614, + "top_sv": 1.4278, + "stable_rank": 23.092 + }, + { + "layer": 2, + "fro_norm": 7.6048, + "top_sv": 1.715, + "stable_rank": 19.663 + }, + { + "layer": 3, + "fro_norm": 7.2869, + "top_sv": 1.5786, + "stable_rank": 21.309 + }, + { + "layer": 4, + "fro_norm": 8.4455, + "top_sv": 1.9094, + "stable_rank": 19.564 + }, + { + "layer": 5, + "fro_norm": 8.6405, + "top_sv": 2.0049, + "stable_rank": 18.573 + }, + { + "layer": 6, + "fro_norm": 8.9963, + "top_sv": 2.0215, + "stable_rank": 19.804 + }, + { + "layer": 7, + "fro_norm": 9.9921, + "top_sv": 2.26, + "stable_rank": 19.548 + }, + { + "layer": 8, + "fro_norm": 11.5258, + "top_sv": 2.8179, + "stable_rank": 16.729 + }, + { + "layer": 9, + "fro_norm": 12.9644, + "top_sv": 3.1027, + "stable_rank": 17.459 + }, + { + "layer": 10, + "fro_norm": 14.784, + "top_sv": 3.4527, + "stable_rank": 18.334 + }, + { + "layer": 11, + "fro_norm": 17.437, + "top_sv": 3.9854, + "stable_rank": 19.142 + }, + { + "layer": 12, + "fro_norm": 19.1988, + "top_sv": 4.1525, + "stable_rank": 21.376 + }, + { + "layer": 13, + "fro_norm": 21.0535, + "top_sv": 4.5432, + "stable_rank": 21.474 + }, + { + "layer": 14, + "fro_norm": 24.3945, + "top_sv": 5.1808, + "stable_rank": 22.171 + }, + { + "layer": 15, + "fro_norm": 27.3284, + "top_sv": 6.0385, + "stable_rank": 20.482 + }, + { + "layer": 16, + "fro_norm": 30.6128, + "top_sv": 6.5172, + "stable_rank": 22.064 + }, + { + "layer": 17, + "fro_norm": 33.04, + "top_sv": 7.0851, + "stable_rank": 21.747 + }, + { + "layer": 18, + "fro_norm": 34.5719, + "top_sv": 7.5609, + "stable_rank": 20.907 + }, + { + "layer": 19, + "fro_norm": 34.7098, + "top_sv": 7.4906, + "stable_rank": 21.472 + }, + { + "layer": 20, + "fro_norm": 34.8733, + "top_sv": 7.0808, + "stable_rank": 24.256 + }, + { + "layer": 21, + "fro_norm": 34.4917, + "top_sv": 5.6092, + "stable_rank": 37.812 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_410m_n50_s2.pt b/results/lenses/e28/e28_Pile-CC_410m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcd05376679eab53033c39936b5e3b39e1827613 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8e5b9cbb43aab61486d074c2ea49cb2b6f618d74dd55a04716102ef76c4cf5d +size 46144865 diff --git a/results/lenses/e28/e28_Pile-CC_410m_n50_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_410m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..cc51cfddf6ae34be2fcb69a0ea22d009cca932b8 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_410m_n50_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Pile-CC_410m_n50_s2.pt", + "sha256": "e8e5b9cbb43aab61486d074c2ea49cb2b6f618d74dd55a04716102ef76c4cf5d", + "bytes": 46144865, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.849818, + "mean_fro": 7.226632, + "rms_per_prompt_fro": 14.179341, + "max_per_prompt_fro": 21.697657, + "heavy_tail_ratio": 1.5302, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.551278, + "mean_fro": 6.806518, + "rms_per_prompt_fro": 12.82677, + "max_per_prompt_fro": 19.671339, + "heavy_tail_ratio": 1.5336, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.033932, + "mean_fro": 7.509592, + "rms_per_prompt_fro": 13.080348, + "max_per_prompt_fro": 19.620565, + "heavy_tail_ratio": 1.5, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.740425, + "mean_fro": 7.222788, + "rms_per_prompt_fro": 11.956769, + "max_per_prompt_fro": 17.48867, + "heavy_tail_ratio": 1.4627, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.41719, + "mean_fro": 8.400042, + "rms_per_prompt_fro": 13.059808, + "max_per_prompt_fro": 18.102705, + "heavy_tail_ratio": 1.3861, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.291266, + "mean_fro": 8.631233, + "rms_per_prompt_fro": 13.065037, + "max_per_prompt_fro": 17.888645, + "heavy_tail_ratio": 1.3692, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.115272, + "mean_fro": 9.003235, + "rms_per_prompt_fro": 13.09428, + "max_per_prompt_fro": 17.235607, + "heavy_tail_ratio": 1.3163, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.95654, + "mean_fro": 10.02952, + "rms_per_prompt_fro": 14.028929, + "max_per_prompt_fro": 17.64271, + "heavy_tail_ratio": 1.2576, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.855878, + "mean_fro": 11.627386, + "rms_per_prompt_fro": 15.84006, + "max_per_prompt_fro": 20.109953, + "heavy_tail_ratio": 1.2696, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.772412, + "mean_fro": 13.131914, + "rms_per_prompt_fro": 17.482774, + "max_per_prompt_fro": 22.150478, + "heavy_tail_ratio": 1.267, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.696305, + "mean_fro": 15.041512, + "rms_per_prompt_fro": 19.590409, + "max_per_prompt_fro": 24.590446, + "heavy_tail_ratio": 1.2552, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.627242, + "mean_fro": 17.802023, + "rms_per_prompt_fro": 22.708867, + "max_per_prompt_fro": 28.712776, + "heavy_tail_ratio": 1.2644, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.48071, + "mean_fro": 19.547678, + "rms_per_prompt_fro": 23.786479, + "max_per_prompt_fro": 26.763897, + "heavy_tail_ratio": 1.1252, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.406462, + "mean_fro": 21.382279, + "rms_per_prompt_fro": 25.358171, + "max_per_prompt_fro": 28.109467, + "heavy_tail_ratio": 1.1085, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.339492, + "mean_fro": 24.70473, + "rms_per_prompt_fro": 28.592373, + "max_per_prompt_fro": 31.434555, + "heavy_tail_ratio": 1.0994, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.28245, + "mean_fro": 27.673556, + "rms_per_prompt_fro": 31.339006, + "max_per_prompt_fro": 34.146469, + "heavy_tail_ratio": 1.0896, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.204781, + "mean_fro": 30.984799, + "rms_per_prompt_fro": 34.009692, + "max_per_prompt_fro": 36.94334, + "heavy_tail_ratio": 1.0863, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.147605, + "mean_fro": 33.346272, + "rms_per_prompt_fro": 35.722632, + "max_per_prompt_fro": 38.267223, + "heavy_tail_ratio": 1.0712, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.106724, + "mean_fro": 34.891898, + "rms_per_prompt_fro": 36.706615, + "max_per_prompt_fro": 38.573795, + "heavy_tail_ratio": 1.0509, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.071485, + "mean_fro": 35.009782, + "rms_per_prompt_fro": 36.239516, + "max_per_prompt_fro": 37.567314, + "heavy_tail_ratio": 1.0366, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.042036, + "mean_fro": 35.040715, + "rms_per_prompt_fro": 35.769624, + "max_per_prompt_fro": 36.963787, + "heavy_tail_ratio": 1.0334, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.019544, + "mean_fro": 34.570748, + "rms_per_prompt_fro": 34.906943, + "max_per_prompt_fro": 35.649303, + "heavy_tail_ratio": 1.0213, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 152.4, + "fit_secs_per_prompt": 3.048, + "total_wall_seconds": 161.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.2265, + "top_sv": 1.4417, + "stable_rank": 25.125 + }, + { + "layer": 1, + "fro_norm": 6.8064, + "top_sv": 1.3707, + "stable_rank": 24.657 + }, + { + "layer": 2, + "fro_norm": 7.5095, + "top_sv": 1.6151, + "stable_rank": 21.618 + }, + { + "layer": 3, + "fro_norm": 7.2227, + "top_sv": 1.503, + "stable_rank": 23.093 + }, + { + "layer": 4, + "fro_norm": 8.3999, + "top_sv": 1.8432, + "stable_rank": 20.77 + }, + { + "layer": 5, + "fro_norm": 8.6311, + "top_sv": 1.959, + "stable_rank": 19.411 + }, + { + "layer": 6, + "fro_norm": 9.0031, + "top_sv": 1.9805, + "stable_rank": 20.664 + }, + { + "layer": 7, + "fro_norm": 10.0294, + "top_sv": 2.2203, + "stable_rank": 20.405 + }, + { + "layer": 8, + "fro_norm": 11.6273, + "top_sv": 2.7829, + "stable_rank": 17.457 + }, + { + "layer": 9, + "fro_norm": 13.1317, + "top_sv": 3.0929, + "stable_rank": 18.026 + }, + { + "layer": 10, + "fro_norm": 15.0413, + "top_sv": 3.4917, + "stable_rank": 18.556 + }, + { + "layer": 11, + "fro_norm": 17.8017, + "top_sv": 4.0802, + "stable_rank": 19.036 + }, + { + "layer": 12, + "fro_norm": 19.5474, + "top_sv": 4.2487, + "stable_rank": 21.167 + }, + { + "layer": 13, + "fro_norm": 21.3819, + "top_sv": 4.6163, + "stable_rank": 21.454 + }, + { + "layer": 14, + "fro_norm": 24.7043, + "top_sv": 5.2173, + "stable_rank": 22.421 + }, + { + "layer": 15, + "fro_norm": 27.673, + "top_sv": 6.1048, + "stable_rank": 20.548 + }, + { + "layer": 16, + "fro_norm": 30.9842, + "top_sv": 6.6927, + "stable_rank": 21.433 + }, + { + "layer": 17, + "fro_norm": 33.3455, + "top_sv": 7.2662, + "stable_rank": 21.06 + }, + { + "layer": 18, + "fro_norm": 34.8909, + "top_sv": 7.8141, + "stable_rank": 19.938 + }, + { + "layer": 19, + "fro_norm": 35.0086, + "top_sv": 7.8138, + "stable_rank": 20.073 + }, + { + "layer": 20, + "fro_norm": 35.0394, + "top_sv": 7.152, + "stable_rank": 24.002 + }, + { + "layer": 21, + "fro_norm": 34.569, + "top_sv": 5.6359, + "stable_rank": 37.622 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n100_s0.pt b/results/lenses/e28/e28_Pile-CC_70m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b43c4045625e9d43df791c9a93597f203184b9a --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ede4b79ebc17a69258af76df58082c6fd9c11449c70b1b2a5d4e382f7565fa0 +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n100_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c9a39d41bf20d31b1143746769bd398c73443e3d --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n100_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n100_s0.pt", + "sha256": "7ede4b79ebc17a69258af76df58082c6fd9c11449c70b1b2a5d4e382f7565fa0", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.24296, + "mean_fro": 33.003623, + "rms_per_prompt_fro": 36.795114, + "max_per_prompt_fro": 42.703876, + "heavy_tail_ratio": 1.1606, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.19081, + "mean_fro": 25.590291, + "rms_per_prompt_fro": 27.925211, + "max_per_prompt_fro": 31.724863, + "heavy_tail_ratio": 1.1361, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.130936, + "mean_fro": 24.338879, + "rms_per_prompt_fro": 25.883295, + "max_per_prompt_fro": 29.227877, + "heavy_tail_ratio": 1.1292, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.063655, + "mean_fro": 22.52952, + "rms_per_prompt_fro": 23.235512, + "max_per_prompt_fro": 24.31151, + "heavy_tail_ratio": 1.0463, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.1, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 16.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.0036, + "top_sv": 5.4248, + "stable_rank": 37.014 + }, + { + "layer": 1, + "fro_norm": 25.5903, + "top_sv": 4.6402, + "stable_rank": 30.414 + }, + { + "layer": 2, + "fro_norm": 24.3388, + "top_sv": 3.7768, + "stable_rank": 41.529 + }, + { + "layer": 3, + "fro_norm": 22.5295, + "top_sv": 4.2907, + "stable_rank": 27.57 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n100_s1.pt b/results/lenses/e28/e28_Pile-CC_70m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9d926446ff4f28dff58e340dceb2818e0d8b368 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a82d874ea184b7517eadcca719de154b062e4c59b5af44c159b38cb7af93f36 +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n100_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9b0d0b8c038d598b5cf35c2912e101666f336142 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n100_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n100_s1.pt", + "sha256": "8a82d874ea184b7517eadcca719de154b062e4c59b5af44c159b38cb7af93f36", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.233726, + "mean_fro": 33.20536, + "rms_per_prompt_fro": 36.882261, + "max_per_prompt_fro": 41.231567, + "heavy_tail_ratio": 1.1179, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.184361, + "mean_fro": 25.721556, + "rms_per_prompt_fro": 27.992344, + "max_per_prompt_fro": 31.077162, + "heavy_tail_ratio": 1.1102, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.127459, + "mean_fro": 24.488666, + "rms_per_prompt_fro": 26.002529, + "max_per_prompt_fro": 29.201994, + "heavy_tail_ratio": 1.123, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.062275, + "mean_fro": 22.659897, + "rms_per_prompt_fro": 23.354814, + "max_per_prompt_fro": 24.807026, + "heavy_tail_ratio": 1.0622, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.1, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 17.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.2053, + "top_sv": 5.5772, + "stable_rank": 35.448 + }, + { + "layer": 1, + "fro_norm": 25.7215, + "top_sv": 4.6243, + "stable_rank": 30.939 + }, + { + "layer": 2, + "fro_norm": 24.4886, + "top_sv": 3.771, + "stable_rank": 42.172 + }, + { + "layer": 3, + "fro_norm": 22.6598, + "top_sv": 4.3022, + "stable_rank": 27.742 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n100_s2.pt b/results/lenses/e28/e28_Pile-CC_70m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..587d4b4b0a3fd7cc49009f77fb31033c562187f2 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2539133caa26f816a0e515ddbd9445a0cc25291b48454f55a9d63381aee328b1 +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n100_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..97f37c3cbde9e927e3a5ad18360b8689b5211836 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n100_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n100_s2.pt", + "sha256": "2539133caa26f816a0e515ddbd9445a0cc25291b48454f55a9d63381aee328b1", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.239462, + "mean_fro": 32.900065, + "rms_per_prompt_fro": 36.628008, + "max_per_prompt_fro": 43.954247, + "heavy_tail_ratio": 1.2, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.189777, + "mean_fro": 25.561032, + "rms_per_prompt_fro": 27.881182, + "max_per_prompt_fro": 33.427799, + "heavy_tail_ratio": 1.1989, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.130024, + "mean_fro": 24.337871, + "rms_per_prompt_fro": 25.871786, + "max_per_prompt_fro": 30.606003, + "heavy_tail_ratio": 1.183, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.063703, + "mean_fro": 22.600807, + "rms_per_prompt_fro": 23.309562, + "max_per_prompt_fro": 25.147326, + "heavy_tail_ratio": 1.0788, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 12.6, + "fit_secs_per_prompt": 0.126, + "total_wall_seconds": 21.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.9, + "top_sv": 5.5434, + "stable_rank": 35.225 + }, + { + "layer": 1, + "fro_norm": 25.561, + "top_sv": 4.6179, + "stable_rank": 30.638 + }, + { + "layer": 2, + "fro_norm": 24.3378, + "top_sv": 3.7739, + "stable_rank": 41.59 + }, + { + "layer": 3, + "fro_norm": 22.6007, + "top_sv": 4.3089, + "stable_rank": 27.512 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n200_s0.pt b/results/lenses/e28/e28_Pile-CC_70m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..be43f45656f95207a98f692584c7902e713485cb --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:116203e39ec6e3ded97c6a73eced00fd5dba5167f9768ec391b0f8fde2b5cc54 +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n200_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..51c69bd0897fc7a23ef72602fca9900b3ddef22c --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n200_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n200_s0.pt", + "sha256": "116203e39ec6e3ded97c6a73eced00fd5dba5167f9768ec391b0f8fde2b5cc54", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.241142, + "mean_fro": 32.827389, + "rms_per_prompt_fro": 36.571859, + "max_per_prompt_fro": 42.856693, + "heavy_tail_ratio": 1.1718, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.190186, + "mean_fro": 25.494545, + "rms_per_prompt_fro": 27.813438, + "max_per_prompt_fro": 34.00552, + "heavy_tail_ratio": 1.2226, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.130241, + "mean_fro": 24.295363, + "rms_per_prompt_fro": 25.829083, + "max_per_prompt_fro": 29.305941, + "heavy_tail_ratio": 1.1346, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.063878, + "mean_fro": 22.574819, + "rms_per_prompt_fro": 23.284681, + "max_per_prompt_fro": 24.643541, + "heavy_tail_ratio": 1.0584, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 22.0, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 27.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.8273, + "top_sv": 5.4388, + "stable_rank": 36.43 + }, + { + "layer": 1, + "fro_norm": 25.4945, + "top_sv": 4.6162, + "stable_rank": 30.501 + }, + { + "layer": 2, + "fro_norm": 24.2953, + "top_sv": 3.765, + "stable_rank": 41.639 + }, + { + "layer": 3, + "fro_norm": 22.5747, + "top_sv": 4.3166, + "stable_rank": 27.351 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n200_s1.pt b/results/lenses/e28/e28_Pile-CC_70m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..18062ea470bfd3297cd7525413bc0af4c265d6ae --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5030d9e08d326066082498ab79a57d63300759302278effd06f7836868363a +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n200_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9b4a3e6c8855735fed2ed4f96a5155fe7e831c1c --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n200_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n200_s1.pt", + "sha256": "fb5030d9e08d326066082498ab79a57d63300759302278effd06f7836868363a", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.241968, + "mean_fro": 32.954418, + "rms_per_prompt_fro": 36.725602, + "max_per_prompt_fro": 44.87648, + "heavy_tail_ratio": 1.2219, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.190969, + "mean_fro": 25.58446, + "rms_per_prompt_fro": 27.920714, + "max_per_prompt_fro": 34.123726, + "heavy_tail_ratio": 1.2222, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.132009, + "mean_fro": 24.395124, + "rms_per_prompt_fro": 25.955416, + "max_per_prompt_fro": 30.594242, + "heavy_tail_ratio": 1.1787, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.06525, + "mean_fro": 22.627896, + "rms_per_prompt_fro": 23.354468, + "max_per_prompt_fro": 24.807026, + "heavy_tail_ratio": 1.0622, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 22.1, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 28.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.9544, + "top_sv": 5.4995, + "stable_rank": 35.907 + }, + { + "layer": 1, + "fro_norm": 25.5844, + "top_sv": 4.6156, + "stable_rank": 30.726 + }, + { + "layer": 2, + "fro_norm": 24.3951, + "top_sv": 3.7652, + "stable_rank": 41.979 + }, + { + "layer": 3, + "fro_norm": 22.6278, + "top_sv": 4.316, + "stable_rank": 27.487 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n200_s2.pt b/results/lenses/e28/e28_Pile-CC_70m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..502084a0e0386f17a4467f5003a433d3c49271ef --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54058be9e1be2d69022b6fd2e195c6f54a67f3448da9027e34d0bb40ee45c83a +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n200_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7bb2bf87c5cb46a42b70faa75c2f31e5d3356432 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n200_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n200_s2.pt", + "sha256": "54058be9e1be2d69022b6fd2e195c6f54a67f3448da9027e34d0bb40ee45c83a", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.240409, + "mean_fro": 32.880981, + "rms_per_prompt_fro": 36.62075, + "max_per_prompt_fro": 43.954247, + "heavy_tail_ratio": 1.2003, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.189706, + "mean_fro": 25.509093, + "rms_per_prompt_fro": 27.823703, + "max_per_prompt_fro": 33.427799, + "heavy_tail_ratio": 1.2014, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.129457, + "mean_fro": 24.307323, + "rms_per_prompt_fro": 25.832826, + "max_per_prompt_fro": 30.606003, + "heavy_tail_ratio": 1.1848, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.064078, + "mean_fro": 22.596626, + "rms_per_prompt_fro": 23.309357, + "max_per_prompt_fro": 25.147326, + "heavy_tail_ratio": 1.0789, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 24.7, + "fit_secs_per_prompt": 0.124, + "total_wall_seconds": 34.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.8809, + "top_sv": 5.6044, + "stable_rank": 34.422 + }, + { + "layer": 1, + "fro_norm": 25.509, + "top_sv": 4.6241, + "stable_rank": 30.433 + }, + { + "layer": 2, + "fro_norm": 24.3073, + "top_sv": 3.7739, + "stable_rank": 41.485 + }, + { + "layer": 3, + "fro_norm": 22.5966, + "top_sv": 4.3371, + "stable_rank": 27.145 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n25_s0.pt b/results/lenses/e28/e28_Pile-CC_70m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bb5e8816b830f67fa9244c6c0f669912685ad85 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab6ad784d04a60a3136cd353c13fcdc53a74f3532fd5070a2a39968514b97dc +size 2099827 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n25_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..838e9bfb4b9e34d7a0ef995d0cc3547c1950cdd9 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n25_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n25_s0.pt", + "sha256": "eab6ad784d04a60a3136cd353c13fcdc53a74f3532fd5070a2a39968514b97dc", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.249016, + "mean_fro": 32.3582, + "rms_per_prompt_fro": 36.163321, + "max_per_prompt_fro": 42.328766, + "heavy_tail_ratio": 1.1705, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.193819, + "mean_fro": 25.204964, + "rms_per_prompt_fro": 27.539457, + "max_per_prompt_fro": 31.724863, + "heavy_tail_ratio": 1.152, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.133373, + "mean_fro": 24.15787, + "rms_per_prompt_fro": 25.718466, + "max_per_prompt_fro": 29.079222, + "heavy_tail_ratio": 1.1307, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.06475, + "mean_fro": 22.370578, + "rms_per_prompt_fro": 23.083464, + "max_per_prompt_fro": 24.128769, + "heavy_tail_ratio": 1.0453, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.0, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 12.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.3582, + "top_sv": 5.1505, + "stable_rank": 39.47 + }, + { + "layer": 1, + "fro_norm": 25.2049, + "top_sv": 4.5722, + "stable_rank": 30.39 + }, + { + "layer": 2, + "fro_norm": 24.1578, + "top_sv": 3.751, + "stable_rank": 41.478 + }, + { + "layer": 3, + "fro_norm": 22.3705, + "top_sv": 4.2767, + "stable_rank": 27.361 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n25_s1.pt b/results/lenses/e28/e28_Pile-CC_70m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..de475664d9979d946b88203f0bfc77014f180920 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50f1a44aae9ea253a2f4398efaab6eab18e703230ecb1f132cb2814cc920138 +size 2099827 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n25_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0000113b2e7520a6908bcf65a7831b5406c41ace --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n25_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n25_s1.pt", + "sha256": "a50f1a44aae9ea253a2f4398efaab6eab18e703230ecb1f132cb2814cc920138", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.234319, + "mean_fro": 33.891713, + "rms_per_prompt_fro": 37.653664, + "max_per_prompt_fro": 41.231567, + "heavy_tail_ratio": 1.095, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.186151, + "mean_fro": 26.264326, + "rms_per_prompt_fro": 28.604618, + "max_per_prompt_fro": 31.077162, + "heavy_tail_ratio": 1.0864, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.13044, + "mean_fro": 24.927702, + "rms_per_prompt_fro": 26.503674, + "max_per_prompt_fro": 29.201994, + "heavy_tail_ratio": 1.1018, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.06189, + "mean_fro": 22.863054, + "rms_per_prompt_fro": 23.559932, + "max_per_prompt_fro": 24.807026, + "heavy_tail_ratio": 1.0529, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.0, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 9.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.8917, + "top_sv": 5.6077, + "stable_rank": 36.527 + }, + { + "layer": 1, + "fro_norm": 26.2643, + "top_sv": 4.6232, + "stable_rank": 32.273 + }, + { + "layer": 2, + "fro_norm": 24.9277, + "top_sv": 3.7985, + "stable_rank": 43.066 + }, + { + "layer": 3, + "fro_norm": 22.863, + "top_sv": 4.2815, + "stable_rank": 28.515 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n25_s2.pt b/results/lenses/e28/e28_Pile-CC_70m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef27ecc97bae3d90f2754eb67badbbb449b55fd6 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa51d2307b90c4714003208a6b71b8999766b3f07f3097daac89ed9176b61dcb +size 2099827 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n25_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..91fbe3efe0263fc6610537731a2d0b2ba2841df0 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n25_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n25_s2.pt", + "sha256": "aa51d2307b90c4714003208a6b71b8999766b3f07f3097daac89ed9176b61dcb", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.221451, + "mean_fro": 33.278057, + "rms_per_prompt_fro": 36.778671, + "max_per_prompt_fro": 40.37582, + "heavy_tail_ratio": 1.0978, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.175104, + "mean_fro": 25.716574, + "rms_per_prompt_fro": 27.87734, + "max_per_prompt_fro": 29.975328, + "heavy_tail_ratio": 1.0753, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.12055, + "mean_fro": 24.496485, + "rms_per_prompt_fro": 25.93101, + "max_per_prompt_fro": 27.576384, + "heavy_tail_ratio": 1.0635, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.059944, + "mean_fro": 22.630557, + "rms_per_prompt_fro": 23.298965, + "max_per_prompt_fro": 23.946051, + "heavy_tail_ratio": 1.0278, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.5, + "fit_secs_per_prompt": 0.138, + "total_wall_seconds": 17.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.278, + "top_sv": 5.6041, + "stable_rank": 35.261 + }, + { + "layer": 1, + "fro_norm": 25.7165, + "top_sv": 4.5545, + "stable_rank": 31.882 + }, + { + "layer": 2, + "fro_norm": 24.4965, + "top_sv": 3.743, + "stable_rank": 42.831 + }, + { + "layer": 3, + "fro_norm": 22.6305, + "top_sv": 4.3144, + "stable_rank": 27.514 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n400_s0.pt b/results/lenses/e28/e28_Pile-CC_70m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..81ebb562ed3c36d3798dcda91e4b568a65810127 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfd9f23e4f0cfd7eb8cb0377910c9623c6aac0719d83941411a7f5cae31f5b79 +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n400_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9862476e2281901a5e2b6f410a7450f7dbe36f7b --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n400_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n400_s0.pt", + "sha256": "cfd9f23e4f0cfd7eb8cb0377910c9623c6aac0719d83941411a7f5cae31f5b79", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.242526, + "mean_fro": 32.813335, + "rms_per_prompt_fro": 36.576582, + "max_per_prompt_fro": 45.162228, + "heavy_tail_ratio": 1.2347, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.191202, + "mean_fro": 25.472217, + "rms_per_prompt_fro": 27.800937, + "max_per_prompt_fro": 34.00552, + "heavy_tail_ratio": 1.2232, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.131791, + "mean_fro": 24.268198, + "rms_per_prompt_fro": 25.817883, + "max_per_prompt_fro": 29.464413, + "heavy_tail_ratio": 1.1412, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.065267, + "mean_fro": 22.579366, + "rms_per_prompt_fro": 23.304561, + "max_per_prompt_fro": 25.019238, + "heavy_tail_ratio": 1.0736, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 44.0, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 49.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.8133, + "top_sv": 5.4673, + "stable_rank": 36.021 + }, + { + "layer": 1, + "fro_norm": 25.4722, + "top_sv": 4.6015, + "stable_rank": 30.643 + }, + { + "layer": 2, + "fro_norm": 24.2682, + "top_sv": 3.7518, + "stable_rank": 41.841 + }, + { + "layer": 3, + "fro_norm": 22.5793, + "top_sv": 4.3241, + "stable_rank": 27.267 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n400_s1.pt b/results/lenses/e28/e28_Pile-CC_70m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..23a31e22384f44c812889c644762c9ad0fc959e9 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:153570cb3400f928ff3d46be8f57e970e3c8cce02bcb7340be3ff13b652a04f6 +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n400_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0cbe4468844d49ad0dc50806833d5edbd5750903 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n400_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n400_s1.pt", + "sha256": "153570cb3400f928ff3d46be8f57e970e3c8cce02bcb7340be3ff13b652a04f6", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.238304, + "mean_fro": 32.959644, + "rms_per_prompt_fro": 36.677201, + "max_per_prompt_fro": 44.87648, + "heavy_tail_ratio": 1.2236, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.18801, + "mean_fro": 25.592321, + "rms_per_prompt_fro": 27.894577, + "max_per_prompt_fro": 34.123726, + "heavy_tail_ratio": 1.2233, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.129616, + "mean_fro": 24.38283, + "rms_per_prompt_fro": 25.914898, + "max_per_prompt_fro": 30.594242, + "heavy_tail_ratio": 1.1806, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.064387, + "mean_fro": 22.640477, + "rms_per_prompt_fro": 23.357982, + "max_per_prompt_fro": 24.807026, + "heavy_tail_ratio": 1.062, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 44.3, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 51.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.9596, + "top_sv": 5.4945, + "stable_rank": 35.984 + }, + { + "layer": 1, + "fro_norm": 25.5923, + "top_sv": 4.6236, + "stable_rank": 30.637 + }, + { + "layer": 2, + "fro_norm": 24.3828, + "top_sv": 3.7683, + "stable_rank": 41.867 + }, + { + "layer": 3, + "fro_norm": 22.6404, + "top_sv": 4.3372, + "stable_rank": 27.249 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n400_s2.pt b/results/lenses/e28/e28_Pile-CC_70m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..75e9175c74de3462f271eb3943379cd8bfc31bd0 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0fc84c5ae790396e6f2d38342dc4c167144891fa1527912a666807fc4ccf4f9 +size 2099837 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n400_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c672f7a82ec491bef753eb1ea09bd85db8ad0525 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n400_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n400_s2.pt", + "sha256": "e0fc84c5ae790396e6f2d38342dc4c167144891fa1527912a666807fc4ccf4f9", + "bytes": 2099837, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.242852, + "mean_fro": 32.892772, + "rms_per_prompt_fro": 36.669944, + "max_per_prompt_fro": 44.496887, + "heavy_tail_ratio": 1.2134, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.191084, + "mean_fro": 25.57079, + "rms_per_prompt_fro": 27.907144, + "max_per_prompt_fro": 33.427799, + "heavy_tail_ratio": 1.1978, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.131536, + "mean_fro": 24.349314, + "rms_per_prompt_fro": 25.90126, + "max_per_prompt_fro": 30.606003, + "heavy_tail_ratio": 1.1816, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.065112, + "mean_fro": 22.614599, + "rms_per_prompt_fro": 23.33923, + "max_per_prompt_fro": 25.51335, + "heavy_tail_ratio": 1.0932, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 47.9, + "fit_secs_per_prompt": 0.12, + "total_wall_seconds": 57.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.8927, + "top_sv": 5.5618, + "stable_rank": 34.976 + }, + { + "layer": 1, + "fro_norm": 25.5707, + "top_sv": 4.6239, + "stable_rank": 30.582 + }, + { + "layer": 2, + "fro_norm": 24.3493, + "top_sv": 3.7669, + "stable_rank": 41.783 + }, + { + "layer": 3, + "fro_norm": 22.6146, + "top_sv": 4.3337, + "stable_rank": 27.23 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n50_s0.pt b/results/lenses/e28/e28_Pile-CC_70m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0c26fd2265283d943885a10e6c13fab622ce8d6 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58672012ff8f98f94a440d51b8821552d8533092e95d4220191d8f7f6eea3398 +size 2099827 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n50_s0_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4a05a819a840b325ae445f2634c81ce630fe21c3 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n50_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n50_s0.pt", + "sha256": "58672012ff8f98f94a440d51b8821552d8533092e95d4220191d8f7f6eea3398", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.247584, + "mean_fro": 33.002602, + "rms_per_prompt_fro": 36.86236, + "max_per_prompt_fro": 42.703876, + "heavy_tail_ratio": 1.1585, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.193187, + "mean_fro": 25.632164, + "rms_per_prompt_fro": 27.998804, + "max_per_prompt_fro": 31.724863, + "heavy_tail_ratio": 1.1331, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.131983, + "mean_fro": 24.417782, + "rms_per_prompt_fro": 25.979218, + "max_per_prompt_fro": 29.079222, + "heavy_tail_ratio": 1.1193, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.064473, + "mean_fro": 22.544235, + "rms_per_prompt_fro": 23.259629, + "max_per_prompt_fro": 24.31151, + "heavy_tail_ratio": 1.0452, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.7, + "fit_secs_per_prompt": 0.114, + "total_wall_seconds": 10.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.0025, + "top_sv": 5.3098, + "stable_rank": 38.631 + }, + { + "layer": 1, + "fro_norm": 25.6321, + "top_sv": 4.6329, + "stable_rank": 30.609 + }, + { + "layer": 2, + "fro_norm": 24.4177, + "top_sv": 3.7851, + "stable_rank": 41.616 + }, + { + "layer": 3, + "fro_norm": 22.5442, + "top_sv": 4.2799, + "stable_rank": 27.746 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n50_s1.pt b/results/lenses/e28/e28_Pile-CC_70m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..af87f0118868a9deb47afa8bca0482677736b7ab --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae020623e076bde097c5ff93d129b93fadb5f5027ace3e70da21f04588fe74f4 +size 2099827 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n50_s1_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e8456fc4f3fd661782414b369c57061b2a7e1ef4 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n50_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n50_s1.pt", + "sha256": "ae020623e076bde097c5ff93d129b93fadb5f5027ace3e70da21f04588fe74f4", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.227883, + "mean_fro": 33.512646, + "rms_per_prompt_fro": 37.13532, + "max_per_prompt_fro": 41.231567, + "heavy_tail_ratio": 1.1103, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.181019, + "mean_fro": 25.998386, + "rms_per_prompt_fro": 28.253664, + "max_per_prompt_fro": 31.077162, + "heavy_tail_ratio": 1.0999, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.125892, + "mean_fro": 24.666474, + "rms_per_prompt_fro": 26.173114, + "max_per_prompt_fro": 29.201994, + "heavy_tail_ratio": 1.1157, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.059078, + "mean_fro": 22.768341, + "rms_per_prompt_fro": 23.431249, + "max_per_prompt_fro": 24.807026, + "heavy_tail_ratio": 1.0587, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.6, + "fit_secs_per_prompt": 0.113, + "total_wall_seconds": 11.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.5126, + "top_sv": 5.726, + "stable_rank": 34.254 + }, + { + "layer": 1, + "fro_norm": 25.9983, + "top_sv": 4.6396, + "stable_rank": 31.4 + }, + { + "layer": 2, + "fro_norm": 24.6664, + "top_sv": 3.7878, + "stable_rank": 42.406 + }, + { + "layer": 3, + "fro_norm": 22.7683, + "top_sv": 4.3104, + "stable_rank": 27.902 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Pile-CC_70m_n50_s2.pt b/results/lenses/e28/e28_Pile-CC_70m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..6da092fd20547256f98ea9c1cb43905dd6d5680b --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faf52c6c09fdc0d0a4bdac4f2e212509663d9177690b243345f0883ccd71674c +size 2099827 diff --git a/results/lenses/e28/e28_Pile-CC_70m_n50_s2_provenance.json b/results/lenses/e28/e28_Pile-CC_70m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0dd4506e03b12e4010cf751956e786076101a540 --- /dev/null +++ b/results/lenses/e28/e28_Pile-CC_70m_n50_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Pile-CC_70m_n50_s2.pt", + "sha256": "faf52c6c09fdc0d0a4bdac4f2e212509663d9177690b243345f0883ccd71674c", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Pile-CC", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Pile-CC.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.249427, + "mean_fro": 32.839329, + "rms_per_prompt_fro": 36.707068, + "max_per_prompt_fro": 43.954247, + "heavy_tail_ratio": 1.1974, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.196054, + "mean_fro": 25.48274, + "rms_per_prompt_fro": 27.869012, + "max_per_prompt_fro": 33.427799, + "heavy_tail_ratio": 1.1995, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.134633, + "mean_fro": 24.344429, + "rms_per_prompt_fro": 25.931476, + "max_per_prompt_fro": 30.606003, + "heavy_tail_ratio": 1.1803, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.067293, + "mean_fro": 22.56671, + "rms_per_prompt_fro": 23.313642, + "max_per_prompt_fro": 25.147326, + "heavy_tail_ratio": 1.0787, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 6.5, + "fit_secs_per_prompt": 0.13, + "total_wall_seconds": 15.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.8393, + "top_sv": 5.5408, + "stable_rank": 35.127 + }, + { + "layer": 1, + "fro_norm": 25.4827, + "top_sv": 4.593, + "stable_rank": 30.783 + }, + { + "layer": 2, + "fro_norm": 24.3444, + "top_sv": 3.7696, + "stable_rank": 41.707 + }, + { + "layer": 3, + "fro_norm": 22.5667, + "top_sv": 4.2947, + "stable_rank": 27.61 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n100_s0.pt b/results/lenses/e28/e28_StackExchange_410m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..441c291b86010a89ab0b2ebbcb00ffa93e111122 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:947a4c69d53ceb80445f452147a531512731b616c9f99ee4d4889d10d8a33f89 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n100_s0_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b73f03a62f144e985d377ca2a03449befd8f80ee --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n100_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n100_s0.pt", + "sha256": "947a4c69d53ceb80445f452147a531512731b616c9f99ee4d4889d10d8a33f89", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.847946, + "mean_fro": 7.993216, + "rms_per_prompt_fro": 13.489239, + "max_per_prompt_fro": 18.450041, + "heavy_tail_ratio": 1.3678, + "n_prompts": 100 + }, + "1": { + "dispersion": 1.655259, + "mean_fro": 7.479604, + "rms_per_prompt_fro": 12.187989, + "max_per_prompt_fro": 16.337557, + "heavy_tail_ratio": 1.3405, + "n_prompts": 100 + }, + "2": { + "dispersion": 1.341129, + "mean_fro": 7.971678, + "rms_per_prompt_fro": 12.197265, + "max_per_prompt_fro": 15.959434, + "heavy_tail_ratio": 1.3084, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.153695, + "mean_fro": 7.688183, + "rms_per_prompt_fro": 11.282773, + "max_per_prompt_fro": 14.45578, + "heavy_tail_ratio": 1.2812, + "n_prompts": 100 + }, + "4": { + "dispersion": 0.951358, + "mean_fro": 8.76896, + "rms_per_prompt_fro": 12.249448, + "max_per_prompt_fro": 15.437881, + "heavy_tail_ratio": 1.2603, + "n_prompts": 100 + }, + "5": { + "dispersion": 0.874229, + "mean_fro": 9.062244, + "rms_per_prompt_fro": 12.406437, + "max_per_prompt_fro": 15.272669, + "heavy_tail_ratio": 1.231, + "n_prompts": 100 + }, + "6": { + "dispersion": 0.780231, + "mean_fro": 9.504091, + "rms_per_prompt_fro": 12.680861, + "max_per_prompt_fro": 15.436184, + "heavy_tail_ratio": 1.2173, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.688226, + "mean_fro": 10.507937, + "rms_per_prompt_fro": 13.653145, + "max_per_prompt_fro": 16.409863, + "heavy_tail_ratio": 1.2019, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.612901, + "mean_fro": 12.116919, + "rms_per_prompt_fro": 15.388494, + "max_per_prompt_fro": 18.278873, + "heavy_tail_ratio": 1.1878, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.559625, + "mean_fro": 13.742304, + "rms_per_prompt_fro": 17.162067, + "max_per_prompt_fro": 20.002867, + "heavy_tail_ratio": 1.1655, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.506584, + "mean_fro": 15.70355, + "rms_per_prompt_fro": 19.275005, + "max_per_prompt_fro": 22.135134, + "heavy_tail_ratio": 1.1484, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.451022, + "mean_fro": 18.621306, + "rms_per_prompt_fro": 22.430924, + "max_per_prompt_fro": 25.839254, + "heavy_tail_ratio": 1.1519, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.362722, + "mean_fro": 20.641809, + "rms_per_prompt_fro": 24.096356, + "max_per_prompt_fro": 27.022219, + "heavy_tail_ratio": 1.1214, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.307563, + "mean_fro": 22.386744, + "rms_per_prompt_fro": 25.598956, + "max_per_prompt_fro": 28.637596, + "heavy_tail_ratio": 1.1187, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.256281, + "mean_fro": 25.684163, + "rms_per_prompt_fro": 28.787822, + "max_per_prompt_fro": 31.647188, + "heavy_tail_ratio": 1.0993, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.214078, + "mean_fro": 28.679974, + "rms_per_prompt_fro": 31.601085, + "max_per_prompt_fro": 34.469643, + "heavy_tail_ratio": 1.0908, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.155753, + "mean_fro": 32.370369, + "rms_per_prompt_fro": 34.800076, + "max_per_prompt_fro": 37.765614, + "heavy_tail_ratio": 1.0852, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.11743, + "mean_fro": 34.67746, + "rms_per_prompt_fro": 36.657048, + "max_per_prompt_fro": 39.224834, + "heavy_tail_ratio": 1.07, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.085623, + "mean_fro": 36.070515, + "rms_per_prompt_fro": 37.583033, + "max_per_prompt_fro": 39.785553, + "heavy_tail_ratio": 1.0586, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.058218, + "mean_fro": 36.110024, + "rms_per_prompt_fro": 37.146275, + "max_per_prompt_fro": 38.707199, + "heavy_tail_ratio": 1.042, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.035381, + "mean_fro": 35.918762, + "rms_per_prompt_fro": 36.548662, + "max_per_prompt_fro": 37.927906, + "heavy_tail_ratio": 1.0377, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.017801, + "mean_fro": 35.107044, + "rms_per_prompt_fro": 35.418131, + "max_per_prompt_fro": 36.156605, + "heavy_tail_ratio": 1.0209, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 308.1, + "fit_secs_per_prompt": 3.081, + "total_wall_seconds": 314.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9931, + "top_sv": 1.4654, + "stable_rank": 29.753 + }, + { + "layer": 1, + "fro_norm": 7.4795, + "top_sv": 1.3782, + "stable_rank": 29.45 + }, + { + "layer": 2, + "fro_norm": 7.9716, + "top_sv": 1.4804, + "stable_rank": 28.996 + }, + { + "layer": 3, + "fro_norm": 7.6881, + "top_sv": 1.4694, + "stable_rank": 27.374 + }, + { + "layer": 4, + "fro_norm": 8.7688, + "top_sv": 1.7263, + "stable_rank": 25.803 + }, + { + "layer": 5, + "fro_norm": 9.0621, + "top_sv": 1.8515, + "stable_rank": 23.957 + }, + { + "layer": 6, + "fro_norm": 9.504, + "top_sv": 1.9519, + "stable_rank": 23.707 + }, + { + "layer": 7, + "fro_norm": 10.5078, + "top_sv": 2.1515, + "stable_rank": 23.852 + }, + { + "layer": 8, + "fro_norm": 12.1168, + "top_sv": 2.6407, + "stable_rank": 21.054 + }, + { + "layer": 9, + "fro_norm": 13.7421, + "top_sv": 2.989, + "stable_rank": 21.138 + }, + { + "layer": 10, + "fro_norm": 15.7034, + "top_sv": 3.4325, + "stable_rank": 20.93 + }, + { + "layer": 11, + "fro_norm": 18.621, + "top_sv": 4.0531, + "stable_rank": 21.107 + }, + { + "layer": 12, + "fro_norm": 20.6415, + "top_sv": 4.482, + "stable_rank": 21.21 + }, + { + "layer": 13, + "fro_norm": 22.3864, + "top_sv": 4.9071, + "stable_rank": 20.812 + }, + { + "layer": 14, + "fro_norm": 25.6837, + "top_sv": 5.5523, + "stable_rank": 21.398 + }, + { + "layer": 15, + "fro_norm": 28.6795, + "top_sv": 6.3873, + "stable_rank": 20.16 + }, + { + "layer": 16, + "fro_norm": 32.3698, + "top_sv": 7.4705, + "stable_rank": 18.775 + }, + { + "layer": 17, + "fro_norm": 34.6767, + "top_sv": 7.9214, + "stable_rank": 19.163 + }, + { + "layer": 18, + "fro_norm": 36.0696, + "top_sv": 8.0982, + "stable_rank": 19.839 + }, + { + "layer": 19, + "fro_norm": 36.109, + "top_sv": 8.0283, + "stable_rank": 20.23 + }, + { + "layer": 20, + "fro_norm": 35.9175, + "top_sv": 7.5947, + "stable_rank": 22.366 + }, + { + "layer": 21, + "fro_norm": 35.1054, + "top_sv": 5.9479, + "stable_rank": 34.835 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n100_s1.pt b/results/lenses/e28/e28_StackExchange_410m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..62020da93eea15f0f61ccd80434951700996a53c --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3a026b1da50466405ffd4d7ad480aec1597e02f1442b7e9320b0f450f8ec9b5 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n100_s1_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0e3dd829a3f30824cd3c21f9f51574ba5681ba0d --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n100_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n100_s1.pt", + "sha256": "f3a026b1da50466405ffd4d7ad480aec1597e02f1442b7e9320b0f450f8ec9b5", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.783063, + "mean_fro": 8.086331, + "rms_per_prompt_fro": 13.490035, + "max_per_prompt_fro": 19.984915, + "heavy_tail_ratio": 1.4815, + "n_prompts": 100 + }, + "1": { + "dispersion": 1.596608, + "mean_fro": 7.558959, + "rms_per_prompt_fro": 12.180502, + "max_per_prompt_fro": 17.600504, + "heavy_tail_ratio": 1.445, + "n_prompts": 100 + }, + "2": { + "dispersion": 1.324573, + "mean_fro": 8.0036, + "rms_per_prompt_fro": 12.202729, + "max_per_prompt_fro": 17.261517, + "heavy_tail_ratio": 1.4146, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.14696, + "mean_fro": 7.695959, + "rms_per_prompt_fro": 11.27651, + "max_per_prompt_fro": 15.296987, + "heavy_tail_ratio": 1.3565, + "n_prompts": 100 + }, + "4": { + "dispersion": 0.960835, + "mean_fro": 8.735059, + "rms_per_prompt_fro": 12.231688, + "max_per_prompt_fro": 16.132614, + "heavy_tail_ratio": 1.3189, + "n_prompts": 100 + }, + "5": { + "dispersion": 0.882882, + "mean_fro": 9.032347, + "rms_per_prompt_fro": 12.394018, + "max_per_prompt_fro": 15.868522, + "heavy_tail_ratio": 1.2803, + "n_prompts": 100 + }, + "6": { + "dispersion": 0.787899, + "mean_fro": 9.476368, + "rms_per_prompt_fro": 12.671072, + "max_per_prompt_fro": 15.966814, + "heavy_tail_ratio": 1.2601, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.695536, + "mean_fro": 10.489526, + "rms_per_prompt_fro": 13.658698, + "max_per_prompt_fro": 16.773069, + "heavy_tail_ratio": 1.228, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.616666, + "mean_fro": 12.061165, + "rms_per_prompt_fro": 15.335549, + "max_per_prompt_fro": 18.029625, + "heavy_tail_ratio": 1.1757, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.563411, + "mean_fro": 13.654472, + "rms_per_prompt_fro": 17.073066, + "max_per_prompt_fro": 20.106562, + "heavy_tail_ratio": 1.1777, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.508812, + "mean_fro": 15.580795, + "rms_per_prompt_fro": 19.13847, + "max_per_prompt_fro": 22.459669, + "heavy_tail_ratio": 1.1735, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.452813, + "mean_fro": 18.426411, + "rms_per_prompt_fro": 22.20985, + "max_per_prompt_fro": 25.981112, + "heavy_tail_ratio": 1.1698, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.361597, + "mean_fro": 20.540051, + "rms_per_prompt_fro": 23.967667, + "max_per_prompt_fro": 27.542673, + "heavy_tail_ratio": 1.1492, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.305324, + "mean_fro": 22.237909, + "rms_per_prompt_fro": 25.406981, + "max_per_prompt_fro": 28.854349, + "heavy_tail_ratio": 1.1357, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.253881, + "mean_fro": 25.478955, + "rms_per_prompt_fro": 28.530521, + "max_per_prompt_fro": 32.247253, + "heavy_tail_ratio": 1.1303, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.210208, + "mean_fro": 28.432962, + "rms_per_prompt_fro": 31.278947, + "max_per_prompt_fro": 35.142059, + "heavy_tail_ratio": 1.1235, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.152868, + "mean_fro": 32.195262, + "rms_per_prompt_fro": 34.568604, + "max_per_prompt_fro": 38.322559, + "heavy_tail_ratio": 1.1086, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.114935, + "mean_fro": 34.584602, + "rms_per_prompt_fro": 36.518053, + "max_per_prompt_fro": 39.639565, + "heavy_tail_ratio": 1.0855, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.084236, + "mean_fro": 35.970046, + "rms_per_prompt_fro": 37.4544, + "max_per_prompt_fro": 40.168762, + "heavy_tail_ratio": 1.0725, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.056898, + "mean_fro": 36.107117, + "rms_per_prompt_fro": 37.120117, + "max_per_prompt_fro": 38.975006, + "heavy_tail_ratio": 1.05, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.033956, + "mean_fro": 35.960254, + "rms_per_prompt_fro": 36.565683, + "max_per_prompt_fro": 38.089752, + "heavy_tail_ratio": 1.0417, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.017055, + "mean_fro": 35.142586, + "rms_per_prompt_fro": 35.441005, + "max_per_prompt_fro": 36.388966, + "heavy_tail_ratio": 1.0267, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 308.0, + "fit_secs_per_prompt": 3.08, + "total_wall_seconds": 314.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.0862, + "top_sv": 1.4622, + "stable_rank": 30.581 + }, + { + "layer": 1, + "fro_norm": 7.5589, + "top_sv": 1.3722, + "stable_rank": 30.346 + }, + { + "layer": 2, + "fro_norm": 8.0035, + "top_sv": 1.4672, + "stable_rank": 29.757 + }, + { + "layer": 3, + "fro_norm": 7.6959, + "top_sv": 1.4463, + "stable_rank": 28.312 + }, + { + "layer": 4, + "fro_norm": 8.7349, + "top_sv": 1.6364, + "stable_rank": 28.491 + }, + { + "layer": 5, + "fro_norm": 9.0322, + "top_sv": 1.7924, + "stable_rank": 25.394 + }, + { + "layer": 6, + "fro_norm": 9.4762, + "top_sv": 1.9158, + "stable_rank": 24.465 + }, + { + "layer": 7, + "fro_norm": 10.4894, + "top_sv": 2.1343, + "stable_rank": 24.155 + }, + { + "layer": 8, + "fro_norm": 12.061, + "top_sv": 2.617, + "stable_rank": 21.241 + }, + { + "layer": 9, + "fro_norm": 13.6543, + "top_sv": 2.9669, + "stable_rank": 21.18 + }, + { + "layer": 10, + "fro_norm": 15.5806, + "top_sv": 3.4259, + "stable_rank": 20.683 + }, + { + "layer": 11, + "fro_norm": 18.4261, + "top_sv": 4.0391, + "stable_rank": 20.811 + }, + { + "layer": 12, + "fro_norm": 20.5398, + "top_sv": 4.5272, + "stable_rank": 20.585 + }, + { + "layer": 13, + "fro_norm": 22.2376, + "top_sv": 4.9393, + "stable_rank": 20.269 + }, + { + "layer": 14, + "fro_norm": 25.4785, + "top_sv": 5.596, + "stable_rank": 20.73 + }, + { + "layer": 15, + "fro_norm": 28.4324, + "top_sv": 6.4066, + "stable_rank": 19.695 + }, + { + "layer": 16, + "fro_norm": 32.1947, + "top_sv": 7.5011, + "stable_rank": 18.421 + }, + { + "layer": 17, + "fro_norm": 34.5838, + "top_sv": 7.9332, + "stable_rank": 19.004 + }, + { + "layer": 18, + "fro_norm": 35.9691, + "top_sv": 8.0588, + "stable_rank": 19.921 + }, + { + "layer": 19, + "fro_norm": 36.106, + "top_sv": 7.9719, + "stable_rank": 20.514 + }, + { + "layer": 20, + "fro_norm": 35.959, + "top_sv": 7.5975, + "stable_rank": 22.401 + }, + { + "layer": 21, + "fro_norm": 35.141, + "top_sv": 5.9552, + "stable_rank": 34.82 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n100_s2.pt b/results/lenses/e28/e28_StackExchange_410m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..8661cb4ba26cda530ebc25710b69874acf209c32 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a2f90dcddc685307ddf94d0671a67bc70983e411aee7e518643cb6dc4336262 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n100_s2_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..1f98066d346227c976424a846185d507ba758d96 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n100_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n100_s2.pt", + "sha256": "4a2f90dcddc685307ddf94d0671a67bc70983e411aee7e518643cb6dc4336262", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.846684, + "mean_fro": 7.816634, + "rms_per_prompt_fro": 13.188317, + "max_per_prompt_fro": 17.906555, + "heavy_tail_ratio": 1.3578, + "n_prompts": 100 + }, + "1": { + "dispersion": 1.654439, + "mean_fro": 7.305065, + "rms_per_prompt_fro": 11.901741, + "max_per_prompt_fro": 16.124765, + "heavy_tail_ratio": 1.3548, + "n_prompts": 100 + }, + "2": { + "dispersion": 1.35887, + "mean_fro": 7.775264, + "rms_per_prompt_fro": 11.941728, + "max_per_prompt_fro": 16.412254, + "heavy_tail_ratio": 1.3744, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.173672, + "mean_fro": 7.495631, + "rms_per_prompt_fro": 11.051093, + "max_per_prompt_fro": 14.864495, + "heavy_tail_ratio": 1.3451, + "n_prompts": 100 + }, + "4": { + "dispersion": 0.969391, + "mean_fro": 8.563139, + "rms_per_prompt_fro": 12.01708, + "max_per_prompt_fro": 16.002729, + "heavy_tail_ratio": 1.3317, + "n_prompts": 100 + }, + "5": { + "dispersion": 0.889593, + "mean_fro": 8.858447, + "rms_per_prompt_fro": 12.17704, + "max_per_prompt_fro": 16.23443, + "heavy_tail_ratio": 1.3332, + "n_prompts": 100 + }, + "6": { + "dispersion": 0.788251, + "mean_fro": 9.303494, + "rms_per_prompt_fro": 12.441144, + "max_per_prompt_fro": 16.501925, + "heavy_tail_ratio": 1.3264, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.693067, + "mean_fro": 10.315278, + "rms_per_prompt_fro": 13.422025, + "max_per_prompt_fro": 17.810621, + "heavy_tail_ratio": 1.327, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.614537, + "mean_fro": 11.903961, + "rms_per_prompt_fro": 15.125701, + "max_per_prompt_fro": 19.751907, + "heavy_tail_ratio": 1.3059, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.558952, + "mean_fro": 13.527061, + "rms_per_prompt_fro": 16.889618, + "max_per_prompt_fro": 21.543802, + "heavy_tail_ratio": 1.2756, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.505019, + "mean_fro": 15.495607, + "rms_per_prompt_fro": 19.009888, + "max_per_prompt_fro": 23.402719, + "heavy_tail_ratio": 1.2311, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.44962, + "mean_fro": 18.385212, + "rms_per_prompt_fro": 22.135826, + "max_per_prompt_fro": 26.593653, + "heavy_tail_ratio": 1.2014, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.361297, + "mean_fro": 20.501909, + "rms_per_prompt_fro": 23.920527, + "max_per_prompt_fro": 28.628778, + "heavy_tail_ratio": 1.1968, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.305904, + "mean_fro": 22.235146, + "rms_per_prompt_fro": 25.409465, + "max_per_prompt_fro": 29.46944, + "heavy_tail_ratio": 1.1598, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.253571, + "mean_fro": 25.510556, + "rms_per_prompt_fro": 28.562381, + "max_per_prompt_fro": 32.413708, + "heavy_tail_ratio": 1.1348, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.210224, + "mean_fro": 28.507171, + "rms_per_prompt_fro": 31.360797, + "max_per_prompt_fro": 35.082249, + "heavy_tail_ratio": 1.1187, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.152822, + "mean_fro": 32.248437, + "rms_per_prompt_fro": 34.624996, + "max_per_prompt_fro": 37.666168, + "heavy_tail_ratio": 1.0878, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.115118, + "mean_fro": 34.617255, + "rms_per_prompt_fro": 36.555524, + "max_per_prompt_fro": 39.150112, + "heavy_tail_ratio": 1.071, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.084778, + "mean_fro": 36.042831, + "rms_per_prompt_fro": 37.53958, + "max_per_prompt_fro": 39.759815, + "heavy_tail_ratio": 1.0591, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.05768, + "mean_fro": 36.155644, + "rms_per_prompt_fro": 37.18375, + "max_per_prompt_fro": 38.615036, + "heavy_tail_ratio": 1.0385, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.035036, + "mean_fro": 35.9692, + "rms_per_prompt_fro": 36.593885, + "max_per_prompt_fro": 37.595169, + "heavy_tail_ratio": 1.0274, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.017646, + "mean_fro": 35.143795, + "rms_per_prompt_fro": 35.452514, + "max_per_prompt_fro": 36.070801, + "heavy_tail_ratio": 1.0174, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 311.3, + "fit_secs_per_prompt": 3.113, + "total_wall_seconds": 319.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.8165, + "top_sv": 1.401, + "stable_rank": 31.127 + }, + { + "layer": 1, + "fro_norm": 7.305, + "top_sv": 1.3231, + "stable_rank": 30.483 + }, + { + "layer": 2, + "fro_norm": 7.7752, + "top_sv": 1.4122, + "stable_rank": 30.314 + }, + { + "layer": 3, + "fro_norm": 7.4955, + "top_sv": 1.4062, + "stable_rank": 28.411 + }, + { + "layer": 4, + "fro_norm": 8.563, + "top_sv": 1.6051, + "stable_rank": 28.462 + }, + { + "layer": 5, + "fro_norm": 8.8583, + "top_sv": 1.7448, + "stable_rank": 25.774 + }, + { + "layer": 6, + "fro_norm": 9.3034, + "top_sv": 1.8599, + "stable_rank": 25.021 + }, + { + "layer": 7, + "fro_norm": 10.3151, + "top_sv": 2.087, + "stable_rank": 24.428 + }, + { + "layer": 8, + "fro_norm": 11.9038, + "top_sv": 2.5695, + "stable_rank": 21.462 + }, + { + "layer": 9, + "fro_norm": 13.5269, + "top_sv": 2.9158, + "stable_rank": 21.522 + }, + { + "layer": 10, + "fro_norm": 15.4954, + "top_sv": 3.3732, + "stable_rank": 21.102 + }, + { + "layer": 11, + "fro_norm": 18.3849, + "top_sv": 3.9826, + "stable_rank": 21.311 + }, + { + "layer": 12, + "fro_norm": 20.5016, + "top_sv": 4.4655, + "stable_rank": 21.079 + }, + { + "layer": 13, + "fro_norm": 22.2348, + "top_sv": 4.885, + "stable_rank": 20.718 + }, + { + "layer": 14, + "fro_norm": 25.5101, + "top_sv": 5.5177, + "stable_rank": 21.375 + }, + { + "layer": 15, + "fro_norm": 28.5066, + "top_sv": 6.3742, + "stable_rank": 20.001 + }, + { + "layer": 16, + "fro_norm": 32.2479, + "top_sv": 7.4731, + "stable_rank": 18.621 + }, + { + "layer": 17, + "fro_norm": 34.6164, + "top_sv": 7.9188, + "stable_rank": 19.109 + }, + { + "layer": 18, + "fro_norm": 36.0419, + "top_sv": 8.0695, + "stable_rank": 19.949 + }, + { + "layer": 19, + "fro_norm": 36.1546, + "top_sv": 8.0012, + "stable_rank": 20.418 + }, + { + "layer": 20, + "fro_norm": 35.9679, + "top_sv": 7.616, + "stable_rank": 22.304 + }, + { + "layer": 21, + "fro_norm": 35.1422, + "top_sv": 5.9671, + "stable_rank": 34.684 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n200_s0.pt b/results/lenses/e28/e28_StackExchange_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c4d37bf282e1a82c70823ecdd9b41ab6c364588 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba4f9a51cbd39b523043bea1286f8efc61ec72df542dc224a419c0e2e0a3a9f +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n200_s0_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a7343e616113b765cd3f2b636e75323710ef34be --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n200_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n200_s0.pt", + "sha256": "fba4f9a51cbd39b523043bea1286f8efc61ec72df542dc224a419c0e2e0a3a9f", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.870418, + "mean_fro": 8.006987, + "rms_per_prompt_fro": 13.565685, + "max_per_prompt_fro": 19.660454, + "heavy_tail_ratio": 1.4493, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.67093, + "mean_fro": 7.493224, + "rms_per_prompt_fro": 12.24616, + "max_per_prompt_fro": 17.510759, + "heavy_tail_ratio": 1.4299, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.360609, + "mean_fro": 7.968367, + "rms_per_prompt_fro": 12.242817, + "max_per_prompt_fro": 17.787825, + "heavy_tail_ratio": 1.4529, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.175558, + "mean_fro": 7.679352, + "rms_per_prompt_fro": 11.326869, + "max_per_prompt_fro": 16.305996, + "heavy_tail_ratio": 1.4396, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.973549, + "mean_fro": 8.765326, + "rms_per_prompt_fro": 12.313798, + "max_per_prompt_fro": 17.9034, + "heavy_tail_ratio": 1.4539, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.896946, + "mean_fro": 9.060084, + "rms_per_prompt_fro": 12.478422, + "max_per_prompt_fro": 17.933376, + "heavy_tail_ratio": 1.4372, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.798941, + "mean_fro": 9.501162, + "rms_per_prompt_fro": 12.743396, + "max_per_prompt_fro": 18.227463, + "heavy_tail_ratio": 1.4303, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.704104, + "mean_fro": 10.505464, + "rms_per_prompt_fro": 13.713972, + "max_per_prompt_fro": 19.273075, + "heavy_tail_ratio": 1.4054, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.627035, + "mean_fro": 12.110025, + "rms_per_prompt_fro": 15.446977, + "max_per_prompt_fro": 21.539688, + "heavy_tail_ratio": 1.3944, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.572409, + "mean_fro": 13.723828, + "rms_per_prompt_fro": 17.209097, + "max_per_prompt_fro": 23.499172, + "heavy_tail_ratio": 1.3655, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.518846, + "mean_fro": 15.674985, + "rms_per_prompt_fro": 19.318081, + "max_per_prompt_fro": 25.770779, + "heavy_tail_ratio": 1.334, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.462289, + "mean_fro": 18.56922, + "rms_per_prompt_fro": 22.454858, + "max_per_prompt_fro": 29.363607, + "heavy_tail_ratio": 1.3077, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.370535, + "mean_fro": 20.615818, + "rms_per_prompt_fro": 24.134909, + "max_per_prompt_fro": 29.734083, + "heavy_tail_ratio": 1.232, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.313846, + "mean_fro": 22.341875, + "rms_per_prompt_fro": 25.60895, + "max_per_prompt_fro": 30.961466, + "heavy_tail_ratio": 1.209, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.260613, + "mean_fro": 25.619236, + "rms_per_prompt_fro": 28.764513, + "max_per_prompt_fro": 33.7141, + "heavy_tail_ratio": 1.1721, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.216134, + "mean_fro": 28.615848, + "rms_per_prompt_fro": 31.557121, + "max_per_prompt_fro": 36.521626, + "heavy_tail_ratio": 1.1573, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.15655, + "mean_fro": 32.310654, + "rms_per_prompt_fro": 34.747852, + "max_per_prompt_fro": 38.571972, + "heavy_tail_ratio": 1.1101, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.117513, + "mean_fro": 34.645192, + "rms_per_prompt_fro": 36.624289, + "max_per_prompt_fro": 39.345181, + "heavy_tail_ratio": 1.0743, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.085475, + "mean_fro": 36.04105, + "rms_per_prompt_fro": 37.549768, + "max_per_prompt_fro": 39.785553, + "heavy_tail_ratio": 1.0595, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.057734, + "mean_fro": 36.109432, + "rms_per_prompt_fro": 37.137185, + "max_per_prompt_fro": 38.823784, + "heavy_tail_ratio": 1.0454, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.034826, + "mean_fro": 35.935929, + "rms_per_prompt_fro": 36.556324, + "max_per_prompt_fro": 38.052616, + "heavy_tail_ratio": 1.0409, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.017502, + "mean_fro": 35.121719, + "rms_per_prompt_fro": 35.427744, + "max_per_prompt_fro": 36.302822, + "heavy_tail_ratio": 1.0247, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 618.6, + "fit_secs_per_prompt": 3.093, + "total_wall_seconds": 624.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.0069, + "top_sv": 1.4474, + "stable_rank": 30.601 + }, + { + "layer": 1, + "fro_norm": 7.4931, + "top_sv": 1.359, + "stable_rank": 30.403 + }, + { + "layer": 2, + "fro_norm": 7.9683, + "top_sv": 1.4436, + "stable_rank": 30.466 + }, + { + "layer": 3, + "fro_norm": 7.6793, + "top_sv": 1.4395, + "stable_rank": 28.46 + }, + { + "layer": 4, + "fro_norm": 8.7652, + "top_sv": 1.6737, + "stable_rank": 27.427 + }, + { + "layer": 5, + "fro_norm": 9.06, + "top_sv": 1.8023, + "stable_rank": 25.271 + }, + { + "layer": 6, + "fro_norm": 9.501, + "top_sv": 1.9114, + "stable_rank": 24.708 + }, + { + "layer": 7, + "fro_norm": 10.5053, + "top_sv": 2.1254, + "stable_rank": 24.43 + }, + { + "layer": 8, + "fro_norm": 12.1099, + "top_sv": 2.6063, + "stable_rank": 21.59 + }, + { + "layer": 9, + "fro_norm": 13.7236, + "top_sv": 2.9506, + "stable_rank": 21.634 + }, + { + "layer": 10, + "fro_norm": 15.6748, + "top_sv": 3.3954, + "stable_rank": 21.312 + }, + { + "layer": 11, + "fro_norm": 18.5689, + "top_sv": 4.0013, + "stable_rank": 21.537 + }, + { + "layer": 12, + "fro_norm": 20.6155, + "top_sv": 4.452, + "stable_rank": 21.443 + }, + { + "layer": 13, + "fro_norm": 22.3416, + "top_sv": 4.8756, + "stable_rank": 20.998 + }, + { + "layer": 14, + "fro_norm": 25.6188, + "top_sv": 5.52, + "stable_rank": 21.539 + }, + { + "layer": 15, + "fro_norm": 28.6153, + "top_sv": 6.3451, + "stable_rank": 20.338 + }, + { + "layer": 16, + "fro_norm": 32.3101, + "top_sv": 7.4401, + "stable_rank": 18.859 + }, + { + "layer": 17, + "fro_norm": 34.6444, + "top_sv": 7.8764, + "stable_rank": 19.347 + }, + { + "layer": 18, + "fro_norm": 36.0401, + "top_sv": 8.0487, + "stable_rank": 20.05 + }, + { + "layer": 19, + "fro_norm": 36.1083, + "top_sv": 7.9827, + "stable_rank": 20.46 + }, + { + "layer": 20, + "fro_norm": 35.9346, + "top_sv": 7.5787, + "stable_rank": 22.482 + }, + { + "layer": 21, + "fro_norm": 35.1201, + "top_sv": 5.9536, + "stable_rank": 34.798 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n200_s1.pt b/results/lenses/e28/e28_StackExchange_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..86fdc2840ed27b9802cc2413f68e01c33592ee51 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7552cfc76417a4b50971284df5944f731e579c196137b8c32a3ae1c2d949d582 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n200_s1_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..61839c1a348e42e59c1b67054e0be113d35e84c6 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n200_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n200_s1.pt", + "sha256": "7552cfc76417a4b50971284df5944f731e579c196137b8c32a3ae1c2d949d582", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.901253, + "mean_fro": 7.880901, + "rms_per_prompt_fro": 13.423589, + "max_per_prompt_fro": 19.984915, + "heavy_tail_ratio": 1.4888, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.700178, + "mean_fro": 7.370725, + "rms_per_prompt_fro": 12.111736, + "max_per_prompt_fro": 17.600504, + "heavy_tail_ratio": 1.4532, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.40236, + "mean_fro": 7.836064, + "rms_per_prompt_fro": 12.145547, + "max_per_prompt_fro": 17.261517, + "heavy_tail_ratio": 1.4212, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.210128, + "mean_fro": 7.557934, + "rms_per_prompt_fro": 11.236004, + "max_per_prompt_fro": 15.296987, + "heavy_tail_ratio": 1.3614, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.010811, + "mean_fro": 8.605534, + "rms_per_prompt_fro": 12.202913, + "max_per_prompt_fro": 16.132614, + "heavy_tail_ratio": 1.322, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.926609, + "mean_fro": 8.908876, + "rms_per_prompt_fro": 12.365731, + "max_per_prompt_fro": 15.87153, + "heavy_tail_ratio": 1.2835, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.825957, + "mean_fro": 9.356067, + "rms_per_prompt_fro": 12.642665, + "max_per_prompt_fro": 16.114813, + "heavy_tail_ratio": 1.2746, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.727687, + "mean_fro": 10.363366, + "rms_per_prompt_fro": 13.621765, + "max_per_prompt_fro": 17.298576, + "heavy_tail_ratio": 1.2699, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.644406, + "mean_fro": 11.941843, + "rms_per_prompt_fro": 15.313551, + "max_per_prompt_fro": 18.925735, + "heavy_tail_ratio": 1.2359, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.58753, + "mean_fro": 13.546573, + "rms_per_prompt_fro": 17.068307, + "max_per_prompt_fro": 20.478926, + "heavy_tail_ratio": 1.1998, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.529523, + "mean_fro": 15.485291, + "rms_per_prompt_fro": 19.151259, + "max_per_prompt_fro": 22.459669, + "heavy_tail_ratio": 1.1728, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.470963, + "mean_fro": 18.346806, + "rms_per_prompt_fro": 22.251606, + "max_per_prompt_fro": 26.241457, + "heavy_tail_ratio": 1.1793, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.375623, + "mean_fro": 20.439171, + "rms_per_prompt_fro": 23.972479, + "max_per_prompt_fro": 27.542673, + "heavy_tail_ratio": 1.1489, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.318239, + "mean_fro": 22.147749, + "rms_per_prompt_fro": 25.428851, + "max_per_prompt_fro": 28.913649, + "heavy_tail_ratio": 1.137, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.264387, + "mean_fro": 25.392483, + "rms_per_prompt_fro": 28.552566, + "max_per_prompt_fro": 32.247253, + "heavy_tail_ratio": 1.1294, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.219255, + "mean_fro": 28.36154, + "rms_per_prompt_fro": 31.316775, + "max_per_prompt_fro": 35.142059, + "heavy_tail_ratio": 1.1221, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.158973, + "mean_fro": 32.108455, + "rms_per_prompt_fro": 34.566554, + "max_per_prompt_fro": 38.322559, + "heavy_tail_ratio": 1.1087, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.119274, + "mean_fro": 34.520385, + "rms_per_prompt_fro": 36.521097, + "max_per_prompt_fro": 39.639565, + "heavy_tail_ratio": 1.0854, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.08754, + "mean_fro": 35.941308, + "rms_per_prompt_fro": 37.481454, + "max_per_prompt_fro": 40.168762, + "heavy_tail_ratio": 1.0717, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.059226, + "mean_fro": 36.090344, + "rms_per_prompt_fro": 37.143712, + "max_per_prompt_fro": 38.975006, + "heavy_tail_ratio": 1.0493, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.03511, + "mean_fro": 35.953667, + "rms_per_prompt_fro": 36.579391, + "max_per_prompt_fro": 38.089752, + "heavy_tail_ratio": 1.0413, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.017491, + "mean_fro": 35.14894, + "rms_per_prompt_fro": 35.455007, + "max_per_prompt_fro": 36.388966, + "heavy_tail_ratio": 1.0263, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 616.9, + "fit_secs_per_prompt": 3.085, + "total_wall_seconds": 623.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.8808, + "top_sv": 1.4187, + "stable_rank": 30.858 + }, + { + "layer": 1, + "fro_norm": 7.3706, + "top_sv": 1.337, + "stable_rank": 30.39 + }, + { + "layer": 2, + "fro_norm": 7.836, + "top_sv": 1.4281, + "stable_rank": 30.108 + }, + { + "layer": 3, + "fro_norm": 7.5578, + "top_sv": 1.4244, + "stable_rank": 28.154 + }, + { + "layer": 4, + "fro_norm": 8.6054, + "top_sv": 1.6306, + "stable_rank": 27.851 + }, + { + "layer": 5, + "fro_norm": 8.9087, + "top_sv": 1.781, + "stable_rank": 25.021 + }, + { + "layer": 6, + "fro_norm": 9.3559, + "top_sv": 1.8973, + "stable_rank": 24.316 + }, + { + "layer": 7, + "fro_norm": 10.3632, + "top_sv": 2.1116, + "stable_rank": 24.086 + }, + { + "layer": 8, + "fro_norm": 11.9417, + "top_sv": 2.601, + "stable_rank": 21.079 + }, + { + "layer": 9, + "fro_norm": 13.5464, + "top_sv": 2.9432, + "stable_rank": 21.184 + }, + { + "layer": 10, + "fro_norm": 15.4851, + "top_sv": 3.4026, + "stable_rank": 20.712 + }, + { + "layer": 11, + "fro_norm": 18.3466, + "top_sv": 4.0058, + "stable_rank": 20.976 + }, + { + "layer": 12, + "fro_norm": 20.4389, + "top_sv": 4.483, + "stable_rank": 20.786 + }, + { + "layer": 13, + "fro_norm": 22.1474, + "top_sv": 4.903, + "stable_rank": 20.404 + }, + { + "layer": 14, + "fro_norm": 25.3921, + "top_sv": 5.5582, + "stable_rank": 20.87 + }, + { + "layer": 15, + "fro_norm": 28.361, + "top_sv": 6.3788, + "stable_rank": 19.768 + }, + { + "layer": 16, + "fro_norm": 32.1079, + "top_sv": 7.4786, + "stable_rank": 18.433 + }, + { + "layer": 17, + "fro_norm": 34.5196, + "top_sv": 7.91, + "stable_rank": 19.045 + }, + { + "layer": 18, + "fro_norm": 35.9403, + "top_sv": 8.074, + "stable_rank": 19.815 + }, + { + "layer": 19, + "fro_norm": 36.0892, + "top_sv": 8.009, + "stable_rank": 20.305 + }, + { + "layer": 20, + "fro_norm": 35.9524, + "top_sv": 7.6066, + "stable_rank": 22.34 + }, + { + "layer": 21, + "fro_norm": 35.1473, + "top_sv": 5.9714, + "stable_rank": 34.644 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n200_s2.pt b/results/lenses/e28/e28_StackExchange_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbb15cf52073c0280f88e1f646419a661f017a31 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20e1867f9f0c7a5528f7b115ea67479c8875ff496ca35ee2323576e2aae81a63 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n200_s2_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..71ff4221774a2f9912c8dedf47a6e3e6cdd4e2c0 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n200_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n200_s2.pt", + "sha256": "20e1867f9f0c7a5528f7b115ea67479c8875ff496ca35ee2323576e2aae81a63", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.884922, + "mean_fro": 7.770883, + "rms_per_prompt_fro": 13.19889, + "max_per_prompt_fro": 22.89805, + "heavy_tail_ratio": 1.7348, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.691242, + "mean_fro": 7.273351, + "rms_per_prompt_fro": 11.931936, + "max_per_prompt_fro": 20.678722, + "heavy_tail_ratio": 1.7331, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.386114, + "mean_fro": 7.73633, + "rms_per_prompt_fro": 11.950349, + "max_per_prompt_fro": 20.684196, + "heavy_tail_ratio": 1.7308, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.197058, + "mean_fro": 7.473542, + "rms_per_prompt_fro": 11.077639, + "max_per_prompt_fro": 18.248838, + "heavy_tail_ratio": 1.6474, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.991881, + "mean_fro": 8.519769, + "rms_per_prompt_fro": 12.024291, + "max_per_prompt_fro": 19.46534, + "heavy_tail_ratio": 1.6188, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.910759, + "mean_fro": 8.822115, + "rms_per_prompt_fro": 12.194827, + "max_per_prompt_fro": 19.447893, + "heavy_tail_ratio": 1.5948, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.809678, + "mean_fro": 9.267667, + "rms_per_prompt_fro": 12.467263, + "max_per_prompt_fro": 19.570419, + "heavy_tail_ratio": 1.5697, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.712729, + "mean_fro": 10.275114, + "rms_per_prompt_fro": 13.447173, + "max_per_prompt_fro": 20.862938, + "heavy_tail_ratio": 1.5515, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.631474, + "mean_fro": 11.850508, + "rms_per_prompt_fro": 15.136557, + "max_per_prompt_fro": 22.600924, + "heavy_tail_ratio": 1.4931, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.5749, + "mean_fro": 13.462808, + "rms_per_prompt_fro": 16.895154, + "max_per_prompt_fro": 24.01169, + "heavy_tail_ratio": 1.4212, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.518298, + "mean_fro": 15.411741, + "rms_per_prompt_fro": 18.990231, + "max_per_prompt_fro": 25.506193, + "heavy_tail_ratio": 1.3431, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.459888, + "mean_fro": 18.274076, + "rms_per_prompt_fro": 22.0798, + "max_per_prompt_fro": 28.124752, + "heavy_tail_ratio": 1.2738, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.368619, + "mean_fro": 20.419632, + "rms_per_prompt_fro": 23.888515, + "max_per_prompt_fro": 30.121614, + "heavy_tail_ratio": 1.2609, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.310964, + "mean_fro": 22.148105, + "rms_per_prompt_fro": 25.358995, + "max_per_prompt_fro": 30.655041, + "heavy_tail_ratio": 1.2088, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.257804, + "mean_fro": 25.412707, + "rms_per_prompt_fro": 28.500827, + "max_per_prompt_fro": 33.430855, + "heavy_tail_ratio": 1.173, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.213255, + "mean_fro": 28.408293, + "rms_per_prompt_fro": 31.291119, + "max_per_prompt_fro": 36.075897, + "heavy_tail_ratio": 1.1529, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.154514, + "mean_fro": 32.184341, + "rms_per_prompt_fro": 34.581535, + "max_per_prompt_fro": 38.575142, + "heavy_tail_ratio": 1.1155, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.115916, + "mean_fro": 34.578148, + "rms_per_prompt_fro": 36.527295, + "max_per_prompt_fro": 39.604698, + "heavy_tail_ratio": 1.0842, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.085099, + "mean_fro": 36.010612, + "rms_per_prompt_fro": 37.511564, + "max_per_prompt_fro": 40.116283, + "heavy_tail_ratio": 1.0694, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.057673, + "mean_fro": 36.134872, + "rms_per_prompt_fro": 37.162266, + "max_per_prompt_fro": 39.08696, + "heavy_tail_ratio": 1.0518, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.034704, + "mean_fro": 35.978088, + "rms_per_prompt_fro": 36.597061, + "max_per_prompt_fro": 37.976398, + "heavy_tail_ratio": 1.0377, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.017357, + "mean_fro": 35.156247, + "rms_per_prompt_fro": 35.460045, + "max_per_prompt_fro": 36.266266, + "heavy_tail_ratio": 1.0227, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 621.7, + "fit_secs_per_prompt": 3.108, + "total_wall_seconds": 630.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.7708, + "top_sv": 1.3955, + "stable_rank": 31.009 + }, + { + "layer": 1, + "fro_norm": 7.2733, + "top_sv": 1.3127, + "stable_rank": 30.699 + }, + { + "layer": 2, + "fro_norm": 7.7362, + "top_sv": 1.4042, + "stable_rank": 30.354 + }, + { + "layer": 3, + "fro_norm": 7.4734, + "top_sv": 1.4052, + "stable_rank": 28.285 + }, + { + "layer": 4, + "fro_norm": 8.5197, + "top_sv": 1.608, + "stable_rank": 28.071 + }, + { + "layer": 5, + "fro_norm": 8.822, + "top_sv": 1.7525, + "stable_rank": 25.342 + }, + { + "layer": 6, + "fro_norm": 9.2675, + "top_sv": 1.8653, + "stable_rank": 24.685 + }, + { + "layer": 7, + "fro_norm": 10.275, + "top_sv": 2.0848, + "stable_rank": 24.29 + }, + { + "layer": 8, + "fro_norm": 11.8503, + "top_sv": 2.5744, + "stable_rank": 21.188 + }, + { + "layer": 9, + "fro_norm": 13.4626, + "top_sv": 2.9231, + "stable_rank": 21.211 + }, + { + "layer": 10, + "fro_norm": 15.4116, + "top_sv": 3.3762, + "stable_rank": 20.837 + }, + { + "layer": 11, + "fro_norm": 18.2738, + "top_sv": 3.9801, + "stable_rank": 21.08 + }, + { + "layer": 12, + "fro_norm": 20.4193, + "top_sv": 4.4701, + "stable_rank": 20.867 + }, + { + "layer": 13, + "fro_norm": 22.1478, + "top_sv": 4.8963, + "stable_rank": 20.461 + }, + { + "layer": 14, + "fro_norm": 25.4123, + "top_sv": 5.5387, + "stable_rank": 21.051 + }, + { + "layer": 15, + "fro_norm": 28.4078, + "top_sv": 6.3842, + "stable_rank": 19.8 + }, + { + "layer": 16, + "fro_norm": 32.1837, + "top_sv": 7.4918, + "stable_rank": 18.455 + }, + { + "layer": 17, + "fro_norm": 34.5774, + "top_sv": 7.9306, + "stable_rank": 19.01 + }, + { + "layer": 18, + "fro_norm": 36.0097, + "top_sv": 8.0884, + "stable_rank": 19.821 + }, + { + "layer": 19, + "fro_norm": 36.1337, + "top_sv": 8.0209, + "stable_rank": 20.294 + }, + { + "layer": 20, + "fro_norm": 35.9768, + "top_sv": 7.6344, + "stable_rank": 22.207 + }, + { + "layer": 21, + "fro_norm": 35.1546, + "top_sv": 5.9883, + "stable_rank": 34.463 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n25_s0.pt b/results/lenses/e28/e28_StackExchange_410m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe9b6b2d3f7410717e0a2e16207338f1eb001b3c --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b85c0ebf98b77ffa4a8cd5cb7558e82cdece403f65662e78cb94d6c0ff8806a3 +size 46145033 diff --git a/results/lenses/e28/e28_StackExchange_410m_n25_s0_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d5c9d5786ab6e37b3d8d9bf6687b73f164130d18 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n25_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n25_s0.pt", + "sha256": "b85c0ebf98b77ffa4a8cd5cb7558e82cdece403f65662e78cb94d6c0ff8806a3", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.951026, + "mean_fro": 7.7218, + "rms_per_prompt_fro": 13.264934, + "max_per_prompt_fro": 16.880209, + "heavy_tail_ratio": 1.2725, + "n_prompts": 25 + }, + "1": { + "dispersion": 1.745367, + "mean_fro": 7.221652, + "rms_per_prompt_fro": 11.965663, + "max_per_prompt_fro": 15.238077, + "heavy_tail_ratio": 1.2735, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.444225, + "mean_fro": 7.682281, + "rms_per_prompt_fro": 12.010492, + "max_per_prompt_fro": 14.753949, + "heavy_tail_ratio": 1.2284, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.256709, + "mean_fro": 7.414226, + "rms_per_prompt_fro": 11.137908, + "max_per_prompt_fro": 13.590579, + "heavy_tail_ratio": 1.2202, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.050913, + "mean_fro": 8.442774, + "rms_per_prompt_fro": 12.090905, + "max_per_prompt_fro": 14.165685, + "heavy_tail_ratio": 1.1716, + "n_prompts": 25 + }, + "5": { + "dispersion": 0.969426, + "mean_fro": 8.72887, + "rms_per_prompt_fro": 12.249767, + "max_per_prompt_fro": 14.322209, + "heavy_tail_ratio": 1.1692, + "n_prompts": 25 + }, + "6": { + "dispersion": 0.865182, + "mean_fro": 9.16611, + "rms_per_prompt_fro": 12.518309, + "max_per_prompt_fro": 14.511678, + "heavy_tail_ratio": 1.1592, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.764956, + "mean_fro": 10.165412, + "rms_per_prompt_fro": 13.504918, + "max_per_prompt_fro": 15.851747, + "heavy_tail_ratio": 1.1738, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.683951, + "mean_fro": 11.699259, + "rms_per_prompt_fro": 15.181794, + "max_per_prompt_fro": 17.421383, + "heavy_tail_ratio": 1.1475, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.623353, + "mean_fro": 13.283904, + "rms_per_prompt_fro": 16.925138, + "max_per_prompt_fro": 19.247288, + "heavy_tail_ratio": 1.1372, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.565641, + "mean_fro": 15.185852, + "rms_per_prompt_fro": 19.001386, + "max_per_prompt_fro": 20.828445, + "heavy_tail_ratio": 1.0962, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.503944, + "mean_fro": 18.000456, + "rms_per_prompt_fro": 22.074927, + "max_per_prompt_fro": 23.860901, + "heavy_tail_ratio": 1.0809, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.401971, + "mean_fro": 20.083782, + "rms_per_prompt_fro": 23.780177, + "max_per_prompt_fro": 25.367682, + "heavy_tail_ratio": 1.0668, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.343253, + "mean_fro": 21.819338, + "rms_per_prompt_fro": 25.288347, + "max_per_prompt_fro": 26.913771, + "heavy_tail_ratio": 1.0643, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.286959, + "mean_fro": 25.015915, + "rms_per_prompt_fro": 28.379113, + "max_per_prompt_fro": 30.478624, + "heavy_tail_ratio": 1.074, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.239221, + "mean_fro": 27.973892, + "rms_per_prompt_fro": 31.14062, + "max_per_prompt_fro": 33.494221, + "heavy_tail_ratio": 1.0756, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.175714, + "mean_fro": 31.679277, + "rms_per_prompt_fro": 34.349943, + "max_per_prompt_fro": 36.22546, + "heavy_tail_ratio": 1.0546, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.131654, + "mean_fro": 34.089485, + "rms_per_prompt_fro": 36.26413, + "max_per_prompt_fro": 37.839737, + "heavy_tail_ratio": 1.0434, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.094926, + "mean_fro": 35.598939, + "rms_per_prompt_fro": 37.250277, + "max_per_prompt_fro": 38.557079, + "heavy_tail_ratio": 1.0351, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.065169, + "mean_fro": 35.792225, + "rms_per_prompt_fro": 36.940095, + "max_per_prompt_fro": 38.000111, + "heavy_tail_ratio": 1.0287, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.039262, + "mean_fro": 35.702744, + "rms_per_prompt_fro": 36.396876, + "max_per_prompt_fro": 37.382595, + "heavy_tail_ratio": 1.0271, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.019693, + "mean_fro": 34.971347, + "rms_per_prompt_fro": 35.314021, + "max_per_prompt_fro": 35.899288, + "heavy_tail_ratio": 1.0166, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 75.7, + "fit_secs_per_prompt": 3.027, + "total_wall_seconds": 89.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.7217, + "top_sv": 1.395, + "stable_rank": 30.641 + }, + { + "layer": 1, + "fro_norm": 7.2216, + "top_sv": 1.3087, + "stable_rank": 30.449 + }, + { + "layer": 2, + "fro_norm": 7.6822, + "top_sv": 1.4115, + "stable_rank": 29.623 + }, + { + "layer": 3, + "fro_norm": 7.4141, + "top_sv": 1.4045, + "stable_rank": 27.868 + }, + { + "layer": 4, + "fro_norm": 8.4427, + "top_sv": 1.6314, + "stable_rank": 26.781 + }, + { + "layer": 5, + "fro_norm": 8.7287, + "top_sv": 1.7398, + "stable_rank": 25.171 + }, + { + "layer": 6, + "fro_norm": 9.166, + "top_sv": 1.8421, + "stable_rank": 24.759 + }, + { + "layer": 7, + "fro_norm": 10.1653, + "top_sv": 2.0457, + "stable_rank": 24.692 + }, + { + "layer": 8, + "fro_norm": 11.6991, + "top_sv": 2.5143, + "stable_rank": 21.651 + }, + { + "layer": 9, + "fro_norm": 13.2837, + "top_sv": 2.8441, + "stable_rank": 21.815 + }, + { + "layer": 10, + "fro_norm": 15.1856, + "top_sv": 3.267, + "stable_rank": 21.605 + }, + { + "layer": 11, + "fro_norm": 18.0002, + "top_sv": 3.866, + "stable_rank": 21.678 + }, + { + "layer": 12, + "fro_norm": 20.0835, + "top_sv": 4.2912, + "stable_rank": 21.904 + }, + { + "layer": 13, + "fro_norm": 21.819, + "top_sv": 4.6974, + "stable_rank": 21.575 + }, + { + "layer": 14, + "fro_norm": 25.0155, + "top_sv": 5.3258, + "stable_rank": 22.062 + }, + { + "layer": 15, + "fro_norm": 27.9734, + "top_sv": 6.1603, + "stable_rank": 20.62 + }, + { + "layer": 16, + "fro_norm": 31.6787, + "top_sv": 7.243, + "stable_rank": 19.129 + }, + { + "layer": 17, + "fro_norm": 34.0887, + "top_sv": 7.6785, + "stable_rank": 19.709 + }, + { + "layer": 18, + "fro_norm": 35.598, + "top_sv": 7.9554, + "stable_rank": 20.023 + }, + { + "layer": 19, + "fro_norm": 35.7911, + "top_sv": 7.9486, + "stable_rank": 20.276 + }, + { + "layer": 20, + "fro_norm": 35.7014, + "top_sv": 7.5585, + "stable_rank": 22.31 + }, + { + "layer": 21, + "fro_norm": 34.9698, + "top_sv": 5.9381, + "stable_rank": 34.681 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n25_s1.pt b/results/lenses/e28/e28_StackExchange_410m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee71fd6f212063024bc84952e4a46aac815e14c5 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2125ffb5668dd2c12e455a8b82321206229b82b77d9fd428ad2a6f08cb59a2d3 +size 46145033 diff --git a/results/lenses/e28/e28_StackExchange_410m_n25_s1_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..33e493e6c6123b57801f0872c6bdf46a4fa8666e --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n25_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n25_s1.pt", + "sha256": "2125ffb5668dd2c12e455a8b82321206229b82b77d9fd428ad2a6f08cb59a2d3", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.531428, + "mean_fro": 8.375944, + "rms_per_prompt_fro": 13.326515, + "max_per_prompt_fro": 16.456812, + "heavy_tail_ratio": 1.2349, + "n_prompts": 25 + }, + "1": { + "dispersion": 1.398897, + "mean_fro": 7.82357, + "rms_per_prompt_fro": 12.117438, + "max_per_prompt_fro": 14.792378, + "heavy_tail_ratio": 1.2208, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.168448, + "mean_fro": 8.270517, + "rms_per_prompt_fro": 12.178874, + "max_per_prompt_fro": 14.786101, + "heavy_tail_ratio": 1.2141, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.021851, + "mean_fro": 7.932536, + "rms_per_prompt_fro": 11.279415, + "max_per_prompt_fro": 13.590834, + "heavy_tail_ratio": 1.2049, + "n_prompts": 25 + }, + "4": { + "dispersion": 0.857467, + "mean_fro": 8.989546, + "rms_per_prompt_fro": 12.251757, + "max_per_prompt_fro": 14.567232, + "heavy_tail_ratio": 1.189, + "n_prompts": 25 + }, + "5": { + "dispersion": 0.794207, + "mean_fro": 9.265763, + "rms_per_prompt_fro": 12.411304, + "max_per_prompt_fro": 14.431639, + "heavy_tail_ratio": 1.1628, + "n_prompts": 25 + }, + "6": { + "dispersion": 0.71514, + "mean_fro": 9.696297, + "rms_per_prompt_fro": 12.698597, + "max_per_prompt_fro": 14.562487, + "heavy_tail_ratio": 1.1468, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.632723, + "mean_fro": 10.68292, + "rms_per_prompt_fro": 13.650428, + "max_per_prompt_fro": 15.221767, + "heavy_tail_ratio": 1.1151, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.567523, + "mean_fro": 12.280724, + "rms_per_prompt_fro": 15.37556, + "max_per_prompt_fro": 16.81068, + "heavy_tail_ratio": 1.0933, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.523325, + "mean_fro": 13.90684, + "rms_per_prompt_fro": 17.164249, + "max_per_prompt_fro": 18.777218, + "heavy_tail_ratio": 1.094, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.477925, + "mean_fro": 15.877459, + "rms_per_prompt_fro": 19.302221, + "max_per_prompt_fro": 21.516895, + "heavy_tail_ratio": 1.1147, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.431453, + "mean_fro": 18.803888, + "rms_per_prompt_fro": 22.497602, + "max_per_prompt_fro": 25.458704, + "heavy_tail_ratio": 1.1316, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.341264, + "mean_fro": 20.791565, + "rms_per_prompt_fro": 24.079326, + "max_per_prompt_fro": 25.808968, + "heavy_tail_ratio": 1.0718, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.289648, + "mean_fro": 22.502714, + "rms_per_prompt_fro": 25.554685, + "max_per_prompt_fro": 27.410515, + "heavy_tail_ratio": 1.0726, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.240733, + "mean_fro": 25.782682, + "rms_per_prompt_fro": 28.718867, + "max_per_prompt_fro": 30.71097, + "heavy_tail_ratio": 1.0694, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.200328, + "mean_fro": 28.734754, + "rms_per_prompt_fro": 31.481652, + "max_per_prompt_fro": 33.62175, + "heavy_tail_ratio": 1.068, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.146511, + "mean_fro": 32.457134, + "rms_per_prompt_fro": 34.753561, + "max_per_prompt_fro": 36.541031, + "heavy_tail_ratio": 1.0514, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.111898, + "mean_fro": 34.713692, + "rms_per_prompt_fro": 36.604392, + "max_per_prompt_fro": 38.086311, + "heavy_tail_ratio": 1.0405, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.08282, + "mean_fro": 36.018253, + "rms_per_prompt_fro": 37.480112, + "max_per_prompt_fro": 39.121746, + "heavy_tail_ratio": 1.0438, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.056666, + "mean_fro": 36.096149, + "rms_per_prompt_fro": 37.104778, + "max_per_prompt_fro": 38.136372, + "heavy_tail_ratio": 1.0278, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.034565, + "mean_fro": 35.932816, + "rms_per_prompt_fro": 36.548545, + "max_per_prompt_fro": 37.457302, + "heavy_tail_ratio": 1.0249, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.017538, + "mean_fro": 35.101682, + "rms_per_prompt_fro": 35.408144, + "max_per_prompt_fro": 36.036442, + "heavy_tail_ratio": 1.0177, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 77.0, + "fit_secs_per_prompt": 3.08, + "total_wall_seconds": 83.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.3758, + "top_sv": 1.5373, + "stable_rank": 29.686 + }, + { + "layer": 1, + "fro_norm": 7.8235, + "top_sv": 1.4281, + "stable_rank": 30.009 + }, + { + "layer": 2, + "fro_norm": 8.2704, + "top_sv": 1.5318, + "stable_rank": 29.152 + }, + { + "layer": 3, + "fro_norm": 7.9324, + "top_sv": 1.5011, + "stable_rank": 27.924 + }, + { + "layer": 4, + "fro_norm": 8.9894, + "top_sv": 1.7411, + "stable_rank": 26.658 + }, + { + "layer": 5, + "fro_norm": 9.2656, + "top_sv": 1.8507, + "stable_rank": 25.066 + }, + { + "layer": 6, + "fro_norm": 9.6962, + "top_sv": 1.945, + "stable_rank": 24.853 + }, + { + "layer": 7, + "fro_norm": 10.6828, + "top_sv": 2.1427, + "stable_rank": 24.858 + }, + { + "layer": 8, + "fro_norm": 12.2806, + "top_sv": 2.6059, + "stable_rank": 22.209 + }, + { + "layer": 9, + "fro_norm": 13.9067, + "top_sv": 2.9504, + "stable_rank": 22.217 + }, + { + "layer": 10, + "fro_norm": 15.8773, + "top_sv": 3.3933, + "stable_rank": 21.893 + }, + { + "layer": 11, + "fro_norm": 18.8036, + "top_sv": 4.0036, + "stable_rank": 22.058 + }, + { + "layer": 12, + "fro_norm": 20.7912, + "top_sv": 4.4447, + "stable_rank": 21.882 + }, + { + "layer": 13, + "fro_norm": 22.5024, + "top_sv": 4.8499, + "stable_rank": 21.528 + }, + { + "layer": 14, + "fro_norm": 25.7822, + "top_sv": 5.4839, + "stable_rank": 22.103 + }, + { + "layer": 15, + "fro_norm": 28.7342, + "top_sv": 6.3062, + "stable_rank": 20.762 + }, + { + "layer": 16, + "fro_norm": 32.4566, + "top_sv": 7.3924, + "stable_rank": 19.277 + }, + { + "layer": 17, + "fro_norm": 34.7129, + "top_sv": 7.8374, + "stable_rank": 19.617 + }, + { + "layer": 18, + "fro_norm": 36.0173, + "top_sv": 7.9374, + "stable_rank": 20.591 + }, + { + "layer": 19, + "fro_norm": 36.095, + "top_sv": 7.8268, + "stable_rank": 21.268 + }, + { + "layer": 20, + "fro_norm": 35.9316, + "top_sv": 7.4859, + "stable_rank": 23.039 + }, + { + "layer": 21, + "fro_norm": 35.1001, + "top_sv": 5.8628, + "stable_rank": 35.843 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n25_s2.pt b/results/lenses/e28/e28_StackExchange_410m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..21ad5126cf6133559b1befe82c7722d3b166cfd9 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:969cd4e7ae1261962c71eafda0cfcd29cf8bf58f9462bae0fe7c1e767a0e71a8 +size 46145033 diff --git a/results/lenses/e28/e28_StackExchange_410m_n25_s2_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7304c548557581b31f92890a4dc4f9ba74dd909b --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n25_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n25_s2.pt", + "sha256": "969cd4e7ae1261962c71eafda0cfcd29cf8bf58f9462bae0fe7c1e767a0e71a8", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.816178, + "mean_fro": 7.902544, + "rms_per_prompt_fro": 13.261632, + "max_per_prompt_fro": 16.937399, + "heavy_tail_ratio": 1.2772, + "n_prompts": 25 + }, + "1": { + "dispersion": 1.617375, + "mean_fro": 7.362907, + "rms_per_prompt_fro": 11.911934, + "max_per_prompt_fro": 14.93219, + "heavy_tail_ratio": 1.2535, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.323336, + "mean_fro": 7.82378, + "rms_per_prompt_fro": 11.925393, + "max_per_prompt_fro": 14.767397, + "heavy_tail_ratio": 1.2383, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.133947, + "mean_fro": 7.516443, + "rms_per_prompt_fro": 10.980047, + "max_per_prompt_fro": 13.139455, + "heavy_tail_ratio": 1.1967, + "n_prompts": 25 + }, + "4": { + "dispersion": 0.936796, + "mean_fro": 8.570957, + "rms_per_prompt_fro": 11.928101, + "max_per_prompt_fro": 14.05892, + "heavy_tail_ratio": 1.1786, + "n_prompts": 25 + }, + "5": { + "dispersion": 0.852362, + "mean_fro": 8.837315, + "rms_per_prompt_fro": 12.027717, + "max_per_prompt_fro": 14.075611, + "heavy_tail_ratio": 1.1703, + "n_prompts": 25 + }, + "6": { + "dispersion": 0.753712, + "mean_fro": 9.266685, + "rms_per_prompt_fro": 12.271667, + "max_per_prompt_fro": 14.389355, + "heavy_tail_ratio": 1.1726, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.66478, + "mean_fro": 10.292609, + "rms_per_prompt_fro": 13.28018, + "max_per_prompt_fro": 15.528144, + "heavy_tail_ratio": 1.1693, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.591372, + "mean_fro": 11.858486, + "rms_per_prompt_fro": 14.959433, + "max_per_prompt_fro": 17.307007, + "heavy_tail_ratio": 1.1569, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.538183, + "mean_fro": 13.463424, + "rms_per_prompt_fro": 16.697812, + "max_per_prompt_fro": 19.107977, + "heavy_tail_ratio": 1.1443, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.4876, + "mean_fro": 15.421356, + "rms_per_prompt_fro": 18.809, + "max_per_prompt_fro": 21.141674, + "heavy_tail_ratio": 1.124, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.434043, + "mean_fro": 18.295475, + "rms_per_prompt_fro": 21.909112, + "max_per_prompt_fro": 24.241796, + "heavy_tail_ratio": 1.1065, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.352441, + "mean_fro": 20.525526, + "rms_per_prompt_fro": 23.870057, + "max_per_prompt_fro": 26.490185, + "heavy_tail_ratio": 1.1098, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.297582, + "mean_fro": 22.243225, + "rms_per_prompt_fro": 25.337581, + "max_per_prompt_fro": 27.747816, + "heavy_tail_ratio": 1.0951, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.247984, + "mean_fro": 25.493113, + "rms_per_prompt_fro": 28.479179, + "max_per_prompt_fro": 30.950033, + "heavy_tail_ratio": 1.0868, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.206369, + "mean_fro": 28.476871, + "rms_per_prompt_fro": 31.277529, + "max_per_prompt_fro": 34.028229, + "heavy_tail_ratio": 1.0879, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.151867, + "mean_fro": 32.17691, + "rms_per_prompt_fro": 34.533883, + "max_per_prompt_fro": 37.222889, + "heavy_tail_ratio": 1.0779, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.114188, + "mean_fro": 34.572743, + "rms_per_prompt_fro": 36.493286, + "max_per_prompt_fro": 39.037834, + "heavy_tail_ratio": 1.0697, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.083028, + "mean_fro": 35.967289, + "rms_per_prompt_fro": 37.430673, + "max_per_prompt_fro": 39.514408, + "heavy_tail_ratio": 1.0557, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.056143, + "mean_fro": 36.127259, + "rms_per_prompt_fro": 37.127558, + "max_per_prompt_fro": 38.615036, + "heavy_tail_ratio": 1.0401, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.03402, + "mean_fro": 35.966018, + "rms_per_prompt_fro": 36.572679, + "max_per_prompt_fro": 37.595169, + "heavy_tail_ratio": 1.028, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.017115, + "mean_fro": 35.124081, + "rms_per_prompt_fro": 35.423376, + "max_per_prompt_fro": 36.070801, + "heavy_tail_ratio": 1.0183, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 77.9, + "fit_secs_per_prompt": 3.114, + "total_wall_seconds": 86.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9025, + "top_sv": 1.4662, + "stable_rank": 29.051 + }, + { + "layer": 1, + "fro_norm": 7.3628, + "top_sv": 1.3851, + "stable_rank": 28.255 + }, + { + "layer": 2, + "fro_norm": 7.8237, + "top_sv": 1.4737, + "stable_rank": 28.184 + }, + { + "layer": 3, + "fro_norm": 7.5164, + "top_sv": 1.4458, + "stable_rank": 27.027 + }, + { + "layer": 4, + "fro_norm": 8.5708, + "top_sv": 1.64, + "stable_rank": 27.311 + }, + { + "layer": 5, + "fro_norm": 8.8372, + "top_sv": 1.7557, + "stable_rank": 25.335 + }, + { + "layer": 6, + "fro_norm": 9.2665, + "top_sv": 1.8544, + "stable_rank": 24.97 + }, + { + "layer": 7, + "fro_norm": 10.2925, + "top_sv": 2.0845, + "stable_rank": 24.379 + }, + { + "layer": 8, + "fro_norm": 11.8583, + "top_sv": 2.5685, + "stable_rank": 21.315 + }, + { + "layer": 9, + "fro_norm": 13.4632, + "top_sv": 2.9095, + "stable_rank": 21.412 + }, + { + "layer": 10, + "fro_norm": 15.4212, + "top_sv": 3.3611, + "stable_rank": 21.051 + }, + { + "layer": 11, + "fro_norm": 18.2952, + "top_sv": 3.9881, + "stable_rank": 21.045 + }, + { + "layer": 12, + "fro_norm": 20.5252, + "top_sv": 4.4869, + "stable_rank": 20.926 + }, + { + "layer": 13, + "fro_norm": 22.2429, + "top_sv": 4.9226, + "stable_rank": 20.417 + }, + { + "layer": 14, + "fro_norm": 25.4927, + "top_sv": 5.5515, + "stable_rank": 21.087 + }, + { + "layer": 15, + "fro_norm": 28.4764, + "top_sv": 6.3898, + "stable_rank": 19.861 + }, + { + "layer": 16, + "fro_norm": 32.1763, + "top_sv": 7.4593, + "stable_rank": 18.607 + }, + { + "layer": 17, + "fro_norm": 34.572, + "top_sv": 7.8876, + "stable_rank": 19.212 + }, + { + "layer": 18, + "fro_norm": 35.9663, + "top_sv": 8.0369, + "stable_rank": 20.027 + }, + { + "layer": 19, + "fro_norm": 36.1261, + "top_sv": 7.984, + "stable_rank": 20.474 + }, + { + "layer": 20, + "fro_norm": 35.9647, + "top_sv": 7.6072, + "stable_rank": 22.351 + }, + { + "layer": 21, + "fro_norm": 35.1225, + "top_sv": 5.9633, + "stable_rank": 34.689 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n400_s0.pt b/results/lenses/e28/e28_StackExchange_410m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d35eb7f1859d7cff140955d4b5b22479707f827 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c3bbbca13ef5891fb19ce2c552334ebf12263c905f25545bc36a831b4bd2a00 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n400_s0_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..82125f78b589934a42dd99a658648a4f403b016a --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n400_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n400_s0.pt", + "sha256": "3c3bbbca13ef5891fb19ce2c552334ebf12263c905f25545bc36a831b4bd2a00", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.899756, + "mean_fro": 7.977382, + "rms_per_prompt_fro": 13.584421, + "max_per_prompt_fro": 21.382202, + "heavy_tail_ratio": 1.574, + "n_prompts": 400 + }, + "1": { + "dispersion": 1.699338, + "mean_fro": 7.464401, + "rms_per_prompt_fro": 12.263759, + "max_per_prompt_fro": 19.421309, + "heavy_tail_ratio": 1.5836, + "n_prompts": 400 + }, + "2": { + "dispersion": 1.385778, + "mean_fro": 7.941705, + "rms_per_prompt_fro": 12.266727, + "max_per_prompt_fro": 19.912868, + "heavy_tail_ratio": 1.6233, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.195705, + "mean_fro": 7.650987, + "rms_per_prompt_fro": 11.337165, + "max_per_prompt_fro": 17.533512, + "heavy_tail_ratio": 1.5466, + "n_prompts": 400 + }, + "4": { + "dispersion": 0.991555, + "mean_fro": 8.728941, + "rms_per_prompt_fro": 12.318497, + "max_per_prompt_fro": 19.180052, + "heavy_tail_ratio": 1.557, + "n_prompts": 400 + }, + "5": { + "dispersion": 0.9143, + "mean_fro": 9.021979, + "rms_per_prompt_fro": 12.482651, + "max_per_prompt_fro": 18.667677, + "heavy_tail_ratio": 1.4955, + "n_prompts": 400 + }, + "6": { + "dispersion": 0.814662, + "mean_fro": 9.464754, + "rms_per_prompt_fro": 12.749914, + "max_per_prompt_fro": 18.677954, + "heavy_tail_ratio": 1.4649, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.717907, + "mean_fro": 10.4713, + "rms_per_prompt_fro": 13.724622, + "max_per_prompt_fro": 19.680288, + "heavy_tail_ratio": 1.4339, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.638875, + "mean_fro": 12.065362, + "rms_per_prompt_fro": 15.445899, + "max_per_prompt_fro": 21.688644, + "heavy_tail_ratio": 1.4042, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.582852, + "mean_fro": 13.673965, + "rms_per_prompt_fro": 17.203411, + "max_per_prompt_fro": 23.499172, + "heavy_tail_ratio": 1.366, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.527384, + "mean_fro": 15.613797, + "rms_per_prompt_fro": 19.296682, + "max_per_prompt_fro": 25.770779, + "heavy_tail_ratio": 1.3355, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.468573, + "mean_fro": 18.486676, + "rms_per_prompt_fro": 22.403018, + "max_per_prompt_fro": 29.363607, + "heavy_tail_ratio": 1.3107, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.374384, + "mean_fro": 20.565485, + "rms_per_prompt_fro": 24.109767, + "max_per_prompt_fro": 29.734083, + "heavy_tail_ratio": 1.2333, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.315931, + "mean_fro": 22.286007, + "rms_per_prompt_fro": 25.56518, + "max_per_prompt_fro": 30.961466, + "heavy_tail_ratio": 1.2111, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.262442, + "mean_fro": 25.556778, + "rms_per_prompt_fro": 28.715193, + "max_per_prompt_fro": 33.7141, + "heavy_tail_ratio": 1.1741, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.217069, + "mean_fro": 28.545009, + "rms_per_prompt_fro": 31.491103, + "max_per_prompt_fro": 36.521626, + "heavy_tail_ratio": 1.1597, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.156702, + "mean_fro": 32.268941, + "rms_per_prompt_fro": 34.705269, + "max_per_prompt_fro": 38.571972, + "heavy_tail_ratio": 1.1114, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.117218, + "mean_fro": 34.635437, + "rms_per_prompt_fro": 36.609155, + "max_per_prompt_fro": 39.345181, + "heavy_tail_ratio": 1.0747, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.085205, + "mean_fro": 36.039261, + "rms_per_prompt_fro": 37.543242, + "max_per_prompt_fro": 39.785553, + "heavy_tail_ratio": 1.0597, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.057604, + "mean_fro": 36.127598, + "rms_per_prompt_fro": 37.153571, + "max_per_prompt_fro": 38.823784, + "heavy_tail_ratio": 1.045, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.034671, + "mean_fro": 35.960388, + "rms_per_prompt_fro": 36.57847, + "max_per_prompt_fro": 38.052616, + "heavy_tail_ratio": 1.0403, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.017302, + "mean_fro": 35.13786, + "rms_per_prompt_fro": 35.440539, + "max_per_prompt_fro": 36.302822, + "heavy_tail_ratio": 1.0243, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1235.4, + "fit_secs_per_prompt": 3.088, + "total_wall_seconds": 1241.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9773, + "top_sv": 1.4327, + "stable_rank": 31.001 + }, + { + "layer": 1, + "fro_norm": 7.4643, + "top_sv": 1.3468, + "stable_rank": 30.716 + }, + { + "layer": 2, + "fro_norm": 7.9416, + "top_sv": 1.4402, + "stable_rank": 30.406 + }, + { + "layer": 3, + "fro_norm": 7.6509, + "top_sv": 1.4374, + "stable_rank": 28.332 + }, + { + "layer": 4, + "fro_norm": 8.7288, + "top_sv": 1.6661, + "stable_rank": 27.449 + }, + { + "layer": 5, + "fro_norm": 9.0219, + "top_sv": 1.7994, + "stable_rank": 25.138 + }, + { + "layer": 6, + "fro_norm": 9.4647, + "top_sv": 1.9102, + "stable_rank": 24.549 + }, + { + "layer": 7, + "fro_norm": 10.4712, + "top_sv": 2.1258, + "stable_rank": 24.262 + }, + { + "layer": 8, + "fro_norm": 12.0652, + "top_sv": 2.6066, + "stable_rank": 21.425 + }, + { + "layer": 9, + "fro_norm": 13.6738, + "top_sv": 2.9531, + "stable_rank": 21.439 + }, + { + "layer": 10, + "fro_norm": 15.6136, + "top_sv": 3.3986, + "stable_rank": 21.106 + }, + { + "layer": 11, + "fro_norm": 18.4864, + "top_sv": 4.0011, + "stable_rank": 21.347 + }, + { + "layer": 12, + "fro_norm": 20.5652, + "top_sv": 4.4609, + "stable_rank": 21.253 + }, + { + "layer": 13, + "fro_norm": 22.2857, + "top_sv": 4.8824, + "stable_rank": 20.835 + }, + { + "layer": 14, + "fro_norm": 25.5563, + "top_sv": 5.5313, + "stable_rank": 21.347 + }, + { + "layer": 15, + "fro_norm": 28.5445, + "top_sv": 6.3581, + "stable_rank": 20.155 + }, + { + "layer": 16, + "fro_norm": 32.2684, + "top_sv": 7.4576, + "stable_rank": 18.722 + }, + { + "layer": 17, + "fro_norm": 34.6347, + "top_sv": 7.8935, + "stable_rank": 19.252 + }, + { + "layer": 18, + "fro_norm": 36.0383, + "top_sv": 8.0571, + "stable_rank": 20.007 + }, + { + "layer": 19, + "fro_norm": 36.1265, + "top_sv": 7.9894, + "stable_rank": 20.447 + }, + { + "layer": 20, + "fro_norm": 35.9591, + "top_sv": 7.5939, + "stable_rank": 22.423 + }, + { + "layer": 21, + "fro_norm": 35.1363, + "top_sv": 5.9683, + "stable_rank": 34.658 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n400_s1.pt b/results/lenses/e28/e28_StackExchange_410m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6b5c4195eea39f6e06e989cfbb387f31eedcbd6 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35924fd85fb01d40f761f59711f0a3e968d3f6bfdcd59a83000cb51888a6ce26 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n400_s1_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..577ebc4be94a7c8a56b7dbd7fea375f7ddea6b2f --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n400_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n400_s1.pt", + "sha256": "35924fd85fb01d40f761f59711f0a3e968d3f6bfdcd59a83000cb51888a6ce26", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.993767, + "mean_fro": 7.788811, + "rms_per_prompt_fro": 13.476595, + "max_per_prompt_fro": 19.984915, + "heavy_tail_ratio": 1.4829, + "n_prompts": 400 + }, + "1": { + "dispersion": 1.780066, + "mean_fro": 7.288688, + "rms_per_prompt_fro": 12.152816, + "max_per_prompt_fro": 17.600504, + "heavy_tail_ratio": 1.4483, + "n_prompts": 400 + }, + "2": { + "dispersion": 1.460003, + "mean_fro": 7.758487, + "rms_per_prompt_fro": 12.168719, + "max_per_prompt_fro": 17.439396, + "heavy_tail_ratio": 1.4331, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.259277, + "mean_fro": 7.485473, + "rms_per_prompt_fro": 11.251334, + "max_per_prompt_fro": 15.296987, + "heavy_tail_ratio": 1.3596, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.049178, + "mean_fro": 8.532041, + "rms_per_prompt_fro": 12.213574, + "max_per_prompt_fro": 16.132614, + "heavy_tail_ratio": 1.3209, + "n_prompts": 400 + }, + "5": { + "dispersion": 0.962447, + "mean_fro": 8.829951, + "rms_per_prompt_fro": 12.369647, + "max_per_prompt_fro": 15.87153, + "heavy_tail_ratio": 1.2831, + "n_prompts": 400 + }, + "6": { + "dispersion": 0.858134, + "mean_fro": 9.273434, + "rms_per_prompt_fro": 12.640931, + "max_per_prompt_fro": 16.114813, + "heavy_tail_ratio": 1.2748, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.755756, + "mean_fro": 10.275847, + "rms_per_prompt_fro": 13.616006, + "max_per_prompt_fro": 17.298576, + "heavy_tail_ratio": 1.2705, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.671035, + "mean_fro": 11.836039, + "rms_per_prompt_fro": 15.30027, + "max_per_prompt_fro": 18.925735, + "heavy_tail_ratio": 1.237, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.611732, + "mean_fro": 13.418599, + "rms_per_prompt_fro": 17.03545, + "max_per_prompt_fro": 20.895554, + "heavy_tail_ratio": 1.2266, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.552042, + "mean_fro": 15.339672, + "rms_per_prompt_fro": 19.110316, + "max_per_prompt_fro": 23.169777, + "heavy_tail_ratio": 1.2124, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.491796, + "mean_fro": 18.173265, + "rms_per_prompt_fro": 22.19666, + "max_per_prompt_fro": 26.688704, + "heavy_tail_ratio": 1.2024, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.394187, + "mean_fro": 20.270019, + "rms_per_prompt_fro": 23.933967, + "max_per_prompt_fro": 27.79759, + "heavy_tail_ratio": 1.1614, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.334158, + "mean_fro": 21.977569, + "rms_per_prompt_fro": 25.385362, + "max_per_prompt_fro": 29.191547, + "heavy_tail_ratio": 1.1499, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.278004, + "mean_fro": 25.199756, + "rms_per_prompt_fro": 28.488031, + "max_per_prompt_fro": 32.273087, + "heavy_tail_ratio": 1.1329, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.230719, + "mean_fro": 28.163074, + "rms_per_prompt_fro": 31.24349, + "max_per_prompt_fro": 35.190823, + "heavy_tail_ratio": 1.1263, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.167908, + "mean_fro": 31.910543, + "rms_per_prompt_fro": 34.485664, + "max_per_prompt_fro": 38.322559, + "heavy_tail_ratio": 1.1113, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.125793, + "mean_fro": 34.332943, + "rms_per_prompt_fro": 36.428423, + "max_per_prompt_fro": 39.639565, + "heavy_tail_ratio": 1.0881, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.091881, + "mean_fro": 35.777627, + "rms_per_prompt_fro": 37.385149, + "max_per_prompt_fro": 40.168762, + "heavy_tail_ratio": 1.0745, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.062265, + "mean_fro": 35.958606, + "rms_per_prompt_fro": 37.061191, + "max_per_prompt_fro": 38.975006, + "heavy_tail_ratio": 1.0516, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.037172, + "mean_fro": 35.85513, + "rms_per_prompt_fro": 36.515455, + "max_per_prompt_fro": 38.089752, + "heavy_tail_ratio": 1.0431, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.018587, + "mean_fro": 35.086655, + "rms_per_prompt_fro": 35.411224, + "max_per_prompt_fro": 36.42778, + "heavy_tail_ratio": 1.0287, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1235.2, + "fit_secs_per_prompt": 3.088, + "total_wall_seconds": 1242.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.7887, + "top_sv": 1.3794, + "stable_rank": 31.881 + }, + { + "layer": 1, + "fro_norm": 7.2886, + "top_sv": 1.3008, + "stable_rank": 31.395 + }, + { + "layer": 2, + "fro_norm": 7.7584, + "top_sv": 1.392, + "stable_rank": 31.064 + }, + { + "layer": 3, + "fro_norm": 7.4854, + "top_sv": 1.3943, + "stable_rank": 28.821 + }, + { + "layer": 4, + "fro_norm": 8.5319, + "top_sv": 1.5904, + "stable_rank": 28.781 + }, + { + "layer": 5, + "fro_norm": 8.8298, + "top_sv": 1.7358, + "stable_rank": 25.878 + }, + { + "layer": 6, + "fro_norm": 9.2733, + "top_sv": 1.8491, + "stable_rank": 25.151 + }, + { + "layer": 7, + "fro_norm": 10.2757, + "top_sv": 2.0701, + "stable_rank": 24.64 + }, + { + "layer": 8, + "fro_norm": 11.8359, + "top_sv": 2.5538, + "stable_rank": 21.479 + }, + { + "layer": 9, + "fro_norm": 13.4184, + "top_sv": 2.8893, + "stable_rank": 21.568 + }, + { + "layer": 10, + "fro_norm": 15.3395, + "top_sv": 3.3397, + "stable_rank": 21.097 + }, + { + "layer": 11, + "fro_norm": 18.173, + "top_sv": 3.9319, + "stable_rank": 21.362 + }, + { + "layer": 12, + "fro_norm": 20.2697, + "top_sv": 4.4076, + "stable_rank": 21.149 + }, + { + "layer": 13, + "fro_norm": 21.9772, + "top_sv": 4.8193, + "stable_rank": 20.796 + }, + { + "layer": 14, + "fro_norm": 25.1993, + "top_sv": 5.4586, + "stable_rank": 21.312 + }, + { + "layer": 15, + "fro_norm": 28.1626, + "top_sv": 6.2818, + "stable_rank": 20.099 + }, + { + "layer": 16, + "fro_norm": 31.9099, + "top_sv": 7.3744, + "stable_rank": 18.724 + }, + { + "layer": 17, + "fro_norm": 34.3321, + "top_sv": 7.8118, + "stable_rank": 19.315 + }, + { + "layer": 18, + "fro_norm": 35.7767, + "top_sv": 7.9828, + "stable_rank": 20.086 + }, + { + "layer": 19, + "fro_norm": 35.9575, + "top_sv": 7.92, + "stable_rank": 20.612 + }, + { + "layer": 20, + "fro_norm": 35.8539, + "top_sv": 7.5597, + "stable_rank": 22.494 + }, + { + "layer": 21, + "fro_norm": 35.0851, + "top_sv": 5.9353, + "stable_rank": 34.943 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n400_s2.pt b/results/lenses/e28/e28_StackExchange_410m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..1edc012f743f911fe1285b710e053d745aae6833 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb267e4d8e622d0fa6bd4039cff285dc01f57a121cf99a7c4c1247d3d68b0c43 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n400_s2_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0a45f621dd204ab8f9719de314a7355b40c9f8fe --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n400_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n400_s2.pt", + "sha256": "eb267e4d8e622d0fa6bd4039cff285dc01f57a121cf99a7c4c1247d3d68b0c43", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.893174, + "mean_fro": 7.757384, + "rms_per_prompt_fro": 13.194793, + "max_per_prompt_fro": 22.89805, + "heavy_tail_ratio": 1.7354, + "n_prompts": 400 + }, + "1": { + "dispersion": 1.692585, + "mean_fro": 7.259492, + "rms_per_prompt_fro": 11.912172, + "max_per_prompt_fro": 20.678722, + "heavy_tail_ratio": 1.7359, + "n_prompts": 400 + }, + "2": { + "dispersion": 1.39374, + "mean_fro": 7.711863, + "rms_per_prompt_fro": 11.931576, + "max_per_prompt_fro": 20.684196, + "heavy_tail_ratio": 1.7336, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.204826, + "mean_fro": 7.453087, + "rms_per_prompt_fro": 11.066833, + "max_per_prompt_fro": 18.248838, + "heavy_tail_ratio": 1.649, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.002237, + "mean_fro": 8.491155, + "rms_per_prompt_fro": 12.015021, + "max_per_prompt_fro": 19.46534, + "heavy_tail_ratio": 1.6201, + "n_prompts": 400 + }, + "5": { + "dispersion": 0.920912, + "mean_fro": 8.796474, + "rms_per_prompt_fro": 12.191648, + "max_per_prompt_fro": 19.447893, + "heavy_tail_ratio": 1.5952, + "n_prompts": 400 + }, + "6": { + "dispersion": 0.819694, + "mean_fro": 9.242973, + "rms_per_prompt_fro": 12.468406, + "max_per_prompt_fro": 19.570419, + "heavy_tail_ratio": 1.5696, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.723087, + "mean_fro": 10.254537, + "rms_per_prompt_fro": 13.460762, + "max_per_prompt_fro": 20.862938, + "heavy_tail_ratio": 1.5499, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.63968, + "mean_fro": 11.827454, + "rms_per_prompt_fro": 15.145053, + "max_per_prompt_fro": 22.600924, + "heavy_tail_ratio": 1.4923, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.581831, + "mean_fro": 13.429228, + "rms_per_prompt_fro": 16.890055, + "max_per_prompt_fro": 24.01169, + "heavy_tail_ratio": 1.4216, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.524621, + "mean_fro": 15.371948, + "rms_per_prompt_fro": 18.980598, + "max_per_prompt_fro": 25.506193, + "heavy_tail_ratio": 1.3438, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.465321, + "mean_fro": 18.229584, + "rms_per_prompt_fro": 22.066994, + "max_per_prompt_fro": 28.124752, + "heavy_tail_ratio": 1.2745, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.373101, + "mean_fro": 20.377983, + "rms_per_prompt_fro": 23.878799, + "max_per_prompt_fro": 30.121614, + "heavy_tail_ratio": 1.2614, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.315082, + "mean_fro": 22.106687, + "rms_per_prompt_fro": 25.351294, + "max_per_prompt_fro": 30.655041, + "heavy_tail_ratio": 1.2092, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.261079, + "mean_fro": 25.373417, + "rms_per_prompt_fro": 28.493783, + "max_per_prompt_fro": 33.430855, + "heavy_tail_ratio": 1.1733, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.215937, + "mean_fro": 28.358448, + "rms_per_prompt_fro": 31.27073, + "max_per_prompt_fro": 36.075897, + "heavy_tail_ratio": 1.1537, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.15686, + "mean_fro": 32.134437, + "rms_per_prompt_fro": 34.562978, + "max_per_prompt_fro": 38.575142, + "heavy_tail_ratio": 1.1161, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.117422, + "mean_fro": 34.557518, + "rms_per_prompt_fro": 36.530125, + "max_per_prompt_fro": 39.604698, + "heavy_tail_ratio": 1.0842, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.085903, + "mean_fro": 35.991554, + "rms_per_prompt_fro": 37.505608, + "max_per_prompt_fro": 40.116283, + "heavy_tail_ratio": 1.0696, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.058222, + "mean_fro": 36.131187, + "rms_per_prompt_fro": 37.168123, + "max_per_prompt_fro": 39.185619, + "heavy_tail_ratio": 1.0543, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.034881, + "mean_fro": 35.976616, + "rms_per_prompt_fro": 36.598693, + "max_per_prompt_fro": 38.274601, + "heavy_tail_ratio": 1.0458, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.017376, + "mean_fro": 35.16335, + "rms_per_prompt_fro": 35.467541, + "max_per_prompt_fro": 36.537865, + "heavy_tail_ratio": 1.0302, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1243.5, + "fit_secs_per_prompt": 3.109, + "total_wall_seconds": 1253.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.7573, + "top_sv": 1.3957, + "stable_rank": 30.893 + }, + { + "layer": 1, + "fro_norm": 7.2594, + "top_sv": 1.3133, + "stable_rank": 30.552 + }, + { + "layer": 2, + "fro_norm": 7.7118, + "top_sv": 1.3951, + "stable_rank": 30.557 + }, + { + "layer": 3, + "fro_norm": 7.453, + "top_sv": 1.4013, + "stable_rank": 28.289 + }, + { + "layer": 4, + "fro_norm": 8.491, + "top_sv": 1.5989, + "stable_rank": 28.202 + }, + { + "layer": 5, + "fro_norm": 8.7964, + "top_sv": 1.7487, + "stable_rank": 25.304 + }, + { + "layer": 6, + "fro_norm": 9.2429, + "top_sv": 1.864, + "stable_rank": 24.588 + }, + { + "layer": 7, + "fro_norm": 10.2544, + "top_sv": 2.0882, + "stable_rank": 24.114 + }, + { + "layer": 8, + "fro_norm": 11.8273, + "top_sv": 2.5764, + "stable_rank": 21.073 + }, + { + "layer": 9, + "fro_norm": 13.429, + "top_sv": 2.9208, + "stable_rank": 21.14 + }, + { + "layer": 10, + "fro_norm": 15.3718, + "top_sv": 3.3783, + "stable_rank": 20.704 + }, + { + "layer": 11, + "fro_norm": 18.2293, + "top_sv": 3.9795, + "stable_rank": 20.984 + }, + { + "layer": 12, + "fro_norm": 20.3777, + "top_sv": 4.4782, + "stable_rank": 20.706 + }, + { + "layer": 13, + "fro_norm": 22.1064, + "top_sv": 4.901, + "stable_rank": 20.346 + }, + { + "layer": 14, + "fro_norm": 25.373, + "top_sv": 5.553, + "stable_rank": 20.878 + }, + { + "layer": 15, + "fro_norm": 28.3579, + "top_sv": 6.3979, + "stable_rank": 19.646 + }, + { + "layer": 16, + "fro_norm": 32.1339, + "top_sv": 7.5032, + "stable_rank": 18.341 + }, + { + "layer": 17, + "fro_norm": 34.5567, + "top_sv": 7.9422, + "stable_rank": 18.932 + }, + { + "layer": 18, + "fro_norm": 35.9905, + "top_sv": 8.0874, + "stable_rank": 19.804 + }, + { + "layer": 19, + "fro_norm": 36.13, + "top_sv": 8.0124, + "stable_rank": 20.334 + }, + { + "layer": 20, + "fro_norm": 35.9753, + "top_sv": 7.6316, + "stable_rank": 22.222 + }, + { + "layer": 21, + "fro_norm": 35.1617, + "top_sv": 5.9869, + "stable_rank": 34.493 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n50_s0.pt b/results/lenses/e28/e28_StackExchange_410m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a29d764bd11e8ab22f219b55b58bc6f2fdc968d7 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b59b1cc3e164aa7ee10f647c45e8c466c0fbdf190064039c5d2740e4a3c0323 +size 46145033 diff --git a/results/lenses/e28/e28_StackExchange_410m_n50_s0_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c6f71b2966dbff455c7c7512e41af83ebaacc97c --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n50_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n50_s0.pt", + "sha256": "0b59b1cc3e164aa7ee10f647c45e8c466c0fbdf190064039c5d2740e4a3c0323", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.889138, + "mean_fro": 7.937179, + "rms_per_prompt_fro": 13.491192, + "max_per_prompt_fro": 18.289335, + "heavy_tail_ratio": 1.3557, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.691006, + "mean_fro": 7.430222, + "rms_per_prompt_fro": 12.188749, + "max_per_prompt_fro": 16.337557, + "heavy_tail_ratio": 1.3404, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.37037, + "mean_fro": 7.927785, + "rms_per_prompt_fro": 12.205624, + "max_per_prompt_fro": 15.959434, + "heavy_tail_ratio": 1.3075, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.180909, + "mean_fro": 7.645574, + "rms_per_prompt_fro": 11.290908, + "max_per_prompt_fro": 14.45578, + "heavy_tail_ratio": 1.2803, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.976745, + "mean_fro": 8.72366, + "rms_per_prompt_fro": 12.265186, + "max_per_prompt_fro": 15.352065, + "heavy_tail_ratio": 1.2517, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.895977, + "mean_fro": 9.014414, + "rms_per_prompt_fro": 12.412349, + "max_per_prompt_fro": 15.272669, + "heavy_tail_ratio": 1.2304, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.797506, + "mean_fro": 9.455665, + "rms_per_prompt_fro": 12.677312, + "max_per_prompt_fro": 15.329688, + "heavy_tail_ratio": 1.2092, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.703315, + "mean_fro": 10.43639, + "rms_per_prompt_fro": 13.620647, + "max_per_prompt_fro": 16.216015, + "heavy_tail_ratio": 1.1905, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.626421, + "mean_fro": 12.033029, + "rms_per_prompt_fro": 15.345867, + "max_per_prompt_fro": 18.131931, + "heavy_tail_ratio": 1.1816, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.571395, + "mean_fro": 13.657416, + "rms_per_prompt_fro": 17.120293, + "max_per_prompt_fro": 19.955544, + "heavy_tail_ratio": 1.1656, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.516281, + "mean_fro": 15.614367, + "rms_per_prompt_fro": 19.227121, + "max_per_prompt_fro": 22.121168, + "heavy_tail_ratio": 1.1505, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.458509, + "mean_fro": 18.517607, + "rms_per_prompt_fro": 22.363478, + "max_per_prompt_fro": 25.839254, + "heavy_tail_ratio": 1.1554, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.366987, + "mean_fro": 20.494861, + "rms_per_prompt_fro": 23.962224, + "max_per_prompt_fro": 27.022219, + "heavy_tail_ratio": 1.1277, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.312557, + "mean_fro": 22.248635, + "rms_per_prompt_fro": 25.489568, + "max_per_prompt_fro": 28.637596, + "heavy_tail_ratio": 1.1235, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.260547, + "mean_fro": 25.512348, + "rms_per_prompt_fro": 28.643755, + "max_per_prompt_fro": 31.647188, + "heavy_tail_ratio": 1.1049, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.218186, + "mean_fro": 28.48481, + "rms_per_prompt_fro": 31.439107, + "max_per_prompt_fro": 34.238823, + "heavy_tail_ratio": 1.0891, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.158936, + "mean_fro": 32.152237, + "rms_per_prompt_fro": 34.613129, + "max_per_prompt_fro": 37.207474, + "heavy_tail_ratio": 1.075, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.119419, + "mean_fro": 34.438797, + "rms_per_prompt_fro": 36.437139, + "max_per_prompt_fro": 38.422569, + "heavy_tail_ratio": 1.0545, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.086719, + "mean_fro": 35.871112, + "rms_per_prompt_fro": 37.39413, + "max_per_prompt_fro": 39.225418, + "heavy_tail_ratio": 1.049, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.059145, + "mean_fro": 35.967197, + "rms_per_prompt_fro": 37.015553, + "max_per_prompt_fro": 38.162895, + "heavy_tail_ratio": 1.031, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.03568, + "mean_fro": 35.793427, + "rms_per_prompt_fro": 36.42638, + "max_per_prompt_fro": 37.382595, + "heavy_tail_ratio": 1.0263, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.017941, + "mean_fro": 35.024132, + "rms_per_prompt_fro": 35.336927, + "max_per_prompt_fro": 35.912029, + "heavy_tail_ratio": 1.0163, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 152.0, + "fit_secs_per_prompt": 3.039, + "total_wall_seconds": 157.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9371, + "top_sv": 1.46, + "stable_rank": 29.555 + }, + { + "layer": 1, + "fro_norm": 7.4301, + "top_sv": 1.3697, + "stable_rank": 29.428 + }, + { + "layer": 2, + "fro_norm": 7.9277, + "top_sv": 1.4913, + "stable_rank": 28.258 + }, + { + "layer": 3, + "fro_norm": 7.6455, + "top_sv": 1.4671, + "stable_rank": 27.158 + }, + { + "layer": 4, + "fro_norm": 8.7235, + "top_sv": 1.7819, + "stable_rank": 23.966 + }, + { + "layer": 5, + "fro_norm": 9.0143, + "top_sv": 1.8921, + "stable_rank": 22.698 + }, + { + "layer": 6, + "fro_norm": 9.4555, + "top_sv": 1.9793, + "stable_rank": 22.822 + }, + { + "layer": 7, + "fro_norm": 10.4363, + "top_sv": 2.1215, + "stable_rank": 24.2 + }, + { + "layer": 8, + "fro_norm": 12.0329, + "top_sv": 2.6027, + "stable_rank": 21.374 + }, + { + "layer": 9, + "fro_norm": 13.6572, + "top_sv": 2.9555, + "stable_rank": 21.353 + }, + { + "layer": 10, + "fro_norm": 15.6142, + "top_sv": 3.3898, + "stable_rank": 21.218 + }, + { + "layer": 11, + "fro_norm": 18.5173, + "top_sv": 4.0055, + "stable_rank": 21.372 + }, + { + "layer": 12, + "fro_norm": 20.4946, + "top_sv": 4.3844, + "stable_rank": 21.85 + }, + { + "layer": 13, + "fro_norm": 22.2483, + "top_sv": 4.8108, + "stable_rank": 21.387 + }, + { + "layer": 14, + "fro_norm": 25.5119, + "top_sv": 5.4464, + "stable_rank": 21.941 + }, + { + "layer": 15, + "fro_norm": 28.4843, + "top_sv": 6.2818, + "stable_rank": 20.561 + }, + { + "layer": 16, + "fro_norm": 32.1517, + "top_sv": 7.3681, + "stable_rank": 19.041 + }, + { + "layer": 17, + "fro_norm": 34.438, + "top_sv": 7.8078, + "stable_rank": 19.454 + }, + { + "layer": 18, + "fro_norm": 35.8702, + "top_sv": 8.028, + "stable_rank": 19.964 + }, + { + "layer": 19, + "fro_norm": 35.9661, + "top_sv": 7.9739, + "stable_rank": 20.344 + }, + { + "layer": 20, + "fro_norm": 35.7922, + "top_sv": 7.5292, + "stable_rank": 22.599 + }, + { + "layer": 21, + "fro_norm": 35.0226, + "top_sv": 5.8998, + "stable_rank": 35.239 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n50_s1.pt b/results/lenses/e28/e28_StackExchange_410m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a0441921e351d423bd7d338846f5c8e9dc495e1 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfe59d22b735a6c975651fa751a0496e470337655b5af5692d1947c7c1eb1c49 +size 46145033 diff --git a/results/lenses/e28/e28_StackExchange_410m_n50_s1_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..81223fc95e2cade0b42fb233caab5a214d84b093 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n50_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n50_s1.pt", + "sha256": "bfe59d22b735a6c975651fa751a0496e470337655b5af5692d1947c7c1eb1c49", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.675894, + "mean_fro": 8.234085, + "rms_per_prompt_fro": 13.469449, + "max_per_prompt_fro": 19.984915, + "heavy_tail_ratio": 1.4837, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.508389, + "mean_fro": 7.696969, + "rms_per_prompt_fro": 12.190379, + "max_per_prompt_fro": 17.600504, + "heavy_tail_ratio": 1.4438, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.256381, + "mean_fro": 8.133922, + "rms_per_prompt_fro": 12.218172, + "max_per_prompt_fro": 17.261517, + "heavy_tail_ratio": 1.4128, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.092728, + "mean_fro": 7.808388, + "rms_per_prompt_fro": 11.29582, + "max_per_prompt_fro": 15.296987, + "heavy_tail_ratio": 1.3542, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.916357, + "mean_fro": 8.858168, + "rms_per_prompt_fro": 12.262586, + "max_per_prompt_fro": 16.132614, + "heavy_tail_ratio": 1.3156, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.845236, + "mean_fro": 9.155297, + "rms_per_prompt_fro": 12.436507, + "max_per_prompt_fro": 15.868522, + "heavy_tail_ratio": 1.276, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.756873, + "mean_fro": 9.597932, + "rms_per_prompt_fro": 12.72178, + "max_per_prompt_fro": 15.966814, + "heavy_tail_ratio": 1.2551, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.66717, + "mean_fro": 10.607206, + "rms_per_prompt_fro": 13.695911, + "max_per_prompt_fro": 16.773069, + "heavy_tail_ratio": 1.2247, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.593432, + "mean_fro": 12.194325, + "rms_per_prompt_fro": 15.393045, + "max_per_prompt_fro": 18.029625, + "heavy_tail_ratio": 1.1713, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.542876, + "mean_fro": 13.80257, + "rms_per_prompt_fro": 17.144524, + "max_per_prompt_fro": 20.106562, + "heavy_tail_ratio": 1.1728, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.492152, + "mean_fro": 15.744114, + "rms_per_prompt_fro": 19.232015, + "max_per_prompt_fro": 22.459669, + "heavy_tail_ratio": 1.1678, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.439897, + "mean_fro": 18.624763, + "rms_per_prompt_fro": 22.34892, + "max_per_prompt_fro": 25.981112, + "heavy_tail_ratio": 1.1625, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.349339, + "mean_fro": 20.692436, + "rms_per_prompt_fro": 24.036553, + "max_per_prompt_fro": 27.542673, + "heavy_tail_ratio": 1.1459, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.294189, + "mean_fro": 22.392807, + "rms_per_prompt_fro": 25.474603, + "max_per_prompt_fro": 28.854349, + "heavy_tail_ratio": 1.1327, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.244619, + "mean_fro": 25.662132, + "rms_per_prompt_fro": 28.629319, + "max_per_prompt_fro": 32.247253, + "heavy_tail_ratio": 1.1264, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.201946, + "mean_fro": 28.613339, + "rms_per_prompt_fro": 31.369752, + "max_per_prompt_fro": 35.142059, + "heavy_tail_ratio": 1.1203, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.146699, + "mean_fro": 32.348477, + "rms_per_prompt_fro": 34.640048, + "max_per_prompt_fro": 38.322559, + "heavy_tail_ratio": 1.1063, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.110651, + "mean_fro": 34.697338, + "rms_per_prompt_fro": 36.566635, + "max_per_prompt_fro": 39.639565, + "heavy_tail_ratio": 1.084, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.081049, + "mean_fro": 36.061116, + "rms_per_prompt_fro": 37.494008, + "max_per_prompt_fro": 40.168762, + "heavy_tail_ratio": 1.0713, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.054841, + "mean_fro": 36.169639, + "rms_per_prompt_fro": 37.148189, + "max_per_prompt_fro": 38.928661, + "heavy_tail_ratio": 1.0479, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.032932, + "mean_fro": 35.999912, + "rms_per_prompt_fro": 36.587882, + "max_per_prompt_fro": 37.961212, + "heavy_tail_ratio": 1.0375, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.016588, + "mean_fro": 35.155189, + "rms_per_prompt_fro": 35.445573, + "max_per_prompt_fro": 36.210915, + "heavy_tail_ratio": 1.0216, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 154.3, + "fit_secs_per_prompt": 3.086, + "total_wall_seconds": 161.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.234, + "top_sv": 1.5105, + "stable_rank": 29.715 + }, + { + "layer": 1, + "fro_norm": 7.6969, + "top_sv": 1.4052, + "stable_rank": 30.0 + }, + { + "layer": 2, + "fro_norm": 8.1338, + "top_sv": 1.5027, + "stable_rank": 29.299 + }, + { + "layer": 3, + "fro_norm": 7.8083, + "top_sv": 1.4805, + "stable_rank": 27.815 + }, + { + "layer": 4, + "fro_norm": 8.858, + "top_sv": 1.6833, + "stable_rank": 27.693 + }, + { + "layer": 5, + "fro_norm": 9.1552, + "top_sv": 1.8225, + "stable_rank": 25.234 + }, + { + "layer": 6, + "fro_norm": 9.5978, + "top_sv": 1.9408, + "stable_rank": 24.457 + }, + { + "layer": 7, + "fro_norm": 10.6071, + "top_sv": 2.1458, + "stable_rank": 24.435 + }, + { + "layer": 8, + "fro_norm": 12.1942, + "top_sv": 2.6216, + "stable_rank": 21.636 + }, + { + "layer": 9, + "fro_norm": 13.8024, + "top_sv": 2.9677, + "stable_rank": 21.631 + }, + { + "layer": 10, + "fro_norm": 15.7439, + "top_sv": 3.4264, + "stable_rank": 21.113 + }, + { + "layer": 11, + "fro_norm": 18.6245, + "top_sv": 4.0315, + "stable_rank": 21.342 + }, + { + "layer": 12, + "fro_norm": 20.6921, + "top_sv": 4.5093, + "stable_rank": 21.056 + }, + { + "layer": 13, + "fro_norm": 22.3925, + "top_sv": 4.9147, + "stable_rank": 20.759 + }, + { + "layer": 14, + "fro_norm": 25.6617, + "top_sv": 5.562, + "stable_rank": 21.287 + }, + { + "layer": 15, + "fro_norm": 28.6128, + "top_sv": 6.3757, + "stable_rank": 20.14 + }, + { + "layer": 16, + "fro_norm": 32.3479, + "top_sv": 7.4787, + "stable_rank": 18.709 + }, + { + "layer": 17, + "fro_norm": 34.6965, + "top_sv": 7.9054, + "stable_rank": 19.263 + }, + { + "layer": 18, + "fro_norm": 36.0601, + "top_sv": 8.0227, + "stable_rank": 20.203 + }, + { + "layer": 19, + "fro_norm": 36.1685, + "top_sv": 7.9454, + "stable_rank": 20.722 + }, + { + "layer": 20, + "fro_norm": 35.9986, + "top_sv": 7.5678, + "stable_rank": 22.627 + }, + { + "layer": 21, + "fro_norm": 35.1536, + "top_sv": 5.934, + "stable_rank": 35.094 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n50_s2.pt b/results/lenses/e28/e28_StackExchange_410m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9eb1e13e62d5a23680eab7d38f17284abd95be8 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54e88c7f45291fbb4056efc5b69049cb997c3c86aa61889931f6bbc060511e5d +size 46145033 diff --git a/results/lenses/e28/e28_StackExchange_410m_n50_s2_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0f4113a8da1276a3cb3d9eee5612384d43685484 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n50_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n50_s2.pt", + "sha256": "54e88c7f45291fbb4056efc5b69049cb997c3c86aa61889931f6bbc060511e5d", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.797916, + "mean_fro": 7.93133, + "rms_per_prompt_fro": 13.266713, + "max_per_prompt_fro": 16.937399, + "heavy_tail_ratio": 1.2767, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.610252, + "mean_fro": 7.407085, + "rms_per_prompt_fro": 11.96709, + "max_per_prompt_fro": 14.93219, + "heavy_tail_ratio": 1.2478, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.315129, + "mean_fro": 7.873556, + "rms_per_prompt_fro": 11.980045, + "max_per_prompt_fro": 14.767397, + "heavy_tail_ratio": 1.2327, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.134, + "mean_fro": 7.577296, + "rms_per_prompt_fro": 11.069078, + "max_per_prompt_fro": 13.292094, + "heavy_tail_ratio": 1.2008, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.932059, + "mean_fro": 8.640821, + "rms_per_prompt_fro": 12.010614, + "max_per_prompt_fro": 14.190033, + "heavy_tail_ratio": 1.1815, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.849606, + "mean_fro": 8.921928, + "rms_per_prompt_fro": 12.133842, + "max_per_prompt_fro": 14.307484, + "heavy_tail_ratio": 1.1791, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.752635, + "mean_fro": 9.362689, + "rms_per_prompt_fro": 12.394993, + "max_per_prompt_fro": 14.638433, + "heavy_tail_ratio": 1.181, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.663036, + "mean_fro": 10.38118, + "rms_per_prompt_fro": 13.387441, + "max_per_prompt_fro": 15.688981, + "heavy_tail_ratio": 1.1719, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.589919, + "mean_fro": 11.960138, + "rms_per_prompt_fro": 15.080776, + "max_per_prompt_fro": 17.307007, + "heavy_tail_ratio": 1.1476, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.537646, + "mean_fro": 13.569071, + "rms_per_prompt_fro": 16.825902, + "max_per_prompt_fro": 19.149542, + "heavy_tail_ratio": 1.1381, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.486766, + "mean_fro": 15.527269, + "rms_per_prompt_fro": 18.932866, + "max_per_prompt_fro": 21.509356, + "heavy_tail_ratio": 1.1361, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.434477, + "mean_fro": 18.388899, + "rms_per_prompt_fro": 22.024317, + "max_per_prompt_fro": 24.927063, + "heavy_tail_ratio": 1.1318, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.350752, + "mean_fro": 20.5242, + "rms_per_prompt_fro": 23.853603, + "max_per_prompt_fro": 26.490185, + "heavy_tail_ratio": 1.1105, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.297014, + "mean_fro": 22.258875, + "rms_per_prompt_fro": 25.349862, + "max_per_prompt_fro": 27.940784, + "heavy_tail_ratio": 1.1022, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.247781, + "mean_fro": 25.515331, + "rms_per_prompt_fro": 28.501673, + "max_per_prompt_fro": 31.251293, + "heavy_tail_ratio": 1.0965, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.206144, + "mean_fro": 28.490475, + "rms_per_prompt_fro": 31.28954, + "max_per_prompt_fro": 34.156677, + "heavy_tail_ratio": 1.0916, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.151128, + "mean_fro": 32.217564, + "rms_per_prompt_fro": 34.566432, + "max_per_prompt_fro": 37.253185, + "heavy_tail_ratio": 1.0777, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.114153, + "mean_fro": 34.58614, + "rms_per_prompt_fro": 36.506861, + "max_per_prompt_fro": 39.037834, + "heavy_tail_ratio": 1.0693, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.084116, + "mean_fro": 36.002087, + "rms_per_prompt_fro": 37.48569, + "max_per_prompt_fro": 39.759815, + "heavy_tail_ratio": 1.0607, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.057059, + "mean_fro": 36.12341, + "rms_per_prompt_fro": 37.139703, + "max_per_prompt_fro": 38.615036, + "heavy_tail_ratio": 1.0397, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.034565, + "mean_fro": 35.961773, + "rms_per_prompt_fro": 36.578009, + "max_per_prompt_fro": 37.595169, + "heavy_tail_ratio": 1.0278, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.01738, + "mean_fro": 35.13203, + "rms_per_prompt_fro": 35.436013, + "max_per_prompt_fro": 36.070801, + "heavy_tail_ratio": 1.0179, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 155.6, + "fit_secs_per_prompt": 3.112, + "total_wall_seconds": 164.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.9313, + "top_sv": 1.4261, + "stable_rank": 30.932 + }, + { + "layer": 1, + "fro_norm": 7.407, + "top_sv": 1.3444, + "stable_rank": 30.356 + }, + { + "layer": 2, + "fro_norm": 7.8735, + "top_sv": 1.4342, + "stable_rank": 30.137 + }, + { + "layer": 3, + "fro_norm": 7.5772, + "top_sv": 1.4148, + "stable_rank": 28.685 + }, + { + "layer": 4, + "fro_norm": 8.6407, + "top_sv": 1.5991, + "stable_rank": 29.197 + }, + { + "layer": 5, + "fro_norm": 8.9218, + "top_sv": 1.7283, + "stable_rank": 26.648 + }, + { + "layer": 6, + "fro_norm": 9.3626, + "top_sv": 1.842, + "stable_rank": 25.834 + }, + { + "layer": 7, + "fro_norm": 10.3811, + "top_sv": 2.0727, + "stable_rank": 25.086 + }, + { + "layer": 8, + "fro_norm": 11.96, + "top_sv": 2.5472, + "stable_rank": 22.047 + }, + { + "layer": 9, + "fro_norm": 13.5689, + "top_sv": 2.8886, + "stable_rank": 22.065 + }, + { + "layer": 10, + "fro_norm": 15.5271, + "top_sv": 3.3389, + "stable_rank": 21.626 + }, + { + "layer": 11, + "fro_norm": 18.3886, + "top_sv": 3.9397, + "stable_rank": 21.785 + }, + { + "layer": 12, + "fro_norm": 20.5239, + "top_sv": 4.4264, + "stable_rank": 21.499 + }, + { + "layer": 13, + "fro_norm": 22.2586, + "top_sv": 4.8537, + "stable_rank": 21.03 + }, + { + "layer": 14, + "fro_norm": 25.5149, + "top_sv": 5.4821, + "stable_rank": 21.661 + }, + { + "layer": 15, + "fro_norm": 28.49, + "top_sv": 6.3188, + "stable_rank": 20.329 + }, + { + "layer": 16, + "fro_norm": 32.217, + "top_sv": 7.4163, + "stable_rank": 18.871 + }, + { + "layer": 17, + "fro_norm": 34.5853, + "top_sv": 7.861, + "stable_rank": 19.357 + }, + { + "layer": 18, + "fro_norm": 36.0011, + "top_sv": 8.0168, + "stable_rank": 20.167 + }, + { + "layer": 19, + "fro_norm": 36.1223, + "top_sv": 7.97, + "stable_rank": 20.541 + }, + { + "layer": 20, + "fro_norm": 35.9604, + "top_sv": 7.5875, + "stable_rank": 22.462 + }, + { + "layer": 21, + "fro_norm": 35.1304, + "top_sv": 5.947, + "stable_rank": 34.895 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n800_s0.pt b/results/lenses/e28/e28_StackExchange_410m_n800_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca4ca06b72bad6bcc74d388864d5f1602fa6e314 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n800_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8daeaf9ab1d0e38698a5bcb0452791802cbd82b35a7173609ed0f669a8f4aa7b +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n800_s0_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n800_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..03456337c073950b4c8e875ae6a5c5601145e3d1 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n800_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n800_s0.pt", + "sha256": "8daeaf9ab1d0e38698a5bcb0452791802cbd82b35a7173609ed0f669a8f4aa7b", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.936555, + "mean_fro": 7.855509, + "rms_per_prompt_fro": 13.461498, + "max_per_prompt_fro": 21.382202, + "heavy_tail_ratio": 1.5884, + "n_prompts": 800 + }, + "1": { + "dispersion": 1.733294, + "mean_fro": 7.354739, + "rms_per_prompt_fro": 12.159353, + "max_per_prompt_fro": 19.421309, + "heavy_tail_ratio": 1.5972, + "n_prompts": 800 + }, + "2": { + "dispersion": 1.41802, + "mean_fro": 7.828161, + "rms_per_prompt_fro": 12.172776, + "max_per_prompt_fro": 19.912868, + "heavy_tail_ratio": 1.6359, + "n_prompts": 800 + }, + "3": { + "dispersion": 1.222931, + "mean_fro": 7.552339, + "rms_per_prompt_fro": 11.260158, + "max_per_prompt_fro": 17.533512, + "heavy_tail_ratio": 1.5571, + "n_prompts": 800 + }, + "4": { + "dispersion": 1.016525, + "mean_fro": 8.614427, + "rms_per_prompt_fro": 12.232867, + "max_per_prompt_fro": 19.180052, + "heavy_tail_ratio": 1.5679, + "n_prompts": 800 + }, + "5": { + "dispersion": 0.93482, + "mean_fro": 8.91195, + "rms_per_prompt_fro": 12.396327, + "max_per_prompt_fro": 18.667677, + "heavy_tail_ratio": 1.5059, + "n_prompts": 800 + }, + "6": { + "dispersion": 0.832005, + "mean_fro": 9.354881, + "rms_per_prompt_fro": 12.661979, + "max_per_prompt_fro": 18.677954, + "heavy_tail_ratio": 1.4751, + "n_prompts": 800 + }, + "7": { + "dispersion": 0.732681, + "mean_fro": 10.355754, + "rms_per_prompt_fro": 13.631416, + "max_per_prompt_fro": 19.680288, + "heavy_tail_ratio": 1.4437, + "n_prompts": 800 + }, + "8": { + "dispersion": 0.650624, + "mean_fro": 11.933381, + "rms_per_prompt_fro": 15.331605, + "max_per_prompt_fro": 21.688644, + "heavy_tail_ratio": 1.4146, + "n_prompts": 800 + }, + "9": { + "dispersion": 0.592619, + "mean_fro": 13.53, + "rms_per_prompt_fro": 17.074727, + "max_per_prompt_fro": 23.499172, + "heavy_tail_ratio": 1.3763, + "n_prompts": 800 + }, + "10": { + "dispersion": 0.534919, + "mean_fro": 15.464909, + "rms_per_prompt_fro": 19.159765, + "max_per_prompt_fro": 25.770779, + "heavy_tail_ratio": 1.345, + "n_prompts": 800 + }, + "11": { + "dispersion": 0.474347, + "mean_fro": 18.31885, + "rms_per_prompt_fro": 22.243242, + "max_per_prompt_fro": 29.363607, + "heavy_tail_ratio": 1.3201, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.379393, + "mean_fro": 20.410296, + "rms_per_prompt_fro": 23.971391, + "max_per_prompt_fro": 29.734083, + "heavy_tail_ratio": 1.2404, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.320358, + "mean_fro": 22.132648, + "rms_per_prompt_fro": 25.431924, + "max_per_prompt_fro": 30.961466, + "heavy_tail_ratio": 1.2174, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.265577, + "mean_fro": 25.38913, + "rms_per_prompt_fro": 28.562227, + "max_per_prompt_fro": 33.7141, + "heavy_tail_ratio": 1.1804, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.21973, + "mean_fro": 28.362936, + "rms_per_prompt_fro": 31.324425, + "max_per_prompt_fro": 36.521626, + "heavy_tail_ratio": 1.1659, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.158878, + "mean_fro": 32.106529, + "rms_per_prompt_fro": 34.563058, + "max_per_prompt_fro": 38.757477, + "heavy_tail_ratio": 1.1214, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.118742, + "mean_fro": 34.504997, + "rms_per_prompt_fro": 36.496149, + "max_per_prompt_fro": 39.671333, + "heavy_tail_ratio": 1.087, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.086633, + "mean_fro": 35.930344, + "rms_per_prompt_fro": 37.454404, + "max_per_prompt_fro": 40.193676, + "heavy_tail_ratio": 1.0731, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.058629, + "mean_fro": 36.058846, + "rms_per_prompt_fro": 37.100841, + "max_per_prompt_fro": 38.823784, + "heavy_tail_ratio": 1.0464, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.035206, + "mean_fro": 35.919578, + "rms_per_prompt_fro": 36.546401, + "max_per_prompt_fro": 38.052616, + "heavy_tail_ratio": 1.0412, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.01761, + "mean_fro": 35.121808, + "rms_per_prompt_fro": 35.429698, + "max_per_prompt_fro": 36.302822, + "heavy_tail_ratio": 1.0246, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2473.5, + "fit_secs_per_prompt": 3.092, + "total_wall_seconds": 2479.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.8554, + "top_sv": 1.411, + "stable_rank": 30.996 + }, + { + "layer": 1, + "fro_norm": 7.3547, + "top_sv": 1.328, + "stable_rank": 30.673 + }, + { + "layer": 2, + "fro_norm": 7.8281, + "top_sv": 1.4193, + "stable_rank": 30.42 + }, + { + "layer": 3, + "fro_norm": 7.5522, + "top_sv": 1.419, + "stable_rank": 28.327 + }, + { + "layer": 4, + "fro_norm": 8.6143, + "top_sv": 1.6394, + "stable_rank": 27.611 + }, + { + "layer": 5, + "fro_norm": 8.9118, + "top_sv": 1.7796, + "stable_rank": 25.079 + }, + { + "layer": 6, + "fro_norm": 9.3548, + "top_sv": 1.891, + "stable_rank": 24.473 + }, + { + "layer": 7, + "fro_norm": 10.3556, + "top_sv": 2.1075, + "stable_rank": 24.145 + }, + { + "layer": 8, + "fro_norm": 11.9332, + "top_sv": 2.5933, + "stable_rank": 21.175 + }, + { + "layer": 9, + "fro_norm": 13.5298, + "top_sv": 2.9385, + "stable_rank": 21.2 + }, + { + "layer": 10, + "fro_norm": 15.4647, + "top_sv": 3.3833, + "stable_rank": 20.893 + }, + { + "layer": 11, + "fro_norm": 18.3186, + "top_sv": 3.9803, + "stable_rank": 21.181 + }, + { + "layer": 12, + "fro_norm": 20.41, + "top_sv": 4.4499, + "stable_rank": 21.037 + }, + { + "layer": 13, + "fro_norm": 22.1324, + "top_sv": 4.8681, + "stable_rank": 20.67 + }, + { + "layer": 14, + "fro_norm": 25.3886, + "top_sv": 5.516, + "stable_rank": 21.185 + }, + { + "layer": 15, + "fro_norm": 28.3624, + "top_sv": 6.3379, + "stable_rank": 20.026 + }, + { + "layer": 16, + "fro_norm": 32.106, + "top_sv": 7.42, + "stable_rank": 18.723 + }, + { + "layer": 17, + "fro_norm": 34.5042, + "top_sv": 7.8536, + "stable_rank": 19.302 + }, + { + "layer": 18, + "fro_norm": 35.9293, + "top_sv": 8.0121, + "stable_rank": 20.11 + }, + { + "layer": 19, + "fro_norm": 36.0578, + "top_sv": 7.9436, + "stable_rank": 20.604 + }, + { + "layer": 20, + "fro_norm": 35.9183, + "top_sv": 7.5746, + "stable_rank": 22.486 + }, + { + "layer": 21, + "fro_norm": 35.1202, + "top_sv": 5.9509, + "stable_rank": 34.83 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n800_s1.pt b/results/lenses/e28/e28_StackExchange_410m_n800_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bba808565f7844bf63f9c9fbf27b86bbd5c9d6b --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n800_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0df04165a7304f0c91270fee186ef7f3a9d1985e161903acce3ffcc5b1eaadbe +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n800_s1_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n800_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e4e99510e628254a2e88b393be44bd073cf46583 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n800_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n800_s1.pt", + "sha256": "0df04165a7304f0c91270fee186ef7f3a9d1985e161903acce3ffcc5b1eaadbe", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.972012, + "mean_fro": 7.781491, + "rms_per_prompt_fro": 13.41492, + "max_per_prompt_fro": 20.959408, + "heavy_tail_ratio": 1.5624, + "n_prompts": 800 + }, + "1": { + "dispersion": 1.764751, + "mean_fro": 7.282606, + "rms_per_prompt_fro": 12.109182, + "max_per_prompt_fro": 18.410004, + "heavy_tail_ratio": 1.5203, + "n_prompts": 800 + }, + "2": { + "dispersion": 1.447436, + "mean_fro": 7.755202, + "rms_per_prompt_fro": 12.132458, + "max_per_prompt_fro": 17.710325, + "heavy_tail_ratio": 1.4597, + "n_prompts": 800 + }, + "3": { + "dispersion": 1.249374, + "mean_fro": 7.488724, + "rms_per_prompt_fro": 11.231522, + "max_per_prompt_fro": 16.373718, + "heavy_tail_ratio": 1.4578, + "n_prompts": 800 + }, + "4": { + "dispersion": 1.03877, + "mean_fro": 8.542708, + "rms_per_prompt_fro": 12.197748, + "max_per_prompt_fro": 17.393656, + "heavy_tail_ratio": 1.426, + "n_prompts": 800 + }, + "5": { + "dispersion": 0.953993, + "mean_fro": 8.840258, + "rms_per_prompt_fro": 12.357381, + "max_per_prompt_fro": 17.09712, + "heavy_tail_ratio": 1.3836, + "n_prompts": 800 + }, + "6": { + "dispersion": 0.849254, + "mean_fro": 9.284436, + "rms_per_prompt_fro": 12.625652, + "max_per_prompt_fro": 17.151518, + "heavy_tail_ratio": 1.3585, + "n_prompts": 800 + }, + "7": { + "dispersion": 0.746797, + "mean_fro": 10.28477, + "rms_per_prompt_fro": 13.593015, + "max_per_prompt_fro": 17.78598, + "heavy_tail_ratio": 1.3085, + "n_prompts": 800 + }, + "8": { + "dispersion": 0.661934, + "mean_fro": 11.855011, + "rms_per_prompt_fro": 15.283009, + "max_per_prompt_fro": 19.440754, + "heavy_tail_ratio": 1.2721, + "n_prompts": 800 + }, + "9": { + "dispersion": 0.602904, + "mean_fro": 13.446174, + "rms_per_prompt_fro": 17.023643, + "max_per_prompt_fro": 21.190538, + "heavy_tail_ratio": 1.2448, + "n_prompts": 800 + }, + "10": { + "dispersion": 0.543924, + "mean_fro": 15.375944, + "rms_per_prompt_fro": 19.105338, + "max_per_prompt_fro": 23.231464, + "heavy_tail_ratio": 1.216, + "n_prompts": 800 + }, + "11": { + "dispersion": 0.482999, + "mean_fro": 18.221201, + "rms_per_prompt_fro": 22.189494, + "max_per_prompt_fro": 26.729731, + "heavy_tail_ratio": 1.2046, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.387327, + "mean_fro": 20.316862, + "rms_per_prompt_fro": 23.930182, + "max_per_prompt_fro": 28.940958, + "heavy_tail_ratio": 1.2094, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.327413, + "mean_fro": 22.029511, + "rms_per_prompt_fro": 25.380954, + "max_per_prompt_fro": 30.333376, + "heavy_tail_ratio": 1.1951, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.271701, + "mean_fro": 25.267989, + "rms_per_prompt_fro": 28.494641, + "max_per_prompt_fro": 33.330925, + "heavy_tail_ratio": 1.1697, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.225191, + "mean_fro": 28.231481, + "rms_per_prompt_fro": 31.248964, + "max_per_prompt_fro": 36.296127, + "heavy_tail_ratio": 1.1615, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.163582, + "mean_fro": 31.990195, + "rms_per_prompt_fro": 34.507644, + "max_per_prompt_fro": 38.567505, + "heavy_tail_ratio": 1.1177, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.122404, + "mean_fro": 34.396545, + "rms_per_prompt_fro": 36.440934, + "max_per_prompt_fro": 39.883991, + "heavy_tail_ratio": 1.0945, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.089548, + "mean_fro": 35.836127, + "rms_per_prompt_fro": 37.406253, + "max_per_prompt_fro": 40.168762, + "heavy_tail_ratio": 1.0739, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.06074, + "mean_fro": 35.999374, + "rms_per_prompt_fro": 37.076553, + "max_per_prompt_fro": 38.975006, + "heavy_tail_ratio": 1.0512, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.036424, + "mean_fro": 35.878971, + "rms_per_prompt_fro": 36.526556, + "max_per_prompt_fro": 38.089752, + "heavy_tail_ratio": 1.0428, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.018227, + "mean_fro": 35.094981, + "rms_per_prompt_fro": 35.413383, + "max_per_prompt_fro": 36.42778, + "heavy_tail_ratio": 1.0286, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2469.2, + "fit_secs_per_prompt": 3.087, + "total_wall_seconds": 2476.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.7814, + "top_sv": 1.3831, + "stable_rank": 31.655 + }, + { + "layer": 1, + "fro_norm": 7.2825, + "top_sv": 1.3013, + "stable_rank": 31.318 + }, + { + "layer": 2, + "fro_norm": 7.7551, + "top_sv": 1.3935, + "stable_rank": 30.972 + }, + { + "layer": 3, + "fro_norm": 7.4886, + "top_sv": 1.3983, + "stable_rank": 28.681 + }, + { + "layer": 4, + "fro_norm": 8.5426, + "top_sv": 1.6025, + "stable_rank": 28.419 + }, + { + "layer": 5, + "fro_norm": 8.8401, + "top_sv": 1.7491, + "stable_rank": 25.544 + }, + { + "layer": 6, + "fro_norm": 9.2843, + "top_sv": 1.8622, + "stable_rank": 24.858 + }, + { + "layer": 7, + "fro_norm": 10.2846, + "top_sv": 2.0846, + "stable_rank": 24.342 + }, + { + "layer": 8, + "fro_norm": 11.8549, + "top_sv": 2.5694, + "stable_rank": 21.287 + }, + { + "layer": 9, + "fro_norm": 13.446, + "top_sv": 2.9111, + "stable_rank": 21.334 + }, + { + "layer": 10, + "fro_norm": 15.3758, + "top_sv": 3.357, + "stable_rank": 20.978 + }, + { + "layer": 11, + "fro_norm": 18.2209, + "top_sv": 3.9529, + "stable_rank": 21.248 + }, + { + "layer": 12, + "fro_norm": 20.3166, + "top_sv": 4.4281, + "stable_rank": 21.051 + }, + { + "layer": 13, + "fro_norm": 22.0292, + "top_sv": 4.8404, + "stable_rank": 20.712 + }, + { + "layer": 14, + "fro_norm": 25.2676, + "top_sv": 5.4818, + "stable_rank": 21.246 + }, + { + "layer": 15, + "fro_norm": 28.2309, + "top_sv": 6.2979, + "stable_rank": 20.094 + }, + { + "layer": 16, + "fro_norm": 31.9896, + "top_sv": 7.3836, + "stable_rank": 18.771 + }, + { + "layer": 17, + "fro_norm": 34.3957, + "top_sv": 7.8173, + "stable_rank": 19.36 + }, + { + "layer": 18, + "fro_norm": 35.8352, + "top_sv": 7.981, + "stable_rank": 20.161 + }, + { + "layer": 19, + "fro_norm": 35.9983, + "top_sv": 7.9156, + "stable_rank": 20.682 + }, + { + "layer": 20, + "fro_norm": 35.8777, + "top_sv": 7.5635, + "stable_rank": 22.501 + }, + { + "layer": 21, + "fro_norm": 35.0934, + "top_sv": 5.9328, + "stable_rank": 34.989 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_410m_n800_s2.pt b/results/lenses/e28/e28_StackExchange_410m_n800_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c8ed8bb1c949db0783411cd1762a0d2646583f6 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n800_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c4fc7abbe22780a9a87b67464bc4eb76000a8767be000a5a3034e0a8dcc0425 +size 46145061 diff --git a/results/lenses/e28/e28_StackExchange_410m_n800_s2_provenance.json b/results/lenses/e28/e28_StackExchange_410m_n800_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ec0dcd9f3437a34f3931f4bc92f18cea78e52049 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_410m_n800_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_StackExchange_410m_n800_s2.pt", + "sha256": "5c4fc7abbe22780a9a87b67464bc4eb76000a8767be000a5a3034e0a8dcc0425", + "bytes": 46145061, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.957157, + "mean_fro": 7.788336, + "rms_per_prompt_fro": 13.393123, + "max_per_prompt_fro": 22.89805, + "heavy_tail_ratio": 1.7097, + "n_prompts": 800 + }, + "1": { + "dispersion": 1.750823, + "mean_fro": 7.290829, + "rms_per_prompt_fro": 12.092281, + "max_per_prompt_fro": 20.678722, + "heavy_tail_ratio": 1.7101, + "n_prompts": 800 + }, + "2": { + "dispersion": 1.436487, + "mean_fro": 7.756364, + "rms_per_prompt_fro": 12.107102, + "max_per_prompt_fro": 20.684196, + "heavy_tail_ratio": 1.7084, + "n_prompts": 800 + }, + "3": { + "dispersion": 1.241131, + "mean_fro": 7.490819, + "rms_per_prompt_fro": 11.21406, + "max_per_prompt_fro": 18.248838, + "heavy_tail_ratio": 1.6273, + "n_prompts": 800 + }, + "4": { + "dispersion": 1.031045, + "mean_fro": 8.542603, + "rms_per_prompt_fro": 12.174468, + "max_per_prompt_fro": 19.46534, + "heavy_tail_ratio": 1.5989, + "n_prompts": 800 + }, + "5": { + "dispersion": 0.946591, + "mean_fro": 8.841109, + "rms_per_prompt_fro": 12.335142, + "max_per_prompt_fro": 19.447893, + "heavy_tail_ratio": 1.5766, + "n_prompts": 800 + }, + "6": { + "dispersion": 0.842676, + "mean_fro": 9.286055, + "rms_per_prompt_fro": 12.605374, + "max_per_prompt_fro": 19.570419, + "heavy_tail_ratio": 1.5525, + "n_prompts": 800 + }, + "7": { + "dispersion": 0.742197, + "mean_fro": 10.294522, + "rms_per_prompt_fro": 13.587978, + "max_per_prompt_fro": 20.862938, + "heavy_tail_ratio": 1.5354, + "n_prompts": 800 + }, + "8": { + "dispersion": 0.657418, + "mean_fro": 11.864026, + "rms_per_prompt_fro": 15.273836, + "max_per_prompt_fro": 22.600924, + "heavy_tail_ratio": 1.4797, + "n_prompts": 800 + }, + "9": { + "dispersion": 0.598257, + "mean_fro": 13.460247, + "rms_per_prompt_fro": 17.01674, + "max_per_prompt_fro": 24.01169, + "heavy_tail_ratio": 1.4111, + "n_prompts": 800 + }, + "10": { + "dispersion": 0.539706, + "mean_fro": 15.391144, + "rms_per_prompt_fro": 19.098081, + "max_per_prompt_fro": 25.506193, + "heavy_tail_ratio": 1.3355, + "n_prompts": 800 + }, + "11": { + "dispersion": 0.47873, + "mean_fro": 18.235516, + "rms_per_prompt_fro": 22.174939, + "max_per_prompt_fro": 28.148739, + "heavy_tail_ratio": 1.2694, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.383644, + "mean_fro": 20.361889, + "rms_per_prompt_fro": 23.951366, + "max_per_prompt_fro": 30.121614, + "heavy_tail_ratio": 1.2576, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.324093, + "mean_fro": 22.085815, + "rms_per_prompt_fro": 25.413975, + "max_per_prompt_fro": 30.655041, + "heavy_tail_ratio": 1.2062, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.268866, + "mean_fro": 25.337173, + "rms_per_prompt_fro": 28.540789, + "max_per_prompt_fro": 33.430855, + "heavy_tail_ratio": 1.1713, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.222401, + "mean_fro": 28.312824, + "rms_per_prompt_fro": 31.303295, + "max_per_prompt_fro": 36.075897, + "heavy_tail_ratio": 1.1525, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.161262, + "mean_fro": 32.071747, + "rms_per_prompt_fro": 34.561117, + "max_per_prompt_fro": 38.687923, + "heavy_tail_ratio": 1.1194, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.120679, + "mean_fro": 34.491909, + "rms_per_prompt_fro": 36.513868, + "max_per_prompt_fro": 39.604698, + "heavy_tail_ratio": 1.0846, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.088087, + "mean_fro": 35.930145, + "rms_per_prompt_fro": 37.479246, + "max_per_prompt_fro": 40.116283, + "heavy_tail_ratio": 1.0704, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.059785, + "mean_fro": 36.068626, + "rms_per_prompt_fro": 37.131157, + "max_per_prompt_fro": 39.185619, + "heavy_tail_ratio": 1.0553, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.035824, + "mean_fro": 35.928458, + "rms_per_prompt_fro": 36.566349, + "max_per_prompt_fro": 38.274601, + "heavy_tail_ratio": 1.0467, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.017884, + "mean_fro": 35.127964, + "rms_per_prompt_fro": 35.440693, + "max_per_prompt_fro": 36.537865, + "heavy_tail_ratio": 1.031, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2484.8, + "fit_secs_per_prompt": 3.106, + "total_wall_seconds": 2493.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.7882, + "top_sv": 1.3916, + "stable_rank": 31.322 + }, + { + "layer": 1, + "fro_norm": 7.2907, + "top_sv": 1.3093, + "stable_rank": 31.007 + }, + { + "layer": 2, + "fro_norm": 7.7563, + "top_sv": 1.3986, + "stable_rank": 30.757 + }, + { + "layer": 3, + "fro_norm": 7.4907, + "top_sv": 1.4006, + "stable_rank": 28.605 + }, + { + "layer": 4, + "fro_norm": 8.5425, + "top_sv": 1.6076, + "stable_rank": 28.238 + }, + { + "layer": 5, + "fro_norm": 8.841, + "top_sv": 1.7539, + "stable_rank": 25.408 + }, + { + "layer": 6, + "fro_norm": 9.2859, + "top_sv": 1.8679, + "stable_rank": 24.715 + }, + { + "layer": 7, + "fro_norm": 10.2944, + "top_sv": 2.0875, + "stable_rank": 24.319 + }, + { + "layer": 8, + "fro_norm": 11.8639, + "top_sv": 2.5741, + "stable_rank": 21.242 + }, + { + "layer": 9, + "fro_norm": 13.46, + "top_sv": 2.9196, + "stable_rank": 21.255 + }, + { + "layer": 10, + "fro_norm": 15.391, + "top_sv": 3.3702, + "stable_rank": 20.856 + }, + { + "layer": 11, + "fro_norm": 18.2352, + "top_sv": 3.965, + "stable_rank": 21.151 + }, + { + "layer": 12, + "fro_norm": 20.3616, + "top_sv": 4.4515, + "stable_rank": 20.922 + }, + { + "layer": 13, + "fro_norm": 22.0855, + "top_sv": 4.873, + "stable_rank": 20.541 + }, + { + "layer": 14, + "fro_norm": 25.3367, + "top_sv": 5.518, + "stable_rank": 21.083 + }, + { + "layer": 15, + "fro_norm": 28.3123, + "top_sv": 6.3557, + "stable_rank": 19.844 + }, + { + "layer": 16, + "fro_norm": 32.0711, + "top_sv": 7.4512, + "stable_rank": 18.526 + }, + { + "layer": 17, + "fro_norm": 34.4911, + "top_sv": 7.8867, + "stable_rank": 19.126 + }, + { + "layer": 18, + "fro_norm": 35.9292, + "top_sv": 8.0493, + "stable_rank": 19.924 + }, + { + "layer": 19, + "fro_norm": 36.0675, + "top_sv": 7.9751, + "stable_rank": 20.453 + }, + { + "layer": 20, + "fro_norm": 35.9272, + "top_sv": 7.5994, + "stable_rank": 22.35 + }, + { + "layer": 21, + "fro_norm": 35.1263, + "top_sv": 5.9626, + "stable_rank": 34.706 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n100_s0.pt b/results/lenses/e28/e28_StackExchange_70m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f623026a877027ad10202ae636e87f854d42783 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6af7ced13a0cabe7d11a7534cd4b4353ac797c91a99c12c597d18a82339a46ff +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n100_s0_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..685592a737f8228a2aa487fa777357367ffaa836 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n100_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n100_s0.pt", + "sha256": "6af7ced13a0cabe7d11a7534cd4b4353ac797c91a99c12c597d18a82339a46ff", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.22312, + "mean_fro": 31.153076, + "rms_per_prompt_fro": 34.453675, + "max_per_prompt_fro": 44.529892, + "heavy_tail_ratio": 1.2925, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.168045, + "mean_fro": 25.120591, + "rms_per_prompt_fro": 27.149367, + "max_per_prompt_fro": 33.477749, + "heavy_tail_ratio": 1.2331, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.118025, + "mean_fro": 24.737921, + "rms_per_prompt_fro": 26.157063, + "max_per_prompt_fro": 30.116081, + "heavy_tail_ratio": 1.1514, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.059605, + "mean_fro": 22.77258, + "rms_per_prompt_fro": 23.44144, + "max_per_prompt_fro": 24.78301, + "heavy_tail_ratio": 1.0572, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.8, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 18.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.153, + "top_sv": 4.7867, + "stable_rank": 42.357 + }, + { + "layer": 1, + "fro_norm": 25.1205, + "top_sv": 4.592, + "stable_rank": 29.926 + }, + { + "layer": 2, + "fro_norm": 24.7379, + "top_sv": 3.7487, + "stable_rank": 43.547 + }, + { + "layer": 3, + "fro_norm": 22.7725, + "top_sv": 4.3844, + "stable_rank": 26.977 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n100_s1.pt b/results/lenses/e28/e28_StackExchange_70m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd822b444811ec6cbd33fa7643936c18f4540f4d --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9540618f31a8c1f59da2ec570bb6645c8267585d4b664898b816c914cd79782 +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n100_s1_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..87a81f602d5228e4f72c0c8233efc3dd458581e4 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n100_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n100_s1.pt", + "sha256": "c9540618f31a8c1f59da2ec570bb6645c8267585d4b664898b816c914cd79782", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.220984, + "mean_fro": 31.300253, + "rms_per_prompt_fro": 34.586199, + "max_per_prompt_fro": 41.470898, + "heavy_tail_ratio": 1.1991, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.166992, + "mean_fro": 25.114389, + "rms_per_prompt_fro": 27.130424, + "max_per_prompt_fro": 31.965729, + "heavy_tail_ratio": 1.1782, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.118694, + "mean_fro": 24.655142, + "rms_per_prompt_fro": 26.077335, + "max_per_prompt_fro": 29.228292, + "heavy_tail_ratio": 1.1208, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.062212, + "mean_fro": 22.756996, + "rms_per_prompt_fro": 23.45419, + "max_per_prompt_fro": 25.595602, + "heavy_tail_ratio": 1.0913, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.0, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 15.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.3002, + "top_sv": 4.8435, + "stable_rank": 41.762 + }, + { + "layer": 1, + "fro_norm": 25.1143, + "top_sv": 4.6139, + "stable_rank": 29.628 + }, + { + "layer": 2, + "fro_norm": 24.6551, + "top_sv": 3.7448, + "stable_rank": 43.347 + }, + { + "layer": 3, + "fro_norm": 22.7569, + "top_sv": 4.4316, + "stable_rank": 26.369 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n100_s2.pt b/results/lenses/e28/e28_StackExchange_70m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..837208bc1c880a5a1c5d9729140fc911489c4101 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b41cf8049c9aedc60bea9640ea68f8af57829dc648d1251e47dcd003c4749d57 +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n100_s2_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..aaac80605802add1f1ca45efdcecec36bfb03a4a --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n100_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n100_s2.pt", + "sha256": "b41cf8049c9aedc60bea9640ea68f8af57829dc648d1251e47dcd003c4749d57", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.223223, + "mean_fro": 31.016357, + "rms_per_prompt_fro": 34.303909, + "max_per_prompt_fro": 41.030972, + "heavy_tail_ratio": 1.1961, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.166212, + "mean_fro": 24.950691, + "rms_per_prompt_fro": 26.944573, + "max_per_prompt_fro": 31.005344, + "heavy_tail_ratio": 1.1507, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.11572, + "mean_fro": 24.582285, + "rms_per_prompt_fro": 25.965684, + "max_per_prompt_fro": 29.210003, + "heavy_tail_ratio": 1.1249, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.060036, + "mean_fro": 22.65894, + "rms_per_prompt_fro": 23.329199, + "max_per_prompt_fro": 25.008461, + "heavy_tail_ratio": 1.072, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.1, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 16.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.0163, + "top_sv": 4.8244, + "stable_rank": 41.332 + }, + { + "layer": 1, + "fro_norm": 24.9506, + "top_sv": 4.6234, + "stable_rank": 29.123 + }, + { + "layer": 2, + "fro_norm": 24.5822, + "top_sv": 3.7597, + "stable_rank": 42.749 + }, + { + "layer": 3, + "fro_norm": 22.6589, + "top_sv": 4.3947, + "stable_rank": 26.584 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n200_s0.pt b/results/lenses/e28/e28_StackExchange_70m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..088908c291cc459ebe9edb5990ebfc5cb7e79362 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b04ea2223f0288ddb706d85180c76938f23f2b586fbe4a90d9d1189c9c1a04 +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n200_s0_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4ca58fde5f0bcb0090dbe2a86897a3e8fdaf6e00 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n200_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n200_s0.pt", + "sha256": "39b04ea2223f0288ddb706d85180c76938f23f2b586fbe4a90d9d1189c9c1a04", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.225439, + "mean_fro": 31.156592, + "rms_per_prompt_fro": 34.490206, + "max_per_prompt_fro": 44.529892, + "heavy_tail_ratio": 1.2911, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.168935, + "mean_fro": 25.085948, + "rms_per_prompt_fro": 27.122249, + "max_per_prompt_fro": 33.477749, + "heavy_tail_ratio": 1.2343, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.118623, + "mean_fro": 24.681692, + "rms_per_prompt_fro": 26.104588, + "max_per_prompt_fro": 30.116081, + "heavy_tail_ratio": 1.1537, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.060903, + "mean_fro": 22.738306, + "rms_per_prompt_fro": 23.420493, + "max_per_prompt_fro": 24.975027, + "heavy_tail_ratio": 1.0664, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 23.6, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 30.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.1565, + "top_sv": 4.8262, + "stable_rank": 41.675 + }, + { + "layer": 1, + "fro_norm": 25.0859, + "top_sv": 4.6217, + "stable_rank": 29.462 + }, + { + "layer": 2, + "fro_norm": 24.6817, + "top_sv": 3.7575, + "stable_rank": 43.146 + }, + { + "layer": 3, + "fro_norm": 22.7383, + "top_sv": 4.3901, + "stable_rank": 26.826 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n200_s1.pt b/results/lenses/e28/e28_StackExchange_70m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc3e58063620dcafa2488460cf20b2b40eb54573 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:894433e6c8294b3bec6ccfca49a2f29a2b3550380c90a698539b92f115a7dd21 +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n200_s1_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..8ba1b2bb234da6624cea7a1c04ac3ce1d969b35a --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n200_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n200_s1.pt", + "sha256": "894433e6c8294b3bec6ccfca49a2f29a2b3550380c90a698539b92f115a7dd21", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.22611, + "mean_fro": 31.221102, + "rms_per_prompt_fro": 34.571081, + "max_per_prompt_fro": 41.470898, + "heavy_tail_ratio": 1.1996, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.16981, + "mean_fro": 25.044837, + "rms_per_prompt_fro": 27.087938, + "max_per_prompt_fro": 31.965729, + "heavy_tail_ratio": 1.1801, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.120645, + "mean_fro": 24.62446, + "rms_per_prompt_fro": 26.067584, + "max_per_prompt_fro": 29.228292, + "heavy_tail_ratio": 1.1213, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.063569, + "mean_fro": 22.722642, + "rms_per_prompt_fro": 23.433748, + "max_per_prompt_fro": 25.718687, + "heavy_tail_ratio": 1.0975, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 21.9, + "fit_secs_per_prompt": 0.109, + "total_wall_seconds": 26.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.2211, + "top_sv": 4.8604, + "stable_rank": 41.262 + }, + { + "layer": 1, + "fro_norm": 25.0448, + "top_sv": 4.6364, + "stable_rank": 29.179 + }, + { + "layer": 2, + "fro_norm": 24.6244, + "top_sv": 3.7632, + "stable_rank": 42.816 + }, + { + "layer": 3, + "fro_norm": 22.7226, + "top_sv": 4.4222, + "stable_rank": 26.402 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n200_s2.pt b/results/lenses/e28/e28_StackExchange_70m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdbc141ff43ca9fbee7893d0b20409250bf7d47f --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8eec1900568e7f1392a93607de7babc4b4b3b73a791860265a6e01cfbeac60f +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n200_s2_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..20170e288e006f9963ce65bfc08822d5e0fc3ec2 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n200_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n200_s2.pt", + "sha256": "d8eec1900568e7f1392a93607de7babc4b4b3b73a791860265a6e01cfbeac60f", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.224737, + "mean_fro": 31.037128, + "rms_per_prompt_fro": 34.348112, + "max_per_prompt_fro": 41.030972, + "heavy_tail_ratio": 1.1946, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.168142, + "mean_fro": 24.978687, + "rms_per_prompt_fro": 26.997122, + "max_per_prompt_fro": 31.424484, + "heavy_tail_ratio": 1.164, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.118062, + "mean_fro": 24.595301, + "rms_per_prompt_fro": 26.006695, + "max_per_prompt_fro": 29.210003, + "heavy_tail_ratio": 1.1232, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.061168, + "mean_fro": 22.682789, + "rms_per_prompt_fro": 23.366223, + "max_per_prompt_fro": 25.382627, + "heavy_tail_ratio": 1.0863, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 22.0, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 27.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.0371, + "top_sv": 4.8236, + "stable_rank": 41.401 + }, + { + "layer": 1, + "fro_norm": 24.9787, + "top_sv": 4.6159, + "stable_rank": 29.283 + }, + { + "layer": 2, + "fro_norm": 24.5953, + "top_sv": 3.7556, + "stable_rank": 42.888 + }, + { + "layer": 3, + "fro_norm": 22.6827, + "top_sv": 4.3967, + "stable_rank": 26.615 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n25_s0.pt b/results/lenses/e28/e28_StackExchange_70m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8025e668c898f6a1dfc643c934207e19232d2140 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df32a3544d11b4b0e44393f92d41d65a5a663c24d02d151459d7defab0a1d83b +size 2099887 diff --git a/results/lenses/e28/e28_StackExchange_70m_n25_s0_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f20a196db9b78bd8d534c529b2f506b859b6d1db --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n25_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n25_s0.pt", + "sha256": "df32a3544d11b4b0e44393f92d41d65a5a663c24d02d151459d7defab0a1d83b", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.247874, + "mean_fro": 30.656111, + "rms_per_prompt_fro": 34.24541, + "max_per_prompt_fro": 40.46941, + "heavy_tail_ratio": 1.1817, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.187883, + "mean_fro": 24.73402, + "rms_per_prompt_fro": 26.957615, + "max_per_prompt_fro": 31.744644, + "heavy_tail_ratio": 1.1776, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.13438, + "mean_fro": 24.458163, + "rms_per_prompt_fro": 26.049721, + "max_per_prompt_fro": 29.785965, + "heavy_tail_ratio": 1.1434, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.071276, + "mean_fro": 22.611256, + "rms_per_prompt_fro": 23.403204, + "max_per_prompt_fro": 24.466993, + "heavy_tail_ratio": 1.0455, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.1, + "fit_secs_per_prompt": 0.125, + "total_wall_seconds": 15.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.6561, + "top_sv": 4.7161, + "stable_rank": 42.254 + }, + { + "layer": 1, + "fro_norm": 24.734, + "top_sv": 4.5475, + "stable_rank": 29.583 + }, + { + "layer": 2, + "fro_norm": 24.4581, + "top_sv": 3.7252, + "stable_rank": 43.107 + }, + { + "layer": 3, + "fro_norm": 22.6112, + "top_sv": 4.3384, + "stable_rank": 27.164 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n25_s1.pt b/results/lenses/e28/e28_StackExchange_70m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..625e41042d61c4514cdbcc232e8cf3831e7b2824 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8057cdb51ba6346fac90ec6e914382a792cae57db9eeaf7b606a4be6e1c1a501 +size 2099887 diff --git a/results/lenses/e28/e28_StackExchange_70m_n25_s1_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4381209aa75ed194ffa3b8d2ef3f30f66592d5bf --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n25_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n25_s1.pt", + "sha256": "8057cdb51ba6346fac90ec6e914382a792cae57db9eeaf7b606a4be6e1c1a501", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.210355, + "mean_fro": 31.379604, + "rms_per_prompt_fro": 34.522623, + "max_per_prompt_fro": 37.208969, + "heavy_tail_ratio": 1.0778, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.157007, + "mean_fro": 25.279199, + "rms_per_prompt_fro": 27.191385, + "max_per_prompt_fro": 29.264046, + "heavy_tail_ratio": 1.0762, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.109399, + "mean_fro": 24.839531, + "rms_per_prompt_fro": 26.162979, + "max_per_prompt_fro": 28.46747, + "heavy_tail_ratio": 1.0881, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.057686, + "mean_fro": 22.823875, + "rms_per_prompt_fro": 23.472952, + "max_per_prompt_fro": 25.56452, + "heavy_tail_ratio": 1.0891, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2.9, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 7.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.3796, + "top_sv": 4.8216, + "stable_rank": 42.356 + }, + { + "layer": 1, + "fro_norm": 25.2792, + "top_sv": 4.5944, + "stable_rank": 30.274 + }, + { + "layer": 2, + "fro_norm": 24.8395, + "top_sv": 3.736, + "stable_rank": 44.205 + }, + { + "layer": 3, + "fro_norm": 22.8238, + "top_sv": 4.3613, + "stable_rank": 27.386 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n25_s2.pt b/results/lenses/e28/e28_StackExchange_70m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..ded15417f2600667eb54f106300832319c046115 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bac3cae3f219841b65dfd31705f8b5e9ec38a2104cd26fe8bb12edd6b4401f03 +size 2099887 diff --git a/results/lenses/e28/e28_StackExchange_70m_n25_s2_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..92e5c02c2d0a50ffce8938625a74756a0ee9fd00 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n25_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n25_s2.pt", + "sha256": "bac3cae3f219841b65dfd31705f8b5e9ec38a2104cd26fe8bb12edd6b4401f03", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.217897, + "mean_fro": 30.975006, + "rms_per_prompt_fro": 34.183513, + "max_per_prompt_fro": 41.030972, + "heavy_tail_ratio": 1.2003, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.164339, + "mean_fro": 24.861578, + "rms_per_prompt_fro": 26.826778, + "max_per_prompt_fro": 31.005344, + "heavy_tail_ratio": 1.1558, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.116806, + "mean_fro": 24.533079, + "rms_per_prompt_fro": 25.926319, + "max_per_prompt_fro": 29.210003, + "heavy_tail_ratio": 1.1267, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.061676, + "mean_fro": 22.601898, + "rms_per_prompt_fro": 23.288466, + "max_per_prompt_fro": 24.261694, + "heavy_tail_ratio": 1.0418, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.0, + "fit_secs_per_prompt": 0.12, + "total_wall_seconds": 8.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.975, + "top_sv": 4.8442, + "stable_rank": 40.886 + }, + { + "layer": 1, + "fro_norm": 24.8615, + "top_sv": 4.6423, + "stable_rank": 28.681 + }, + { + "layer": 2, + "fro_norm": 24.533, + "top_sv": 3.7812, + "stable_rank": 42.096 + }, + { + "layer": 3, + "fro_norm": 22.6019, + "top_sv": 4.3991, + "stable_rank": 26.397 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n400_s0.pt b/results/lenses/e28/e28_StackExchange_70m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d709da54cd8ded8cbbb6197b385832eaadeed89f --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13dd95f375ae43663f35ae06db99627030a090d827e2b4fb6806003006d7ec8 +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n400_s0_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..85bd31aa77df2c09afb753f1f9cd094aeb2e9f97 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n400_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n400_s0.pt", + "sha256": "d13dd95f375ae43663f35ae06db99627030a090d827e2b4fb6806003006d7ec8", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.226536, + "mean_fro": 31.218696, + "rms_per_prompt_fro": 34.574423, + "max_per_prompt_fro": 49.458347, + "heavy_tail_ratio": 1.4305, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.169825, + "mean_fro": 25.112092, + "rms_per_prompt_fro": 27.160846, + "max_per_prompt_fro": 36.863297, + "heavy_tail_ratio": 1.3572, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.119229, + "mean_fro": 24.696028, + "rms_per_prompt_fro": 26.126827, + "max_per_prompt_fro": 32.447609, + "heavy_tail_ratio": 1.2419, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.060933, + "mean_fro": 22.721363, + "rms_per_prompt_fro": 23.403366, + "max_per_prompt_fro": 25.173361, + "heavy_tail_ratio": 1.0756, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 46.3, + "fit_secs_per_prompt": 0.116, + "total_wall_seconds": 52.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.2186, + "top_sv": 4.827, + "stable_rank": 41.828 + }, + { + "layer": 1, + "fro_norm": 25.1121, + "top_sv": 4.6183, + "stable_rank": 29.567 + }, + { + "layer": 2, + "fro_norm": 24.696, + "top_sv": 3.7574, + "stable_rank": 43.2 + }, + { + "layer": 3, + "fro_norm": 22.7213, + "top_sv": 4.3936, + "stable_rank": 26.744 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n400_s1.pt b/results/lenses/e28/e28_StackExchange_70m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bf78d4bfd1a01f5746b865869d221ce5729ef5b --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58ad9fc98ed1823b12921b5fc30a222a80b1ca91ee036fb01f8d35955cd1521c +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n400_s1_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a152c5f75ebb573d6eedcdc5135280f605beced3 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n400_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n400_s1.pt", + "sha256": "58ad9fc98ed1823b12921b5fc30a222a80b1ca91ee036fb01f8d35955cd1521c", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.234269, + "mean_fro": 31.097213, + "rms_per_prompt_fro": 34.54827, + "max_per_prompt_fro": 43.678513, + "heavy_tail_ratio": 1.2643, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.176729, + "mean_fro": 24.986757, + "rms_per_prompt_fro": 27.104919, + "max_per_prompt_fro": 32.577568, + "heavy_tail_ratio": 1.2019, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.125874, + "mean_fro": 24.598719, + "rms_per_prompt_fro": 26.101011, + "max_per_prompt_fro": 29.88217, + "heavy_tail_ratio": 1.1449, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.066845, + "mean_fro": 22.69622, + "rms_per_prompt_fro": 23.442519, + "max_per_prompt_fro": 25.718687, + "heavy_tail_ratio": 1.0971, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 43.8, + "fit_secs_per_prompt": 0.109, + "total_wall_seconds": 48.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.0972, + "top_sv": 4.8327, + "stable_rank": 41.407 + }, + { + "layer": 1, + "fro_norm": 24.9867, + "top_sv": 4.6151, + "stable_rank": 29.313 + }, + { + "layer": 2, + "fro_norm": 24.5987, + "top_sv": 3.7557, + "stable_rank": 42.898 + }, + { + "layer": 3, + "fro_norm": 22.6962, + "top_sv": 4.3993, + "stable_rank": 26.616 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n400_s2.pt b/results/lenses/e28/e28_StackExchange_70m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..975a1cd1fac422844723c031dd2e28a08f663f36 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ed1b087708fa07fc5f83308e4a13104827891494f46f6262a43b138fd2c8542 +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n400_s2_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a9a7249409b916e3dc0b14cc416e95d31dc3ea54 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n400_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n400_s2.pt", + "sha256": "8ed1b087708fa07fc5f83308e4a13104827891494f46f6262a43b138fd2c8542", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.22559, + "mean_fro": 30.933315, + "rms_per_prompt_fro": 34.245146, + "max_per_prompt_fro": 42.679546, + "heavy_tail_ratio": 1.2463, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.168921, + "mean_fro": 24.890336, + "rms_per_prompt_fro": 26.910595, + "max_per_prompt_fro": 33.249985, + "heavy_tail_ratio": 1.2356, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.119043, + "mean_fro": 24.508821, + "rms_per_prompt_fro": 25.926617, + "max_per_prompt_fro": 31.016733, + "heavy_tail_ratio": 1.1963, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.06198, + "mean_fro": 22.666603, + "rms_per_prompt_fro": 23.358485, + "max_per_prompt_fro": 25.424313, + "heavy_tail_ratio": 1.0884, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 43.9, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 49.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.9333, + "top_sv": 4.8027, + "stable_rank": 41.483 + }, + { + "layer": 1, + "fro_norm": 24.8903, + "top_sv": 4.5918, + "stable_rank": 29.383 + }, + { + "layer": 2, + "fro_norm": 24.5088, + "top_sv": 3.7307, + "stable_rank": 43.159 + }, + { + "layer": 3, + "fro_norm": 22.6665, + "top_sv": 4.4149, + "stable_rank": 26.358 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n50_s0.pt b/results/lenses/e28/e28_StackExchange_70m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d4a87bd7ca8df5c7dc715bcbe22da11df8741fa --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cba6014f368782ffe274fbf359aeafb5850b9f5b349709a38f581637f7faee89 +size 2099887 diff --git a/results/lenses/e28/e28_StackExchange_70m_n50_s0_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..2967dd78627a06d9053269b0d1b44604d35352f3 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n50_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n50_s0.pt", + "sha256": "cba6014f368782ffe274fbf359aeafb5850b9f5b349709a38f581637f7faee89", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.229183, + "mean_fro": 30.829944, + "rms_per_prompt_fro": 34.180698, + "max_per_prompt_fro": 40.46941, + "heavy_tail_ratio": 1.184, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.173317, + "mean_fro": 24.884993, + "rms_per_prompt_fro": 26.95537, + "max_per_prompt_fro": 31.744644, + "heavy_tail_ratio": 1.1777, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.12228, + "mean_fro": 24.565464, + "rms_per_prompt_fro": 26.024087, + "max_per_prompt_fro": 29.785965, + "heavy_tail_ratio": 1.1446, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.063221, + "mean_fro": 22.704161, + "rms_per_prompt_fro": 23.410858, + "max_per_prompt_fro": 24.466993, + "heavy_tail_ratio": 1.0451, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 6.0, + "fit_secs_per_prompt": 0.12, + "total_wall_seconds": 12.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.8299, + "top_sv": 4.7288, + "stable_rank": 42.506 + }, + { + "layer": 1, + "fro_norm": 24.885, + "top_sv": 4.5451, + "stable_rank": 29.977 + }, + { + "layer": 2, + "fro_norm": 24.5654, + "top_sv": 3.7103, + "stable_rank": 43.836 + }, + { + "layer": 3, + "fro_norm": 22.7041, + "top_sv": 4.3676, + "stable_rank": 27.022 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n50_s1.pt b/results/lenses/e28/e28_StackExchange_70m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..744bc9fa9157fcc57a37d638fe0d07dd63a1c9c0 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c48d88b881f4a63e1a9e78983f11626c723ea1bdb66879a34998fe2bd2153774 +size 2099887 diff --git a/results/lenses/e28/e28_StackExchange_70m_n50_s1_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9e9a1763dd35ee0c92b04edae06b136256309697 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n50_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n50_s1.pt", + "sha256": "c48d88b881f4a63e1a9e78983f11626c723ea1bdb66879a34998fe2bd2153774", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.211491, + "mean_fro": 31.399101, + "rms_per_prompt_fro": 34.560289, + "max_per_prompt_fro": 40.140759, + "heavy_tail_ratio": 1.1615, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.159227, + "mean_fro": 25.187504, + "rms_per_prompt_fro": 27.118732, + "max_per_prompt_fro": 30.999601, + "heavy_tail_ratio": 1.1431, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.112821, + "mean_fro": 24.704199, + "rms_per_prompt_fro": 26.060539, + "max_per_prompt_fro": 28.892212, + "heavy_tail_ratio": 1.1087, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.059461, + "mean_fro": 22.750904, + "rms_per_prompt_fro": 23.417533, + "max_per_prompt_fro": 25.56452, + "heavy_tail_ratio": 1.0917, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.7, + "fit_secs_per_prompt": 0.113, + "total_wall_seconds": 10.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.399, + "top_sv": 4.8531, + "stable_rank": 41.859 + }, + { + "layer": 1, + "fro_norm": 25.1875, + "top_sv": 4.6189, + "stable_rank": 29.737 + }, + { + "layer": 2, + "fro_norm": 24.7041, + "top_sv": 3.7424, + "stable_rank": 43.576 + }, + { + "layer": 3, + "fro_norm": 22.7508, + "top_sv": 4.4159, + "stable_rank": 26.543 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n50_s2.pt b/results/lenses/e28/e28_StackExchange_70m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..49c8b1d49bad28c6134abd1538c8fedb3ce024a0 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:746f1b29f48b998c2b88ef6b302afb5d617fec1e9e0791da3efdfd92b352878f +size 2099887 diff --git a/results/lenses/e28/e28_StackExchange_70m_n50_s2_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..676d120d3b4c72f72ee82b3d2294fccb58cae5f6 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n50_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n50_s2.pt", + "sha256": "746f1b29f48b998c2b88ef6b302afb5d617fec1e9e0791da3efdfd92b352878f", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.215747, + "mean_fro": 31.073736, + "rms_per_prompt_fro": 34.262187, + "max_per_prompt_fro": 41.030972, + "heavy_tail_ratio": 1.1976, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.16166, + "mean_fro": 24.920088, + "rms_per_prompt_fro": 26.858959, + "max_per_prompt_fro": 31.005344, + "heavy_tail_ratio": 1.1544, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.112514, + "mean_fro": 24.550856, + "rms_per_prompt_fro": 25.895205, + "max_per_prompt_fro": 29.210003, + "heavy_tail_ratio": 1.128, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.058084, + "mean_fro": 22.646981, + "rms_per_prompt_fro": 23.295417, + "max_per_prompt_fro": 24.261694, + "heavy_tail_ratio": 1.0415, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.6, + "fit_secs_per_prompt": 0.113, + "total_wall_seconds": 10.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.0737, + "top_sv": 4.8099, + "stable_rank": 41.736 + }, + { + "layer": 1, + "fro_norm": 24.92, + "top_sv": 4.6023, + "stable_rank": 29.319 + }, + { + "layer": 2, + "fro_norm": 24.5508, + "top_sv": 3.7403, + "stable_rank": 43.083 + }, + { + "layer": 3, + "fro_norm": 22.6469, + "top_sv": 4.4125, + "stable_rank": 26.342 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n800_s0.pt b/results/lenses/e28/e28_StackExchange_70m_n800_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..06dddae962a3dba555278d1f078e69b1e20a147a --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n800_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b1bd5a6391796d8f2c853a013cbca76e4174b8bbf03a36d2c41d1c32acc74dc +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n800_s0_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n800_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..dca50126b385fdbf4f6d06aba1e2c19f1486b52a --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n800_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n800_s0.pt", + "sha256": "8b1bd5a6391796d8f2c853a013cbca76e4174b8bbf03a36d2c41d1c32acc74dc", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.227846, + "mean_fro": 31.183529, + "rms_per_prompt_fro": 34.553907, + "max_per_prompt_fro": 49.458347, + "heavy_tail_ratio": 1.4313, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.171478, + "mean_fro": 25.080711, + "rms_per_prompt_fro": 27.146066, + "max_per_prompt_fro": 36.863297, + "heavy_tail_ratio": 1.358, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.120932, + "mean_fro": 24.660363, + "rms_per_prompt_fro": 26.108927, + "max_per_prompt_fro": 32.447609, + "heavy_tail_ratio": 1.2428, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.061936, + "mean_fro": 22.729714, + "rms_per_prompt_fro": 23.42303, + "max_per_prompt_fro": 25.507322, + "heavy_tail_ratio": 1.089, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 91.8, + "fit_secs_per_prompt": 0.115, + "total_wall_seconds": 98.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.1835, + "top_sv": 4.8298, + "stable_rank": 41.686 + }, + { + "layer": 1, + "fro_norm": 25.0807, + "top_sv": 4.6186, + "stable_rank": 29.489 + }, + { + "layer": 2, + "fro_norm": 24.6603, + "top_sv": 3.7544, + "stable_rank": 43.144 + }, + { + "layer": 3, + "fro_norm": 22.7297, + "top_sv": 4.4033, + "stable_rank": 26.645 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n800_s1.pt b/results/lenses/e28/e28_StackExchange_70m_n800_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..92d7104170d457b4f8d54b50746de15d1ad0614f --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n800_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:557f07a3c09ad86da08a7ee2cc71ddea139056c7b2e17262431b42745a7ae710 +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n800_s1_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n800_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7fb106898642c70fb306e4823b2e1430e95b4e1a --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n800_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n800_s1.pt", + "sha256": "557f07a3c09ad86da08a7ee2cc71ddea139056c7b2e17262431b42745a7ae710", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.23338, + "mean_fro": 31.15873, + "rms_per_prompt_fro": 34.604152, + "max_per_prompt_fro": 45.98196, + "heavy_tail_ratio": 1.3288, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.176603, + "mean_fro": 25.058245, + "rms_per_prompt_fro": 27.181014, + "max_per_prompt_fro": 34.997395, + "heavy_tail_ratio": 1.2876, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.125818, + "mean_fro": 24.643255, + "rms_per_prompt_fro": 26.147615, + "max_per_prompt_fro": 31.965462, + "heavy_tail_ratio": 1.2225, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.065629, + "mean_fro": 22.713451, + "rms_per_prompt_fro": 23.446935, + "max_per_prompt_fro": 25.718687, + "heavy_tail_ratio": 1.0969, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 87.2, + "fit_secs_per_prompt": 0.109, + "total_wall_seconds": 91.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 31.1587, + "top_sv": 4.8302, + "stable_rank": 41.613 + }, + { + "layer": 1, + "fro_norm": 25.0582, + "top_sv": 4.614, + "stable_rank": 29.495 + }, + { + "layer": 2, + "fro_norm": 24.6432, + "top_sv": 3.7493, + "stable_rank": 43.201 + }, + { + "layer": 3, + "fro_norm": 22.7134, + "top_sv": 4.391, + "stable_rank": 26.757 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_StackExchange_70m_n800_s2.pt b/results/lenses/e28/e28_StackExchange_70m_n800_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..0eb7210cc5d3f3517122d13902083f4881c90c47 --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n800_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec32cd8954e8d39f7bfb9bf74bfea6fec52baef31d0462399e0d47d183c95c5a +size 2099897 diff --git a/results/lenses/e28/e28_StackExchange_70m_n800_s2_provenance.json b/results/lenses/e28/e28_StackExchange_70m_n800_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c7fe3571d50f17e1ed8e6c4d236a355a1d9065fc --- /dev/null +++ b/results/lenses/e28/e28_StackExchange_70m_n800_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_StackExchange_70m_n800_s2.pt", + "sha256": "ec32cd8954e8d39f7bfb9bf74bfea6fec52baef31d0462399e0d47d183c95c5a", + "bytes": 2099897, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "StackExchange", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/StackExchange.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.232398, + "mean_fro": 30.958929, + "rms_per_prompt_fro": 34.368564, + "max_per_prompt_fro": 43.676731, + "heavy_tail_ratio": 1.2708, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.174612, + "mean_fro": 24.914429, + "rms_per_prompt_fro": 27.002134, + "max_per_prompt_fro": 33.249985, + "heavy_tail_ratio": 1.2314, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.123427, + "mean_fro": 24.528707, + "rms_per_prompt_fro": 25.998433, + "max_per_prompt_fro": 31.016733, + "heavy_tail_ratio": 1.193, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.064512, + "mean_fro": 22.666948, + "rms_per_prompt_fro": 23.386661, + "max_per_prompt_fro": 25.424313, + "heavy_tail_ratio": 1.0871, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 88.1, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 93.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.9589, + "top_sv": 4.8119, + "stable_rank": 41.393 + }, + { + "layer": 1, + "fro_norm": 24.9144, + "top_sv": 4.6037, + "stable_rank": 29.288 + }, + { + "layer": 2, + "fro_norm": 24.5287, + "top_sv": 3.7406, + "stable_rank": 43.0 + }, + { + "layer": 3, + "fro_norm": 22.6669, + "top_sv": 4.4024, + "stable_rank": 26.509 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c1e804f02a7397edaa8d626e0c09699370a7ca1 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:257c8c60058eafbfb51b97fb1fee2e97b05910316d887fc65b33457d68700a29 +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..5c20e666e4407e4f6507ae2403b60cf790a26072 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n100_s0.pt", + "sha256": "257c8c60058eafbfb51b97fb1fee2e97b05910316d887fc65b33457d68700a29", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.634915, + "mean_fro": 9.538297, + "rms_per_prompt_fro": 15.482967, + "max_per_prompt_fro": 19.348158, + "heavy_tail_ratio": 1.2496, + "n_prompts": 100 + }, + "1": { + "dispersion": 1.456216, + "mean_fro": 8.808498, + "rms_per_prompt_fro": 13.804959, + "max_per_prompt_fro": 17.370386, + "heavy_tail_ratio": 1.2583, + "n_prompts": 100 + }, + "2": { + "dispersion": 1.205437, + "mean_fro": 9.513458, + "rms_per_prompt_fro": 14.128163, + "max_per_prompt_fro": 17.830729, + "heavy_tail_ratio": 1.2621, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.07401, + "mean_fro": 9.020938, + "rms_per_prompt_fro": 12.991435, + "max_per_prompt_fro": 16.243773, + "heavy_tail_ratio": 1.2503, + "n_prompts": 100 + }, + "4": { + "dispersion": 0.894961, + "mean_fro": 10.135005, + "rms_per_prompt_fro": 13.951601, + "max_per_prompt_fro": 17.349812, + "heavy_tail_ratio": 1.2436, + "n_prompts": 100 + }, + "5": { + "dispersion": 0.819693, + "mean_fro": 10.212481, + "rms_per_prompt_fro": 13.776227, + "max_per_prompt_fro": 17.102009, + "heavy_tail_ratio": 1.2414, + "n_prompts": 100 + }, + "6": { + "dispersion": 0.741413, + "mean_fro": 10.513221, + "rms_per_prompt_fro": 13.873521, + "max_per_prompt_fro": 17.416403, + "heavy_tail_ratio": 1.2554, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.661705, + "mean_fro": 11.527343, + "rms_per_prompt_fro": 14.859567, + "max_per_prompt_fro": 18.555748, + "heavy_tail_ratio": 1.2487, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.606856, + "mean_fro": 13.136835, + "rms_per_prompt_fro": 16.652491, + "max_per_prompt_fro": 20.628031, + "heavy_tail_ratio": 1.2387, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.563018, + "mean_fro": 14.652411, + "rms_per_prompt_fro": 18.318552, + "max_per_prompt_fro": 22.51523, + "heavy_tail_ratio": 1.2291, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.516702, + "mean_fro": 16.457705, + "rms_per_prompt_fro": 20.268394, + "max_per_prompt_fro": 24.403898, + "heavy_tail_ratio": 1.204, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.469888, + "mean_fro": 19.085247, + "rms_per_prompt_fro": 23.138749, + "max_per_prompt_fro": 27.447655, + "heavy_tail_ratio": 1.1862, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.384993, + "mean_fro": 20.8636, + "rms_per_prompt_fro": 24.553478, + "max_per_prompt_fro": 29.091446, + "heavy_tail_ratio": 1.1848, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.34426, + "mean_fro": 22.791711, + "rms_per_prompt_fro": 26.425217, + "max_per_prompt_fro": 30.741051, + "heavy_tail_ratio": 1.1633, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.296577, + "mean_fro": 26.210724, + "rms_per_prompt_fro": 29.845448, + "max_per_prompt_fro": 34.060944, + "heavy_tail_ratio": 1.1412, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.256577, + "mean_fro": 29.239235, + "rms_per_prompt_fro": 32.776344, + "max_per_prompt_fro": 36.90963, + "heavy_tail_ratio": 1.1261, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.18722, + "mean_fro": 32.319992, + "rms_per_prompt_fro": 35.215745, + "max_per_prompt_fro": 39.270016, + "heavy_tail_ratio": 1.1151, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.146961, + "mean_fro": 34.469421, + "rms_per_prompt_fro": 36.915461, + "max_per_prompt_fro": 39.907074, + "heavy_tail_ratio": 1.081, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.113676, + "mean_fro": 35.792503, + "rms_per_prompt_fro": 37.772129, + "max_per_prompt_fro": 40.455715, + "heavy_tail_ratio": 1.071, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.07958, + "mean_fro": 35.503937, + "rms_per_prompt_fro": 36.889594, + "max_per_prompt_fro": 38.525658, + "heavy_tail_ratio": 1.0444, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.047971, + "mean_fro": 35.542141, + "rms_per_prompt_fro": 36.384648, + "max_per_prompt_fro": 37.682678, + "heavy_tail_ratio": 1.0357, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.023536, + "mean_fro": 34.863689, + "rms_per_prompt_fro": 35.271582, + "max_per_prompt_fro": 36.347958, + "heavy_tail_ratio": 1.0305, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 308.0, + "fit_secs_per_prompt": 3.08, + "total_wall_seconds": 315.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.5382, + "top_sv": 2.0703, + "stable_rank": 21.226 + }, + { + "layer": 1, + "fro_norm": 8.8084, + "top_sv": 1.8868, + "stable_rank": 21.795 + }, + { + "layer": 2, + "fro_norm": 9.5133, + "top_sv": 2.0795, + "stable_rank": 20.929 + }, + { + "layer": 3, + "fro_norm": 9.0208, + "top_sv": 2.002, + "stable_rank": 20.303 + }, + { + "layer": 4, + "fro_norm": 10.1349, + "top_sv": 2.2126, + "stable_rank": 20.982 + }, + { + "layer": 5, + "fro_norm": 10.2124, + "top_sv": 2.3622, + "stable_rank": 18.691 + }, + { + "layer": 6, + "fro_norm": 10.5131, + "top_sv": 2.2087, + "stable_rank": 22.657 + }, + { + "layer": 7, + "fro_norm": 11.5272, + "top_sv": 2.4753, + "stable_rank": 21.687 + }, + { + "layer": 8, + "fro_norm": 13.1366, + "top_sv": 2.9414, + "stable_rank": 19.946 + }, + { + "layer": 9, + "fro_norm": 14.6522, + "top_sv": 3.3241, + "stable_rank": 19.429 + }, + { + "layer": 10, + "fro_norm": 16.4575, + "top_sv": 3.8124, + "stable_rank": 18.635 + }, + { + "layer": 11, + "fro_norm": 19.085, + "top_sv": 4.4405, + "stable_rank": 18.473 + }, + { + "layer": 12, + "fro_norm": 20.8633, + "top_sv": 4.6773, + "stable_rank": 19.897 + }, + { + "layer": 13, + "fro_norm": 22.7914, + "top_sv": 5.1506, + "stable_rank": 19.581 + }, + { + "layer": 14, + "fro_norm": 26.2103, + "top_sv": 5.8725, + "stable_rank": 19.92 + }, + { + "layer": 15, + "fro_norm": 29.2387, + "top_sv": 6.7013, + "stable_rank": 19.037 + }, + { + "layer": 16, + "fro_norm": 32.3194, + "top_sv": 7.5743, + "stable_rank": 18.207 + }, + { + "layer": 17, + "fro_norm": 34.4686, + "top_sv": 8.0813, + "stable_rank": 18.192 + }, + { + "layer": 18, + "fro_norm": 35.7916, + "top_sv": 8.331, + "stable_rank": 18.457 + }, + { + "layer": 19, + "fro_norm": 35.5029, + "top_sv": 7.9852, + "stable_rank": 19.768 + }, + { + "layer": 20, + "fro_norm": 35.5409, + "top_sv": 7.4216, + "stable_rank": 22.933 + }, + { + "layer": 21, + "fro_norm": 34.8622, + "top_sv": 5.7711, + "stable_rank": 36.491 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..e76fa854915162d0b24a7088cdb8b04435fe5c91 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4826bb37268b663e21b44e65ff7b6cfda452612034020814245a1cc806007d44 +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..5dfeaf0142fa0582f515c1cd04c35eac4ff91d2d --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n100_s1.pt", + "sha256": "4826bb37268b663e21b44e65ff7b6cfda452612034020814245a1cc806007d44", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.610767, + "mean_fro": 9.782696, + "rms_per_prompt_fro": 15.806751, + "max_per_prompt_fro": 21.750072, + "heavy_tail_ratio": 1.376, + "n_prompts": 100 + }, + "1": { + "dispersion": 1.429976, + "mean_fro": 9.036693, + "rms_per_prompt_fro": 14.086741, + "max_per_prompt_fro": 19.534609, + "heavy_tail_ratio": 1.3867, + "n_prompts": 100 + }, + "2": { + "dispersion": 1.167786, + "mean_fro": 9.779174, + "rms_per_prompt_fro": 14.398273, + "max_per_prompt_fro": 18.922707, + "heavy_tail_ratio": 1.3142, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.034908, + "mean_fro": 9.24154, + "rms_per_prompt_fro": 13.183074, + "max_per_prompt_fro": 18.290125, + "heavy_tail_ratio": 1.3874, + "n_prompts": 100 + }, + "4": { + "dispersion": 0.852908, + "mean_fro": 10.379379, + "rms_per_prompt_fro": 14.128572, + "max_per_prompt_fro": 19.738934, + "heavy_tail_ratio": 1.3971, + "n_prompts": 100 + }, + "5": { + "dispersion": 0.778811, + "mean_fro": 10.470999, + "rms_per_prompt_fro": 13.965389, + "max_per_prompt_fro": 19.655035, + "heavy_tail_ratio": 1.4074, + "n_prompts": 100 + }, + "6": { + "dispersion": 0.695914, + "mean_fro": 10.767634, + "rms_per_prompt_fro": 14.022395, + "max_per_prompt_fro": 17.305576, + "heavy_tail_ratio": 1.2341, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.616709, + "mean_fro": 11.757374, + "rms_per_prompt_fro": 14.949486, + "max_per_prompt_fro": 18.091373, + "heavy_tail_ratio": 1.2102, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.564174, + "mean_fro": 13.403694, + "rms_per_prompt_fro": 16.76359, + "max_per_prompt_fro": 19.955797, + "heavy_tail_ratio": 1.1904, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.522974, + "mean_fro": 14.951101, + "rms_per_prompt_fro": 18.450977, + "max_per_prompt_fro": 21.773565, + "heavy_tail_ratio": 1.1801, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.479616, + "mean_fro": 16.796364, + "rms_per_prompt_fro": 20.431009, + "max_per_prompt_fro": 23.691769, + "heavy_tail_ratio": 1.1596, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.435815, + "mean_fro": 19.468886, + "rms_per_prompt_fro": 23.328689, + "max_per_prompt_fro": 26.592684, + "heavy_tail_ratio": 1.1399, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.354374, + "mean_fro": 21.118038, + "rms_per_prompt_fro": 24.57666, + "max_per_prompt_fro": 28.123104, + "heavy_tail_ratio": 1.1443, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.316371, + "mean_fro": 23.063605, + "rms_per_prompt_fro": 26.461619, + "max_per_prompt_fro": 29.974928, + "heavy_tail_ratio": 1.1328, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.273277, + "mean_fro": 26.471893, + "rms_per_prompt_fro": 29.870774, + "max_per_prompt_fro": 33.409733, + "heavy_tail_ratio": 1.1185, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.236857, + "mean_fro": 29.492178, + "rms_per_prompt_fro": 32.799459, + "max_per_prompt_fro": 36.172672, + "heavy_tail_ratio": 1.1028, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.171612, + "mean_fro": 32.45156, + "rms_per_prompt_fro": 35.125902, + "max_per_prompt_fro": 38.707355, + "heavy_tail_ratio": 1.102, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.13522, + "mean_fro": 34.546269, + "rms_per_prompt_fro": 36.807911, + "max_per_prompt_fro": 40.091644, + "heavy_tail_ratio": 1.0892, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.105178, + "mean_fro": 35.852382, + "rms_per_prompt_fro": 37.690691, + "max_per_prompt_fro": 40.512966, + "heavy_tail_ratio": 1.0749, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.073448, + "mean_fro": 35.546319, + "rms_per_prompt_fro": 36.828587, + "max_per_prompt_fro": 38.84742, + "heavy_tail_ratio": 1.0548, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.044254, + "mean_fro": 35.566718, + "rms_per_prompt_fro": 36.345179, + "max_per_prompt_fro": 37.550762, + "heavy_tail_ratio": 1.0332, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.021691, + "mean_fro": 34.869628, + "rms_per_prompt_fro": 35.245772, + "max_per_prompt_fro": 36.007542, + "heavy_tail_ratio": 1.0216, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 312.0, + "fit_secs_per_prompt": 3.12, + "total_wall_seconds": 323.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.7826, + "top_sv": 2.2221, + "stable_rank": 19.382 + }, + { + "layer": 1, + "fro_norm": 9.0366, + "top_sv": 1.9773, + "stable_rank": 20.886 + }, + { + "layer": 2, + "fro_norm": 9.779, + "top_sv": 2.2156, + "stable_rank": 19.481 + }, + { + "layer": 3, + "fro_norm": 9.2414, + "top_sv": 2.0583, + "stable_rank": 20.159 + }, + { + "layer": 4, + "fro_norm": 10.3793, + "top_sv": 2.3063, + "stable_rank": 20.254 + }, + { + "layer": 5, + "fro_norm": 10.4709, + "top_sv": 2.4529, + "stable_rank": 18.223 + }, + { + "layer": 6, + "fro_norm": 10.7675, + "top_sv": 2.3181, + "stable_rank": 21.576 + }, + { + "layer": 7, + "fro_norm": 11.7572, + "top_sv": 2.5299, + "stable_rank": 21.597 + }, + { + "layer": 8, + "fro_norm": 13.4035, + "top_sv": 3.0134, + "stable_rank": 19.784 + }, + { + "layer": 9, + "fro_norm": 14.9509, + "top_sv": 3.4116, + "stable_rank": 19.205 + }, + { + "layer": 10, + "fro_norm": 16.7961, + "top_sv": 3.901, + "stable_rank": 18.538 + }, + { + "layer": 11, + "fro_norm": 19.4686, + "top_sv": 4.5476, + "stable_rank": 18.328 + }, + { + "layer": 12, + "fro_norm": 21.1178, + "top_sv": 4.7195, + "stable_rank": 20.022 + }, + { + "layer": 13, + "fro_norm": 23.0632, + "top_sv": 5.2161, + "stable_rank": 19.55 + }, + { + "layer": 14, + "fro_norm": 26.4715, + "top_sv": 5.9394, + "stable_rank": 19.864 + }, + { + "layer": 15, + "fro_norm": 29.4917, + "top_sv": 6.7623, + "stable_rank": 19.02 + }, + { + "layer": 16, + "fro_norm": 32.451, + "top_sv": 7.6232, + "stable_rank": 18.121 + }, + { + "layer": 17, + "fro_norm": 34.5455, + "top_sv": 8.126, + "stable_rank": 18.073 + }, + { + "layer": 18, + "fro_norm": 35.8515, + "top_sv": 8.3415, + "stable_rank": 18.473 + }, + { + "layer": 19, + "fro_norm": 35.5452, + "top_sv": 8.0021, + "stable_rank": 19.731 + }, + { + "layer": 20, + "fro_norm": 35.5655, + "top_sv": 7.4196, + "stable_rank": 22.977 + }, + { + "layer": 21, + "fro_norm": 34.8681, + "top_sv": 5.7693, + "stable_rank": 36.527 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fe7b873c008b88722060a237626d2ef3bbd3652 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8142513be6cf969377e2b9671f7a23a1130fea92bcfcb13c0c3a2fcdca6ece83 +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..5b845392390db1ba4968985089da8479819cf708 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n100_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n100_s2.pt", + "sha256": "8142513be6cf969377e2b9671f7a23a1130fea92bcfcb13c0c3a2fcdca6ece83", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.598143, + "mean_fro": 9.839368, + "rms_per_prompt_fro": 15.859836, + "max_per_prompt_fro": 21.137394, + "heavy_tail_ratio": 1.3328, + "n_prompts": 100 + }, + "1": { + "dispersion": 1.419517, + "mean_fro": 9.067164, + "rms_per_prompt_fro": 14.103789, + "max_per_prompt_fro": 18.507341, + "heavy_tail_ratio": 1.3122, + "n_prompts": 100 + }, + "2": { + "dispersion": 1.167151, + "mean_fro": 9.780684, + "rms_per_prompt_fro": 14.398387, + "max_per_prompt_fro": 18.640663, + "heavy_tail_ratio": 1.2946, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.037051, + "mean_fro": 9.266561, + "rms_per_prompt_fro": 13.225726, + "max_per_prompt_fro": 16.709667, + "heavy_tail_ratio": 1.2634, + "n_prompts": 100 + }, + "4": { + "dispersion": 0.857889, + "mean_fro": 10.414429, + "rms_per_prompt_fro": 14.195323, + "max_per_prompt_fro": 17.622261, + "heavy_tail_ratio": 1.2414, + "n_prompts": 100 + }, + "5": { + "dispersion": 0.78371, + "mean_fro": 10.507812, + "rms_per_prompt_fro": 14.033774, + "max_per_prompt_fro": 17.23596, + "heavy_tail_ratio": 1.2282, + "n_prompts": 100 + }, + "6": { + "dispersion": 0.710251, + "mean_fro": 10.820804, + "rms_per_prompt_fro": 14.151075, + "max_per_prompt_fro": 17.290504, + "heavy_tail_ratio": 1.2219, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.630398, + "mean_fro": 11.826817, + "rms_per_prompt_fro": 15.101313, + "max_per_prompt_fro": 18.175869, + "heavy_tail_ratio": 1.2036, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.577571, + "mean_fro": 13.47599, + "rms_per_prompt_fro": 16.92603, + "max_per_prompt_fro": 20.239119, + "heavy_tail_ratio": 1.1957, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.535781, + "mean_fro": 15.0378, + "rms_per_prompt_fro": 18.635836, + "max_per_prompt_fro": 22.413094, + "heavy_tail_ratio": 1.2027, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.492617, + "mean_fro": 16.876562, + "rms_per_prompt_fro": 20.618556, + "max_per_prompt_fro": 24.555136, + "heavy_tail_ratio": 1.1909, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.447826, + "mean_fro": 19.576206, + "rms_per_prompt_fro": 23.555197, + "max_per_prompt_fro": 27.345827, + "heavy_tail_ratio": 1.1609, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.364687, + "mean_fro": 21.268606, + "rms_per_prompt_fro": 24.845947, + "max_per_prompt_fro": 28.499649, + "heavy_tail_ratio": 1.1471, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.325926, + "mean_fro": 23.217093, + "rms_per_prompt_fro": 26.734218, + "max_per_prompt_fro": 30.125408, + "heavy_tail_ratio": 1.1268, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.280429, + "mean_fro": 26.628222, + "rms_per_prompt_fro": 30.131446, + "max_per_prompt_fro": 33.699787, + "heavy_tail_ratio": 1.1184, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.242703, + "mean_fro": 29.65414, + "rms_per_prompt_fro": 33.057422, + "max_per_prompt_fro": 36.839882, + "heavy_tail_ratio": 1.1144, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.176772, + "mean_fro": 32.731662, + "rms_per_prompt_fro": 35.507024, + "max_per_prompt_fro": 39.002068, + "heavy_tail_ratio": 1.0984, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.139171, + "mean_fro": 34.81213, + "rms_per_prompt_fro": 37.15567, + "max_per_prompt_fro": 39.963058, + "heavy_tail_ratio": 1.0756, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.107511, + "mean_fro": 36.073665, + "rms_per_prompt_fro": 37.963333, + "max_per_prompt_fro": 40.203117, + "heavy_tail_ratio": 1.059, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.075666, + "mean_fro": 35.686561, + "rms_per_prompt_fro": 37.012081, + "max_per_prompt_fro": 38.339561, + "heavy_tail_ratio": 1.0359, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.045556, + "mean_fro": 35.64492, + "rms_per_prompt_fro": 36.447794, + "max_per_prompt_fro": 37.516258, + "heavy_tail_ratio": 1.0293, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.022629, + "mean_fro": 34.918533, + "rms_per_prompt_fro": 35.311411, + "max_per_prompt_fro": 36.064835, + "heavy_tail_ratio": 1.0213, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 304.9, + "fit_secs_per_prompt": 3.049, + "total_wall_seconds": 314.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.8392, + "top_sv": 2.2027, + "stable_rank": 19.953 + }, + { + "layer": 1, + "fro_norm": 9.067, + "top_sv": 1.977, + "stable_rank": 21.034 + }, + { + "layer": 2, + "fro_norm": 9.7805, + "top_sv": 2.1853, + "stable_rank": 20.031 + }, + { + "layer": 3, + "fro_norm": 9.2664, + "top_sv": 2.0724, + "stable_rank": 19.993 + }, + { + "layer": 4, + "fro_norm": 10.4143, + "top_sv": 2.3184, + "stable_rank": 20.178 + }, + { + "layer": 5, + "fro_norm": 10.5077, + "top_sv": 2.4678, + "stable_rank": 18.13 + }, + { + "layer": 6, + "fro_norm": 10.8207, + "top_sv": 2.3245, + "stable_rank": 21.669 + }, + { + "layer": 7, + "fro_norm": 11.8267, + "top_sv": 2.5677, + "stable_rank": 21.214 + }, + { + "layer": 8, + "fro_norm": 13.4758, + "top_sv": 3.0404, + "stable_rank": 19.645 + }, + { + "layer": 9, + "fro_norm": 15.0376, + "top_sv": 3.4436, + "stable_rank": 19.069 + }, + { + "layer": 10, + "fro_norm": 16.8763, + "top_sv": 3.9564, + "stable_rank": 18.196 + }, + { + "layer": 11, + "fro_norm": 19.5759, + "top_sv": 4.6021, + "stable_rank": 18.094 + }, + { + "layer": 12, + "fro_norm": 21.2683, + "top_sv": 4.7914, + "stable_rank": 19.703 + }, + { + "layer": 13, + "fro_norm": 23.2168, + "top_sv": 5.2715, + "stable_rank": 19.397 + }, + { + "layer": 14, + "fro_norm": 26.6278, + "top_sv": 5.9924, + "stable_rank": 19.746 + }, + { + "layer": 15, + "fro_norm": 29.6536, + "top_sv": 6.8033, + "stable_rank": 18.999 + }, + { + "layer": 16, + "fro_norm": 32.731, + "top_sv": 7.7271, + "stable_rank": 17.943 + }, + { + "layer": 17, + "fro_norm": 34.8114, + "top_sv": 8.2209, + "stable_rank": 17.931 + }, + { + "layer": 18, + "fro_norm": 36.0728, + "top_sv": 8.4314, + "stable_rank": 18.305 + }, + { + "layer": 19, + "fro_norm": 35.6855, + "top_sv": 8.0737, + "stable_rank": 19.536 + }, + { + "layer": 20, + "fro_norm": 35.6437, + "top_sv": 7.4539, + "stable_rank": 22.867 + }, + { + "layer": 21, + "fro_norm": 34.917, + "top_sv": 5.8038, + "stable_rank": 36.195 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4f27dca45365cc777e5da7d9d7b2dd54993f06e --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f8f06ad3ce7ef6c3a0dc44afa63838df505ac8cd1617b25fda0aa9747a31524 +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..26f875abaeb148089a05ebbe4d832b30e4f059ca --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "6f8f06ad3ce7ef6c3a0dc44afa63838df505ac8cd1617b25fda0aa9747a31524", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.687575, + "mean_fro": 9.546088, + "rms_per_prompt_fro": 15.649691, + "max_per_prompt_fro": 27.007643, + "heavy_tail_ratio": 1.7258, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.500843, + "mean_fro": 8.822303, + "rms_per_prompt_fro": 13.951638, + "max_per_prompt_fro": 22.984131, + "heavy_tail_ratio": 1.6474, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.244424, + "mean_fro": 9.519076, + "rms_per_prompt_fro": 14.26091, + "max_per_prompt_fro": 23.135078, + "heavy_tail_ratio": 1.6223, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.107224, + "mean_fro": 9.032018, + "rms_per_prompt_fro": 13.111131, + "max_per_prompt_fro": 20.889572, + "heavy_tail_ratio": 1.5933, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.924452, + "mean_fro": 10.145487, + "rms_per_prompt_fro": 14.074288, + "max_per_prompt_fro": 23.325769, + "heavy_tail_ratio": 1.6573, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.841388, + "mean_fro": 10.227048, + "rms_per_prompt_fro": 13.877874, + "max_per_prompt_fro": 21.056416, + "heavy_tail_ratio": 1.5173, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.752951, + "mean_fro": 10.528097, + "rms_per_prompt_fro": 13.9391, + "max_per_prompt_fro": 20.742741, + "heavy_tail_ratio": 1.4881, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.670298, + "mean_fro": 11.539109, + "rms_per_prompt_fro": 14.913148, + "max_per_prompt_fro": 21.689234, + "heavy_tail_ratio": 1.4544, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.613304, + "mean_fro": 13.147351, + "rms_per_prompt_fro": 16.69923, + "max_per_prompt_fro": 23.220459, + "heavy_tail_ratio": 1.3905, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.567808, + "mean_fro": 14.669437, + "rms_per_prompt_fro": 18.367918, + "max_per_prompt_fro": 24.576418, + "heavy_tail_ratio": 1.338, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.520673, + "mean_fro": 16.475122, + "rms_per_prompt_fro": 20.316388, + "max_per_prompt_fro": 25.775553, + "heavy_tail_ratio": 1.2687, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.473447, + "mean_fro": 19.103962, + "rms_per_prompt_fro": 23.189468, + "max_per_prompt_fro": 27.704704, + "heavy_tail_ratio": 1.1947, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.388199, + "mean_fro": 20.904106, + "rms_per_prompt_fro": 24.62961, + "max_per_prompt_fro": 29.31465, + "heavy_tail_ratio": 1.1902, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.348074, + "mean_fro": 22.830717, + "rms_per_prompt_fro": 26.507967, + "max_per_prompt_fro": 30.741051, + "heavy_tail_ratio": 1.1597, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.300236, + "mean_fro": 26.228956, + "rms_per_prompt_fro": 29.908323, + "max_per_prompt_fro": 34.060944, + "heavy_tail_ratio": 1.1388, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.260629, + "mean_fro": 29.255806, + "rms_per_prompt_fro": 32.847757, + "max_per_prompt_fro": 36.90963, + "heavy_tail_ratio": 1.1237, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.190816, + "mean_fro": 32.330802, + "rms_per_prompt_fro": 35.280825, + "max_per_prompt_fro": 39.270016, + "heavy_tail_ratio": 1.1131, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.15024, + "mean_fro": 34.465207, + "rms_per_prompt_fro": 36.963681, + "max_per_prompt_fro": 39.907074, + "heavy_tail_ratio": 1.0796, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.116732, + "mean_fro": 35.789375, + "rms_per_prompt_fro": 37.82061, + "max_per_prompt_fro": 40.455715, + "heavy_tail_ratio": 1.0697, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.081921, + "mean_fro": 35.508552, + "rms_per_prompt_fro": 36.934377, + "max_per_prompt_fro": 38.525658, + "heavy_tail_ratio": 1.0431, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.049447, + "mean_fro": 35.555298, + "rms_per_prompt_fro": 36.423749, + "max_per_prompt_fro": 37.682678, + "heavy_tail_ratio": 1.0346, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.024281, + "mean_fro": 34.870449, + "rms_per_prompt_fro": 35.291257, + "max_per_prompt_fro": 36.347958, + "heavy_tail_ratio": 1.0299, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 616.9, + "fit_secs_per_prompt": 3.084, + "total_wall_seconds": 624.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.546, + "top_sv": 2.0681, + "stable_rank": 21.306 + }, + { + "layer": 1, + "fro_norm": 8.8222, + "top_sv": 1.8898, + "stable_rank": 21.793 + }, + { + "layer": 2, + "fro_norm": 9.519, + "top_sv": 2.0812, + "stable_rank": 20.919 + }, + { + "layer": 3, + "fro_norm": 9.0319, + "top_sv": 2.0099, + "stable_rank": 20.194 + }, + { + "layer": 4, + "fro_norm": 10.1454, + "top_sv": 2.2222, + "stable_rank": 20.844 + }, + { + "layer": 5, + "fro_norm": 10.2269, + "top_sv": 2.374, + "stable_rank": 18.557 + }, + { + "layer": 6, + "fro_norm": 10.528, + "top_sv": 2.2084, + "stable_rank": 22.728 + }, + { + "layer": 7, + "fro_norm": 11.539, + "top_sv": 2.4832, + "stable_rank": 21.592 + }, + { + "layer": 8, + "fro_norm": 13.1472, + "top_sv": 2.9464, + "stable_rank": 19.911 + }, + { + "layer": 9, + "fro_norm": 14.6692, + "top_sv": 3.3338, + "stable_rank": 19.362 + }, + { + "layer": 10, + "fro_norm": 16.4749, + "top_sv": 3.8178, + "stable_rank": 18.622 + }, + { + "layer": 11, + "fro_norm": 19.1037, + "top_sv": 4.4463, + "stable_rank": 18.461 + }, + { + "layer": 12, + "fro_norm": 20.9038, + "top_sv": 4.69, + "stable_rank": 19.866 + }, + { + "layer": 13, + "fro_norm": 22.8304, + "top_sv": 5.1613, + "stable_rank": 19.566 + }, + { + "layer": 14, + "fro_norm": 26.2285, + "top_sv": 5.8773, + "stable_rank": 19.915 + }, + { + "layer": 15, + "fro_norm": 29.2553, + "top_sv": 6.7098, + "stable_rank": 19.01 + }, + { + "layer": 16, + "fro_norm": 32.3302, + "top_sv": 7.5749, + "stable_rank": 18.216 + }, + { + "layer": 17, + "fro_norm": 34.4645, + "top_sv": 8.0779, + "stable_rank": 18.203 + }, + { + "layer": 18, + "fro_norm": 35.7885, + "top_sv": 8.3135, + "stable_rank": 18.532 + }, + { + "layer": 19, + "fro_norm": 35.5075, + "top_sv": 7.9752, + "stable_rank": 19.822 + }, + { + "layer": 20, + "fro_norm": 35.554, + "top_sv": 7.4164, + "stable_rank": 22.982 + }, + { + "layer": 21, + "fro_norm": 34.8689, + "top_sv": 5.77, + "stable_rank": 36.52 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f028bbd9c38ae8bef2db88b1b3f2be5bdfd7692 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a2a3f4524ac4db804501e412d4ebc188935058687f1a468e2b5bfe0cdfbbfcd +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c3509ce74774745f0e32c9cdb8eb208f3a4f2410 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "2a2a3f4524ac4db804501e412d4ebc188935058687f1a468e2b5bfe0cdfbbfcd", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.695329, + "mean_fro": 9.586534, + "rms_per_prompt_fro": 15.738651, + "max_per_prompt_fro": 21.750072, + "heavy_tail_ratio": 1.382, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.503615, + "mean_fro": 8.871353, + "rms_per_prompt_fro": 14.036979, + "max_per_prompt_fro": 19.534609, + "heavy_tail_ratio": 1.3917, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.229101, + "mean_fro": 9.605462, + "rms_per_prompt_fro": 14.341123, + "max_per_prompt_fro": 19.733892, + "heavy_tail_ratio": 1.376, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.087694, + "mean_fro": 9.094661, + "rms_per_prompt_fro": 13.140742, + "max_per_prompt_fro": 18.290125, + "heavy_tail_ratio": 1.3919, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.899555, + "mean_fro": 10.224495, + "rms_per_prompt_fro": 14.091844, + "max_per_prompt_fro": 19.738934, + "heavy_tail_ratio": 1.4007, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.822064, + "mean_fro": 10.319009, + "rms_per_prompt_fro": 13.928995, + "max_per_prompt_fro": 19.655035, + "heavy_tail_ratio": 1.4111, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.737482, + "mean_fro": 10.61828, + "rms_per_prompt_fro": 13.996336, + "max_per_prompt_fro": 19.287895, + "heavy_tail_ratio": 1.3781, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.651368, + "mean_fro": 11.608234, + "rms_per_prompt_fro": 14.917225, + "max_per_prompt_fro": 20.31992, + "heavy_tail_ratio": 1.3622, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.596165, + "mean_fro": 13.252058, + "rms_per_prompt_fro": 16.742571, + "max_per_prompt_fro": 22.782795, + "heavy_tail_ratio": 1.3608, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.551545, + "mean_fro": 14.800219, + "rms_per_prompt_fro": 18.435308, + "max_per_prompt_fro": 24.730282, + "heavy_tail_ratio": 1.3415, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.504794, + "mean_fro": 16.640455, + "rms_per_prompt_fro": 20.412851, + "max_per_prompt_fro": 26.740112, + "heavy_tail_ratio": 1.31, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.458572, + "mean_fro": 19.307318, + "rms_per_prompt_fro": 23.317706, + "max_per_prompt_fro": 29.689667, + "heavy_tail_ratio": 1.2733, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.371101, + "mean_fro": 20.974508, + "rms_per_prompt_fro": 24.559898, + "max_per_prompt_fro": 30.227711, + "heavy_tail_ratio": 1.2308, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.330737, + "mean_fro": 22.917713, + "rms_per_prompt_fro": 26.437321, + "max_per_prompt_fro": 32.066608, + "heavy_tail_ratio": 1.2129, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.28437, + "mean_fro": 26.337292, + "rms_per_prompt_fro": 29.848063, + "max_per_prompt_fro": 35.79073, + "heavy_tail_ratio": 1.1991, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.245919, + "mean_fro": 29.37314, + "rms_per_prompt_fro": 32.786522, + "max_per_prompt_fro": 38.912979, + "heavy_tail_ratio": 1.1869, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.178064, + "mean_fro": 32.396619, + "rms_per_prompt_fro": 35.162851, + "max_per_prompt_fro": 40.017754, + "heavy_tail_ratio": 1.1381, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.139579, + "mean_fro": 34.507774, + "rms_per_prompt_fro": 36.837425, + "max_per_prompt_fro": 40.112587, + "heavy_tail_ratio": 1.0889, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.108215, + "mean_fro": 35.827507, + "rms_per_prompt_fro": 37.71625, + "max_per_prompt_fro": 40.691536, + "heavy_tail_ratio": 1.0789, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.075499, + "mean_fro": 35.540889, + "rms_per_prompt_fro": 36.858138, + "max_per_prompt_fro": 38.84742, + "heavy_tail_ratio": 1.054, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.045281, + "mean_fro": 35.552635, + "rms_per_prompt_fro": 36.348645, + "max_per_prompt_fro": 37.574535, + "heavy_tail_ratio": 1.0337, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.022343, + "mean_fro": 34.864914, + "rms_per_prompt_fro": 35.252254, + "max_per_prompt_fro": 36.007542, + "heavy_tail_ratio": 1.0214, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 623.3, + "fit_secs_per_prompt": 3.117, + "total_wall_seconds": 633.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.5864, + "top_sv": 2.1371, + "stable_rank": 20.121 + }, + { + "layer": 1, + "fro_norm": 8.8712, + "top_sv": 1.9287, + "stable_rank": 21.156 + }, + { + "layer": 2, + "fro_norm": 9.6053, + "top_sv": 2.1506, + "stable_rank": 19.949 + }, + { + "layer": 3, + "fro_norm": 9.0946, + "top_sv": 2.0339, + "stable_rank": 19.995 + }, + { + "layer": 4, + "fro_norm": 10.2244, + "top_sv": 2.2608, + "stable_rank": 20.453 + }, + { + "layer": 5, + "fro_norm": 10.3189, + "top_sv": 2.4135, + "stable_rank": 18.28 + }, + { + "layer": 6, + "fro_norm": 10.6182, + "top_sv": 2.2694, + "stable_rank": 21.892 + }, + { + "layer": 7, + "fro_norm": 11.6081, + "top_sv": 2.4962, + "stable_rank": 21.626 + }, + { + "layer": 8, + "fro_norm": 13.2519, + "top_sv": 2.9703, + "stable_rank": 19.905 + }, + { + "layer": 9, + "fro_norm": 14.8, + "top_sv": 3.368, + "stable_rank": 19.31 + }, + { + "layer": 10, + "fro_norm": 16.6402, + "top_sv": 3.8589, + "stable_rank": 18.595 + }, + { + "layer": 11, + "fro_norm": 19.307, + "top_sv": 4.4985, + "stable_rank": 18.42 + }, + { + "layer": 12, + "fro_norm": 20.9742, + "top_sv": 4.6879, + "stable_rank": 20.017 + }, + { + "layer": 13, + "fro_norm": 22.9174, + "top_sv": 5.1739, + "stable_rank": 19.62 + }, + { + "layer": 14, + "fro_norm": 26.3368, + "top_sv": 5.8989, + "stable_rank": 19.934 + }, + { + "layer": 15, + "fro_norm": 29.3726, + "top_sv": 6.7298, + "stable_rank": 19.049 + }, + { + "layer": 16, + "fro_norm": 32.396, + "top_sv": 7.6138, + "stable_rank": 18.104 + }, + { + "layer": 17, + "fro_norm": 34.5069, + "top_sv": 8.1182, + "stable_rank": 18.067 + }, + { + "layer": 18, + "fro_norm": 35.8266, + "top_sv": 8.3389, + "stable_rank": 18.459 + }, + { + "layer": 19, + "fro_norm": 35.5398, + "top_sv": 8.0164, + "stable_rank": 19.655 + }, + { + "layer": 20, + "fro_norm": 35.5514, + "top_sv": 7.4097, + "stable_rank": 23.021 + }, + { + "layer": 21, + "fro_norm": 34.8634, + "top_sv": 5.7636, + "stable_rank": 36.589 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcb9eb59785597ea235b83fb90e795e705070de6 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e5dfd9f19145467500b542c74c1103bd90a44d430e4e3528aca22cdb589922 +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9ece7fd019e7d5ea0126a20109903ae56f232d5d --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n200_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "a1e5dfd9f19145467500b542c74c1103bd90a44d430e4e3528aca22cdb589922", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.592598, + "mean_fro": 10.012433, + "rms_per_prompt_fro": 16.121567, + "max_per_prompt_fro": 21.654562, + "heavy_tail_ratio": 1.3432, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.415301, + "mean_fro": 9.235359, + "rms_per_prompt_fro": 14.352891, + "max_per_prompt_fro": 18.9706, + "heavy_tail_ratio": 1.3217, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.160429, + "mean_fro": 9.949005, + "rms_per_prompt_fro": 14.623445, + "max_per_prompt_fro": 19.53392, + "heavy_tail_ratio": 1.3358, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.032584, + "mean_fro": 9.418343, + "rms_per_prompt_fro": 13.427612, + "max_per_prompt_fro": 17.337177, + "heavy_tail_ratio": 1.2912, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.856445, + "mean_fro": 10.551456, + "rms_per_prompt_fro": 14.376509, + "max_per_prompt_fro": 18.679291, + "heavy_tail_ratio": 1.2993, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.784229, + "mean_fro": 10.635675, + "rms_per_prompt_fro": 14.206606, + "max_per_prompt_fro": 18.57575, + "heavy_tail_ratio": 1.3075, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.705468, + "mean_fro": 10.930763, + "rms_per_prompt_fro": 14.274874, + "max_per_prompt_fro": 18.039284, + "heavy_tail_ratio": 1.2637, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.626076, + "mean_fro": 11.936239, + "rms_per_prompt_fro": 15.220816, + "max_per_prompt_fro": 18.870104, + "heavy_tail_ratio": 1.2398, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.573061, + "mean_fro": 13.574806, + "rms_per_prompt_fro": 17.025755, + "max_per_prompt_fro": 21.238649, + "heavy_tail_ratio": 1.2474, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.53237, + "mean_fro": 15.119249, + "rms_per_prompt_fro": 18.71596, + "max_per_prompt_fro": 23.496769, + "heavy_tail_ratio": 1.2554, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.489207, + "mean_fro": 16.939096, + "rms_per_prompt_fro": 20.671303, + "max_per_prompt_fro": 25.47921, + "heavy_tail_ratio": 1.2326, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.44491, + "mean_fro": 19.607754, + "rms_per_prompt_fro": 23.569387, + "max_per_prompt_fro": 28.637785, + "heavy_tail_ratio": 1.215, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.362318, + "mean_fro": 21.295542, + "rms_per_prompt_fro": 24.85581, + "max_per_prompt_fro": 28.499649, + "heavy_tail_ratio": 1.1466, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.323907, + "mean_fro": 23.244041, + "rms_per_prompt_fro": 26.74486, + "max_per_prompt_fro": 30.125408, + "heavy_tail_ratio": 1.1264, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.27906, + "mean_fro": 26.667693, + "rms_per_prompt_fro": 30.159965, + "max_per_prompt_fro": 33.699787, + "heavy_tail_ratio": 1.1174, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.241442, + "mean_fro": 29.698103, + "rms_per_prompt_fro": 33.089624, + "max_per_prompt_fro": 36.839882, + "heavy_tail_ratio": 1.1133, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.176285, + "mean_fro": 32.721878, + "rms_per_prompt_fro": 35.489064, + "max_per_prompt_fro": 39.002068, + "heavy_tail_ratio": 1.099, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.138985, + "mean_fro": 34.804241, + "rms_per_prompt_fro": 37.144206, + "max_per_prompt_fro": 39.963058, + "heavy_tail_ratio": 1.0759, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.107884, + "mean_fro": 36.073486, + "rms_per_prompt_fro": 37.969538, + "max_per_prompt_fro": 40.415257, + "heavy_tail_ratio": 1.0644, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.076035, + "mean_fro": 35.687397, + "rms_per_prompt_fro": 37.019288, + "max_per_prompt_fro": 38.691559, + "heavy_tail_ratio": 1.0452, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.045725, + "mean_fro": 35.665817, + "rms_per_prompt_fro": 36.472117, + "max_per_prompt_fro": 37.801369, + "heavy_tail_ratio": 1.0364, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.022664, + "mean_fro": 34.93251, + "rms_per_prompt_fro": 35.326145, + "max_per_prompt_fro": 36.17366, + "heavy_tail_ratio": 1.024, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 609.3, + "fit_secs_per_prompt": 3.047, + "total_wall_seconds": 618.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 10.0123, + "top_sv": 2.2435, + "stable_rank": 19.917 + }, + { + "layer": 1, + "fro_norm": 9.2352, + "top_sv": 2.0216, + "stable_rank": 20.87 + }, + { + "layer": 2, + "fro_norm": 9.9489, + "top_sv": 2.2261, + "stable_rank": 19.974 + }, + { + "layer": 3, + "fro_norm": 9.4182, + "top_sv": 2.1185, + "stable_rank": 19.764 + }, + { + "layer": 4, + "fro_norm": 10.5513, + "top_sv": 2.3563, + "stable_rank": 20.053 + }, + { + "layer": 5, + "fro_norm": 10.6356, + "top_sv": 2.5163, + "stable_rank": 17.865 + }, + { + "layer": 6, + "fro_norm": 10.9306, + "top_sv": 2.3463, + "stable_rank": 21.703 + }, + { + "layer": 7, + "fro_norm": 11.9361, + "top_sv": 2.5918, + "stable_rank": 21.209 + }, + { + "layer": 8, + "fro_norm": 13.5746, + "top_sv": 3.0613, + "stable_rank": 19.662 + }, + { + "layer": 9, + "fro_norm": 15.119, + "top_sv": 3.4656, + "stable_rank": 19.032 + }, + { + "layer": 10, + "fro_norm": 16.9389, + "top_sv": 3.9755, + "stable_rank": 18.155 + }, + { + "layer": 11, + "fro_norm": 19.6075, + "top_sv": 4.6174, + "stable_rank": 18.032 + }, + { + "layer": 12, + "fro_norm": 21.2952, + "top_sv": 4.8109, + "stable_rank": 19.593 + }, + { + "layer": 13, + "fro_norm": 23.2437, + "top_sv": 5.3107, + "stable_rank": 19.156 + }, + { + "layer": 14, + "fro_norm": 26.6672, + "top_sv": 6.0468, + "stable_rank": 19.449 + }, + { + "layer": 15, + "fro_norm": 29.6977, + "top_sv": 6.8633, + "stable_rank": 18.723 + }, + { + "layer": 16, + "fro_norm": 32.7213, + "top_sv": 7.7425, + "stable_rank": 17.861 + }, + { + "layer": 17, + "fro_norm": 34.8035, + "top_sv": 8.2454, + "stable_rank": 17.817 + }, + { + "layer": 18, + "fro_norm": 36.0726, + "top_sv": 8.4639, + "stable_rank": 18.164 + }, + { + "layer": 19, + "fro_norm": 35.6863, + "top_sv": 8.1, + "stable_rank": 19.41 + }, + { + "layer": 20, + "fro_norm": 35.6646, + "top_sv": 7.4625, + "stable_rank": 22.841 + }, + { + "layer": 21, + "fro_norm": 34.931, + "top_sv": 5.8076, + "stable_rank": 36.176 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8210cdd52e9b8ec80522dd0b38bab45a27bb57eb --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d070039c4d7a38016e6cb2d50e302368b1a3207ed87f1f4bb534e789e70890d5 +size 46145145 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..16cc45860cc1d249d793f61197404639b153dadb --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n25_s0.pt", + "sha256": "d070039c4d7a38016e6cb2d50e302368b1a3207ed87f1f4bb534e789e70890d5", + "bytes": 46145145, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.483285, + "mean_fro": 9.819909, + "rms_per_prompt_fro": 15.474648, + "max_per_prompt_fro": 19.246235, + "heavy_tail_ratio": 1.2437, + "n_prompts": 25 + }, + "1": { + "dispersion": 1.34195, + "mean_fro": 9.077571, + "rms_per_prompt_fro": 13.891799, + "max_per_prompt_fro": 17.370386, + "heavy_tail_ratio": 1.2504, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.112491, + "mean_fro": 9.720875, + "rms_per_prompt_fro": 14.128717, + "max_per_prompt_fro": 17.396793, + "heavy_tail_ratio": 1.2313, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.994306, + "mean_fro": 9.21746, + "rms_per_prompt_fro": 13.016887, + "max_per_prompt_fro": 16.22654, + "heavy_tail_ratio": 1.2466, + "n_prompts": 25 + }, + "4": { + "dispersion": 0.839541, + "mean_fro": 10.244792, + "rms_per_prompt_fro": 13.894977, + "max_per_prompt_fro": 17.349812, + "heavy_tail_ratio": 1.2486, + "n_prompts": 25 + }, + "5": { + "dispersion": 0.77388, + "mean_fro": 10.307815, + "rms_per_prompt_fro": 13.728679, + "max_per_prompt_fro": 17.102009, + "heavy_tail_ratio": 1.2457, + "n_prompts": 25 + }, + "6": { + "dispersion": 0.698782, + "mean_fro": 10.607173, + "rms_per_prompt_fro": 13.825107, + "max_per_prompt_fro": 17.416403, + "heavy_tail_ratio": 1.2598, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.62425, + "mean_fro": 11.570344, + "rms_per_prompt_fro": 14.74595, + "max_per_prompt_fro": 18.555748, + "heavy_tail_ratio": 1.2584, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.575386, + "mean_fro": 13.134637, + "rms_per_prompt_fro": 16.485858, + "max_per_prompt_fro": 20.628031, + "heavy_tail_ratio": 1.2513, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.536184, + "mean_fro": 14.646382, + "rms_per_prompt_fro": 18.153148, + "max_per_prompt_fro": 22.51523, + "heavy_tail_ratio": 1.2403, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.494074, + "mean_fro": 16.446375, + "rms_per_prompt_fro": 20.102786, + "max_per_prompt_fro": 24.403898, + "heavy_tail_ratio": 1.214, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.450253, + "mean_fro": 19.052693, + "rms_per_prompt_fro": 22.94448, + "max_per_prompt_fro": 27.447655, + "heavy_tail_ratio": 1.1963, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.368089, + "mean_fro": 20.90151, + "rms_per_prompt_fro": 24.447526, + "max_per_prompt_fro": 29.091446, + "heavy_tail_ratio": 1.19, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.326698, + "mean_fro": 22.81825, + "rms_per_prompt_fro": 26.282603, + "max_per_prompt_fro": 30.741051, + "heavy_tail_ratio": 1.1696, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.280547, + "mean_fro": 26.22225, + "rms_per_prompt_fro": 29.673427, + "max_per_prompt_fro": 34.060944, + "heavy_tail_ratio": 1.1479, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.24345, + "mean_fro": 29.248478, + "rms_per_prompt_fro": 32.615009, + "max_per_prompt_fro": 36.90963, + "heavy_tail_ratio": 1.1317, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.175447, + "mean_fro": 32.372958, + "rms_per_prompt_fro": 35.098125, + "max_per_prompt_fro": 39.270016, + "heavy_tail_ratio": 1.1189, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.136223, + "mean_fro": 34.484917, + "rms_per_prompt_fro": 36.758774, + "max_per_prompt_fro": 39.831615, + "heavy_tail_ratio": 1.0836, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.105649, + "mean_fro": 35.849907, + "rms_per_prompt_fro": 37.696117, + "max_per_prompt_fro": 40.455715, + "heavy_tail_ratio": 1.0732, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.073981, + "mean_fro": 35.550388, + "rms_per_prompt_fro": 36.841945, + "max_per_prompt_fro": 38.525658, + "heavy_tail_ratio": 1.0457, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.04446, + "mean_fro": 35.624366, + "rms_per_prompt_fro": 36.407686, + "max_per_prompt_fro": 37.682678, + "heavy_tail_ratio": 1.035, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.021928, + "mean_fro": 34.88263, + "rms_per_prompt_fro": 35.263004, + "max_per_prompt_fro": 35.923981, + "heavy_tail_ratio": 1.0187, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 77.0, + "fit_secs_per_prompt": 3.081, + "total_wall_seconds": 84.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.8198, + "top_sv": 2.1625, + "stable_rank": 20.621 + }, + { + "layer": 1, + "fro_norm": 9.0774, + "top_sv": 1.9461, + "stable_rank": 21.757 + }, + { + "layer": 2, + "fro_norm": 9.7208, + "top_sv": 2.1301, + "stable_rank": 20.826 + }, + { + "layer": 3, + "fro_norm": 9.2173, + "top_sv": 2.0292, + "stable_rank": 20.632 + }, + { + "layer": 4, + "fro_norm": 10.2447, + "top_sv": 2.2497, + "stable_rank": 20.736 + }, + { + "layer": 5, + "fro_norm": 10.3077, + "top_sv": 2.3824, + "stable_rank": 18.72 + }, + { + "layer": 6, + "fro_norm": 10.6071, + "top_sv": 2.2809, + "stable_rank": 21.626 + }, + { + "layer": 7, + "fro_norm": 11.5702, + "top_sv": 2.4915, + "stable_rank": 21.566 + }, + { + "layer": 8, + "fro_norm": 13.1344, + "top_sv": 2.9503, + "stable_rank": 19.819 + }, + { + "layer": 9, + "fro_norm": 14.6462, + "top_sv": 3.3549, + "stable_rank": 19.059 + }, + { + "layer": 10, + "fro_norm": 16.4462, + "top_sv": 3.8515, + "stable_rank": 18.234 + }, + { + "layer": 11, + "fro_norm": 19.0524, + "top_sv": 4.5046, + "stable_rank": 17.889 + }, + { + "layer": 12, + "fro_norm": 20.9012, + "top_sv": 4.7361, + "stable_rank": 19.476 + }, + { + "layer": 13, + "fro_norm": 22.8179, + "top_sv": 5.2188, + "stable_rank": 19.116 + }, + { + "layer": 14, + "fro_norm": 26.2218, + "top_sv": 5.9334, + "stable_rank": 19.531 + }, + { + "layer": 15, + "fro_norm": 29.248, + "top_sv": 6.7279, + "stable_rank": 18.899 + }, + { + "layer": 16, + "fro_norm": 32.3723, + "top_sv": 7.6074, + "stable_rank": 18.108 + }, + { + "layer": 17, + "fro_norm": 34.4842, + "top_sv": 8.0922, + "stable_rank": 18.16 + }, + { + "layer": 18, + "fro_norm": 35.849, + "top_sv": 8.3518, + "stable_rank": 18.424 + }, + { + "layer": 19, + "fro_norm": 35.5494, + "top_sv": 8.0195, + "stable_rank": 19.65 + }, + { + "layer": 20, + "fro_norm": 35.6232, + "top_sv": 7.4572, + "stable_rank": 22.82 + }, + { + "layer": 21, + "fro_norm": 34.8811, + "top_sv": 5.7898, + "stable_rank": 36.295 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..67c3dbbd684ce40186a5fdceeeab28ccf64cb07f --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:207874b5cc9417c1b2c8c8dca5ac4393d4bf0a696c3fd6b4dcc97b5f65de3d88 +size 46145145 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e2c0c62c87f268ea41326d91ebf4eba08d7c8999 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n25_s1.pt", + "sha256": "207874b5cc9417c1b2c8c8dca5ac4393d4bf0a696c3fd6b4dcc97b5f65de3d88", + "bytes": 46145145, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.510561, + "mean_fro": 10.078457, + "rms_per_prompt_fro": 15.969062, + "max_per_prompt_fro": 21.536518, + "heavy_tail_ratio": 1.3486, + "n_prompts": 25 + }, + "1": { + "dispersion": 1.346622, + "mean_fro": 9.217212, + "rms_per_prompt_fro": 14.11956, + "max_per_prompt_fro": 18.718309, + "heavy_tail_ratio": 1.3257, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.127449, + "mean_fro": 9.87745, + "rms_per_prompt_fro": 14.40703, + "max_per_prompt_fro": 18.897516, + "heavy_tail_ratio": 1.3117, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.024979, + "mean_fro": 9.317437, + "rms_per_prompt_fro": 13.258878, + "max_per_prompt_fro": 18.290125, + "heavy_tail_ratio": 1.3795, + "n_prompts": 25 + }, + "4": { + "dispersion": 0.861828, + "mean_fro": 10.433612, + "rms_per_prompt_fro": 14.236539, + "max_per_prompt_fro": 19.738934, + "heavy_tail_ratio": 1.3865, + "n_prompts": 25 + }, + "5": { + "dispersion": 0.797982, + "mean_fro": 10.525276, + "rms_per_prompt_fro": 14.113221, + "max_per_prompt_fro": 19.655035, + "heavy_tail_ratio": 1.3927, + "n_prompts": 25 + }, + "6": { + "dispersion": 0.697415, + "mean_fro": 10.827838, + "rms_per_prompt_fro": 14.107034, + "max_per_prompt_fro": 17.305576, + "heavy_tail_ratio": 1.2267, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.61853, + "mean_fro": 11.822229, + "rms_per_prompt_fro": 15.040415, + "max_per_prompt_fro": 18.091373, + "heavy_tail_ratio": 1.2029, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.565044, + "mean_fro": 13.491568, + "rms_per_prompt_fro": 16.878186, + "max_per_prompt_fro": 19.955797, + "heavy_tail_ratio": 1.1823, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.524893, + "mean_fro": 15.067619, + "rms_per_prompt_fro": 18.606488, + "max_per_prompt_fro": 21.773565, + "heavy_tail_ratio": 1.1702, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.483794, + "mean_fro": 16.932018, + "rms_per_prompt_fro": 20.625073, + "max_per_prompt_fro": 23.691769, + "heavy_tail_ratio": 1.1487, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.437794, + "mean_fro": 19.626579, + "rms_per_prompt_fro": 23.533846, + "max_per_prompt_fro": 26.581158, + "heavy_tail_ratio": 1.1295, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.357134, + "mean_fro": 21.261993, + "rms_per_prompt_fro": 24.769388, + "max_per_prompt_fro": 28.123104, + "heavy_tail_ratio": 1.1354, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.320553, + "mean_fro": 23.195562, + "rms_per_prompt_fro": 26.655251, + "max_per_prompt_fro": 29.974928, + "heavy_tail_ratio": 1.1245, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.277814, + "mean_fro": 26.654471, + "rms_per_prompt_fro": 30.130326, + "max_per_prompt_fro": 33.409733, + "heavy_tail_ratio": 1.1088, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.240113, + "mean_fro": 29.656189, + "rms_per_prompt_fro": 33.025237, + "max_per_prompt_fro": 36.172672, + "heavy_tail_ratio": 1.0953, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.174763, + "mean_fro": 32.629222, + "rms_per_prompt_fro": 35.365663, + "max_per_prompt_fro": 38.707355, + "heavy_tail_ratio": 1.0945, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.138912, + "mean_fro": 34.768571, + "rms_per_prompt_fro": 37.104961, + "max_per_prompt_fro": 40.091644, + "heavy_tail_ratio": 1.0805, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.108302, + "mean_fro": 36.042247, + "rms_per_prompt_fro": 37.943802, + "max_per_prompt_fro": 40.512966, + "heavy_tail_ratio": 1.0677, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.075698, + "mean_fro": 35.688241, + "rms_per_prompt_fro": 37.01437, + "max_per_prompt_fro": 38.84742, + "heavy_tail_ratio": 1.0495, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.044868, + "mean_fro": 35.690934, + "rms_per_prompt_fro": 36.482837, + "max_per_prompt_fro": 37.550762, + "heavy_tail_ratio": 1.0293, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.021781, + "mean_fro": 34.953682, + "rms_per_prompt_fro": 35.33229, + "max_per_prompt_fro": 36.007542, + "heavy_tail_ratio": 1.0191, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 78.5, + "fit_secs_per_prompt": 3.14, + "total_wall_seconds": 88.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 10.0783, + "top_sv": 2.4246, + "stable_rank": 17.278 + }, + { + "layer": 1, + "fro_norm": 9.2171, + "top_sv": 2.0907, + "stable_rank": 19.436 + }, + { + "layer": 2, + "fro_norm": 9.8773, + "top_sv": 2.2879, + "stable_rank": 18.638 + }, + { + "layer": 3, + "fro_norm": 9.3173, + "top_sv": 2.1138, + "stable_rank": 19.429 + }, + { + "layer": 4, + "fro_norm": 10.4335, + "top_sv": 2.3586, + "stable_rank": 19.569 + }, + { + "layer": 5, + "fro_norm": 10.5251, + "top_sv": 2.4851, + "stable_rank": 17.937 + }, + { + "layer": 6, + "fro_norm": 10.8277, + "top_sv": 2.3804, + "stable_rank": 20.69 + }, + { + "layer": 7, + "fro_norm": 11.8221, + "top_sv": 2.5885, + "stable_rank": 20.858 + }, + { + "layer": 8, + "fro_norm": 13.4914, + "top_sv": 3.0845, + "stable_rank": 19.131 + }, + { + "layer": 9, + "fro_norm": 15.0674, + "top_sv": 3.5005, + "stable_rank": 18.527 + }, + { + "layer": 10, + "fro_norm": 16.9318, + "top_sv": 3.9869, + "stable_rank": 18.035 + }, + { + "layer": 11, + "fro_norm": 19.6263, + "top_sv": 4.6635, + "stable_rank": 17.711 + }, + { + "layer": 12, + "fro_norm": 21.2617, + "top_sv": 4.8138, + "stable_rank": 19.508 + }, + { + "layer": 13, + "fro_norm": 23.1952, + "top_sv": 5.3254, + "stable_rank": 18.971 + }, + { + "layer": 14, + "fro_norm": 26.654, + "top_sv": 6.0696, + "stable_rank": 19.284 + }, + { + "layer": 15, + "fro_norm": 29.6557, + "top_sv": 6.8905, + "stable_rank": 18.523 + }, + { + "layer": 16, + "fro_norm": 32.6287, + "top_sv": 7.7799, + "stable_rank": 17.59 + }, + { + "layer": 17, + "fro_norm": 34.7678, + "top_sv": 8.2564, + "stable_rank": 17.732 + }, + { + "layer": 18, + "fro_norm": 36.0413, + "top_sv": 8.4211, + "stable_rank": 18.317 + }, + { + "layer": 19, + "fro_norm": 35.6872, + "top_sv": 8.0607, + "stable_rank": 19.601 + }, + { + "layer": 20, + "fro_norm": 35.6897, + "top_sv": 7.5139, + "stable_rank": 22.561 + }, + { + "layer": 21, + "fro_norm": 34.9522, + "top_sv": 5.831, + "stable_rank": 35.93 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e5e7a4714536a0adbf7cd4b7ff8e2ee403b9c00 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7fc83d478f118f991670728226f22b31a88356400cb8f5ac7d1ddb9c44560df +size 46145145 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..73cf5fc3485964b7a72171c6d982ee20fe767711 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n25_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n25_s2.pt", + "sha256": "d7fc83d478f118f991670728226f22b31a88356400cb8f5ac7d1ddb9c44560df", + "bytes": 46145145, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.539926, + "mean_fro": 10.384283, + "rms_per_prompt_fro": 16.549584, + "max_per_prompt_fro": 19.7409, + "heavy_tail_ratio": 1.1928, + "n_prompts": 25 + }, + "1": { + "dispersion": 1.375063, + "mean_fro": 9.549656, + "rms_per_prompt_fro": 14.717203, + "max_per_prompt_fro": 17.352787, + "heavy_tail_ratio": 1.1791, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.128759, + "mean_fro": 10.25089, + "rms_per_prompt_fro": 14.956323, + "max_per_prompt_fro": 17.142986, + "heavy_tail_ratio": 1.1462, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.999646, + "mean_fro": 9.659883, + "rms_per_prompt_fro": 13.659928, + "max_per_prompt_fro": 15.707211, + "heavy_tail_ratio": 1.1499, + "n_prompts": 25 + }, + "4": { + "dispersion": 0.835143, + "mean_fro": 10.793253, + "rms_per_prompt_fro": 14.621346, + "max_per_prompt_fro": 16.908823, + "heavy_tail_ratio": 1.1564, + "n_prompts": 25 + }, + "5": { + "dispersion": 0.766674, + "mean_fro": 10.846044, + "rms_per_prompt_fro": 14.416159, + "max_per_prompt_fro": 16.575871, + "heavy_tail_ratio": 1.1498, + "n_prompts": 25 + }, + "6": { + "dispersion": 0.699316, + "mean_fro": 11.153839, + "rms_per_prompt_fro": 14.5399, + "max_per_prompt_fro": 16.873854, + "heavy_tail_ratio": 1.1605, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.627078, + "mean_fro": 12.12402, + "rms_per_prompt_fro": 15.465033, + "max_per_prompt_fro": 17.967571, + "heavy_tail_ratio": 1.1618, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.579093, + "mean_fro": 13.738029, + "rms_per_prompt_fro": 17.263478, + "max_per_prompt_fro": 20.239119, + "heavy_tail_ratio": 1.1724, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.539179, + "mean_fro": 15.264644, + "rms_per_prompt_fro": 18.937877, + "max_per_prompt_fro": 22.413094, + "heavy_tail_ratio": 1.1835, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.497073, + "mean_fro": 17.080492, + "rms_per_prompt_fro": 20.898821, + "max_per_prompt_fro": 24.555136, + "heavy_tail_ratio": 1.175, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.450593, + "mean_fro": 19.75427, + "rms_per_prompt_fro": 23.792153, + "max_per_prompt_fro": 27.345827, + "heavy_tail_ratio": 1.1494, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.366166, + "mean_fro": 21.358328, + "rms_per_prompt_fro": 24.964276, + "max_per_prompt_fro": 28.499649, + "heavy_tail_ratio": 1.1416, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.328962, + "mean_fro": 23.328852, + "rms_per_prompt_fro": 26.893643, + "max_per_prompt_fro": 30.125408, + "heavy_tail_ratio": 1.1202, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.281691, + "mean_fro": 26.709108, + "rms_per_prompt_fro": 30.237855, + "max_per_prompt_fro": 33.699787, + "heavy_tail_ratio": 1.1145, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.243433, + "mean_fro": 29.639797, + "rms_per_prompt_fro": 33.051138, + "max_per_prompt_fro": 36.839882, + "heavy_tail_ratio": 1.1146, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.177337, + "mean_fro": 32.652648, + "rms_per_prompt_fro": 35.429806, + "max_per_prompt_fro": 39.002068, + "heavy_tail_ratio": 1.1008, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.139514, + "mean_fro": 34.743076, + "rms_per_prompt_fro": 37.087548, + "max_per_prompt_fro": 39.963058, + "heavy_tail_ratio": 1.0775, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.108325, + "mean_fro": 35.974796, + "rms_per_prompt_fro": 37.873191, + "max_per_prompt_fro": 40.203117, + "heavy_tail_ratio": 1.0615, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.076633, + "mean_fro": 35.590638, + "rms_per_prompt_fro": 36.929183, + "max_per_prompt_fro": 38.337688, + "heavy_tail_ratio": 1.0381, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.044987, + "mean_fro": 35.653751, + "rms_per_prompt_fro": 36.446902, + "max_per_prompt_fro": 37.362934, + "heavy_tail_ratio": 1.0251, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.022479, + "mean_fro": 34.945694, + "rms_per_prompt_fro": 35.33629, + "max_per_prompt_fro": 36.064835, + "heavy_tail_ratio": 1.0206, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 76.3, + "fit_secs_per_prompt": 3.052, + "total_wall_seconds": 85.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 10.3842, + "top_sv": 2.3918, + "stable_rank": 18.85 + }, + { + "layer": 1, + "fro_norm": 9.5495, + "top_sv": 2.1155, + "stable_rank": 20.378 + }, + { + "layer": 2, + "fro_norm": 10.2508, + "top_sv": 2.3186, + "stable_rank": 19.547 + }, + { + "layer": 3, + "fro_norm": 9.6598, + "top_sv": 2.1722, + "stable_rank": 19.775 + }, + { + "layer": 4, + "fro_norm": 10.7931, + "top_sv": 2.4161, + "stable_rank": 19.956 + }, + { + "layer": 5, + "fro_norm": 10.8459, + "top_sv": 2.5346, + "stable_rank": 18.311 + }, + { + "layer": 6, + "fro_norm": 11.1537, + "top_sv": 2.4752, + "stable_rank": 20.306 + }, + { + "layer": 7, + "fro_norm": 12.1239, + "top_sv": 2.6823, + "stable_rank": 20.429 + }, + { + "layer": 8, + "fro_norm": 13.7378, + "top_sv": 3.1492, + "stable_rank": 19.03 + }, + { + "layer": 9, + "fro_norm": 15.2644, + "top_sv": 3.5583, + "stable_rank": 18.403 + }, + { + "layer": 10, + "fro_norm": 17.0803, + "top_sv": 4.0752, + "stable_rank": 17.567 + }, + { + "layer": 11, + "fro_norm": 19.754, + "top_sv": 4.7463, + "stable_rank": 17.322 + }, + { + "layer": 12, + "fro_norm": 21.358, + "top_sv": 4.9094, + "stable_rank": 18.926 + }, + { + "layer": 13, + "fro_norm": 23.3285, + "top_sv": 5.4183, + "stable_rank": 18.538 + }, + { + "layer": 14, + "fro_norm": 26.7087, + "top_sv": 6.1476, + "stable_rank": 18.875 + }, + { + "layer": 15, + "fro_norm": 29.6393, + "top_sv": 6.9109, + "stable_rank": 18.394 + }, + { + "layer": 16, + "fro_norm": 32.652, + "top_sv": 7.8231, + "stable_rank": 17.421 + }, + { + "layer": 17, + "fro_norm": 34.7423, + "top_sv": 8.3173, + "stable_rank": 17.448 + }, + { + "layer": 18, + "fro_norm": 35.9739, + "top_sv": 8.4881, + "stable_rank": 17.962 + }, + { + "layer": 19, + "fro_norm": 35.5896, + "top_sv": 8.0711, + "stable_rank": 19.444 + }, + { + "layer": 20, + "fro_norm": 35.6525, + "top_sv": 7.4825, + "stable_rank": 22.703 + }, + { + "layer": 21, + "fro_norm": 34.9442, + "top_sv": 5.8066, + "stable_rank": 36.216 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..163f00e770efe6ef36ac2f54b2e8bca72bc69dc1 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bc59371a760c85c5570f68e00a6e41a65a1ff7168079fc9ab5a695e2d70987b +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..8cffb0813595ebf38f9d5ee96fe85318f26b6ba9 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n400_s0.pt", + "sha256": "4bc59371a760c85c5570f68e00a6e41a65a1ff7168079fc9ab5a695e2d70987b", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.634971, + "mean_fro": 9.642245, + "rms_per_prompt_fro": 15.651865, + "max_per_prompt_fro": 27.007643, + "heavy_tail_ratio": 1.7255, + "n_prompts": 400 + }, + "1": { + "dispersion": 1.45359, + "mean_fro": 8.919035, + "rms_per_prompt_fro": 13.970721, + "max_per_prompt_fro": 22.984131, + "heavy_tail_ratio": 1.6452, + "n_prompts": 400 + }, + "2": { + "dispersion": 1.195675, + "mean_fro": 9.636432, + "rms_per_prompt_fro": 14.27908, + "max_per_prompt_fro": 23.135078, + "heavy_tail_ratio": 1.6202, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.060475, + "mean_fro": 9.130834, + "rms_per_prompt_fro": 13.106724, + "max_per_prompt_fro": 20.889572, + "heavy_tail_ratio": 1.5938, + "n_prompts": 400 + }, + "4": { + "dispersion": 0.878298, + "mean_fro": 10.272536, + "rms_per_prompt_fro": 14.078613, + "max_per_prompt_fro": 23.325769, + "heavy_tail_ratio": 1.6568, + "n_prompts": 400 + }, + "5": { + "dispersion": 0.800158, + "mean_fro": 10.360139, + "rms_per_prompt_fro": 13.900196, + "max_per_prompt_fro": 21.056416, + "heavy_tail_ratio": 1.5148, + "n_prompts": 400 + }, + "6": { + "dispersion": 0.719233, + "mean_fro": 10.657027, + "rms_per_prompt_fro": 13.973445, + "max_per_prompt_fro": 20.742741, + "heavy_tail_ratio": 1.4844, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.639165, + "mean_fro": 11.673607, + "rms_per_prompt_fro": 14.945707, + "max_per_prompt_fro": 21.689234, + "heavy_tail_ratio": 1.4512, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.585246, + "mean_fro": 13.320406, + "rms_per_prompt_fro": 16.771262, + "max_per_prompt_fro": 23.220459, + "heavy_tail_ratio": 1.3845, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.542501, + "mean_fro": 14.860232, + "rms_per_prompt_fro": 18.456033, + "max_per_prompt_fro": 24.576418, + "heavy_tail_ratio": 1.3316, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.497653, + "mean_fro": 16.690975, + "rms_per_prompt_fro": 20.426185, + "max_per_prompt_fro": 25.775553, + "heavy_tail_ratio": 1.2619, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.452957, + "mean_fro": 19.366762, + "rms_per_prompt_fro": 23.344441, + "max_per_prompt_fro": 27.912014, + "heavy_tail_ratio": 1.1957, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.368859, + "mean_fro": 21.115539, + "rms_per_prompt_fro": 24.704807, + "max_per_prompt_fro": 29.31465, + "heavy_tail_ratio": 1.1866, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.329524, + "mean_fro": 23.055703, + "rms_per_prompt_fro": 26.584374, + "max_per_prompt_fro": 30.741051, + "heavy_tail_ratio": 1.1564, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.284085, + "mean_fro": 26.488595, + "rms_per_prompt_fro": 30.01621, + "max_per_prompt_fro": 34.354977, + "heavy_tail_ratio": 1.1445, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.246557, + "mean_fro": 29.540358, + "rms_per_prompt_fro": 32.981614, + "max_per_prompt_fro": 37.698631, + "heavy_tail_ratio": 1.143, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.179317, + "mean_fro": 32.559872, + "rms_per_prompt_fro": 35.358831, + "max_per_prompt_fro": 39.270016, + "heavy_tail_ratio": 1.1106, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.141072, + "mean_fro": 34.658631, + "rms_per_prompt_fro": 37.022685, + "max_per_prompt_fro": 40.173393, + "heavy_tail_ratio": 1.0851, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.10944, + "mean_fro": 35.951638, + "rms_per_prompt_fro": 37.86784, + "max_per_prompt_fro": 40.500912, + "heavy_tail_ratio": 1.0695, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.076857, + "mean_fro": 35.61573, + "rms_per_prompt_fro": 36.959063, + "max_per_prompt_fro": 38.931988, + "heavy_tail_ratio": 1.0534, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.046679, + "mean_fro": 35.605294, + "rms_per_prompt_fro": 36.426821, + "max_per_prompt_fro": 37.803265, + "heavy_tail_ratio": 1.0378, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.022975, + "mean_fro": 34.894592, + "rms_per_prompt_fro": 35.293169, + "max_per_prompt_fro": 36.347958, + "heavy_tail_ratio": 1.0299, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1236.2, + "fit_secs_per_prompt": 3.091, + "total_wall_seconds": 1244.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.6421, + "top_sv": 2.1257, + "stable_rank": 20.575 + }, + { + "layer": 1, + "fro_norm": 8.9189, + "top_sv": 1.937, + "stable_rank": 21.201 + }, + { + "layer": 2, + "fro_norm": 9.6363, + "top_sv": 2.1418, + "stable_rank": 20.243 + }, + { + "layer": 3, + "fro_norm": 9.1307, + "top_sv": 2.0563, + "stable_rank": 19.716 + }, + { + "layer": 4, + "fro_norm": 10.2724, + "top_sv": 2.2805, + "stable_rank": 20.29 + }, + { + "layer": 5, + "fro_norm": 10.36, + "top_sv": 2.441, + "stable_rank": 18.012 + }, + { + "layer": 6, + "fro_norm": 10.6569, + "top_sv": 2.2551, + "stable_rank": 22.333 + }, + { + "layer": 7, + "fro_norm": 11.6735, + "top_sv": 2.5213, + "stable_rank": 21.436 + }, + { + "layer": 8, + "fro_norm": 13.3202, + "top_sv": 2.9997, + "stable_rank": 19.718 + }, + { + "layer": 9, + "fro_norm": 14.86, + "top_sv": 3.3884, + "stable_rank": 19.233 + }, + { + "layer": 10, + "fro_norm": 16.6907, + "top_sv": 3.8883, + "stable_rank": 18.426 + }, + { + "layer": 11, + "fro_norm": 19.3665, + "top_sv": 4.5188, + "stable_rank": 18.368 + }, + { + "layer": 12, + "fro_norm": 21.1152, + "top_sv": 4.7421, + "stable_rank": 19.827 + }, + { + "layer": 13, + "fro_norm": 23.0554, + "top_sv": 5.224, + "stable_rank": 19.478 + }, + { + "layer": 14, + "fro_norm": 26.4881, + "top_sv": 5.9524, + "stable_rank": 19.803 + }, + { + "layer": 15, + "fro_norm": 29.5399, + "top_sv": 6.7941, + "stable_rank": 18.904 + }, + { + "layer": 16, + "fro_norm": 32.5593, + "top_sv": 7.6374, + "stable_rank": 18.175 + }, + { + "layer": 17, + "fro_norm": 34.6579, + "top_sv": 8.1399, + "stable_rank": 18.129 + }, + { + "layer": 18, + "fro_norm": 35.9508, + "top_sv": 8.3796, + "stable_rank": 18.407 + }, + { + "layer": 19, + "fro_norm": 35.6147, + "top_sv": 8.0451, + "stable_rank": 19.597 + }, + { + "layer": 20, + "fro_norm": 35.6041, + "top_sv": 7.4382, + "stable_rank": 22.912 + }, + { + "layer": 21, + "fro_norm": 34.8931, + "top_sv": 5.794, + "stable_rank": 36.268 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f043cb4dc88bcd206337c2323b6b0bac5bfe14f --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec859ffabba4ee9fd77e271d3468a6a48543e460059a20372a417472f28984d6 +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..fa6567cf87492c3486459b688f35336dd22f0458 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n400_s1.pt", + "sha256": "ec859ffabba4ee9fd77e271d3468a6a48543e460059a20372a417472f28984d6", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.688143, + "mean_fro": 9.584403, + "rms_per_prompt_fro": 15.714162, + "max_per_prompt_fro": 21.750072, + "heavy_tail_ratio": 1.3841, + "n_prompts": 400 + }, + "1": { + "dispersion": 1.498511, + "mean_fro": 8.87854, + "rms_per_prompt_fro": 14.034025, + "max_per_prompt_fro": 19.534609, + "heavy_tail_ratio": 1.3919, + "n_prompts": 400 + }, + "2": { + "dispersion": 1.229673, + "mean_fro": 9.606856, + "rms_per_prompt_fro": 14.345045, + "max_per_prompt_fro": 19.886108, + "heavy_tail_ratio": 1.3863, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.0919, + "mean_fro": 9.103269, + "rms_per_prompt_fro": 13.166426, + "max_per_prompt_fro": 18.290125, + "heavy_tail_ratio": 1.3891, + "n_prompts": 400 + }, + "4": { + "dispersion": 0.903986, + "mean_fro": 10.241653, + "rms_per_prompt_fro": 14.131943, + "max_per_prompt_fro": 19.738934, + "heavy_tail_ratio": 1.3968, + "n_prompts": 400 + }, + "5": { + "dispersion": 0.825785, + "mean_fro": 10.331997, + "rms_per_prompt_fro": 13.960762, + "max_per_prompt_fro": 19.655035, + "heavy_tail_ratio": 1.4079, + "n_prompts": 400 + }, + "6": { + "dispersion": 0.741495, + "mean_fro": 10.630028, + "rms_per_prompt_fro": 14.027992, + "max_per_prompt_fro": 19.287895, + "heavy_tail_ratio": 1.375, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.657962, + "mean_fro": 11.637711, + "rms_per_prompt_fro": 14.984937, + "max_per_prompt_fro": 20.31992, + "heavy_tail_ratio": 1.356, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.601983, + "mean_fro": 13.277873, + "rms_per_prompt_fro": 16.805732, + "max_per_prompt_fro": 22.782795, + "heavy_tail_ratio": 1.3557, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.556637, + "mean_fro": 14.821616, + "rms_per_prompt_fro": 18.492229, + "max_per_prompt_fro": 24.730282, + "heavy_tail_ratio": 1.3373, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.509809, + "mean_fro": 16.652561, + "rms_per_prompt_fro": 20.461714, + "max_per_prompt_fro": 26.740112, + "heavy_tail_ratio": 1.3068, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.463317, + "mean_fro": 19.317697, + "rms_per_prompt_fro": 23.368166, + "max_per_prompt_fro": 29.689667, + "heavy_tail_ratio": 1.2705, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.376979, + "mean_fro": 21.011038, + "rms_per_prompt_fro": 24.655354, + "max_per_prompt_fro": 30.227711, + "heavy_tail_ratio": 1.226, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.336233, + "mean_fro": 22.951242, + "rms_per_prompt_fro": 26.530618, + "max_per_prompt_fro": 32.066608, + "heavy_tail_ratio": 1.2087, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.289194, + "mean_fro": 26.372855, + "rms_per_prompt_fro": 29.944449, + "max_per_prompt_fro": 35.79073, + "heavy_tail_ratio": 1.1952, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.250309, + "mean_fro": 29.405827, + "rms_per_prompt_fro": 32.880777, + "max_per_prompt_fro": 38.912979, + "heavy_tail_ratio": 1.1835, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.181472, + "mean_fro": 32.414062, + "rms_per_prompt_fro": 35.232641, + "max_per_prompt_fro": 40.017754, + "heavy_tail_ratio": 1.1358, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.142582, + "mean_fro": 34.526863, + "rms_per_prompt_fro": 36.906321, + "max_per_prompt_fro": 40.410805, + "heavy_tail_ratio": 1.095, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.110399, + "mean_fro": 35.835892, + "rms_per_prompt_fro": 37.762244, + "max_per_prompt_fro": 40.691536, + "heavy_tail_ratio": 1.0776, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.077232, + "mean_fro": 35.552117, + "rms_per_prompt_fro": 36.899469, + "max_per_prompt_fro": 39.029053, + "heavy_tail_ratio": 1.0577, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.046585, + "mean_fro": 35.556134, + "rms_per_prompt_fro": 36.374898, + "max_per_prompt_fro": 37.79361, + "heavy_tail_ratio": 1.039, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.022959, + "mean_fro": 34.872429, + "rms_per_prompt_fro": 35.270471, + "max_per_prompt_fro": 36.037338, + "heavy_tail_ratio": 1.0217, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1246.6, + "fit_secs_per_prompt": 3.116, + "total_wall_seconds": 1256.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.5843, + "top_sv": 2.1195, + "stable_rank": 20.449 + }, + { + "layer": 1, + "fro_norm": 8.8784, + "top_sv": 1.9231, + "stable_rank": 21.314 + }, + { + "layer": 2, + "fro_norm": 9.6067, + "top_sv": 2.1334, + "stable_rank": 20.277 + }, + { + "layer": 3, + "fro_norm": 9.1032, + "top_sv": 2.0369, + "stable_rank": 19.974 + }, + { + "layer": 4, + "fro_norm": 10.2415, + "top_sv": 2.2593, + "stable_rank": 20.549 + }, + { + "layer": 5, + "fro_norm": 10.3319, + "top_sv": 2.4158, + "stable_rank": 18.291 + }, + { + "layer": 6, + "fro_norm": 10.6299, + "top_sv": 2.2579, + "stable_rank": 22.164 + }, + { + "layer": 7, + "fro_norm": 11.6376, + "top_sv": 2.5087, + "stable_rank": 21.519 + }, + { + "layer": 8, + "fro_norm": 13.2777, + "top_sv": 2.9871, + "stable_rank": 19.758 + }, + { + "layer": 9, + "fro_norm": 14.8214, + "top_sv": 3.3797, + "stable_rank": 19.232 + }, + { + "layer": 10, + "fro_norm": 16.6523, + "top_sv": 3.8696, + "stable_rank": 18.519 + }, + { + "layer": 11, + "fro_norm": 19.3174, + "top_sv": 4.5045, + "stable_rank": 18.391 + }, + { + "layer": 12, + "fro_norm": 21.0107, + "top_sv": 4.7074, + "stable_rank": 19.921 + }, + { + "layer": 13, + "fro_norm": 22.9509, + "top_sv": 5.194, + "stable_rank": 19.525 + }, + { + "layer": 14, + "fro_norm": 26.3724, + "top_sv": 5.9222, + "stable_rank": 19.831 + }, + { + "layer": 15, + "fro_norm": 29.4053, + "top_sv": 6.7601, + "stable_rank": 18.921 + }, + { + "layer": 16, + "fro_norm": 32.4135, + "top_sv": 7.6139, + "stable_rank": 18.123 + }, + { + "layer": 17, + "fro_norm": 34.5261, + "top_sv": 8.1147, + "stable_rank": 18.103 + }, + { + "layer": 18, + "fro_norm": 35.835, + "top_sv": 8.3455, + "stable_rank": 18.438 + }, + { + "layer": 19, + "fro_norm": 35.5511, + "top_sv": 8.0247, + "stable_rank": 19.627 + }, + { + "layer": 20, + "fro_norm": 35.5549, + "top_sv": 7.4068, + "stable_rank": 23.043 + }, + { + "layer": 21, + "fro_norm": 34.8709, + "top_sv": 5.7674, + "stable_rank": 36.557 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cf96e69809b02c94606a325f8e203b9c0954a78 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8ac3e2549cda84008e03a18d1cb7033688247b61df83eb101519832580aa2bd +size 46145173 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..2efa907e38ea1cd6bc6462d9682ae08284925d94 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n400_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n400_s2.pt", + "sha256": "f8ac3e2549cda84008e03a18d1cb7033688247b61df83eb101519832580aa2bd", + "bytes": 46145173, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.654085, + "mean_fro": 9.761081, + "rms_per_prompt_fro": 15.902131, + "max_per_prompt_fro": 24.165665, + "heavy_tail_ratio": 1.5196, + "n_prompts": 400 + }, + "1": { + "dispersion": 1.463844, + "mean_fro": 9.020115, + "rms_per_prompt_fro": 14.158546, + "max_per_prompt_fro": 20.986425, + "heavy_tail_ratio": 1.4822, + "n_prompts": 400 + }, + "2": { + "dispersion": 1.202497, + "mean_fro": 9.737053, + "rms_per_prompt_fro": 14.450575, + "max_per_prompt_fro": 21.855257, + "heavy_tail_ratio": 1.5124, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.069119, + "mean_fro": 9.232337, + "rms_per_prompt_fro": 13.280192, + "max_per_prompt_fro": 20.380758, + "heavy_tail_ratio": 1.5347, + "n_prompts": 400 + }, + "4": { + "dispersion": 0.885968, + "mean_fro": 10.371903, + "rms_per_prompt_fro": 14.24379, + "max_per_prompt_fro": 22.429878, + "heavy_tail_ratio": 1.5747, + "n_prompts": 400 + }, + "5": { + "dispersion": 0.809909, + "mean_fro": 10.46371, + "rms_per_prompt_fro": 14.077127, + "max_per_prompt_fro": 21.079952, + "heavy_tail_ratio": 1.4975, + "n_prompts": 400 + }, + "6": { + "dispersion": 0.72629, + "mean_fro": 10.759747, + "rms_per_prompt_fro": 14.137057, + "max_per_prompt_fro": 18.668427, + "heavy_tail_ratio": 1.3205, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.64493, + "mean_fro": 11.774016, + "rms_per_prompt_fro": 15.100744, + "max_per_prompt_fro": 19.924604, + "heavy_tail_ratio": 1.3194, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.590628, + "mean_fro": 13.417539, + "rms_per_prompt_fro": 16.922214, + "max_per_prompt_fro": 22.063967, + "heavy_tail_ratio": 1.3038, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.547444, + "mean_fro": 14.962037, + "rms_per_prompt_fro": 18.612219, + "max_per_prompt_fro": 23.765785, + "heavy_tail_ratio": 1.2769, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.502149, + "mean_fro": 16.791817, + "rms_per_prompt_fro": 20.580421, + "max_per_prompt_fro": 25.62677, + "heavy_tail_ratio": 1.2452, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.45713, + "mean_fro": 19.463722, + "rms_per_prompt_fro": 23.494981, + "max_per_prompt_fro": 28.817194, + "heavy_tail_ratio": 1.2265, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.372146, + "mean_fro": 21.176496, + "rms_per_prompt_fro": 24.805859, + "max_per_prompt_fro": 29.755514, + "heavy_tail_ratio": 1.1995, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.332494, + "mean_fro": 23.120052, + "rms_per_prompt_fro": 26.688336, + "max_per_prompt_fro": 31.628721, + "heavy_tail_ratio": 1.1851, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.286539, + "mean_fro": 26.545998, + "rms_per_prompt_fro": 30.109988, + "max_per_prompt_fro": 34.860329, + "heavy_tail_ratio": 1.1578, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.248008, + "mean_fro": 29.584453, + "rms_per_prompt_fro": 33.050059, + "max_per_prompt_fro": 38.147495, + "heavy_tail_ratio": 1.1542, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.180501, + "mean_fro": 32.61213, + "rms_per_prompt_fro": 35.433357, + "max_per_prompt_fro": 39.78809, + "heavy_tail_ratio": 1.1229, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.142127, + "mean_fro": 34.712894, + "rms_per_prompt_fro": 37.097786, + "max_per_prompt_fro": 40.477703, + "heavy_tail_ratio": 1.0911, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.110218, + "mean_fro": 36.005508, + "rms_per_prompt_fro": 37.937886, + "max_per_prompt_fro": 41.1273, + "heavy_tail_ratio": 1.0841, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.07755, + "mean_fro": 35.653896, + "rms_per_prompt_fro": 37.010564, + "max_per_prompt_fro": 39.040413, + "heavy_tail_ratio": 1.0548, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.046688, + "mean_fro": 35.633678, + "rms_per_prompt_fro": 36.456015, + "max_per_prompt_fro": 37.814735, + "heavy_tail_ratio": 1.0373, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.023079, + "mean_fro": 34.915523, + "rms_per_prompt_fro": 35.316131, + "max_per_prompt_fro": 36.17366, + "heavy_tail_ratio": 1.0243, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1218.6, + "fit_secs_per_prompt": 3.046, + "total_wall_seconds": 1227.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.7609, + "top_sv": 2.1883, + "stable_rank": 19.897 + }, + { + "layer": 1, + "fro_norm": 9.02, + "top_sv": 1.9894, + "stable_rank": 20.557 + }, + { + "layer": 2, + "fro_norm": 9.7369, + "top_sv": 2.198, + "stable_rank": 19.624 + }, + { + "layer": 3, + "fro_norm": 9.2322, + "top_sv": 2.0981, + "stable_rank": 19.363 + }, + { + "layer": 4, + "fro_norm": 10.3718, + "top_sv": 2.3386, + "stable_rank": 19.67 + }, + { + "layer": 5, + "fro_norm": 10.4636, + "top_sv": 2.4954, + "stable_rank": 17.582 + }, + { + "layer": 6, + "fro_norm": 10.7596, + "top_sv": 2.3173, + "stable_rank": 21.559 + }, + { + "layer": 7, + "fro_norm": 11.7739, + "top_sv": 2.5701, + "stable_rank": 20.986 + }, + { + "layer": 8, + "fro_norm": 13.4174, + "top_sv": 3.0472, + "stable_rank": 19.388 + }, + { + "layer": 9, + "fro_norm": 14.9618, + "top_sv": 3.4409, + "stable_rank": 18.907 + }, + { + "layer": 10, + "fro_norm": 16.7916, + "top_sv": 3.9454, + "stable_rank": 18.114 + }, + { + "layer": 11, + "fro_norm": 19.4634, + "top_sv": 4.5857, + "stable_rank": 18.015 + }, + { + "layer": 12, + "fro_norm": 21.1762, + "top_sv": 4.7888, + "stable_rank": 19.554 + }, + { + "layer": 13, + "fro_norm": 23.1197, + "top_sv": 5.28, + "stable_rank": 19.173 + }, + { + "layer": 14, + "fro_norm": 26.5456, + "top_sv": 6.0069, + "stable_rank": 19.529 + }, + { + "layer": 15, + "fro_norm": 29.584, + "top_sv": 6.8355, + "stable_rank": 18.731 + }, + { + "layer": 16, + "fro_norm": 32.6115, + "top_sv": 7.7156, + "stable_rank": 17.865 + }, + { + "layer": 17, + "fro_norm": 34.7121, + "top_sv": 8.223, + "stable_rank": 17.82 + }, + { + "layer": 18, + "fro_norm": 36.0046, + "top_sv": 8.4535, + "stable_rank": 18.14 + }, + { + "layer": 19, + "fro_norm": 35.6528, + "top_sv": 8.1113, + "stable_rank": 19.32 + }, + { + "layer": 20, + "fro_norm": 35.6325, + "top_sv": 7.457, + "stable_rank": 22.833 + }, + { + "layer": 21, + "fro_norm": 34.914, + "top_sv": 5.8042, + "stable_rank": 36.184 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e850b8b18460b9dc0d924cbb02245cc576d3481 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5da1a5918c276fbaf3534076f6ba77954be94549d894c15badc78bf830ce5399 +size 46145145 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f33d9978be164c511c435823ad193d7d84578b3f --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n50_s0.pt", + "sha256": "5da1a5918c276fbaf3534076f6ba77954be94549d894c15badc78bf830ce5399", + "bytes": 46145145, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.528686, + "mean_fro": 9.662331, + "rms_per_prompt_fro": 15.364888, + "max_per_prompt_fro": 19.34812, + "heavy_tail_ratio": 1.2592, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.364466, + "mean_fro": 8.913322, + "rms_per_prompt_fro": 13.705855, + "max_per_prompt_fro": 17.370386, + "heavy_tail_ratio": 1.2674, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.131519, + "mean_fro": 9.613492, + "rms_per_prompt_fro": 14.035432, + "max_per_prompt_fro": 17.734081, + "heavy_tail_ratio": 1.2635, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.007356, + "mean_fro": 9.121597, + "rms_per_prompt_fro": 12.923588, + "max_per_prompt_fro": 16.22654, + "heavy_tail_ratio": 1.2556, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.839044, + "mean_fro": 10.227427, + "rms_per_prompt_fro": 13.869554, + "max_per_prompt_fro": 17.349812, + "heavy_tail_ratio": 1.2509, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.768995, + "mean_fro": 10.294233, + "rms_per_prompt_fro": 13.691697, + "max_per_prompt_fro": 17.102009, + "heavy_tail_ratio": 1.2491, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.694066, + "mean_fro": 10.590757, + "rms_per_prompt_fro": 13.784535, + "max_per_prompt_fro": 17.416403, + "heavy_tail_ratio": 1.2635, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.61768, + "mean_fro": 11.598801, + "rms_per_prompt_fro": 14.75229, + "max_per_prompt_fro": 18.555748, + "heavy_tail_ratio": 1.2578, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.566819, + "mean_fro": 13.194737, + "rms_per_prompt_fro": 16.516201, + "max_per_prompt_fro": 20.628031, + "heavy_tail_ratio": 1.249, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.526607, + "mean_fro": 14.718738, + "rms_per_prompt_fro": 18.185877, + "max_per_prompt_fro": 22.51523, + "heavy_tail_ratio": 1.2381, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.483813, + "mean_fro": 16.528713, + "rms_per_prompt_fro": 20.133932, + "max_per_prompt_fro": 24.403898, + "heavy_tail_ratio": 1.2121, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.44082, + "mean_fro": 19.164709, + "rms_per_prompt_fro": 23.004197, + "max_per_prompt_fro": 27.447655, + "heavy_tail_ratio": 1.1932, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.361085, + "mean_fro": 20.988447, + "rms_per_prompt_fro": 24.486284, + "max_per_prompt_fro": 29.091446, + "heavy_tail_ratio": 1.1881, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.321737, + "mean_fro": 22.925918, + "rms_per_prompt_fro": 26.357196, + "max_per_prompt_fro": 30.741051, + "heavy_tail_ratio": 1.1663, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.276733, + "mean_fro": 26.36133, + "rms_per_prompt_fro": 29.786358, + "max_per_prompt_fro": 34.060944, + "heavy_tail_ratio": 1.1435, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.240111, + "mean_fro": 29.422914, + "rms_per_prompt_fro": 32.765438, + "max_per_prompt_fro": 36.90963, + "heavy_tail_ratio": 1.1265, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.173271, + "mean_fro": 32.51548, + "rms_per_prompt_fro": 35.219998, + "max_per_prompt_fro": 39.270016, + "heavy_tail_ratio": 1.115, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.135584, + "mean_fro": 34.635913, + "rms_per_prompt_fro": 36.909344, + "max_per_prompt_fro": 39.831615, + "heavy_tail_ratio": 1.0792, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.105111, + "mean_fro": 35.934883, + "rms_per_prompt_fro": 37.776273, + "max_per_prompt_fro": 40.455715, + "heavy_tail_ratio": 1.0709, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.074012, + "mean_fro": 35.582973, + "rms_per_prompt_fro": 36.87625, + "max_per_prompt_fro": 38.525658, + "heavy_tail_ratio": 1.0447, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.044952, + "mean_fro": 35.586983, + "rms_per_prompt_fro": 36.378044, + "max_per_prompt_fro": 37.682678, + "heavy_tail_ratio": 1.0359, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.022197, + "mean_fro": 34.87587, + "rms_per_prompt_fro": 35.260815, + "max_per_prompt_fro": 35.923981, + "heavy_tail_ratio": 1.0188, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 153.5, + "fit_secs_per_prompt": 3.069, + "total_wall_seconds": 160.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.6622, + "top_sv": 2.0647, + "stable_rank": 21.9 + }, + { + "layer": 1, + "fro_norm": 8.9132, + "top_sv": 1.8871, + "stable_rank": 22.31 + }, + { + "layer": 2, + "fro_norm": 9.6134, + "top_sv": 2.0813, + "stable_rank": 21.335 + }, + { + "layer": 3, + "fro_norm": 9.1215, + "top_sv": 2.0143, + "stable_rank": 20.506 + }, + { + "layer": 4, + "fro_norm": 10.2273, + "top_sv": 2.2191, + "stable_rank": 21.241 + }, + { + "layer": 5, + "fro_norm": 10.2941, + "top_sv": 2.3648, + "stable_rank": 18.949 + }, + { + "layer": 6, + "fro_norm": 10.5906, + "top_sv": 2.2212, + "stable_rank": 22.734 + }, + { + "layer": 7, + "fro_norm": 11.5987, + "top_sv": 2.4827, + "stable_rank": 21.826 + }, + { + "layer": 8, + "fro_norm": 13.1946, + "top_sv": 2.9423, + "stable_rank": 20.111 + }, + { + "layer": 9, + "fro_norm": 14.7185, + "top_sv": 3.3379, + "stable_rank": 19.444 + }, + { + "layer": 10, + "fro_norm": 16.5285, + "top_sv": 3.847, + "stable_rank": 18.46 + }, + { + "layer": 11, + "fro_norm": 19.1644, + "top_sv": 4.4932, + "stable_rank": 18.192 + }, + { + "layer": 12, + "fro_norm": 20.9881, + "top_sv": 4.7415, + "stable_rank": 19.594 + }, + { + "layer": 13, + "fro_norm": 22.9256, + "top_sv": 5.2216, + "stable_rank": 19.277 + }, + { + "layer": 14, + "fro_norm": 26.3609, + "top_sv": 5.9493, + "stable_rank": 19.633 + }, + { + "layer": 15, + "fro_norm": 29.4224, + "top_sv": 6.7879, + "stable_rank": 18.788 + }, + { + "layer": 16, + "fro_norm": 32.5149, + "top_sv": 7.6843, + "stable_rank": 17.904 + }, + { + "layer": 17, + "fro_norm": 34.6351, + "top_sv": 8.1782, + "stable_rank": 17.936 + }, + { + "layer": 18, + "fro_norm": 35.934, + "top_sv": 8.428, + "stable_rank": 18.179 + }, + { + "layer": 19, + "fro_norm": 35.5819, + "top_sv": 8.0681, + "stable_rank": 19.45 + }, + { + "layer": 20, + "fro_norm": 35.5858, + "top_sv": 7.4724, + "stable_rank": 22.68 + }, + { + "layer": 21, + "fro_norm": 34.8743, + "top_sv": 5.8161, + "stable_rank": 35.954 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..b93b5c32be62c662f4c0d445fc37031901220ad2 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02e2b60567d2a6578ce15cec0d0c1bcde20fc7831b6df4df0914c15c944d5f70 +size 46145145 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e431fea1bc08873b53f4dad84867182e5db5de81 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n50_s1.pt", + "sha256": "02e2b60567d2a6578ce15cec0d0c1bcde20fc7831b6df4df0914c15c944d5f70", + "bytes": 46145145, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.550163, + "mean_fro": 9.92529, + "rms_per_prompt_fro": 15.849925, + "max_per_prompt_fro": 21.536518, + "heavy_tail_ratio": 1.3588, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.380351, + "mean_fro": 9.142516, + "rms_per_prompt_fro": 14.105427, + "max_per_prompt_fro": 18.718309, + "heavy_tail_ratio": 1.327, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.143236, + "mean_fro": 9.849177, + "rms_per_prompt_fro": 14.418995, + "max_per_prompt_fro": 18.897516, + "heavy_tail_ratio": 1.3106, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.027999, + "mean_fro": 9.308519, + "rms_per_prompt_fro": 13.256059, + "max_per_prompt_fro": 18.290125, + "heavy_tail_ratio": 1.3798, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.855217, + "mean_fro": 10.449302, + "rms_per_prompt_fro": 14.232611, + "max_per_prompt_fro": 19.738934, + "heavy_tail_ratio": 1.3869, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.784444, + "mean_fro": 10.540156, + "rms_per_prompt_fro": 14.079866, + "max_per_prompt_fro": 19.655035, + "heavy_tail_ratio": 1.396, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.694673, + "mean_fro": 10.84265, + "rms_per_prompt_fro": 14.114918, + "max_per_prompt_fro": 17.305576, + "heavy_tail_ratio": 1.226, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.615873, + "mean_fro": 11.851773, + "rms_per_prompt_fro": 15.065618, + "max_per_prompt_fro": 18.091373, + "heavy_tail_ratio": 1.2008, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.563398, + "mean_fro": 13.515463, + "rms_per_prompt_fro": 16.899182, + "max_per_prompt_fro": 19.955797, + "heavy_tail_ratio": 1.1809, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.522204, + "mean_fro": 15.072774, + "rms_per_prompt_fro": 18.596433, + "max_per_prompt_fro": 21.773565, + "heavy_tail_ratio": 1.1708, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.479791, + "mean_fro": 16.923584, + "rms_per_prompt_fro": 20.586975, + "max_per_prompt_fro": 23.691769, + "heavy_tail_ratio": 1.1508, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.435216, + "mean_fro": 19.614803, + "rms_per_prompt_fro": 23.498635, + "max_per_prompt_fro": 26.581158, + "heavy_tail_ratio": 1.1312, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.356264, + "mean_fro": 21.288727, + "rms_per_prompt_fro": 24.792584, + "max_per_prompt_fro": 28.123104, + "heavy_tail_ratio": 1.1343, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.319296, + "mean_fro": 23.226304, + "rms_per_prompt_fro": 26.677876, + "max_per_prompt_fro": 29.974928, + "heavy_tail_ratio": 1.1236, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.277101, + "mean_fro": 26.661721, + "rms_per_prompt_fro": 30.130112, + "max_per_prompt_fro": 33.409733, + "heavy_tail_ratio": 1.1088, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.240209, + "mean_fro": 29.706296, + "rms_per_prompt_fro": 33.082314, + "max_per_prompt_fro": 36.172672, + "heavy_tail_ratio": 1.0934, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.174126, + "mean_fro": 32.657166, + "rms_per_prompt_fro": 35.386352, + "max_per_prompt_fro": 38.707355, + "heavy_tail_ratio": 1.0938, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.137476, + "mean_fro": 34.750214, + "rms_per_prompt_fro": 37.061983, + "max_per_prompt_fro": 40.091644, + "heavy_tail_ratio": 1.0817, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.106992, + "mean_fro": 36.037751, + "rms_per_prompt_fro": 37.916648, + "max_per_prompt_fro": 40.512966, + "heavy_tail_ratio": 1.0685, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.07485, + "mean_fro": 35.657284, + "rms_per_prompt_fro": 36.967686, + "max_per_prompt_fro": 38.84742, + "heavy_tail_ratio": 1.0508, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.045164, + "mean_fro": 35.630497, + "rms_per_prompt_fro": 36.426228, + "max_per_prompt_fro": 37.550762, + "heavy_tail_ratio": 1.0309, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.02213, + "mean_fro": 34.916381, + "rms_per_prompt_fro": 35.30062, + "max_per_prompt_fro": 36.007542, + "heavy_tail_ratio": 1.02, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 156.0, + "fit_secs_per_prompt": 3.12, + "total_wall_seconds": 165.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.9252, + "top_sv": 2.2298, + "stable_rank": 19.812 + }, + { + "layer": 1, + "fro_norm": 9.1424, + "top_sv": 1.9825, + "stable_rank": 21.266 + }, + { + "layer": 2, + "fro_norm": 9.8491, + "top_sv": 2.1877, + "stable_rank": 20.267 + }, + { + "layer": 3, + "fro_norm": 9.3084, + "top_sv": 2.058, + "stable_rank": 20.458 + }, + { + "layer": 4, + "fro_norm": 10.4492, + "top_sv": 2.2964, + "stable_rank": 20.704 + }, + { + "layer": 5, + "fro_norm": 10.54, + "top_sv": 2.4375, + "stable_rank": 18.698 + }, + { + "layer": 6, + "fro_norm": 10.8425, + "top_sv": 2.3184, + "stable_rank": 21.872 + }, + { + "layer": 7, + "fro_norm": 11.8516, + "top_sv": 2.5434, + "stable_rank": 21.713 + }, + { + "layer": 8, + "fro_norm": 13.5153, + "top_sv": 3.0257, + "stable_rank": 19.953 + }, + { + "layer": 9, + "fro_norm": 15.0726, + "top_sv": 3.4229, + "stable_rank": 19.39 + }, + { + "layer": 10, + "fro_norm": 16.9233, + "top_sv": 3.9141, + "stable_rank": 18.694 + }, + { + "layer": 11, + "fro_norm": 19.6145, + "top_sv": 4.5741, + "stable_rank": 18.388 + }, + { + "layer": 12, + "fro_norm": 21.2885, + "top_sv": 4.751, + "stable_rank": 20.078 + }, + { + "layer": 13, + "fro_norm": 23.226, + "top_sv": 5.2385, + "stable_rank": 19.658 + }, + { + "layer": 14, + "fro_norm": 26.6613, + "top_sv": 5.9674, + "stable_rank": 19.962 + }, + { + "layer": 15, + "fro_norm": 29.7059, + "top_sv": 6.803, + "stable_rank": 19.067 + }, + { + "layer": 16, + "fro_norm": 32.6566, + "top_sv": 7.6789, + "stable_rank": 18.086 + }, + { + "layer": 17, + "fro_norm": 34.7495, + "top_sv": 8.1844, + "stable_rank": 18.027 + }, + { + "layer": 18, + "fro_norm": 36.0369, + "top_sv": 8.4011, + "stable_rank": 18.4 + }, + { + "layer": 19, + "fro_norm": 35.6562, + "top_sv": 8.0552, + "stable_rank": 19.594 + }, + { + "layer": 20, + "fro_norm": 35.6293, + "top_sv": 7.4506, + "stable_rank": 22.868 + }, + { + "layer": 21, + "fro_norm": 34.9148, + "top_sv": 5.8, + "stable_rank": 36.238 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..23a11c0e31d6c03b49f6a89acb8105b6aa13a8f3 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36e82a189185caa8b3b0d97605c5118bb554b516f78a2fd919c9dfc51acd2ef6 +size 46145145 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9f2917fb0d3b43fcfc0dd0c9ba31afec6277a16a --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_410m_n50_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_410m_n50_s2.pt", + "sha256": "36e82a189185caa8b3b0d97605c5118bb554b516f78a2fd919c9dfc51acd2ef6", + "bytes": 46145145, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 1.639921, + "mean_fro": 10.002806, + "rms_per_prompt_fro": 16.252391, + "max_per_prompt_fro": 21.137394, + "heavy_tail_ratio": 1.3006, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.459667, + "mean_fro": 9.227162, + "rms_per_prompt_fro": 14.47126, + "max_per_prompt_fro": 18.507341, + "heavy_tail_ratio": 1.2789, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.20046, + "mean_fro": 9.92445, + "rms_per_prompt_fro": 14.721876, + "max_per_prompt_fro": 18.640663, + "heavy_tail_ratio": 1.2662, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.063695, + "mean_fro": 9.376935, + "rms_per_prompt_fro": 13.470497, + "max_per_prompt_fro": 16.709667, + "heavy_tail_ratio": 1.2405, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.881439, + "mean_fro": 10.509025, + "rms_per_prompt_fro": 14.414764, + "max_per_prompt_fro": 17.622261, + "heavy_tail_ratio": 1.2225, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.803375, + "mean_fro": 10.588451, + "rms_per_prompt_fro": 14.219211, + "max_per_prompt_fro": 17.23596, + "heavy_tail_ratio": 1.2122, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.728426, + "mean_fro": 10.898966, + "rms_per_prompt_fro": 14.328829, + "max_per_prompt_fro": 17.290504, + "heavy_tail_ratio": 1.2067, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.649899, + "mean_fro": 11.882256, + "rms_per_prompt_fro": 15.262568, + "max_per_prompt_fro": 18.175869, + "heavy_tail_ratio": 1.1909, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.597631, + "mean_fro": 13.494507, + "rms_per_prompt_fro": 17.05671, + "max_per_prompt_fro": 20.239119, + "heavy_tail_ratio": 1.1866, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.555204, + "mean_fro": 15.017488, + "rms_per_prompt_fro": 18.727984, + "max_per_prompt_fro": 22.413094, + "heavy_tail_ratio": 1.1968, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.511751, + "mean_fro": 16.831108, + "rms_per_prompt_fro": 20.694399, + "max_per_prompt_fro": 24.555136, + "heavy_tail_ratio": 1.1866, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.465234, + "mean_fro": 19.486353, + "rms_per_prompt_fro": 23.587613, + "max_per_prompt_fro": 27.345827, + "heavy_tail_ratio": 1.1593, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.381533, + "mean_fro": 21.171262, + "rms_per_prompt_fro": 24.884414, + "max_per_prompt_fro": 28.499649, + "heavy_tail_ratio": 1.1453, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.342623, + "mean_fro": 23.132332, + "rms_per_prompt_fro": 26.803805, + "max_per_prompt_fro": 30.125408, + "heavy_tail_ratio": 1.1239, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.294981, + "mean_fro": 26.487161, + "rms_per_prompt_fro": 30.141654, + "max_per_prompt_fro": 33.699787, + "heavy_tail_ratio": 1.118, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.255568, + "mean_fro": 29.471664, + "rms_per_prompt_fro": 33.023631, + "max_per_prompt_fro": 36.839882, + "heavy_tail_ratio": 1.1156, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.186741, + "mean_fro": 32.490905, + "rms_per_prompt_fro": 35.394822, + "max_per_prompt_fro": 39.002068, + "heavy_tail_ratio": 1.1019, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.146679, + "mean_fro": 34.561508, + "rms_per_prompt_fro": 37.009535, + "max_per_prompt_fro": 39.963058, + "heavy_tail_ratio": 1.0798, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.112741, + "mean_fro": 35.832912, + "rms_per_prompt_fro": 37.798906, + "max_per_prompt_fro": 40.203117, + "heavy_tail_ratio": 1.0636, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.079124, + "mean_fro": 35.511199, + "rms_per_prompt_fro": 36.889354, + "max_per_prompt_fro": 38.337688, + "heavy_tail_ratio": 1.0393, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.047182, + "mean_fro": 35.549323, + "rms_per_prompt_fro": 36.378303, + "max_per_prompt_fro": 37.362934, + "heavy_tail_ratio": 1.0271, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.023515, + "mean_fro": 34.876685, + "rms_per_prompt_fro": 35.284365, + "max_per_prompt_fro": 36.064835, + "heavy_tail_ratio": 1.0221, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 152.7, + "fit_secs_per_prompt": 3.055, + "total_wall_seconds": 162.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 10.0027, + "top_sv": 2.2194, + "stable_rank": 20.313 + }, + { + "layer": 1, + "fro_norm": 9.227, + "top_sv": 1.9732, + "stable_rank": 21.867 + }, + { + "layer": 2, + "fro_norm": 9.9243, + "top_sv": 2.1682, + "stable_rank": 20.95 + }, + { + "layer": 3, + "fro_norm": 9.3768, + "top_sv": 2.0284, + "stable_rank": 21.369 + }, + { + "layer": 4, + "fro_norm": 10.5089, + "top_sv": 2.2602, + "stable_rank": 21.617 + }, + { + "layer": 5, + "fro_norm": 10.5883, + "top_sv": 2.3922, + "stable_rank": 19.592 + }, + { + "layer": 6, + "fro_norm": 10.8988, + "top_sv": 2.3136, + "stable_rank": 22.192 + }, + { + "layer": 7, + "fro_norm": 11.8821, + "top_sv": 2.5448, + "stable_rank": 21.801 + }, + { + "layer": 8, + "fro_norm": 13.4943, + "top_sv": 3.0098, + "stable_rank": 20.102 + }, + { + "layer": 9, + "fro_norm": 15.0173, + "top_sv": 3.3994, + "stable_rank": 19.515 + }, + { + "layer": 10, + "fro_norm": 16.8309, + "top_sv": 3.9006, + "stable_rank": 18.619 + }, + { + "layer": 11, + "fro_norm": 19.486, + "top_sv": 4.5618, + "stable_rank": 18.246 + }, + { + "layer": 12, + "fro_norm": 21.171, + "top_sv": 4.7605, + "stable_rank": 19.778 + }, + { + "layer": 13, + "fro_norm": 23.132, + "top_sv": 5.2411, + "stable_rank": 19.48 + }, + { + "layer": 14, + "fro_norm": 26.4867, + "top_sv": 5.9418, + "stable_rank": 19.871 + }, + { + "layer": 15, + "fro_norm": 29.4712, + "top_sv": 6.7449, + "stable_rank": 19.092 + }, + { + "layer": 16, + "fro_norm": 32.4903, + "top_sv": 7.663, + "stable_rank": 17.977 + }, + { + "layer": 17, + "fro_norm": 34.5607, + "top_sv": 8.1486, + "stable_rank": 17.989 + }, + { + "layer": 18, + "fro_norm": 35.832, + "top_sv": 8.3737, + "stable_rank": 18.311 + }, + { + "layer": 19, + "fro_norm": 35.5101, + "top_sv": 8.0023, + "stable_rank": 19.691 + }, + { + "layer": 20, + "fro_norm": 35.5481, + "top_sv": 7.4219, + "stable_rank": 22.941 + }, + { + "layer": 21, + "fro_norm": 34.8751, + "top_sv": 5.7782, + "stable_rank": 36.429 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b640f5b2c8218dfc38f5fd94418d27ccaa41af5 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dce47af9d07fca3c7ea3ab5f28f95f5d1d2403a9cd43d6daff2181f6268eff3 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..1fe4dbd8b53e27f1d6439102a90e5beae1d49354 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n100_s0.pt", + "sha256": "4dce47af9d07fca3c7ea3ab5f28f95f5d1d2403a9cd43d6daff2181f6268eff3", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.193779, + "mean_fro": 33.292756, + "rms_per_prompt_fro": 36.375733, + "max_per_prompt_fro": 46.730221, + "heavy_tail_ratio": 1.2847, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.142143, + "mean_fro": 26.024441, + "rms_per_prompt_fro": 27.812604, + "max_per_prompt_fro": 31.047367, + "heavy_tail_ratio": 1.1163, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.100582, + "mean_fro": 24.943307, + "rms_per_prompt_fro": 26.167676, + "max_per_prompt_fro": 29.04269, + "heavy_tail_ratio": 1.1099, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.050461, + "mean_fro": 22.809385, + "rms_per_prompt_fro": 23.3778, + "max_per_prompt_fro": 24.286463, + "heavy_tail_ratio": 1.0389, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.2, + "fit_secs_per_prompt": 0.112, + "total_wall_seconds": 16.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.2927, + "top_sv": 5.6936, + "stable_rank": 34.192 + }, + { + "layer": 1, + "fro_norm": 26.0244, + "top_sv": 4.4826, + "stable_rank": 33.706 + }, + { + "layer": 2, + "fro_norm": 24.9433, + "top_sv": 3.6818, + "stable_rank": 45.897 + }, + { + "layer": 3, + "fro_norm": 22.8093, + "top_sv": 4.4203, + "stable_rank": 26.627 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..42a60b25793008461e58492ae7c6ef5b9242639d --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55269aaec6ca8db12a9d7767688d397003dbde3b5b74fcc469d51696e0175bd0 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7ee468788c76df50fa5f9b7c6a64b13bd89c272c --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n100_s1.pt", + "sha256": "55269aaec6ca8db12a9d7767688d397003dbde3b5b74fcc469d51696e0175bd0", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.17837, + "mean_fro": 32.918134, + "rms_per_prompt_fro": 35.733534, + "max_per_prompt_fro": 40.525749, + "heavy_tail_ratio": 1.1341, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.13294, + "mean_fro": 25.856655, + "rms_per_prompt_fro": 27.52173, + "max_per_prompt_fro": 30.952721, + "heavy_tail_ratio": 1.1247, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.093018, + "mean_fro": 24.837102, + "rms_per_prompt_fro": 25.966567, + "max_per_prompt_fro": 28.702723, + "heavy_tail_ratio": 1.1054, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.04588, + "mean_fro": 22.794845, + "rms_per_prompt_fro": 23.311894, + "max_per_prompt_fro": 24.535908, + "heavy_tail_ratio": 1.0525, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.1, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 16.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.9181, + "top_sv": 5.7853, + "stable_rank": 32.376 + }, + { + "layer": 1, + "fro_norm": 25.8566, + "top_sv": 4.4825, + "stable_rank": 33.273 + }, + { + "layer": 2, + "fro_norm": 24.8371, + "top_sv": 3.6928, + "stable_rank": 45.238 + }, + { + "layer": 3, + "fro_norm": 22.7948, + "top_sv": 4.4131, + "stable_rank": 26.68 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd10658335f1cf663389e27131b9d99f84bcae19 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30dbd33314dc2e0adc3fcc0b59b9c2f012cbf45afaea5a409e4270633083edce +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e9c45381f7fb0d669405913e3a08f68163682b7c --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n100_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n100_s2.pt", + "sha256": "30dbd33314dc2e0adc3fcc0b59b9c2f012cbf45afaea5a409e4270633083edce", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.182175, + "mean_fro": 33.368286, + "rms_per_prompt_fro": 36.280629, + "max_per_prompt_fro": 42.426418, + "heavy_tail_ratio": 1.1694, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.136672, + "mean_fro": 26.167131, + "rms_per_prompt_fro": 27.898043, + "max_per_prompt_fro": 31.799965, + "heavy_tail_ratio": 1.1399, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.09566, + "mean_fro": 25.060742, + "rms_per_prompt_fro": 26.232019, + "max_per_prompt_fro": 28.937706, + "heavy_tail_ratio": 1.1031, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.047129, + "mean_fro": 22.856408, + "rms_per_prompt_fro": 23.388804, + "max_per_prompt_fro": 24.356796, + "heavy_tail_ratio": 1.0414, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.9, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 18.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.3682, + "top_sv": 5.802, + "stable_rank": 33.076 + }, + { + "layer": 1, + "fro_norm": 26.1671, + "top_sv": 4.5267, + "stable_rank": 33.415 + }, + { + "layer": 2, + "fro_norm": 25.0607, + "top_sv": 3.7146, + "stable_rank": 45.516 + }, + { + "layer": 3, + "fro_norm": 22.8564, + "top_sv": 4.4069, + "stable_rank": 26.9 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0695adfffbe7b72f96cc456470c75c2533b634b --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39a7873e0c4d9609a6aec275305586c7508f7f049b0430c5b04e5d58bc2b6657 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ac7d86557a1f50de81b1c747d128cbb5f561c5b8 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n200_s0.pt", + "sha256": "39a7873e0c4d9609a6aec275305586c7508f7f049b0430c5b04e5d58bc2b6657", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.187063, + "mean_fro": 33.26806, + "rms_per_prompt_fro": 36.246362, + "max_per_prompt_fro": 46.730221, + "heavy_tail_ratio": 1.2892, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.139269, + "mean_fro": 26.025003, + "rms_per_prompt_fro": 27.77819, + "max_per_prompt_fro": 31.52582, + "heavy_tail_ratio": 1.1349, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.098657, + "mean_fro": 24.935274, + "rms_per_prompt_fro": 26.136364, + "max_per_prompt_fro": 29.04269, + "heavy_tail_ratio": 1.1112, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.049102, + "mean_fro": 22.821285, + "rms_per_prompt_fro": 23.37486, + "max_per_prompt_fro": 24.936337, + "heavy_tail_ratio": 1.0668, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 22.0, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 27.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.268, + "top_sv": 5.7064, + "stable_rank": 33.988 + }, + { + "layer": 1, + "fro_norm": 26.025, + "top_sv": 4.4742, + "stable_rank": 33.834 + }, + { + "layer": 2, + "fro_norm": 24.9352, + "top_sv": 3.6739, + "stable_rank": 46.065 + }, + { + "layer": 3, + "fro_norm": 22.8212, + "top_sv": 4.4293, + "stable_rank": 26.546 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..52e10b623586513a4971646aac64a44b6d3c6d14 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd2b8d183898a85123f5c3bba871bb07147a3cab16ea56d08397f3894b2f278 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f2eb15fa01936076aacef381c1b4b8b5011d1b01 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n200_s1.pt", + "sha256": "9cd2b8d183898a85123f5c3bba871bb07147a3cab16ea56d08397f3894b2f278", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.185175, + "mean_fro": 32.884625, + "rms_per_prompt_fro": 35.800096, + "max_per_prompt_fro": 42.516273, + "heavy_tail_ratio": 1.1876, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.137994, + "mean_fro": 25.820141, + "rms_per_prompt_fro": 27.544102, + "max_per_prompt_fro": 32.264908, + "heavy_tail_ratio": 1.1714, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.096952, + "mean_fro": 24.822392, + "rms_per_prompt_fro": 25.99785, + "max_per_prompt_fro": 29.580742, + "heavy_tail_ratio": 1.1378, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.048087, + "mean_fro": 22.769587, + "rms_per_prompt_fro": 23.310618, + "max_per_prompt_fro": 24.990871, + "heavy_tail_ratio": 1.0721, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 22.0, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 27.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.8846, + "top_sv": 5.7186, + "stable_rank": 33.067 + }, + { + "layer": 1, + "fro_norm": 25.8201, + "top_sv": 4.4809, + "stable_rank": 33.203 + }, + { + "layer": 2, + "fro_norm": 24.8224, + "top_sv": 3.6909, + "stable_rank": 45.23 + }, + { + "layer": 3, + "fro_norm": 22.7695, + "top_sv": 4.4073, + "stable_rank": 26.691 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc289159d554dbc055530534145c66c638e797bc --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58a24bc75783b4c955660a48dab07303177ebb283ed9371e25a09a06acacb4d1 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..955117f8f63e491bb58ccf69550f060ea3b0ca6b --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n200_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n200_s2.pt", + "sha256": "58a24bc75783b4c955660a48dab07303177ebb283ed9371e25a09a06acacb4d1", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.180217, + "mean_fro": 33.414849, + "rms_per_prompt_fro": 36.301147, + "max_per_prompt_fro": 42.426418, + "heavy_tail_ratio": 1.1687, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.134245, + "mean_fro": 26.199702, + "rms_per_prompt_fro": 27.902925, + "max_per_prompt_fro": 32.261349, + "heavy_tail_ratio": 1.1562, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.094195, + "mean_fro": 25.080532, + "rms_per_prompt_fro": 26.23518, + "max_per_prompt_fro": 29.832989, + "heavy_tail_ratio": 1.1371, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.046111, + "mean_fro": 22.868864, + "rms_per_prompt_fro": 23.390173, + "max_per_prompt_fro": 24.639008, + "heavy_tail_ratio": 1.0534, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 23.4, + "fit_secs_per_prompt": 0.117, + "total_wall_seconds": 30.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.4148, + "top_sv": 5.8866, + "stable_rank": 32.221 + }, + { + "layer": 1, + "fro_norm": 26.1997, + "top_sv": 4.5182, + "stable_rank": 33.625 + }, + { + "layer": 2, + "fro_norm": 25.0805, + "top_sv": 3.7123, + "stable_rank": 45.645 + }, + { + "layer": 3, + "fro_norm": 22.8688, + "top_sv": 4.4213, + "stable_rank": 26.754 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..82a81be2a774bafd42a68fce9b028427153c6c9f --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda0a62dc263c99a2b0a0d068acbe049a1a9461092db0ca10016d07c05bd8bb0 +size 2099927 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a359ba6ec77e834ef64b6f6fdea8cfd92e0aa60d --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n25_s0.pt", + "sha256": "eda0a62dc263c99a2b0a0d068acbe049a1a9461092db0ca10016d07c05bd8bb0", + "bytes": 2099927, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.19006, + "mean_fro": 32.718037, + "rms_per_prompt_fro": 35.692068, + "max_per_prompt_fro": 39.87569, + "heavy_tail_ratio": 1.1172, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.145377, + "mean_fro": 25.674314, + "rms_per_prompt_fro": 27.477234, + "max_per_prompt_fro": 30.609146, + "heavy_tail_ratio": 1.114, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.104153, + "mean_fro": 24.720814, + "rms_per_prompt_fro": 25.976304, + "max_per_prompt_fro": 28.256298, + "heavy_tail_ratio": 1.0878, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.053676, + "mean_fro": 22.760631, + "rms_per_prompt_fro": 23.363499, + "max_per_prompt_fro": 24.236881, + "heavy_tail_ratio": 1.0374, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.0, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 7.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.718, + "top_sv": 5.8109, + "stable_rank": 31.702 + }, + { + "layer": 1, + "fro_norm": 25.6743, + "top_sv": 4.43, + "stable_rank": 33.588 + }, + { + "layer": 2, + "fro_norm": 24.7208, + "top_sv": 3.7303, + "stable_rank": 43.918 + }, + { + "layer": 3, + "fro_norm": 22.7606, + "top_sv": 4.4734, + "stable_rank": 25.887 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..6706fb5cbe589bd0ca8501ef0d4c78f6af13bb99 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbdaa281c2cf024ab8ed65fb5c056ff5c36a00931288b7d7fe31bbc5b1e29e54 +size 2099927 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d70f67fddbc3dc7109f449634255c553c2b4e1f7 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n25_s1.pt", + "sha256": "cbdaa281c2cf024ab8ed65fb5c056ff5c36a00931288b7d7fe31bbc5b1e29e54", + "bytes": 2099927, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.165253, + "mean_fro": 33.195641, + "rms_per_prompt_fro": 35.833654, + "max_per_prompt_fro": 40.30677, + "heavy_tail_ratio": 1.1248, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.124359, + "mean_fro": 26.084238, + "rms_per_prompt_fro": 27.65863, + "max_per_prompt_fro": 30.952721, + "heavy_tail_ratio": 1.1191, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.08808, + "mean_fro": 24.99236, + "rms_per_prompt_fro": 26.069803, + "max_per_prompt_fro": 28.702723, + "heavy_tail_ratio": 1.101, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.041686, + "mean_fro": 22.9172, + "rms_per_prompt_fro": 23.389986, + "max_per_prompt_fro": 24.253162, + "heavy_tail_ratio": 1.0369, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.0, + "fit_secs_per_prompt": 0.12, + "total_wall_seconds": 8.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.1956, + "top_sv": 6.0723, + "stable_rank": 29.885 + }, + { + "layer": 1, + "fro_norm": 26.0842, + "top_sv": 4.5165, + "stable_rank": 33.355 + }, + { + "layer": 2, + "fro_norm": 24.9923, + "top_sv": 3.7562, + "stable_rank": 44.27 + }, + { + "layer": 3, + "fro_norm": 22.9172, + "top_sv": 4.475, + "stable_rank": 26.226 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..b08df1356e5933499cd1612fbab680499d3df5a6 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87f0395b1b967ca436bb3b601317f478b3a60eaf78f7a4d3fe50c45fb1287eb5 +size 2099927 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a18d9402bf3075852c70aabfbfb8210e6db44460 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n25_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n25_s2.pt", + "sha256": "87f0395b1b967ca436bb3b601317f478b3a60eaf78f7a4d3fe50c45fb1287eb5", + "bytes": 2099927, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.17703, + "mean_fro": 33.271764, + "rms_per_prompt_fro": 36.096878, + "max_per_prompt_fro": 40.319595, + "heavy_tail_ratio": 1.117, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.132841, + "mean_fro": 26.214056, + "rms_per_prompt_fro": 27.900936, + "max_per_prompt_fro": 30.90568, + "heavy_tail_ratio": 1.1077, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.093272, + "mean_fro": 25.108057, + "rms_per_prompt_fro": 26.252895, + "max_per_prompt_fro": 28.937706, + "heavy_tail_ratio": 1.1023, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.045204, + "mean_fro": 22.857938, + "rms_per_prompt_fro": 23.368865, + "max_per_prompt_fro": 24.334276, + "heavy_tail_ratio": 1.0413, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.3, + "fit_secs_per_prompt": 0.131, + "total_wall_seconds": 9.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.2717, + "top_sv": 5.8733, + "stable_rank": 32.091 + }, + { + "layer": 1, + "fro_norm": 26.214, + "top_sv": 4.5284, + "stable_rank": 33.511 + }, + { + "layer": 2, + "fro_norm": 25.108, + "top_sv": 3.724, + "stable_rank": 45.459 + }, + { + "layer": 3, + "fro_norm": 22.8579, + "top_sv": 4.4197, + "stable_rank": 26.747 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d84bbc49f0b2397089194c074836e592cd99f3ba --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574ae9e09c17e469c14104999d66fe98b86491cb4d4d7b44c969ebd5d0f8a246 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..852be89c9ded64bcb3779f07c2b75657b96e53b8 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n400_s0.pt", + "sha256": "574ae9e09c17e469c14104999d66fe98b86491cb4d4d7b44c969ebd5d0f8a246", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.183626, + "mean_fro": 33.222911, + "rms_per_prompt_fro": 36.144721, + "max_per_prompt_fro": 46.730221, + "heavy_tail_ratio": 1.2929, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.137246, + "mean_fro": 26.015402, + "rms_per_prompt_fro": 27.743271, + "max_per_prompt_fro": 31.52582, + "heavy_tail_ratio": 1.1363, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.096553, + "mean_fro": 24.936687, + "rms_per_prompt_fro": 26.112809, + "max_per_prompt_fro": 29.04269, + "heavy_tail_ratio": 1.1122, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.047754, + "mean_fro": 22.816902, + "rms_per_prompt_fro": 23.35535, + "max_per_prompt_fro": 24.936337, + "heavy_tail_ratio": 1.0677, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 43.7, + "fit_secs_per_prompt": 0.109, + "total_wall_seconds": 48.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.2228, + "top_sv": 5.7287, + "stable_rank": 33.632 + }, + { + "layer": 1, + "fro_norm": 26.0154, + "top_sv": 4.4862, + "stable_rank": 33.628 + }, + { + "layer": 2, + "fro_norm": 24.9366, + "top_sv": 3.6862, + "stable_rank": 45.762 + }, + { + "layer": 3, + "fro_norm": 22.8169, + "top_sv": 4.4087, + "stable_rank": 26.784 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..0416a219bc8a02dec2db0718f72594e072294c6c --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a573e328a5f61071be015d456fd6ff84fbbdaa15825538bb58f4f002dd28d056 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..1e1de65014fd4e430c907c66c84dd5963413d781 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n400_s1.pt", + "sha256": "a573e328a5f61071be015d456fd6ff84fbbdaa15825538bb58f4f002dd28d056", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.190252, + "mean_fro": 32.930004, + "rms_per_prompt_fro": 35.926203, + "max_per_prompt_fro": 42.573498, + "heavy_tail_ratio": 1.185, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.142064, + "mean_fro": 25.816459, + "rms_per_prompt_fro": 27.589379, + "max_per_prompt_fro": 32.264908, + "heavy_tail_ratio": 1.1695, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.099653, + "mean_fro": 24.808544, + "rms_per_prompt_fro": 26.015315, + "max_per_prompt_fro": 29.753614, + "heavy_tail_ratio": 1.1437, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.049559, + "mean_fro": 22.759204, + "rms_per_prompt_fro": 23.316351, + "max_per_prompt_fro": 24.990871, + "heavy_tail_ratio": 1.0718, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 44.2, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 50.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.93, + "top_sv": 5.6916, + "stable_rank": 33.474 + }, + { + "layer": 1, + "fro_norm": 25.8164, + "top_sv": 4.4742, + "stable_rank": 33.294 + }, + { + "layer": 2, + "fro_norm": 24.8085, + "top_sv": 3.6817, + "stable_rank": 45.406 + }, + { + "layer": 3, + "fro_norm": 22.7592, + "top_sv": 4.4025, + "stable_rank": 26.725 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..34bbcc4e90f6a59214d907fc6897ca2fc7daf1e6 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db924e59c27de003785f19abc31810c70613d1d1d924044bd3dea6baaf484ec0 +size 2099937 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..477ea8562a92d8c0e5494051205c4a45cc7b9182 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n400_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n400_s2.pt", + "sha256": "db924e59c27de003785f19abc31810c70613d1d1d924044bd3dea6baaf484ec0", + "bytes": 2099937, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.18567, + "mean_fro": 33.212574, + "rms_per_prompt_fro": 36.164667, + "max_per_prompt_fro": 45.803135, + "heavy_tail_ratio": 1.2665, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.137593, + "mean_fro": 26.039346, + "rms_per_prompt_fro": 27.773049, + "max_per_prompt_fro": 32.730389, + "heavy_tail_ratio": 1.1785, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.096851, + "mean_fro": 24.982487, + "rms_per_prompt_fro": 26.164316, + "max_per_prompt_fro": 29.832989, + "heavy_tail_ratio": 1.1402, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.048325, + "mean_fro": 22.832222, + "rms_per_prompt_fro": 23.377395, + "max_per_prompt_fro": 24.850088, + "heavy_tail_ratio": 1.063, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 46.9, + "fit_secs_per_prompt": 0.117, + "total_wall_seconds": 53.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.2125, + "top_sv": 5.8101, + "stable_rank": 32.677 + }, + { + "layer": 1, + "fro_norm": 26.0393, + "top_sv": 4.4953, + "stable_rank": 33.553 + }, + { + "layer": 2, + "fro_norm": 24.9824, + "top_sv": 3.6996, + "stable_rank": 45.6 + }, + { + "layer": 3, + "fro_norm": 22.8322, + "top_sv": 4.4124, + "stable_rank": 26.776 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s0.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..643fca80ede467ea3414b57aa1518be3da34efcd --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e22172c87840636861d0e9376dd37f0434fdc7e3db35f25d331321c9e6de695 +size 2099927 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s0_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a97b60c0579464e9fd6bfacf051c165058e9d4a1 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n50_s0.pt", + "sha256": "4e22172c87840636861d0e9376dd37f0434fdc7e3db35f25d331321c9e6de695", + "bytes": 2099927, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.181109, + "mean_fro": 33.320889, + "rms_per_prompt_fro": 36.212761, + "max_per_prompt_fro": 40.244698, + "heavy_tail_ratio": 1.1113, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.137616, + "mean_fro": 26.060498, + "rms_per_prompt_fro": 27.795886, + "max_per_prompt_fro": 30.609146, + "heavy_tail_ratio": 1.1012, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.097288, + "mean_fro": 24.997942, + "rms_per_prompt_fro": 26.18572, + "max_per_prompt_fro": 28.289171, + "heavy_tail_ratio": 1.0803, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.049344, + "mean_fro": 22.833998, + "rms_per_prompt_fro": 23.39057, + "max_per_prompt_fro": 24.286463, + "heavy_tail_ratio": 1.0383, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.6, + "fit_secs_per_prompt": 0.113, + "total_wall_seconds": 10.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.3208, + "top_sv": 5.7605, + "stable_rank": 33.459 + }, + { + "layer": 1, + "fro_norm": 26.0604, + "top_sv": 4.5062, + "stable_rank": 33.446 + }, + { + "layer": 2, + "fro_norm": 24.9979, + "top_sv": 3.7128, + "stable_rank": 45.332 + }, + { + "layer": 3, + "fro_norm": 22.8339, + "top_sv": 4.4343, + "stable_rank": 26.516 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s1.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..b78450e01e9e38199d2be7522d55863975f1af71 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2578682de58f4808d7a2786f0cabe96d9686bd55f4fd4a0ead55e2755e0c9d1f +size 2099927 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s1_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..60264c50a7b192a5ac5555114ee0c6e3d86a9355 --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n50_s1.pt", + "sha256": "2578682de58f4808d7a2786f0cabe96d9686bd55f4fd4a0ead55e2755e0c9d1f", + "bytes": 2099927, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.17256, + "mean_fro": 33.321178, + "rms_per_prompt_fro": 36.081773, + "max_per_prompt_fro": 40.525749, + "heavy_tail_ratio": 1.1232, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.129175, + "mean_fro": 26.129227, + "rms_per_prompt_fro": 27.765611, + "max_per_prompt_fro": 30.952721, + "heavy_tail_ratio": 1.1148, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.09075, + "mean_fro": 25.017239, + "rms_per_prompt_fro": 26.127744, + "max_per_prompt_fro": 28.702723, + "heavy_tail_ratio": 1.0986, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.043958, + "mean_fro": 22.900102, + "rms_per_prompt_fro": 23.398007, + "max_per_prompt_fro": 24.535908, + "heavy_tail_ratio": 1.0486, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.7, + "fit_secs_per_prompt": 0.114, + "total_wall_seconds": 11.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.3211, + "top_sv": 5.9161, + "stable_rank": 31.723 + }, + { + "layer": 1, + "fro_norm": 26.1292, + "top_sv": 4.5121, + "stable_rank": 33.535 + }, + { + "layer": 2, + "fro_norm": 25.0172, + "top_sv": 3.7107, + "stable_rank": 45.454 + }, + { + "layer": 3, + "fro_norm": 22.9001, + "top_sv": 4.4357, + "stable_rank": 26.653 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s2.pt b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..b41487a1fd739a9a3e95750344ae81f19ce5d56f --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:795a74b02d7b3590fcbbd89f2cdc55eb36676e8785bc4d4923ded1cf359a2b29 +size 2099927 diff --git a/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s2_provenance.json b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a5e61344b406897620afa23d83e03ce481dd143d --- /dev/null +++ b/results/lenses/e28/e28_USPTO_Backgrounds_70m_n50_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_USPTO_Backgrounds_70m_n50_s2.pt", + "sha256": "795a74b02d7b3590fcbbd89f2cdc55eb36676e8785bc4d4923ded1cf359a2b29", + "bytes": 2099927, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "USPTO_Backgrounds", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/USPTO_Backgrounds.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.185698, + "mean_fro": 33.112861, + "rms_per_prompt_fro": 36.056518, + "max_per_prompt_fro": 40.319595, + "heavy_tail_ratio": 1.1182, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.140305, + "mean_fro": 26.009798, + "rms_per_prompt_fro": 27.774584, + "max_per_prompt_fro": 30.90568, + "heavy_tail_ratio": 1.1127, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.098811, + "mean_fro": 24.925345, + "rms_per_prompt_fro": 26.127792, + "max_per_prompt_fro": 28.937706, + "heavy_tail_ratio": 1.1075, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.048667, + "mean_fro": 22.847183, + "rms_per_prompt_fro": 23.396525, + "max_per_prompt_fro": 24.334276, + "heavy_tail_ratio": 1.0401, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 6.2, + "fit_secs_per_prompt": 0.124, + "total_wall_seconds": 12.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 33.1128, + "top_sv": 5.6669, + "stable_rank": 34.143 + }, + { + "layer": 1, + "fro_norm": 26.0098, + "top_sv": 4.4891, + "stable_rank": 33.571 + }, + { + "layer": 2, + "fro_norm": 24.9253, + "top_sv": 3.6776, + "stable_rank": 45.935 + }, + { + "layer": 3, + "fro_norm": 22.8471, + "top_sv": 4.4116, + "stable_rank": 26.82 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n100_s0.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..3551cfd269032103e3a8b3d0c4cbe50202ea0e19 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48e9a4ef08fc7636956b2b70791e71ce128c3a52ee2560b21a0109f3ca302c38 +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n100_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..026b85d3ea4f6056e14896049729f0e5076560ad --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n100_s0.pt", + "sha256": "48e9a4ef08fc7636956b2b70791e71ce128c3a52ee2560b21a0109f3ca302c38", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.698925, + "mean_fro": 6.821853, + "rms_per_prompt_fro": 13.12019, + "max_per_prompt_fro": 18.362366, + "heavy_tail_ratio": 1.3996, + "n_prompts": 100 + }, + "1": { + "dispersion": 2.412581, + "mean_fro": 6.436424, + "rms_per_prompt_fro": 11.890117, + "max_per_prompt_fro": 16.383207, + "heavy_tail_ratio": 1.3779, + "n_prompts": 100 + }, + "2": { + "dispersion": 1.957433, + "mean_fro": 7.028405, + "rms_per_prompt_fro": 12.086879, + "max_per_prompt_fro": 16.309441, + "heavy_tail_ratio": 1.3494, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.66845, + "mean_fro": 6.836937, + "rms_per_prompt_fro": 11.168405, + "max_per_prompt_fro": 15.609672, + "heavy_tail_ratio": 1.3977, + "n_prompts": 100 + }, + "4": { + "dispersion": 1.36366, + "mean_fro": 7.778315, + "rms_per_prompt_fro": 11.958537, + "max_per_prompt_fro": 16.447428, + "heavy_tail_ratio": 1.3754, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.209114, + "mean_fro": 8.0498, + "rms_per_prompt_fro": 11.964488, + "max_per_prompt_fro": 16.488127, + "heavy_tail_ratio": 1.3781, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.034312, + "mean_fro": 8.349298, + "rms_per_prompt_fro": 11.908546, + "max_per_prompt_fro": 14.691499, + "heavy_tail_ratio": 1.2337, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.91254, + "mean_fro": 9.237271, + "rms_per_prompt_fro": 12.774648, + "max_per_prompt_fro": 15.402933, + "heavy_tail_ratio": 1.2057, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.82067, + "mean_fro": 10.574228, + "rms_per_prompt_fro": 14.268041, + "max_per_prompt_fro": 17.245813, + "heavy_tail_ratio": 1.2087, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.752133, + "mean_fro": 11.914587, + "rms_per_prompt_fro": 15.77112, + "max_per_prompt_fro": 18.73131, + "heavy_tail_ratio": 1.1877, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.68568, + "mean_fro": 13.618384, + "rms_per_prompt_fro": 17.681255, + "max_per_prompt_fro": 20.712793, + "heavy_tail_ratio": 1.1715, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.62434, + "mean_fro": 16.053179, + "rms_per_prompt_fro": 20.459713, + "max_per_prompt_fro": 23.832846, + "heavy_tail_ratio": 1.1649, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.517804, + "mean_fro": 18.250843, + "rms_per_prompt_fro": 22.484891, + "max_per_prompt_fro": 25.334635, + "heavy_tail_ratio": 1.1267, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.452323, + "mean_fro": 19.900642, + "rms_per_prompt_fro": 23.982738, + "max_per_prompt_fro": 27.170097, + "heavy_tail_ratio": 1.1329, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.388499, + "mean_fro": 23.294609, + "rms_per_prompt_fro": 27.449106, + "max_per_prompt_fro": 30.900297, + "heavy_tail_ratio": 1.1257, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.332201, + "mean_fro": 26.319242, + "rms_per_prompt_fro": 30.37793, + "max_per_prompt_fro": 33.765656, + "heavy_tail_ratio": 1.1115, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.252081, + "mean_fro": 29.751001, + "rms_per_prompt_fro": 33.290311, + "max_per_prompt_fro": 36.382008, + "heavy_tail_ratio": 1.0929, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.187863, + "mean_fro": 32.125405, + "rms_per_prompt_fro": 35.013202, + "max_per_prompt_fro": 37.829273, + "heavy_tail_ratio": 1.0804, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.140696, + "mean_fro": 33.896451, + "rms_per_prompt_fro": 36.202559, + "max_per_prompt_fro": 38.206364, + "heavy_tail_ratio": 1.0553, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.096401, + "mean_fro": 34.385749, + "rms_per_prompt_fro": 36.005033, + "max_per_prompt_fro": 37.266689, + "heavy_tail_ratio": 1.035, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.054609, + "mean_fro": 34.814648, + "rms_per_prompt_fro": 35.752607, + "max_per_prompt_fro": 36.651428, + "heavy_tail_ratio": 1.0251, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.024963, + "mean_fro": 34.468175, + "rms_per_prompt_fro": 34.895741, + "max_per_prompt_fro": 35.54113, + "heavy_tail_ratio": 1.0185, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 304.2, + "fit_secs_per_prompt": 3.042, + "total_wall_seconds": 312.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.8218, + "top_sv": 1.3737, + "stable_rank": 24.66 + }, + { + "layer": 1, + "fro_norm": 6.4363, + "top_sv": 1.3246, + "stable_rank": 23.612 + }, + { + "layer": 2, + "fro_norm": 7.0283, + "top_sv": 1.5215, + "stable_rank": 21.338 + }, + { + "layer": 3, + "fro_norm": 6.8368, + "top_sv": 1.4125, + "stable_rank": 23.43 + }, + { + "layer": 4, + "fro_norm": 7.7782, + "top_sv": 1.7085, + "stable_rank": 20.726 + }, + { + "layer": 5, + "fro_norm": 8.0497, + "top_sv": 1.7781, + "stable_rank": 20.494 + }, + { + "layer": 6, + "fro_norm": 8.3492, + "top_sv": 1.721, + "stable_rank": 23.534 + }, + { + "layer": 7, + "fro_norm": 9.2372, + "top_sv": 1.8658, + "stable_rank": 24.51 + }, + { + "layer": 8, + "fro_norm": 10.5741, + "top_sv": 2.3617, + "stable_rank": 20.047 + }, + { + "layer": 9, + "fro_norm": 11.9144, + "top_sv": 2.6907, + "stable_rank": 19.607 + }, + { + "layer": 10, + "fro_norm": 13.6182, + "top_sv": 3.0431, + "stable_rank": 20.027 + }, + { + "layer": 11, + "fro_norm": 16.053, + "top_sv": 3.52, + "stable_rank": 20.799 + }, + { + "layer": 12, + "fro_norm": 18.2505, + "top_sv": 3.9297, + "stable_rank": 21.569 + }, + { + "layer": 13, + "fro_norm": 19.9003, + "top_sv": 4.2263, + "stable_rank": 22.172 + }, + { + "layer": 14, + "fro_norm": 23.2943, + "top_sv": 4.917, + "stable_rank": 22.444 + }, + { + "layer": 15, + "fro_norm": 26.3187, + "top_sv": 5.7713, + "stable_rank": 20.796 + }, + { + "layer": 16, + "fro_norm": 29.7505, + "top_sv": 6.3772, + "stable_rank": 21.763 + }, + { + "layer": 17, + "fro_norm": 32.1247, + "top_sv": 6.902, + "stable_rank": 21.664 + }, + { + "layer": 18, + "fro_norm": 33.8956, + "top_sv": 7.4304, + "stable_rank": 20.809 + }, + { + "layer": 19, + "fro_norm": 34.3847, + "top_sv": 7.3336, + "stable_rank": 21.983 + }, + { + "layer": 20, + "fro_norm": 34.8134, + "top_sv": 7.0, + "stable_rank": 24.734 + }, + { + "layer": 21, + "fro_norm": 34.4666, + "top_sv": 5.5753, + "stable_rank": 38.217 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n100_s1.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..c34ecb9a7ad225260fceb51c64c2d5339fb9859e --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e29405b8043a5564eabbadf48f310a05e6cf758c060201d6d33395b28e60892a +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n100_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..24726baa83c21324e1f72dfa5012a241998078ae --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n100_s1.pt", + "sha256": "e29405b8043a5564eabbadf48f310a05e6cf758c060201d6d33395b28e60892a", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.867106, + "mean_fro": 6.758536, + "rms_per_prompt_fro": 13.290632, + "max_per_prompt_fro": 22.19487, + "heavy_tail_ratio": 1.67, + "n_prompts": 100 + }, + "1": { + "dispersion": 2.555744, + "mean_fro": 6.371321, + "rms_per_prompt_fro": 12.014196, + "max_per_prompt_fro": 19.147593, + "heavy_tail_ratio": 1.5937, + "n_prompts": 100 + }, + "2": { + "dispersion": 2.051036, + "mean_fro": 6.978811, + "rms_per_prompt_fro": 12.190039, + "max_per_prompt_fro": 19.291433, + "heavy_tail_ratio": 1.5826, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.724687, + "mean_fro": 6.781738, + "rms_per_prompt_fro": 11.194361, + "max_per_prompt_fro": 17.139297, + "heavy_tail_ratio": 1.5311, + "n_prompts": 100 + }, + "4": { + "dispersion": 1.402455, + "mean_fro": 7.727315, + "rms_per_prompt_fro": 11.977226, + "max_per_prompt_fro": 17.873886, + "heavy_tail_ratio": 1.4923, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.245585, + "mean_fro": 7.987161, + "rms_per_prompt_fro": 11.968982, + "max_per_prompt_fro": 18.013294, + "heavy_tail_ratio": 1.505, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.067552, + "mean_fro": 8.292724, + "rms_per_prompt_fro": 11.924096, + "max_per_prompt_fro": 16.665794, + "heavy_tail_ratio": 1.3977, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.937324, + "mean_fro": 9.175485, + "rms_per_prompt_fro": 12.771154, + "max_per_prompt_fro": 16.177975, + "heavy_tail_ratio": 1.2668, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.843556, + "mean_fro": 10.507011, + "rms_per_prompt_fro": 14.266168, + "max_per_prompt_fro": 18.541143, + "heavy_tail_ratio": 1.2997, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.771581, + "mean_fro": 11.849755, + "rms_per_prompt_fro": 15.772115, + "max_per_prompt_fro": 20.537996, + "heavy_tail_ratio": 1.3022, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.702204, + "mean_fro": 13.556181, + "rms_per_prompt_fro": 17.686551, + "max_per_prompt_fro": 22.904638, + "heavy_tail_ratio": 1.295, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.639767, + "mean_fro": 15.99525, + "rms_per_prompt_fro": 20.482461, + "max_per_prompt_fro": 27.136269, + "heavy_tail_ratio": 1.3249, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.528611, + "mean_fro": 18.213119, + "rms_per_prompt_fro": 22.518157, + "max_per_prompt_fro": 27.989729, + "heavy_tail_ratio": 1.243, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.459157, + "mean_fro": 19.874159, + "rms_per_prompt_fro": 24.007102, + "max_per_prompt_fro": 29.441811, + "heavy_tail_ratio": 1.2264, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.390324, + "mean_fro": 23.247201, + "rms_per_prompt_fro": 27.41124, + "max_per_prompt_fro": 32.696583, + "heavy_tail_ratio": 1.1928, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.331003, + "mean_fro": 26.290905, + "rms_per_prompt_fro": 30.331579, + "max_per_prompt_fro": 35.626503, + "heavy_tail_ratio": 1.1746, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.251127, + "mean_fro": 29.769766, + "rms_per_prompt_fro": 33.298613, + "max_per_prompt_fro": 36.620102, + "heavy_tail_ratio": 1.0997, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.185508, + "mean_fro": 32.11486, + "rms_per_prompt_fro": 34.966996, + "max_per_prompt_fro": 37.982109, + "heavy_tail_ratio": 1.0862, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.137581, + "mean_fro": 33.871575, + "rms_per_prompt_fro": 36.126547, + "max_per_prompt_fro": 38.831398, + "heavy_tail_ratio": 1.0749, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.093496, + "mean_fro": 34.369808, + "rms_per_prompt_fro": 35.940628, + "max_per_prompt_fro": 37.647743, + "heavy_tail_ratio": 1.0475, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.053516, + "mean_fro": 34.799134, + "rms_per_prompt_fro": 35.718152, + "max_per_prompt_fro": 36.623009, + "heavy_tail_ratio": 1.0253, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.024407, + "mean_fro": 34.45575, + "rms_per_prompt_fro": 34.873705, + "max_per_prompt_fro": 35.595573, + "heavy_tail_ratio": 1.0207, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 309.3, + "fit_secs_per_prompt": 3.093, + "total_wall_seconds": 315.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7584, + "top_sv": 1.3248, + "stable_rank": 26.025 + }, + { + "layer": 1, + "fro_norm": 6.3712, + "top_sv": 1.2812, + "stable_rank": 24.729 + }, + { + "layer": 2, + "fro_norm": 6.9787, + "top_sv": 1.4659, + "stable_rank": 22.665 + }, + { + "layer": 3, + "fro_norm": 6.7816, + "top_sv": 1.3818, + "stable_rank": 24.087 + }, + { + "layer": 4, + "fro_norm": 7.7272, + "top_sv": 1.6469, + "stable_rank": 22.015 + }, + { + "layer": 5, + "fro_norm": 7.9871, + "top_sv": 1.7318, + "stable_rank": 21.272 + }, + { + "layer": 6, + "fro_norm": 8.2926, + "top_sv": 1.7175, + "stable_rank": 23.313 + }, + { + "layer": 7, + "fro_norm": 9.1754, + "top_sv": 1.8549, + "stable_rank": 24.469 + }, + { + "layer": 8, + "fro_norm": 10.5069, + "top_sv": 2.3458, + "stable_rank": 20.062 + }, + { + "layer": 9, + "fro_norm": 11.8496, + "top_sv": 2.6673, + "stable_rank": 19.736 + }, + { + "layer": 10, + "fro_norm": 13.556, + "top_sv": 3.0175, + "stable_rank": 20.182 + }, + { + "layer": 11, + "fro_norm": 15.995, + "top_sv": 3.4862, + "stable_rank": 21.051 + }, + { + "layer": 12, + "fro_norm": 18.2128, + "top_sv": 3.8873, + "stable_rank": 21.951 + }, + { + "layer": 13, + "fro_norm": 19.8738, + "top_sv": 4.1834, + "stable_rank": 22.568 + }, + { + "layer": 14, + "fro_norm": 23.2469, + "top_sv": 4.8355, + "stable_rank": 23.113 + }, + { + "layer": 15, + "fro_norm": 26.2904, + "top_sv": 5.6831, + "stable_rank": 21.401 + }, + { + "layer": 16, + "fro_norm": 29.7692, + "top_sv": 6.3069, + "stable_rank": 22.279 + }, + { + "layer": 17, + "fro_norm": 32.1142, + "top_sv": 6.8251, + "stable_rank": 22.14 + }, + { + "layer": 18, + "fro_norm": 33.8707, + "top_sv": 7.388, + "stable_rank": 21.018 + }, + { + "layer": 19, + "fro_norm": 34.3687, + "top_sv": 7.2875, + "stable_rank": 22.242 + }, + { + "layer": 20, + "fro_norm": 34.7979, + "top_sv": 6.9979, + "stable_rank": 24.727 + }, + { + "layer": 21, + "fro_norm": 34.4542, + "top_sv": 5.5994, + "stable_rank": 37.862 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n100_s2.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4c744a9927cff3b0612290f53205b2bbbd5f933 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1187f5d000b701566a366acc23a44da8a44f9030dd1844782fdb976429a34ace +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n100_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0a3f45d00ef49be1671c7fb471e97dcd3545d98d --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n100_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n100_s2.pt", + "sha256": "1187f5d000b701566a366acc23a44da8a44f9030dd1844782fdb976429a34ace", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.922895, + "mean_fro": 6.726267, + "rms_per_prompt_fro": 13.322245, + "max_per_prompt_fro": 24.485331, + "heavy_tail_ratio": 1.8379, + "n_prompts": 100 + }, + "1": { + "dispersion": 2.622747, + "mean_fro": 6.350078, + "rms_per_prompt_fro": 12.08643, + "max_per_prompt_fro": 21.961533, + "heavy_tail_ratio": 1.817, + "n_prompts": 100 + }, + "2": { + "dispersion": 2.112274, + "mean_fro": 6.947008, + "rms_per_prompt_fro": 12.255661, + "max_per_prompt_fro": 21.365519, + "heavy_tail_ratio": 1.7433, + "n_prompts": 100 + }, + "3": { + "dispersion": 1.779305, + "mean_fro": 6.757496, + "rms_per_prompt_fro": 11.265588, + "max_per_prompt_fro": 18.536762, + "heavy_tail_ratio": 1.6454, + "n_prompts": 100 + }, + "4": { + "dispersion": 1.449297, + "mean_fro": 7.687161, + "rms_per_prompt_fro": 12.030585, + "max_per_prompt_fro": 18.625755, + "heavy_tail_ratio": 1.5482, + "n_prompts": 100 + }, + "5": { + "dispersion": 1.287141, + "mean_fro": 7.956005, + "rms_per_prompt_fro": 12.032103, + "max_per_prompt_fro": 18.531601, + "heavy_tail_ratio": 1.5402, + "n_prompts": 100 + }, + "6": { + "dispersion": 1.109546, + "mean_fro": 8.261541, + "rms_per_prompt_fro": 11.999289, + "max_per_prompt_fro": 18.148132, + "heavy_tail_ratio": 1.5124, + "n_prompts": 100 + }, + "7": { + "dispersion": 0.977651, + "mean_fro": 9.146355, + "rms_per_prompt_fro": 12.862426, + "max_per_prompt_fro": 18.539761, + "heavy_tail_ratio": 1.4414, + "n_prompts": 100 + }, + "8": { + "dispersion": 0.876829, + "mean_fro": 10.469473, + "rms_per_prompt_fro": 14.342905, + "max_per_prompt_fro": 19.655499, + "heavy_tail_ratio": 1.3704, + "n_prompts": 100 + }, + "9": { + "dispersion": 0.801039, + "mean_fro": 11.801556, + "rms_per_prompt_fro": 15.838019, + "max_per_prompt_fro": 20.738667, + "heavy_tail_ratio": 1.3094, + "n_prompts": 100 + }, + "10": { + "dispersion": 0.725545, + "mean_fro": 13.498515, + "rms_per_prompt_fro": 17.731652, + "max_per_prompt_fro": 22.484716, + "heavy_tail_ratio": 1.2681, + "n_prompts": 100 + }, + "11": { + "dispersion": 0.654994, + "mean_fro": 15.916233, + "rms_per_prompt_fro": 20.475688, + "max_per_prompt_fro": 25.420773, + "heavy_tail_ratio": 1.2415, + "n_prompts": 100 + }, + "12": { + "dispersion": 0.54397, + "mean_fro": 18.135904, + "rms_per_prompt_fro": 22.535053, + "max_per_prompt_fro": 27.391455, + "heavy_tail_ratio": 1.2155, + "n_prompts": 100 + }, + "13": { + "dispersion": 0.469964, + "mean_fro": 19.778167, + "rms_per_prompt_fro": 23.979458, + "max_per_prompt_fro": 29.221926, + "heavy_tail_ratio": 1.2186, + "n_prompts": 100 + }, + "14": { + "dispersion": 0.397251, + "mean_fro": 23.127588, + "rms_per_prompt_fro": 27.338054, + "max_per_prompt_fro": 32.134945, + "heavy_tail_ratio": 1.1755, + "n_prompts": 100 + }, + "15": { + "dispersion": 0.336103, + "mean_fro": 26.154537, + "rms_per_prompt_fro": 30.232008, + "max_per_prompt_fro": 34.935623, + "heavy_tail_ratio": 1.1556, + "n_prompts": 100 + }, + "16": { + "dispersion": 0.254198, + "mean_fro": 29.652054, + "rms_per_prompt_fro": 33.207629, + "max_per_prompt_fro": 36.653419, + "heavy_tail_ratio": 1.1038, + "n_prompts": 100 + }, + "17": { + "dispersion": 0.187096, + "mean_fro": 32.019908, + "rms_per_prompt_fro": 34.886949, + "max_per_prompt_fro": 37.880894, + "heavy_tail_ratio": 1.0858, + "n_prompts": 100 + }, + "18": { + "dispersion": 0.139153, + "mean_fro": 33.785138, + "rms_per_prompt_fro": 36.05925, + "max_per_prompt_fro": 38.560974, + "heavy_tail_ratio": 1.0694, + "n_prompts": 100 + }, + "19": { + "dispersion": 0.094789, + "mean_fro": 34.323292, + "rms_per_prompt_fro": 35.913202, + "max_per_prompt_fro": 37.993721, + "heavy_tail_ratio": 1.0579, + "n_prompts": 100 + }, + "20": { + "dispersion": 0.053698, + "mean_fro": 34.749157, + "rms_per_prompt_fro": 35.669943, + "max_per_prompt_fro": 37.379372, + "heavy_tail_ratio": 1.0479, + "n_prompts": 100 + }, + "21": { + "dispersion": 0.024595, + "mean_fro": 34.429954, + "rms_per_prompt_fro": 34.850778, + "max_per_prompt_fro": 35.963951, + "heavy_tail_ratio": 1.0319, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 308.0, + "fit_secs_per_prompt": 3.08, + "total_wall_seconds": 316.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7262, + "top_sv": 1.2535, + "stable_rank": 28.793 + }, + { + "layer": 1, + "fro_norm": 6.35, + "top_sv": 1.2209, + "stable_rank": 27.053 + }, + { + "layer": 2, + "fro_norm": 6.9469, + "top_sv": 1.401, + "stable_rank": 24.589 + }, + { + "layer": 3, + "fro_norm": 6.7574, + "top_sv": 1.3316, + "stable_rank": 25.751 + }, + { + "layer": 4, + "fro_norm": 7.6871, + "top_sv": 1.5827, + "stable_rank": 23.59 + }, + { + "layer": 5, + "fro_norm": 7.9559, + "top_sv": 1.682, + "stable_rank": 22.373 + }, + { + "layer": 6, + "fro_norm": 8.2614, + "top_sv": 1.6613, + "stable_rank": 24.729 + }, + { + "layer": 7, + "fro_norm": 9.1462, + "top_sv": 1.8382, + "stable_rank": 24.756 + }, + { + "layer": 8, + "fro_norm": 10.4693, + "top_sv": 2.3163, + "stable_rank": 20.428 + }, + { + "layer": 9, + "fro_norm": 11.8014, + "top_sv": 2.6393, + "stable_rank": 19.994 + }, + { + "layer": 10, + "fro_norm": 13.4983, + "top_sv": 2.9968, + "stable_rank": 20.289 + }, + { + "layer": 11, + "fro_norm": 15.916, + "top_sv": 3.4582, + "stable_rank": 21.183 + }, + { + "layer": 12, + "fro_norm": 18.1356, + "top_sv": 3.8699, + "stable_rank": 21.962 + }, + { + "layer": 13, + "fro_norm": 19.7778, + "top_sv": 4.1768, + "stable_rank": 22.422 + }, + { + "layer": 14, + "fro_norm": 23.1272, + "top_sv": 4.8329, + "stable_rank": 22.9 + }, + { + "layer": 15, + "fro_norm": 26.154, + "top_sv": 5.6746, + "stable_rank": 21.243 + }, + { + "layer": 16, + "fro_norm": 29.6514, + "top_sv": 6.337, + "stable_rank": 21.894 + }, + { + "layer": 17, + "fro_norm": 32.0193, + "top_sv": 6.8435, + "stable_rank": 21.891 + }, + { + "layer": 18, + "fro_norm": 33.7843, + "top_sv": 7.3833, + "stable_rank": 20.938 + }, + { + "layer": 19, + "fro_norm": 34.3223, + "top_sv": 7.3005, + "stable_rank": 22.103 + }, + { + "layer": 20, + "fro_norm": 34.748, + "top_sv": 6.9849, + "stable_rank": 24.748 + }, + { + "layer": 21, + "fro_norm": 34.4284, + "top_sv": 5.5598, + "stable_rank": 38.345 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n200_s0.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4479d4e25fe800f43dc63f34365ca1644ab202f --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efcf8b669825016233690a19162ff645dc58ed2e2628a2e113f26713e27cf998 +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n200_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..1bac6fc2b93fd4b0784723e7acd4ad6fc87153c2 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n200_s0.pt", + "sha256": "efcf8b669825016233690a19162ff645dc58ed2e2628a2e113f26713e27cf998", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.861487, + "mean_fro": 6.796736, + "rms_per_prompt_fro": 13.356038, + "max_per_prompt_fro": 24.010969, + "heavy_tail_ratio": 1.7978, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.549657, + "mean_fro": 6.416556, + "rms_per_prompt_fro": 12.089135, + "max_per_prompt_fro": 21.187668, + "heavy_tail_ratio": 1.7526, + "n_prompts": 200 + }, + "2": { + "dispersion": 2.044423, + "mean_fro": 7.023286, + "rms_per_prompt_fro": 12.254422, + "max_per_prompt_fro": 19.695543, + "heavy_tail_ratio": 1.6072, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.735997, + "mean_fro": 6.819405, + "rms_per_prompt_fro": 11.279874, + "max_per_prompt_fro": 16.917124, + "heavy_tail_ratio": 1.4998, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.413242, + "mean_fro": 7.766029, + "rms_per_prompt_fro": 12.064226, + "max_per_prompt_fro": 17.250628, + "heavy_tail_ratio": 1.4299, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.255817, + "mean_fro": 8.036547, + "rms_per_prompt_fro": 12.070392, + "max_per_prompt_fro": 16.66309, + "heavy_tail_ratio": 1.3805, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.074254, + "mean_fro": 8.342312, + "rms_per_prompt_fro": 12.014823, + "max_per_prompt_fro": 15.841203, + "heavy_tail_ratio": 1.3185, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.947963, + "mean_fro": 9.229598, + "rms_per_prompt_fro": 12.881698, + "max_per_prompt_fro": 16.463778, + "heavy_tail_ratio": 1.2781, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.851697, + "mean_fro": 10.566246, + "rms_per_prompt_fro": 14.37824, + "max_per_prompt_fro": 18.002295, + "heavy_tail_ratio": 1.2521, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.779446, + "mean_fro": 11.901339, + "rms_per_prompt_fro": 15.875894, + "max_per_prompt_fro": 19.832401, + "heavy_tail_ratio": 1.2492, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.710127, + "mean_fro": 13.600033, + "rms_per_prompt_fro": 17.785012, + "max_per_prompt_fro": 21.981054, + "heavy_tail_ratio": 1.2359, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.645265, + "mean_fro": 16.029085, + "rms_per_prompt_fro": 20.56017, + "max_per_prompt_fro": 25.598974, + "heavy_tail_ratio": 1.2451, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.532094, + "mean_fro": 18.220495, + "rms_per_prompt_fro": 22.552929, + "max_per_prompt_fro": 26.207283, + "heavy_tail_ratio": 1.162, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.464208, + "mean_fro": 19.878103, + "rms_per_prompt_fro": 24.053393, + "max_per_prompt_fro": 28.151562, + "heavy_tail_ratio": 1.1704, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.395801, + "mean_fro": 23.266611, + "rms_per_prompt_fro": 27.488113, + "max_per_prompt_fro": 31.418961, + "heavy_tail_ratio": 1.143, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.336721, + "mean_fro": 26.293068, + "rms_per_prompt_fro": 30.399168, + "max_per_prompt_fro": 34.432747, + "heavy_tail_ratio": 1.1327, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.255101, + "mean_fro": 29.739409, + "rms_per_prompt_fro": 33.317441, + "max_per_prompt_fro": 36.798679, + "heavy_tail_ratio": 1.1045, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.189352, + "mean_fro": 32.117416, + "rms_per_prompt_fro": 35.026417, + "max_per_prompt_fro": 38.78054, + "heavy_tail_ratio": 1.1072, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.141344, + "mean_fro": 33.895927, + "rms_per_prompt_fro": 36.21227, + "max_per_prompt_fro": 39.442684, + "heavy_tail_ratio": 1.0892, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.096491, + "mean_fro": 34.379364, + "rms_per_prompt_fro": 35.999827, + "max_per_prompt_fro": 38.283794, + "heavy_tail_ratio": 1.0634, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.054483, + "mean_fro": 34.806698, + "rms_per_prompt_fro": 35.742311, + "max_per_prompt_fro": 37.033623, + "heavy_tail_ratio": 1.0361, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.024938, + "mean_fro": 34.467484, + "rms_per_prompt_fro": 34.89461, + "max_per_prompt_fro": 35.604694, + "heavy_tail_ratio": 1.0203, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 607.8, + "fit_secs_per_prompt": 3.039, + "total_wall_seconds": 616.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7966, + "top_sv": 1.3151, + "stable_rank": 26.711 + }, + { + "layer": 1, + "fro_norm": 6.4165, + "top_sv": 1.2722, + "stable_rank": 25.437 + }, + { + "layer": 2, + "fro_norm": 7.0232, + "top_sv": 1.4729, + "stable_rank": 22.736 + }, + { + "layer": 3, + "fro_norm": 6.8193, + "top_sv": 1.3762, + "stable_rank": 24.555 + }, + { + "layer": 4, + "fro_norm": 7.7659, + "top_sv": 1.6578, + "stable_rank": 21.944 + }, + { + "layer": 5, + "fro_norm": 8.0365, + "top_sv": 1.7397, + "stable_rank": 21.338 + }, + { + "layer": 6, + "fro_norm": 8.3422, + "top_sv": 1.7104, + "stable_rank": 23.79 + }, + { + "layer": 7, + "fro_norm": 9.2295, + "top_sv": 1.8554, + "stable_rank": 24.746 + }, + { + "layer": 8, + "fro_norm": 10.5661, + "top_sv": 2.3535, + "stable_rank": 20.156 + }, + { + "layer": 9, + "fro_norm": 11.9012, + "top_sv": 2.677, + "stable_rank": 19.764 + }, + { + "layer": 10, + "fro_norm": 13.5998, + "top_sv": 3.0262, + "stable_rank": 20.196 + }, + { + "layer": 11, + "fro_norm": 16.0289, + "top_sv": 3.4982, + "stable_rank": 20.995 + }, + { + "layer": 12, + "fro_norm": 18.2202, + "top_sv": 3.901, + "stable_rank": 21.815 + }, + { + "layer": 13, + "fro_norm": 19.8778, + "top_sv": 4.1912, + "stable_rank": 22.494 + }, + { + "layer": 14, + "fro_norm": 23.2662, + "top_sv": 4.869, + "stable_rank": 22.833 + }, + { + "layer": 15, + "fro_norm": 26.2926, + "top_sv": 5.7087, + "stable_rank": 21.213 + }, + { + "layer": 16, + "fro_norm": 29.7388, + "top_sv": 6.3437, + "stable_rank": 21.977 + }, + { + "layer": 17, + "fro_norm": 32.1167, + "top_sv": 6.8712, + "stable_rank": 21.847 + }, + { + "layer": 18, + "fro_norm": 33.895, + "top_sv": 7.3944, + "stable_rank": 21.012 + }, + { + "layer": 19, + "fro_norm": 34.3783, + "top_sv": 7.2839, + "stable_rank": 22.276 + }, + { + "layer": 20, + "fro_norm": 34.8055, + "top_sv": 6.9712, + "stable_rank": 24.928 + }, + { + "layer": 21, + "fro_norm": 34.4659, + "top_sv": 5.5533, + "stable_rank": 38.519 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n200_s1.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c0d63efafb091f1a7cbc1c017d95ed977ca4b63 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfa5aee55a338b05ccb9fc7d66a631da83b97c6e48cfa387105689bd30805033 +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n200_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0e217db0b31d53f6690581dd15bb90e1fcfd3ec0 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n200_s1.pt", + "sha256": "bfa5aee55a338b05ccb9fc7d66a631da83b97c6e48cfa387105689bd30805033", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.831926, + "mean_fro": 6.729943, + "rms_per_prompt_fro": 13.174068, + "max_per_prompt_fro": 22.19487, + "heavy_tail_ratio": 1.6847, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.524456, + "mean_fro": 6.350707, + "rms_per_prompt_fro": 11.922522, + "max_per_prompt_fro": 19.147593, + "heavy_tail_ratio": 1.606, + "n_prompts": 200 + }, + "2": { + "dispersion": 2.022247, + "mean_fro": 6.961231, + "rms_per_prompt_fro": 12.101829, + "max_per_prompt_fro": 19.291433, + "heavy_tail_ratio": 1.5941, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.706953, + "mean_fro": 6.766281, + "rms_per_prompt_fro": 11.13244, + "max_per_prompt_fro": 17.139297, + "heavy_tail_ratio": 1.5396, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.386158, + "mean_fro": 7.716629, + "rms_per_prompt_fro": 11.920028, + "max_per_prompt_fro": 17.873886, + "heavy_tail_ratio": 1.4995, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.231854, + "mean_fro": 7.976846, + "rms_per_prompt_fro": 11.916921, + "max_per_prompt_fro": 18.013294, + "heavy_tail_ratio": 1.5116, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.059499, + "mean_fro": 8.289101, + "rms_per_prompt_fro": 11.895649, + "max_per_prompt_fro": 16.665794, + "heavy_tail_ratio": 1.401, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.933222, + "mean_fro": 9.17646, + "rms_per_prompt_fro": 12.758983, + "max_per_prompt_fro": 16.177975, + "heavy_tail_ratio": 1.268, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.839211, + "mean_fro": 10.508351, + "rms_per_prompt_fro": 14.251164, + "max_per_prompt_fro": 18.541143, + "heavy_tail_ratio": 1.301, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.769014, + "mean_fro": 11.849492, + "rms_per_prompt_fro": 15.760333, + "max_per_prompt_fro": 20.537996, + "heavy_tail_ratio": 1.3031, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.69965, + "mean_fro": 13.551729, + "rms_per_prompt_fro": 17.667476, + "max_per_prompt_fro": 22.904638, + "heavy_tail_ratio": 1.2964, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.63522, + "mean_fro": 15.9899, + "rms_per_prompt_fro": 20.447199, + "max_per_prompt_fro": 27.136269, + "heavy_tail_ratio": 1.3271, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.525368, + "mean_fro": 18.196835, + "rms_per_prompt_fro": 22.474141, + "max_per_prompt_fro": 27.989729, + "heavy_tail_ratio": 1.2454, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.45499, + "mean_fro": 19.854189, + "rms_per_prompt_fro": 23.948708, + "max_per_prompt_fro": 29.441811, + "heavy_tail_ratio": 1.2294, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.386598, + "mean_fro": 23.225058, + "rms_per_prompt_fro": 27.348411, + "max_per_prompt_fro": 32.696583, + "heavy_tail_ratio": 1.1956, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.328206, + "mean_fro": 26.256426, + "rms_per_prompt_fro": 30.259962, + "max_per_prompt_fro": 35.626503, + "heavy_tail_ratio": 1.1773, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.248268, + "mean_fro": 29.743684, + "rms_per_prompt_fro": 33.231402, + "max_per_prompt_fro": 36.620102, + "heavy_tail_ratio": 1.102, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.18298, + "mean_fro": 32.118709, + "rms_per_prompt_fro": 34.933876, + "max_per_prompt_fro": 37.982109, + "heavy_tail_ratio": 1.0873, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.13538, + "mean_fro": 33.866199, + "rms_per_prompt_fro": 36.085863, + "max_per_prompt_fro": 38.831398, + "heavy_tail_ratio": 1.0761, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.091701, + "mean_fro": 34.362267, + "rms_per_prompt_fro": 35.903249, + "max_per_prompt_fro": 37.647743, + "heavy_tail_ratio": 1.0486, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.052534, + "mean_fro": 34.778935, + "rms_per_prompt_fro": 35.680783, + "max_per_prompt_fro": 36.680122, + "heavy_tail_ratio": 1.028, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.024092, + "mean_fro": 34.44262, + "rms_per_prompt_fro": 34.855053, + "max_per_prompt_fro": 35.595573, + "heavy_tail_ratio": 1.0212, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 618.6, + "fit_secs_per_prompt": 3.093, + "total_wall_seconds": 625.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7299, + "top_sv": 1.2595, + "stable_rank": 28.551 + }, + { + "layer": 1, + "fro_norm": 6.3506, + "top_sv": 1.2217, + "stable_rank": 27.019 + }, + { + "layer": 2, + "fro_norm": 6.9611, + "top_sv": 1.404, + "stable_rank": 24.584 + }, + { + "layer": 3, + "fro_norm": 6.7662, + "top_sv": 1.3348, + "stable_rank": 25.694 + }, + { + "layer": 4, + "fro_norm": 7.7165, + "top_sv": 1.5926, + "stable_rank": 23.477 + }, + { + "layer": 5, + "fro_norm": 7.9768, + "top_sv": 1.6806, + "stable_rank": 22.527 + }, + { + "layer": 6, + "fro_norm": 8.289, + "top_sv": 1.6805, + "stable_rank": 24.329 + }, + { + "layer": 7, + "fro_norm": 9.1764, + "top_sv": 1.8451, + "stable_rank": 24.735 + }, + { + "layer": 8, + "fro_norm": 10.5082, + "top_sv": 2.3346, + "stable_rank": 20.26 + }, + { + "layer": 9, + "fro_norm": 11.8493, + "top_sv": 2.657, + "stable_rank": 19.888 + }, + { + "layer": 10, + "fro_norm": 13.5515, + "top_sv": 3.0104, + "stable_rank": 20.264 + }, + { + "layer": 11, + "fro_norm": 15.9897, + "top_sv": 3.4825, + "stable_rank": 21.081 + }, + { + "layer": 12, + "fro_norm": 18.1965, + "top_sv": 3.8872, + "stable_rank": 21.913 + }, + { + "layer": 13, + "fro_norm": 19.8538, + "top_sv": 4.1837, + "stable_rank": 22.52 + }, + { + "layer": 14, + "fro_norm": 23.2247, + "top_sv": 4.839, + "stable_rank": 23.035 + }, + { + "layer": 15, + "fro_norm": 26.2559, + "top_sv": 5.6885, + "stable_rank": 21.304 + }, + { + "layer": 16, + "fro_norm": 29.7431, + "top_sv": 6.3226, + "stable_rank": 22.13 + }, + { + "layer": 17, + "fro_norm": 32.118, + "top_sv": 6.8416, + "stable_rank": 22.039 + }, + { + "layer": 18, + "fro_norm": 33.8653, + "top_sv": 7.3832, + "stable_rank": 21.039 + }, + { + "layer": 19, + "fro_norm": 34.3612, + "top_sv": 7.2868, + "stable_rank": 22.236 + }, + { + "layer": 20, + "fro_norm": 34.7777, + "top_sv": 6.9992, + "stable_rank": 24.689 + }, + { + "layer": 21, + "fro_norm": 34.4411, + "top_sv": 5.5865, + "stable_rank": 38.008 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n200_s2.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5186a7690a2ba7f659042937cdeb162e7b6034e3 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d97ade0445171e2329dc86da6ef93a218f178e3390f182e511c237ca2f1ed6 +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n200_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..73f874436dafb690b71fcef0d32759276248350f --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n200_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n200_s2.pt", + "sha256": "33d97ade0445171e2329dc86da6ef93a218f178e3390f182e511c237ca2f1ed6", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.87708, + "mean_fro": 6.770955, + "rms_per_prompt_fro": 13.332215, + "max_per_prompt_fro": 24.485331, + "heavy_tail_ratio": 1.8366, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.564365, + "mean_fro": 6.389493, + "rms_per_prompt_fro": 12.06306, + "max_per_prompt_fro": 21.961533, + "heavy_tail_ratio": 1.8206, + "n_prompts": 200 + }, + "2": { + "dispersion": 2.045691, + "mean_fro": 7.00127, + "rms_per_prompt_fro": 12.218551, + "max_per_prompt_fro": 21.365519, + "heavy_tail_ratio": 1.7486, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.715811, + "mean_fro": 6.802894, + "rms_per_prompt_fro": 11.210978, + "max_per_prompt_fro": 18.536762, + "heavy_tail_ratio": 1.6534, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.390481, + "mean_fro": 7.74823, + "rms_per_prompt_fro": 11.979678, + "max_per_prompt_fro": 18.625755, + "heavy_tail_ratio": 1.5548, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.232127, + "mean_fro": 8.013132, + "rms_per_prompt_fro": 11.971863, + "max_per_prompt_fro": 18.531601, + "heavy_tail_ratio": 1.5479, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.059671, + "mean_fro": 8.319357, + "rms_per_prompt_fro": 11.939571, + "max_per_prompt_fro": 18.148132, + "heavy_tail_ratio": 1.52, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.932198, + "mean_fro": 9.206266, + "rms_per_prompt_fro": 12.797035, + "max_per_prompt_fro": 18.539761, + "heavy_tail_ratio": 1.4488, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.83646, + "mean_fro": 10.530466, + "rms_per_prompt_fro": 14.270471, + "max_per_prompt_fro": 19.655499, + "heavy_tail_ratio": 1.3774, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.765797, + "mean_fro": 11.865776, + "rms_per_prompt_fro": 15.767636, + "max_per_prompt_fro": 20.738667, + "heavy_tail_ratio": 1.3153, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.696256, + "mean_fro": 13.560488, + "rms_per_prompt_fro": 17.66123, + "max_per_prompt_fro": 22.484716, + "heavy_tail_ratio": 1.2731, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.631565, + "mean_fro": 15.988402, + "rms_per_prompt_fro": 20.42242, + "max_per_prompt_fro": 25.420773, + "heavy_tail_ratio": 1.2447, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.524603, + "mean_fro": 18.209681, + "rms_per_prompt_fro": 22.484367, + "max_per_prompt_fro": 27.391455, + "heavy_tail_ratio": 1.2182, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.453857, + "mean_fro": 19.864886, + "rms_per_prompt_fro": 23.952283, + "max_per_prompt_fro": 29.221926, + "heavy_tail_ratio": 1.22, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.384785, + "mean_fro": 23.232942, + "rms_per_prompt_fro": 27.339802, + "max_per_prompt_fro": 32.134945, + "heavy_tail_ratio": 1.1754, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.326395, + "mean_fro": 26.271355, + "rms_per_prompt_fro": 30.256518, + "max_per_prompt_fro": 34.935623, + "heavy_tail_ratio": 1.1546, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.246508, + "mean_fro": 29.732918, + "rms_per_prompt_fro": 33.195942, + "max_per_prompt_fro": 36.653419, + "heavy_tail_ratio": 1.1042, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.181912, + "mean_fro": 32.095764, + "rms_per_prompt_fro": 34.893166, + "max_per_prompt_fro": 37.880894, + "heavy_tail_ratio": 1.0856, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.135255, + "mean_fro": 33.860359, + "rms_per_prompt_fro": 36.077651, + "max_per_prompt_fro": 38.560974, + "heavy_tail_ratio": 1.0688, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.092008, + "mean_fro": 34.361816, + "rms_per_prompt_fro": 35.907821, + "max_per_prompt_fro": 37.993721, + "heavy_tail_ratio": 1.0581, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.052168, + "mean_fro": 34.784688, + "rms_per_prompt_fro": 35.680485, + "max_per_prompt_fro": 37.379372, + "heavy_tail_ratio": 1.0476, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.023823, + "mean_fro": 34.440311, + "rms_per_prompt_fro": 34.848126, + "max_per_prompt_fro": 35.963951, + "heavy_tail_ratio": 1.032, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 617.5, + "fit_secs_per_prompt": 3.087, + "total_wall_seconds": 625.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7709, + "top_sv": 1.2531, + "stable_rank": 29.197 + }, + { + "layer": 1, + "fro_norm": 6.3894, + "top_sv": 1.2135, + "stable_rank": 27.725 + }, + { + "layer": 2, + "fro_norm": 7.0012, + "top_sv": 1.3991, + "stable_rank": 25.04 + }, + { + "layer": 3, + "fro_norm": 6.8028, + "top_sv": 1.3244, + "stable_rank": 26.385 + }, + { + "layer": 4, + "fro_norm": 7.7481, + "top_sv": 1.5748, + "stable_rank": 24.207 + }, + { + "layer": 5, + "fro_norm": 8.013, + "top_sv": 1.673, + "stable_rank": 22.94 + }, + { + "layer": 6, + "fro_norm": 8.3193, + "top_sv": 1.6582, + "stable_rank": 25.171 + }, + { + "layer": 7, + "fro_norm": 9.2061, + "top_sv": 1.84, + "stable_rank": 25.033 + }, + { + "layer": 8, + "fro_norm": 10.5303, + "top_sv": 2.3245, + "stable_rank": 20.523 + }, + { + "layer": 9, + "fro_norm": 11.8656, + "top_sv": 2.6439, + "stable_rank": 20.142 + }, + { + "layer": 10, + "fro_norm": 13.5603, + "top_sv": 2.9977, + "stable_rank": 20.463 + }, + { + "layer": 11, + "fro_norm": 15.9882, + "top_sv": 3.4631, + "stable_rank": 21.314 + }, + { + "layer": 12, + "fro_norm": 18.2094, + "top_sv": 3.8724, + "stable_rank": 22.112 + }, + { + "layer": 13, + "fro_norm": 19.8645, + "top_sv": 4.1811, + "stable_rank": 22.572 + }, + { + "layer": 14, + "fro_norm": 23.2326, + "top_sv": 4.8474, + "stable_rank": 22.971 + }, + { + "layer": 15, + "fro_norm": 26.2708, + "top_sv": 5.6974, + "stable_rank": 21.261 + }, + { + "layer": 16, + "fro_norm": 29.7324, + "top_sv": 6.3424, + "stable_rank": 21.976 + }, + { + "layer": 17, + "fro_norm": 32.0951, + "top_sv": 6.8558, + "stable_rank": 21.916 + }, + { + "layer": 18, + "fro_norm": 33.8595, + "top_sv": 7.4012, + "stable_rank": 20.929 + }, + { + "layer": 19, + "fro_norm": 34.3608, + "top_sv": 7.3081, + "stable_rank": 22.106 + }, + { + "layer": 20, + "fro_norm": 34.7835, + "top_sv": 6.9998, + "stable_rank": 24.693 + }, + { + "layer": 21, + "fro_norm": 34.4388, + "top_sv": 5.5737, + "stable_rank": 38.178 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n25_s0.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..51aab18a000569ff686aef39d32c3bb7843239bf --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88bc80e8f4ca4c18f0053c96d5e73c80418ffa2583a78d76e430eb5d72f4a80f +size 46145005 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n25_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..3700f3a5b8ae9a9e72992abf4b2d900cb3cf811c --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n25_s0.pt", + "sha256": "88bc80e8f4ca4c18f0053c96d5e73c80418ffa2583a78d76e430eb5d72f4a80f", + "bytes": 46145005, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.506607, + "mean_fro": 7.194975, + "rms_per_prompt_fro": 13.473265, + "max_per_prompt_fro": 17.955008, + "heavy_tail_ratio": 1.3326, + "n_prompts": 25 + }, + "1": { + "dispersion": 2.257224, + "mean_fro": 6.754592, + "rms_per_prompt_fro": 12.190539, + "max_per_prompt_fro": 16.050516, + "heavy_tail_ratio": 1.3166, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.874566, + "mean_fro": 7.34324, + "rms_per_prompt_fro": 12.45013, + "max_per_prompt_fro": 16.309441, + "heavy_tail_ratio": 1.31, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.60917, + "mean_fro": 7.078091, + "rms_per_prompt_fro": 11.433187, + "max_per_prompt_fro": 15.609672, + "heavy_tail_ratio": 1.3653, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.339228, + "mean_fro": 8.02494, + "rms_per_prompt_fro": 12.273773, + "max_per_prompt_fro": 16.447428, + "heavy_tail_ratio": 1.34, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.19757, + "mean_fro": 8.267809, + "rms_per_prompt_fro": 12.256369, + "max_per_prompt_fro": 16.488127, + "heavy_tail_ratio": 1.3453, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.021359, + "mean_fro": 8.548001, + "rms_per_prompt_fro": 12.15308, + "max_per_prompt_fro": 14.61259, + "heavy_tail_ratio": 1.2024, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.90037, + "mean_fro": 9.419993, + "rms_per_prompt_fro": 12.985827, + "max_per_prompt_fro": 15.402933, + "heavy_tail_ratio": 1.1861, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.814601, + "mean_fro": 10.751284, + "rms_per_prompt_fro": 14.482745, + "max_per_prompt_fro": 16.903938, + "heavy_tail_ratio": 1.1672, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.749966, + "mean_fro": 12.077935, + "rms_per_prompt_fro": 15.977452, + "max_per_prompt_fro": 18.171692, + "heavy_tail_ratio": 1.1373, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.687865, + "mean_fro": 13.768097, + "rms_per_prompt_fro": 17.887218, + "max_per_prompt_fro": 20.219521, + "heavy_tail_ratio": 1.1304, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.628346, + "mean_fro": 16.190282, + "rms_per_prompt_fro": 20.65988, + "max_per_prompt_fro": 23.372515, + "heavy_tail_ratio": 1.1313, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.515942, + "mean_fro": 18.348354, + "rms_per_prompt_fro": 22.591155, + "max_per_prompt_fro": 24.542681, + "heavy_tail_ratio": 1.0864, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.446083, + "mean_fro": 19.987223, + "rms_per_prompt_fro": 24.035277, + "max_per_prompt_fro": 25.828203, + "heavy_tail_ratio": 1.0746, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.380571, + "mean_fro": 23.306221, + "rms_per_prompt_fro": 27.384277, + "max_per_prompt_fro": 29.367434, + "heavy_tail_ratio": 1.0724, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.323674, + "mean_fro": 26.263573, + "rms_per_prompt_fro": 30.216508, + "max_per_prompt_fro": 32.558727, + "heavy_tail_ratio": 1.0775, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.242837, + "mean_fro": 29.659412, + "rms_per_prompt_fro": 33.065084, + "max_per_prompt_fro": 35.419628, + "heavy_tail_ratio": 1.0712, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.178746, + "mean_fro": 32.031665, + "rms_per_prompt_fro": 34.776805, + "max_per_prompt_fro": 36.317493, + "heavy_tail_ratio": 1.0443, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.134069, + "mean_fro": 33.778905, + "rms_per_prompt_fro": 35.972052, + "max_per_prompt_fro": 37.271492, + "heavy_tail_ratio": 1.0361, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.091057, + "mean_fro": 34.264209, + "rms_per_prompt_fro": 35.790229, + "max_per_prompt_fro": 36.984135, + "heavy_tail_ratio": 1.0334, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.05003, + "mean_fro": 34.758995, + "rms_per_prompt_fro": 35.617886, + "max_per_prompt_fro": 36.309116, + "heavy_tail_ratio": 1.0194, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.02283, + "mean_fro": 34.396156, + "rms_per_prompt_fro": 34.786572, + "max_per_prompt_fro": 35.322987, + "heavy_tail_ratio": 1.0154, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 76.2, + "fit_secs_per_prompt": 3.049, + "total_wall_seconds": 86.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.1949, + "top_sv": 1.6269, + "stable_rank": 19.557 + }, + { + "layer": 1, + "fro_norm": 6.7545, + "top_sv": 1.5308, + "stable_rank": 19.47 + }, + { + "layer": 2, + "fro_norm": 7.3432, + "top_sv": 1.7249, + "stable_rank": 18.123 + }, + { + "layer": 3, + "fro_norm": 7.078, + "top_sv": 1.5591, + "stable_rank": 20.61 + }, + { + "layer": 4, + "fro_norm": 8.0248, + "top_sv": 1.835, + "stable_rank": 19.126 + }, + { + "layer": 5, + "fro_norm": 8.2677, + "top_sv": 1.8764, + "stable_rank": 19.415 + }, + { + "layer": 6, + "fro_norm": 8.5479, + "top_sv": 1.8154, + "stable_rank": 22.171 + }, + { + "layer": 7, + "fro_norm": 9.4199, + "top_sv": 1.9427, + "stable_rank": 23.512 + }, + { + "layer": 8, + "fro_norm": 10.7511, + "top_sv": 2.4577, + "stable_rank": 19.136 + }, + { + "layer": 9, + "fro_norm": 12.0778, + "top_sv": 2.7574, + "stable_rank": 19.186 + }, + { + "layer": 10, + "fro_norm": 13.7679, + "top_sv": 3.0976, + "stable_rank": 19.755 + }, + { + "layer": 11, + "fro_norm": 16.1901, + "top_sv": 3.6025, + "stable_rank": 20.198 + }, + { + "layer": 12, + "fro_norm": 18.348, + "top_sv": 4.0049, + "stable_rank": 20.99 + }, + { + "layer": 13, + "fro_norm": 19.9869, + "top_sv": 4.3355, + "stable_rank": 21.253 + }, + { + "layer": 14, + "fro_norm": 23.3058, + "top_sv": 4.9964, + "stable_rank": 21.758 + }, + { + "layer": 15, + "fro_norm": 26.2631, + "top_sv": 5.7777, + "stable_rank": 20.662 + }, + { + "layer": 16, + "fro_norm": 29.6589, + "top_sv": 6.4026, + "stable_rank": 21.458 + }, + { + "layer": 17, + "fro_norm": 32.0311, + "top_sv": 6.8884, + "stable_rank": 21.622 + }, + { + "layer": 18, + "fro_norm": 33.778, + "top_sv": 7.4066, + "stable_rank": 20.798 + }, + { + "layer": 19, + "fro_norm": 34.2632, + "top_sv": 7.2916, + "stable_rank": 22.081 + }, + { + "layer": 20, + "fro_norm": 34.7578, + "top_sv": 6.9909, + "stable_rank": 24.719 + }, + { + "layer": 21, + "fro_norm": 34.3945, + "top_sv": 5.5261, + "stable_rank": 38.739 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n25_s1.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3427e0ef32c5d75dcfe93df222eea1781ae7eb98 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:640063636e75a733a5220793dbfe6bfd266319f701f1bfbebc8439dcaf37049f +size 46145005 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n25_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..492ce26021ee2be6048501c65e36bde6a284b4fd --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n25_s1.pt", + "sha256": "640063636e75a733a5220793dbfe6bfd266319f701f1bfbebc8439dcaf37049f", + "bytes": 46145005, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.222138, + "mean_fro": 7.225977, + "rms_per_prompt_fro": 12.970856, + "max_per_prompt_fro": 16.242861, + "heavy_tail_ratio": 1.2523, + "n_prompts": 25 + }, + "1": { + "dispersion": 2.008496, + "mean_fro": 6.781982, + "rms_per_prompt_fro": 11.763359, + "max_per_prompt_fro": 14.562891, + "heavy_tail_ratio": 1.238, + "n_prompts": 25 + }, + "2": { + "dispersion": 1.651349, + "mean_fro": 7.370093, + "rms_per_prompt_fro": 12.000695, + "max_per_prompt_fro": 14.774945, + "heavy_tail_ratio": 1.2312, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.39822, + "mean_fro": 7.109544, + "rms_per_prompt_fro": 11.009974, + "max_per_prompt_fro": 13.285714, + "heavy_tail_ratio": 1.2067, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.153516, + "mean_fro": 8.055918, + "rms_per_prompt_fro": 11.821948, + "max_per_prompt_fro": 14.094844, + "heavy_tail_ratio": 1.1923, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.028811, + "mean_fro": 8.307494, + "rms_per_prompt_fro": 11.832889, + "max_per_prompt_fro": 13.955465, + "heavy_tail_ratio": 1.1794, + "n_prompts": 25 + }, + "6": { + "dispersion": 0.898413, + "mean_fro": 8.599084, + "rms_per_prompt_fro": 11.848067, + "max_per_prompt_fro": 13.863462, + "heavy_tail_ratio": 1.1701, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.805232, + "mean_fro": 9.501445, + "rms_per_prompt_fro": 12.766037, + "max_per_prompt_fro": 14.802132, + "heavy_tail_ratio": 1.1595, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.727712, + "mean_fro": 10.856272, + "rms_per_prompt_fro": 14.269749, + "max_per_prompt_fro": 16.264036, + "heavy_tail_ratio": 1.1398, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.670488, + "mean_fro": 12.213966, + "rms_per_prompt_fro": 15.78623, + "max_per_prompt_fro": 17.716604, + "heavy_tail_ratio": 1.1223, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.614803, + "mean_fro": 13.905867, + "rms_per_prompt_fro": 17.670864, + "max_per_prompt_fro": 19.718908, + "heavy_tail_ratio": 1.1159, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.564854, + "mean_fro": 16.333856, + "rms_per_prompt_fro": 20.432697, + "max_per_prompt_fro": 22.907913, + "heavy_tail_ratio": 1.1211, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.474117, + "mean_fro": 18.577875, + "rms_per_prompt_fro": 22.555995, + "max_per_prompt_fro": 24.408339, + "heavy_tail_ratio": 1.0821, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.414123, + "mean_fro": 20.210934, + "rms_per_prompt_fro": 24.034214, + "max_per_prompt_fro": 26.159382, + "heavy_tail_ratio": 1.0884, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.355412, + "mean_fro": 23.601566, + "rms_per_prompt_fro": 27.477452, + "max_per_prompt_fro": 29.792509, + "heavy_tail_ratio": 1.0843, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.302812, + "mean_fro": 26.621027, + "rms_per_prompt_fro": 30.385448, + "max_per_prompt_fro": 33.088852, + "heavy_tail_ratio": 1.089, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.229733, + "mean_fro": 30.014166, + "rms_per_prompt_fro": 33.283702, + "max_per_prompt_fro": 35.631195, + "heavy_tail_ratio": 1.0705, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.170514, + "mean_fro": 32.289676, + "rms_per_prompt_fro": 34.934291, + "max_per_prompt_fro": 37.413486, + "heavy_tail_ratio": 1.071, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.126972, + "mean_fro": 34.028488, + "rms_per_prompt_fro": 36.124278, + "max_per_prompt_fro": 38.020199, + "heavy_tail_ratio": 1.0525, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.086431, + "mean_fro": 34.453362, + "rms_per_prompt_fro": 35.911436, + "max_per_prompt_fro": 37.108562, + "heavy_tail_ratio": 1.0333, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.050007, + "mean_fro": 34.876309, + "rms_per_prompt_fro": 35.737693, + "max_per_prompt_fro": 36.388874, + "heavy_tail_ratio": 1.0182, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.022991, + "mean_fro": 34.475742, + "rms_per_prompt_fro": 34.869799, + "max_per_prompt_fro": 35.316956, + "heavy_tail_ratio": 1.0128, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 77.3, + "fit_secs_per_prompt": 3.091, + "total_wall_seconds": 83.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.2259, + "top_sv": 1.4682, + "stable_rank": 24.222 + }, + { + "layer": 1, + "fro_norm": 6.7819, + "top_sv": 1.3922, + "stable_rank": 23.73 + }, + { + "layer": 2, + "fro_norm": 7.37, + "top_sv": 1.5681, + "stable_rank": 22.091 + }, + { + "layer": 3, + "fro_norm": 7.1095, + "top_sv": 1.4447, + "stable_rank": 24.217 + }, + { + "layer": 4, + "fro_norm": 8.0558, + "top_sv": 1.6885, + "stable_rank": 22.763 + }, + { + "layer": 5, + "fro_norm": 8.3074, + "top_sv": 1.7766, + "stable_rank": 21.864 + }, + { + "layer": 6, + "fro_norm": 8.599, + "top_sv": 1.7326, + "stable_rank": 24.633 + }, + { + "layer": 7, + "fro_norm": 9.5013, + "top_sv": 1.8842, + "stable_rank": 25.428 + }, + { + "layer": 8, + "fro_norm": 10.8561, + "top_sv": 2.3791, + "stable_rank": 20.822 + }, + { + "layer": 9, + "fro_norm": 12.2138, + "top_sv": 2.711, + "stable_rank": 20.298 + }, + { + "layer": 10, + "fro_norm": 13.9056, + "top_sv": 3.0694, + "stable_rank": 20.525 + }, + { + "layer": 11, + "fro_norm": 16.3336, + "top_sv": 3.5432, + "stable_rank": 21.251 + }, + { + "layer": 12, + "fro_norm": 18.5776, + "top_sv": 3.9569, + "stable_rank": 22.043 + }, + { + "layer": 13, + "fro_norm": 20.2106, + "top_sv": 4.2854, + "stable_rank": 22.242 + }, + { + "layer": 14, + "fro_norm": 23.6012, + "top_sv": 4.9469, + "stable_rank": 22.762 + }, + { + "layer": 15, + "fro_norm": 26.6205, + "top_sv": 5.7691, + "stable_rank": 21.292 + }, + { + "layer": 16, + "fro_norm": 30.0136, + "top_sv": 6.3761, + "stable_rank": 22.158 + }, + { + "layer": 17, + "fro_norm": 32.289, + "top_sv": 6.9021, + "stable_rank": 21.885 + }, + { + "layer": 18, + "fro_norm": 34.0276, + "top_sv": 7.4918, + "stable_rank": 20.629 + }, + { + "layer": 19, + "fro_norm": 34.4523, + "top_sv": 7.3368, + "stable_rank": 22.051 + }, + { + "layer": 20, + "fro_norm": 34.8751, + "top_sv": 7.0478, + "stable_rank": 24.486 + }, + { + "layer": 21, + "fro_norm": 34.4742, + "top_sv": 5.6682, + "stable_rank": 36.991 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n25_s2.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b806d65e3f2cc1ac644d8bb8b0e3afba8a7c9fa --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13827efcdc248d16ec92e4fb117d48a1b776f93bd8fb46fdd7b4bd569f01a03 +size 46145005 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n25_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ab4e6d31eaf8bf5f42920de555cd8d4a9b18900e --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n25_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n25_s2.pt", + "sha256": "d13827efcdc248d16ec92e4fb117d48a1b776f93bd8fb46fdd7b4bd569f01a03", + "bytes": 46145005, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.739858, + "mean_fro": 6.939857, + "rms_per_prompt_fro": 13.420788, + "max_per_prompt_fro": 19.645554, + "heavy_tail_ratio": 1.4638, + "n_prompts": 25 + }, + "1": { + "dispersion": 2.48661, + "mean_fro": 6.525881, + "rms_per_prompt_fro": 12.18543, + "max_per_prompt_fro": 18.059361, + "heavy_tail_ratio": 1.482, + "n_prompts": 25 + }, + "2": { + "dispersion": 2.017352, + "mean_fro": 7.089646, + "rms_per_prompt_fro": 12.315089, + "max_per_prompt_fro": 17.360987, + "heavy_tail_ratio": 1.4097, + "n_prompts": 25 + }, + "3": { + "dispersion": 1.706954, + "mean_fro": 6.863744, + "rms_per_prompt_fro": 11.292797, + "max_per_prompt_fro": 15.802229, + "heavy_tail_ratio": 1.3993, + "n_prompts": 25 + }, + "4": { + "dispersion": 1.38305, + "mean_fro": 7.785406, + "rms_per_prompt_fro": 12.018433, + "max_per_prompt_fro": 15.854173, + "heavy_tail_ratio": 1.3192, + "n_prompts": 25 + }, + "5": { + "dispersion": 1.227163, + "mean_fro": 8.037275, + "rms_per_prompt_fro": 11.994574, + "max_per_prompt_fro": 15.6437, + "heavy_tail_ratio": 1.3042, + "n_prompts": 25 + }, + "6": { + "dispersion": 1.059316, + "mean_fro": 8.348357, + "rms_per_prompt_fro": 11.980157, + "max_per_prompt_fro": 15.298518, + "heavy_tail_ratio": 1.277, + "n_prompts": 25 + }, + "7": { + "dispersion": 0.932019, + "mean_fro": 9.245423, + "rms_per_prompt_fro": 12.85087, + "max_per_prompt_fro": 15.987034, + "heavy_tail_ratio": 1.244, + "n_prompts": 25 + }, + "8": { + "dispersion": 0.83093, + "mean_fro": 10.582844, + "rms_per_prompt_fro": 14.319844, + "max_per_prompt_fro": 17.441622, + "heavy_tail_ratio": 1.218, + "n_prompts": 25 + }, + "9": { + "dispersion": 0.757303, + "mean_fro": 11.923746, + "rms_per_prompt_fro": 15.806514, + "max_per_prompt_fro": 19.016964, + "heavy_tail_ratio": 1.2031, + "n_prompts": 25 + }, + "10": { + "dispersion": 0.68399, + "mean_fro": 13.645411, + "rms_per_prompt_fro": 17.707465, + "max_per_prompt_fro": 20.77474, + "heavy_tail_ratio": 1.1732, + "n_prompts": 25 + }, + "11": { + "dispersion": 0.620094, + "mean_fro": 16.124992, + "rms_per_prompt_fro": 20.524358, + "max_per_prompt_fro": 23.907835, + "heavy_tail_ratio": 1.1649, + "n_prompts": 25 + }, + "12": { + "dispersion": 0.516597, + "mean_fro": 18.342201, + "rms_per_prompt_fro": 22.588457, + "max_per_prompt_fro": 25.347282, + "heavy_tail_ratio": 1.1221, + "n_prompts": 25 + }, + "13": { + "dispersion": 0.445115, + "mean_fro": 19.973406, + "rms_per_prompt_fro": 24.010615, + "max_per_prompt_fro": 27.265087, + "heavy_tail_ratio": 1.1355, + "n_prompts": 25 + }, + "14": { + "dispersion": 0.37672, + "mean_fro": 23.338212, + "rms_per_prompt_fro": 27.383589, + "max_per_prompt_fro": 30.383566, + "heavy_tail_ratio": 1.1096, + "n_prompts": 25 + }, + "15": { + "dispersion": 0.319273, + "mean_fro": 26.410322, + "rms_per_prompt_fro": 30.334788, + "max_per_prompt_fro": 33.130489, + "heavy_tail_ratio": 1.0922, + "n_prompts": 25 + }, + "16": { + "dispersion": 0.240855, + "mean_fro": 29.758822, + "rms_per_prompt_fro": 33.149438, + "max_per_prompt_fro": 35.40237, + "heavy_tail_ratio": 1.068, + "n_prompts": 25 + }, + "17": { + "dispersion": 0.178868, + "mean_fro": 32.237882, + "rms_per_prompt_fro": 35.002499, + "max_per_prompt_fro": 36.947639, + "heavy_tail_ratio": 1.0556, + "n_prompts": 25 + }, + "18": { + "dispersion": 0.132094, + "mean_fro": 33.92072, + "rms_per_prompt_fro": 36.091612, + "max_per_prompt_fro": 37.915199, + "heavy_tail_ratio": 1.0505, + "n_prompts": 25 + }, + "19": { + "dispersion": 0.089708, + "mean_fro": 34.370721, + "rms_per_prompt_fro": 35.879284, + "max_per_prompt_fro": 36.923241, + "heavy_tail_ratio": 1.0291, + "n_prompts": 25 + }, + "20": { + "dispersion": 0.052151, + "mean_fro": 34.798371, + "rms_per_prompt_fro": 35.694228, + "max_per_prompt_fro": 36.726715, + "heavy_tail_ratio": 1.0289, + "n_prompts": 25 + }, + "21": { + "dispersion": 0.023721, + "mean_fro": 34.448776, + "rms_per_prompt_fro": 34.854955, + "max_per_prompt_fro": 35.574158, + "heavy_tail_ratio": 1.0206, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 77.2, + "fit_secs_per_prompt": 3.089, + "total_wall_seconds": 85.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9398, + "top_sv": 1.3574, + "stable_rank": 26.138 + }, + { + "layer": 1, + "fro_norm": 6.5258, + "top_sv": 1.2952, + "stable_rank": 25.387 + }, + { + "layer": 2, + "fro_norm": 7.0896, + "top_sv": 1.4538, + "stable_rank": 23.781 + }, + { + "layer": 3, + "fro_norm": 6.8637, + "top_sv": 1.3803, + "stable_rank": 24.725 + }, + { + "layer": 4, + "fro_norm": 7.7853, + "top_sv": 1.6056, + "stable_rank": 23.513 + }, + { + "layer": 5, + "fro_norm": 8.0372, + "top_sv": 1.6884, + "stable_rank": 22.66 + }, + { + "layer": 6, + "fro_norm": 8.3483, + "top_sv": 1.7165, + "stable_rank": 23.654 + }, + { + "layer": 7, + "fro_norm": 9.2453, + "top_sv": 1.9104, + "stable_rank": 23.421 + }, + { + "layer": 8, + "fro_norm": 10.5827, + "top_sv": 2.3986, + "stable_rank": 19.467 + }, + { + "layer": 9, + "fro_norm": 11.9236, + "top_sv": 2.7098, + "stable_rank": 19.362 + }, + { + "layer": 10, + "fro_norm": 13.6452, + "top_sv": 3.0551, + "stable_rank": 19.949 + }, + { + "layer": 11, + "fro_norm": 16.1248, + "top_sv": 3.5665, + "stable_rank": 20.441 + }, + { + "layer": 12, + "fro_norm": 18.3419, + "top_sv": 3.9743, + "stable_rank": 21.3 + }, + { + "layer": 13, + "fro_norm": 19.9731, + "top_sv": 4.2771, + "stable_rank": 21.807 + }, + { + "layer": 14, + "fro_norm": 23.3378, + "top_sv": 4.9252, + "stable_rank": 22.453 + }, + { + "layer": 15, + "fro_norm": 26.4098, + "top_sv": 5.8169, + "stable_rank": 20.613 + }, + { + "layer": 16, + "fro_norm": 29.7582, + "top_sv": 6.3935, + "stable_rank": 21.664 + }, + { + "layer": 17, + "fro_norm": 32.2372, + "top_sv": 6.9309, + "stable_rank": 21.634 + }, + { + "layer": 18, + "fro_norm": 33.9198, + "top_sv": 7.4203, + "stable_rank": 20.896 + }, + { + "layer": 19, + "fro_norm": 34.3697, + "top_sv": 7.2899, + "stable_rank": 22.228 + }, + { + "layer": 20, + "fro_norm": 34.7971, + "top_sv": 6.9998, + "stable_rank": 24.712 + }, + { + "layer": 21, + "fro_norm": 34.4472, + "top_sv": 5.5686, + "stable_rank": 38.267 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n400_s0.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6cb1d7d7c73be86777ba2bde327a7068b4430ce3 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e2af80098d34c57e6cdbfcf680a9c9303e18daae91b1577d3e9222e7eb4952b +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n400_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..277f228fd658be5aed4476503342d875f0ede2d1 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n400_s0.pt", + "sha256": "5e2af80098d34c57e6cdbfcf680a9c9303e18daae91b1577d3e9222e7eb4952b", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.915317, + "mean_fro": 6.735492, + "rms_per_prompt_fro": 13.327625, + "max_per_prompt_fro": 24.010969, + "heavy_tail_ratio": 1.8016, + "n_prompts": 400 + }, + "1": { + "dispersion": 2.590846, + "mean_fro": 6.36133, + "rms_per_prompt_fro": 12.054419, + "max_per_prompt_fro": 21.187668, + "heavy_tail_ratio": 1.7577, + "n_prompts": 400 + }, + "2": { + "dispersion": 2.075144, + "mean_fro": 6.971453, + "rms_per_prompt_fro": 12.225202, + "max_per_prompt_fro": 21.357693, + "heavy_tail_ratio": 1.747, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.756463, + "mean_fro": 6.777601, + "rms_per_prompt_fro": 11.252579, + "max_per_prompt_fro": 18.622065, + "heavy_tail_ratio": 1.6549, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.426337, + "mean_fro": 7.724647, + "rms_per_prompt_fro": 12.032455, + "max_per_prompt_fro": 18.780937, + "heavy_tail_ratio": 1.5609, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.266597, + "mean_fro": 7.995206, + "rms_per_prompt_fro": 12.036962, + "max_per_prompt_fro": 18.145401, + "heavy_tail_ratio": 1.5075, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.083787, + "mean_fro": 8.304218, + "rms_per_prompt_fro": 11.987411, + "max_per_prompt_fro": 17.646355, + "heavy_tail_ratio": 1.4721, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.956017, + "mean_fro": 9.194169, + "rms_per_prompt_fro": 12.858751, + "max_per_prompt_fro": 18.226198, + "heavy_tail_ratio": 1.4174, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.85803, + "mean_fro": 10.531087, + "rms_per_prompt_fro": 14.354879, + "max_per_prompt_fro": 19.531025, + "heavy_tail_ratio": 1.3606, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.784765, + "mean_fro": 11.872157, + "rms_per_prompt_fro": 15.860618, + "max_per_prompt_fro": 20.859287, + "heavy_tail_ratio": 1.3152, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.714478, + "mean_fro": 13.574479, + "rms_per_prompt_fro": 17.774162, + "max_per_prompt_fro": 22.48172, + "heavy_tail_ratio": 1.2649, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.648904, + "mean_fro": 16.007792, + "rms_per_prompt_fro": 20.555547, + "max_per_prompt_fro": 25.598974, + "heavy_tail_ratio": 1.2454, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.536267, + "mean_fro": 18.211653, + "rms_per_prompt_fro": 22.572658, + "max_per_prompt_fro": 26.548603, + "heavy_tail_ratio": 1.1761, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.467097, + "mean_fro": 19.86532, + "rms_per_prompt_fro": 24.061626, + "max_per_prompt_fro": 28.151562, + "heavy_tail_ratio": 1.17, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.397767, + "mean_fro": 23.250042, + "rms_per_prompt_fro": 27.487869, + "max_per_prompt_fro": 31.418961, + "heavy_tail_ratio": 1.143, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.338225, + "mean_fro": 26.280074, + "rms_per_prompt_fro": 30.401232, + "max_per_prompt_fro": 34.432747, + "heavy_tail_ratio": 1.1326, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.255793, + "mean_fro": 29.732382, + "rms_per_prompt_fro": 33.318751, + "max_per_prompt_fro": 36.798679, + "heavy_tail_ratio": 1.1044, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.189463, + "mean_fro": 32.117871, + "rms_per_prompt_fro": 35.028554, + "max_per_prompt_fro": 38.78054, + "heavy_tail_ratio": 1.1071, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.141129, + "mean_fro": 33.897426, + "rms_per_prompt_fro": 36.210459, + "max_per_prompt_fro": 39.442684, + "heavy_tail_ratio": 1.0893, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.095972, + "mean_fro": 34.385045, + "rms_per_prompt_fro": 35.997246, + "max_per_prompt_fro": 38.283794, + "heavy_tail_ratio": 1.0635, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.054406, + "mean_fro": 34.813606, + "rms_per_prompt_fro": 35.748091, + "max_per_prompt_fro": 37.069771, + "heavy_tail_ratio": 1.037, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.024875, + "mean_fro": 34.464367, + "rms_per_prompt_fro": 34.890388, + "max_per_prompt_fro": 35.838272, + "heavy_tail_ratio": 1.0272, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1214.8, + "fit_secs_per_prompt": 3.037, + "total_wall_seconds": 1223.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7354, + "top_sv": 1.2902, + "stable_rank": 27.251 + }, + { + "layer": 1, + "fro_norm": 6.3612, + "top_sv": 1.2512, + "stable_rank": 25.847 + }, + { + "layer": 2, + "fro_norm": 6.9714, + "top_sv": 1.4538, + "stable_rank": 22.996 + }, + { + "layer": 3, + "fro_norm": 6.7775, + "top_sv": 1.3648, + "stable_rank": 24.66 + }, + { + "layer": 4, + "fro_norm": 7.7245, + "top_sv": 1.644, + "stable_rank": 22.077 + }, + { + "layer": 5, + "fro_norm": 7.9951, + "top_sv": 1.7306, + "stable_rank": 21.342 + }, + { + "layer": 6, + "fro_norm": 8.3041, + "top_sv": 1.7064, + "stable_rank": 23.682 + }, + { + "layer": 7, + "fro_norm": 9.194, + "top_sv": 1.8525, + "stable_rank": 24.631 + }, + { + "layer": 8, + "fro_norm": 10.531, + "top_sv": 2.3435, + "stable_rank": 20.194 + }, + { + "layer": 9, + "fro_norm": 11.872, + "top_sv": 2.6739, + "stable_rank": 19.713 + }, + { + "layer": 10, + "fro_norm": 13.5743, + "top_sv": 3.0246, + "stable_rank": 20.142 + }, + { + "layer": 11, + "fro_norm": 16.0076, + "top_sv": 3.492, + "stable_rank": 21.014 + }, + { + "layer": 12, + "fro_norm": 18.2114, + "top_sv": 3.8899, + "stable_rank": 21.919 + }, + { + "layer": 13, + "fro_norm": 19.865, + "top_sv": 4.1787, + "stable_rank": 22.6 + }, + { + "layer": 14, + "fro_norm": 23.2497, + "top_sv": 4.8527, + "stable_rank": 22.955 + }, + { + "layer": 15, + "fro_norm": 26.2796, + "top_sv": 5.6916, + "stable_rank": 21.319 + }, + { + "layer": 16, + "fro_norm": 29.7318, + "top_sv": 6.3261, + "stable_rank": 22.089 + }, + { + "layer": 17, + "fro_norm": 32.1172, + "top_sv": 6.8542, + "stable_rank": 21.957 + }, + { + "layer": 18, + "fro_norm": 33.8966, + "top_sv": 7.3866, + "stable_rank": 21.058 + }, + { + "layer": 19, + "fro_norm": 34.384, + "top_sv": 7.2686, + "stable_rank": 22.378 + }, + { + "layer": 20, + "fro_norm": 34.8124, + "top_sv": 6.9743, + "stable_rank": 24.916 + }, + { + "layer": 21, + "fro_norm": 34.4628, + "top_sv": 5.5611, + "stable_rank": 38.405 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n400_s1.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f01fe2400832dc7b426aec7591cf2fcb9dac72d --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7c60a21498755c793c8d3206a0d805ae5348572d7a51ce393c1574bf218217 +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n400_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..2e9d622e88142a4a21d8f01d30e79f4c9ccefba8 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n400_s1.pt", + "sha256": "6e7c60a21498755c793c8d3206a0d805ae5348572d7a51ce393c1574bf218217", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.899967, + "mean_fro": 6.729803, + "rms_per_prompt_fro": 13.29024, + "max_per_prompt_fro": 22.19487, + "heavy_tail_ratio": 1.67, + "n_prompts": 400 + }, + "1": { + "dispersion": 2.580004, + "mean_fro": 6.355502, + "rms_per_prompt_fro": 12.025181, + "max_per_prompt_fro": 19.147593, + "heavy_tail_ratio": 1.5923, + "n_prompts": 400 + }, + "2": { + "dispersion": 2.063728, + "mean_fro": 6.966048, + "rms_per_prompt_fro": 12.193028, + "max_per_prompt_fro": 19.291433, + "heavy_tail_ratio": 1.5822, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.74134, + "mean_fro": 6.770717, + "rms_per_prompt_fro": 11.210271, + "max_per_prompt_fro": 17.139297, + "heavy_tail_ratio": 1.5289, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.412676, + "mean_fro": 7.720067, + "rms_per_prompt_fro": 11.99142, + "max_per_prompt_fro": 17.873886, + "heavy_tail_ratio": 1.4906, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.253424, + "mean_fro": 7.984528, + "rms_per_prompt_fro": 11.985901, + "max_per_prompt_fro": 18.013294, + "heavy_tail_ratio": 1.5029, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.077544, + "mean_fro": 8.291444, + "rms_per_prompt_fro": 11.951027, + "max_per_prompt_fro": 16.665794, + "heavy_tail_ratio": 1.3945, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.951967, + "mean_fro": 9.173398, + "rms_per_prompt_fro": 12.816413, + "max_per_prompt_fro": 16.387278, + "heavy_tail_ratio": 1.2786, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.85669, + "mean_fro": 10.497439, + "rms_per_prompt_fro": 14.303854, + "max_per_prompt_fro": 18.541143, + "heavy_tail_ratio": 1.2962, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.785378, + "mean_fro": 11.832817, + "rms_per_prompt_fro": 15.81078, + "max_per_prompt_fro": 20.537996, + "heavy_tail_ratio": 1.299, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.715133, + "mean_fro": 13.522357, + "rms_per_prompt_fro": 17.709297, + "max_per_prompt_fro": 22.904638, + "heavy_tail_ratio": 1.2934, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.650957, + "mean_fro": 15.93948, + "rms_per_prompt_fro": 20.480568, + "max_per_prompt_fro": 27.136269, + "heavy_tail_ratio": 1.325, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.539375, + "mean_fro": 18.153971, + "rms_per_prompt_fro": 22.523913, + "max_per_prompt_fro": 27.989729, + "heavy_tail_ratio": 1.2427, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.468544, + "mean_fro": 19.812563, + "rms_per_prompt_fro": 24.009553, + "max_per_prompt_fro": 29.441811, + "heavy_tail_ratio": 1.2263, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.398765, + "mean_fro": 23.180953, + "rms_per_prompt_fro": 27.41597, + "max_per_prompt_fro": 32.696583, + "heavy_tail_ratio": 1.1926, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.33925, + "mean_fro": 26.220154, + "rms_per_prompt_fro": 30.343525, + "max_per_prompt_fro": 35.626503, + "heavy_tail_ratio": 1.1741, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.255836, + "mean_fro": 29.666147, + "rms_per_prompt_fro": 33.245091, + "max_per_prompt_fro": 36.620102, + "heavy_tail_ratio": 1.1015, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.188991, + "mean_fro": 32.044361, + "rms_per_prompt_fro": 34.941441, + "max_per_prompt_fro": 37.982109, + "heavy_tail_ratio": 1.087, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.14033, + "mean_fro": 33.814666, + "rms_per_prompt_fro": 36.109413, + "max_per_prompt_fro": 38.831398, + "heavy_tail_ratio": 1.0754, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.095381, + "mean_fro": 34.320621, + "rms_per_prompt_fro": 35.92012, + "max_per_prompt_fro": 37.647743, + "heavy_tail_ratio": 1.0481, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.054029, + "mean_fro": 34.771625, + "rms_per_prompt_fro": 35.698604, + "max_per_prompt_fro": 36.747364, + "heavy_tail_ratio": 1.0294, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.024731, + "mean_fro": 34.437665, + "rms_per_prompt_fro": 34.860904, + "max_per_prompt_fro": 35.596199, + "heavy_tail_ratio": 1.0211, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1237.0, + "fit_secs_per_prompt": 3.093, + "total_wall_seconds": 1243.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7297, + "top_sv": 1.287, + "stable_rank": 27.342 + }, + { + "layer": 1, + "fro_norm": 6.3554, + "top_sv": 1.2521, + "stable_rank": 25.762 + }, + { + "layer": 2, + "fro_norm": 6.966, + "top_sv": 1.4471, + "stable_rank": 23.171 + }, + { + "layer": 3, + "fro_norm": 6.7706, + "top_sv": 1.358, + "stable_rank": 24.858 + }, + { + "layer": 4, + "fro_norm": 7.72, + "top_sv": 1.6283, + "stable_rank": 22.478 + }, + { + "layer": 5, + "fro_norm": 7.9844, + "top_sv": 1.716, + "stable_rank": 21.649 + }, + { + "layer": 6, + "fro_norm": 8.2913, + "top_sv": 1.6946, + "stable_rank": 23.94 + }, + { + "layer": 7, + "fro_norm": 9.1733, + "top_sv": 1.8415, + "stable_rank": 24.814 + }, + { + "layer": 8, + "fro_norm": 10.4973, + "top_sv": 2.3245, + "stable_rank": 20.394 + }, + { + "layer": 9, + "fro_norm": 11.8326, + "top_sv": 2.6555, + "stable_rank": 19.855 + }, + { + "layer": 10, + "fro_norm": 13.5221, + "top_sv": 3.006, + "stable_rank": 20.235 + }, + { + "layer": 11, + "fro_norm": 15.9393, + "top_sv": 3.4693, + "stable_rank": 21.108 + }, + { + "layer": 12, + "fro_norm": 18.1537, + "top_sv": 3.8669, + "stable_rank": 22.039 + }, + { + "layer": 13, + "fro_norm": 19.8122, + "top_sv": 4.1713, + "stable_rank": 22.56 + }, + { + "layer": 14, + "fro_norm": 23.1806, + "top_sv": 4.8388, + "stable_rank": 22.949 + }, + { + "layer": 15, + "fro_norm": 26.2197, + "top_sv": 5.6862, + "stable_rank": 21.262 + }, + { + "layer": 16, + "fro_norm": 29.6656, + "top_sv": 6.3016, + "stable_rank": 22.162 + }, + { + "layer": 17, + "fro_norm": 32.0437, + "top_sv": 6.8216, + "stable_rank": 22.066 + }, + { + "layer": 18, + "fro_norm": 33.8138, + "top_sv": 7.37, + "stable_rank": 21.05 + }, + { + "layer": 19, + "fro_norm": 34.3195, + "top_sv": 7.2491, + "stable_rank": 22.414 + }, + { + "layer": 20, + "fro_norm": 34.7705, + "top_sv": 6.9837, + "stable_rank": 24.789 + }, + { + "layer": 21, + "fro_norm": 34.4361, + "top_sv": 5.5738, + "stable_rank": 38.17 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n400_s2.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3c2087db21d700535e2e2266a024a731fff9fa1 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb1664fa377f21fc868166697c03a31592782ba9c0eac0fc67ecf52b357a01b +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n400_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..317c650ccd4e4a4abc5c98fcb7dcb30ef187d79d --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n400_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n400_s2.pt", + "sha256": "8eb1664fa377f21fc868166697c03a31592782ba9c0eac0fc67ecf52b357a01b", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.899784, + "mean_fro": 6.772687, + "rms_per_prompt_fro": 13.374614, + "max_per_prompt_fro": 24.485331, + "heavy_tail_ratio": 1.8307, + "n_prompts": 400 + }, + "1": { + "dispersion": 2.576866, + "mean_fro": 6.392537, + "rms_per_prompt_fro": 12.089952, + "max_per_prompt_fro": 21.961533, + "heavy_tail_ratio": 1.8165, + "n_prompts": 400 + }, + "2": { + "dispersion": 2.058196, + "mean_fro": 7.006353, + "rms_per_prompt_fro": 12.252498, + "max_per_prompt_fro": 21.365519, + "heavy_tail_ratio": 1.7438, + "n_prompts": 400 + }, + "3": { + "dispersion": 1.732656, + "mean_fro": 6.807176, + "rms_per_prompt_fro": 11.252769, + "max_per_prompt_fro": 18.536762, + "heavy_tail_ratio": 1.6473, + "n_prompts": 400 + }, + "4": { + "dispersion": 1.403024, + "mean_fro": 7.755164, + "rms_per_prompt_fro": 12.021814, + "max_per_prompt_fro": 18.625755, + "heavy_tail_ratio": 1.5493, + "n_prompts": 400 + }, + "5": { + "dispersion": 1.243531, + "mean_fro": 8.019528, + "rms_per_prompt_fro": 12.011987, + "max_per_prompt_fro": 18.531601, + "heavy_tail_ratio": 1.5428, + "n_prompts": 400 + }, + "6": { + "dispersion": 1.069718, + "mean_fro": 8.326926, + "rms_per_prompt_fro": 11.979543, + "max_per_prompt_fro": 18.148132, + "heavy_tail_ratio": 1.5149, + "n_prompts": 400 + }, + "7": { + "dispersion": 0.942696, + "mean_fro": 9.215399, + "rms_per_prompt_fro": 12.844481, + "max_per_prompt_fro": 18.539761, + "heavy_tail_ratio": 1.4434, + "n_prompts": 400 + }, + "8": { + "dispersion": 0.848117, + "mean_fro": 10.540104, + "rms_per_prompt_fro": 14.328793, + "max_per_prompt_fro": 19.655499, + "heavy_tail_ratio": 1.3717, + "n_prompts": 400 + }, + "9": { + "dispersion": 0.776971, + "mean_fro": 11.874729, + "rms_per_prompt_fro": 15.829379, + "max_per_prompt_fro": 20.738667, + "heavy_tail_ratio": 1.3101, + "n_prompts": 400 + }, + "10": { + "dispersion": 0.706832, + "mean_fro": 13.561929, + "rms_per_prompt_fro": 17.718088, + "max_per_prompt_fro": 22.484716, + "heavy_tail_ratio": 1.269, + "n_prompts": 400 + }, + "11": { + "dispersion": 0.641966, + "mean_fro": 15.983174, + "rms_per_prompt_fro": 20.480712, + "max_per_prompt_fro": 25.420773, + "heavy_tail_ratio": 1.2412, + "n_prompts": 400 + }, + "12": { + "dispersion": 0.532025, + "mean_fro": 18.190523, + "rms_per_prompt_fro": 22.515317, + "max_per_prompt_fro": 27.391455, + "heavy_tail_ratio": 1.2166, + "n_prompts": 400 + }, + "13": { + "dispersion": 0.461847, + "mean_fro": 19.8479, + "rms_per_prompt_fro": 23.997472, + "max_per_prompt_fro": 29.221926, + "heavy_tail_ratio": 1.2177, + "n_prompts": 400 + }, + "14": { + "dispersion": 0.392045, + "mean_fro": 23.206385, + "rms_per_prompt_fro": 27.380047, + "max_per_prompt_fro": 32.134945, + "heavy_tail_ratio": 1.1737, + "n_prompts": 400 + }, + "15": { + "dispersion": 0.332938, + "mean_fro": 26.246161, + "rms_per_prompt_fro": 30.301965, + "max_per_prompt_fro": 34.935623, + "heavy_tail_ratio": 1.1529, + "n_prompts": 400 + }, + "16": { + "dispersion": 0.251261, + "mean_fro": 29.699265, + "rms_per_prompt_fro": 33.221532, + "max_per_prompt_fro": 36.653419, + "heavy_tail_ratio": 1.1033, + "n_prompts": 400 + }, + "17": { + "dispersion": 0.185296, + "mean_fro": 32.074005, + "rms_per_prompt_fro": 34.919391, + "max_per_prompt_fro": 37.9259, + "heavy_tail_ratio": 1.0861, + "n_prompts": 400 + }, + "18": { + "dispersion": 0.13772, + "mean_fro": 33.848228, + "rms_per_prompt_fro": 36.103859, + "max_per_prompt_fro": 38.865517, + "heavy_tail_ratio": 1.0765, + "n_prompts": 400 + }, + "19": { + "dispersion": 0.093506, + "mean_fro": 34.350277, + "rms_per_prompt_fro": 35.920371, + "max_per_prompt_fro": 38.035507, + "heavy_tail_ratio": 1.0589, + "n_prompts": 400 + }, + "20": { + "dispersion": 0.053077, + "mean_fro": 34.77826, + "rms_per_prompt_fro": 35.689283, + "max_per_prompt_fro": 37.379372, + "heavy_tail_ratio": 1.0474, + "n_prompts": 400 + }, + "21": { + "dispersion": 0.024198, + "mean_fro": 34.435162, + "rms_per_prompt_fro": 34.849297, + "max_per_prompt_fro": 35.963951, + "heavy_tail_ratio": 1.032, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 1236.2, + "fit_secs_per_prompt": 3.091, + "total_wall_seconds": 1243.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7726, + "top_sv": 1.2352, + "stable_rank": 30.065 + }, + { + "layer": 1, + "fro_norm": 6.3925, + "top_sv": 1.199, + "stable_rank": 28.425 + }, + { + "layer": 2, + "fro_norm": 7.0063, + "top_sv": 1.3838, + "stable_rank": 25.635 + }, + { + "layer": 3, + "fro_norm": 6.8071, + "top_sv": 1.3095, + "stable_rank": 27.023 + }, + { + "layer": 4, + "fro_norm": 7.7551, + "top_sv": 1.561, + "stable_rank": 24.681 + }, + { + "layer": 5, + "fro_norm": 8.0194, + "top_sv": 1.6579, + "stable_rank": 23.399 + }, + { + "layer": 6, + "fro_norm": 8.3268, + "top_sv": 1.6437, + "stable_rank": 25.664 + }, + { + "layer": 7, + "fro_norm": 9.2153, + "top_sv": 1.8233, + "stable_rank": 25.545 + }, + { + "layer": 8, + "fro_norm": 10.54, + "top_sv": 2.2991, + "stable_rank": 21.017 + }, + { + "layer": 9, + "fro_norm": 11.8746, + "top_sv": 2.6247, + "stable_rank": 20.468 + }, + { + "layer": 10, + "fro_norm": 13.5617, + "top_sv": 2.9766, + "stable_rank": 20.758 + }, + { + "layer": 11, + "fro_norm": 15.9829, + "top_sv": 3.4412, + "stable_rank": 21.572 + }, + { + "layer": 12, + "fro_norm": 18.1902, + "top_sv": 3.8428, + "stable_rank": 22.407 + }, + { + "layer": 13, + "fro_norm": 19.8476, + "top_sv": 4.1439, + "stable_rank": 22.941 + }, + { + "layer": 14, + "fro_norm": 23.206, + "top_sv": 4.8123, + "stable_rank": 23.254 + }, + { + "layer": 15, + "fro_norm": 26.2457, + "top_sv": 5.6544, + "stable_rank": 21.545 + }, + { + "layer": 16, + "fro_norm": 29.6986, + "top_sv": 6.3079, + "stable_rank": 22.167 + }, + { + "layer": 17, + "fro_norm": 32.0734, + "top_sv": 6.8268, + "stable_rank": 22.073 + }, + { + "layer": 18, + "fro_norm": 33.8474, + "top_sv": 7.3756, + "stable_rank": 21.06 + }, + { + "layer": 19, + "fro_norm": 34.3493, + "top_sv": 7.2809, + "stable_rank": 22.257 + }, + { + "layer": 20, + "fro_norm": 34.777, + "top_sv": 6.9894, + "stable_rank": 24.758 + }, + { + "layer": 21, + "fro_norm": 34.4336, + "top_sv": 5.5769, + "stable_rank": 38.122 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n50_s0.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b99fe1ecb42c13a75c45f06ebfbaa63bc9afe83 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d160e9c92c8679c2a25f99a4c47381fe48c930294d70420349be74acca805f +size 46145005 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n50_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9e1b8b1aba0953c11cf6da5b1f3eccb1986c1d9b --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n50_s0.pt", + "sha256": "60d160e9c92c8679c2a25f99a4c47381fe48c930294d70420349be74acca805f", + "bytes": 46145005, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.665911, + "mean_fro": 6.987834, + "rms_per_prompt_fro": 13.379304, + "max_per_prompt_fro": 17.955008, + "heavy_tail_ratio": 1.342, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.39345, + "mean_fro": 6.583368, + "rms_per_prompt_fro": 12.127432, + "max_per_prompt_fro": 16.050516, + "heavy_tail_ratio": 1.3235, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.945371, + "mean_fro": 7.169757, + "rms_per_prompt_fro": 12.304797, + "max_per_prompt_fro": 16.309441, + "heavy_tail_ratio": 1.3255, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.658429, + "mean_fro": 6.9639, + "rms_per_prompt_fro": 11.354422, + "max_per_prompt_fro": 15.609672, + "heavy_tail_ratio": 1.3748, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.363014, + "mean_fro": 7.911997, + "rms_per_prompt_fro": 12.1624, + "max_per_prompt_fro": 16.447428, + "heavy_tail_ratio": 1.3523, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.210182, + "mean_fro": 8.168513, + "rms_per_prompt_fro": 12.143869, + "max_per_prompt_fro": 16.488127, + "heavy_tail_ratio": 1.3577, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.041237, + "mean_fro": 8.453677, + "rms_per_prompt_fro": 12.077928, + "max_per_prompt_fro": 14.61259, + "heavy_tail_ratio": 1.2099, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.92111, + "mean_fro": 9.330147, + "rms_per_prompt_fro": 12.931968, + "max_per_prompt_fro": 15.402933, + "heavy_tail_ratio": 1.1911, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.832668, + "mean_fro": 10.663131, + "rms_per_prompt_fro": 14.435329, + "max_per_prompt_fro": 17.245813, + "heavy_tail_ratio": 1.1947, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.763904, + "mean_fro": 12.002168, + "rms_per_prompt_fro": 15.940327, + "max_per_prompt_fro": 18.73131, + "heavy_tail_ratio": 1.1751, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.699406, + "mean_fro": 13.696271, + "rms_per_prompt_fro": 17.854633, + "max_per_prompt_fro": 20.712793, + "heavy_tail_ratio": 1.1601, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.640576, + "mean_fro": 16.114153, + "rms_per_prompt_fro": 20.639806, + "max_per_prompt_fro": 23.832846, + "heavy_tail_ratio": 1.1547, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.528896, + "mean_fro": 18.287966, + "rms_per_prompt_fro": 22.612799, + "max_per_prompt_fro": 25.334635, + "heavy_tail_ratio": 1.1204, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.462567, + "mean_fro": 19.935457, + "rms_per_prompt_fro": 24.109268, + "max_per_prompt_fro": 27.170097, + "heavy_tail_ratio": 1.127, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.397087, + "mean_fro": 23.314164, + "rms_per_prompt_fro": 27.556976, + "max_per_prompt_fro": 30.900297, + "heavy_tail_ratio": 1.1213, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.340406, + "mean_fro": 26.319563, + "rms_per_prompt_fro": 30.471709, + "max_per_prompt_fro": 33.765656, + "heavy_tail_ratio": 1.1081, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.25796, + "mean_fro": 29.727499, + "rms_per_prompt_fro": 33.342007, + "max_per_prompt_fro": 36.382008, + "heavy_tail_ratio": 1.0912, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.192081, + "mean_fro": 32.036842, + "rms_per_prompt_fro": 34.978614, + "max_per_prompt_fro": 37.829273, + "heavy_tail_ratio": 1.0815, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.143828, + "mean_fro": 33.807305, + "rms_per_prompt_fro": 36.156881, + "max_per_prompt_fro": 38.206364, + "heavy_tail_ratio": 1.0567, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.098766, + "mean_fro": 34.285396, + "rms_per_prompt_fro": 35.938652, + "max_per_prompt_fro": 37.266689, + "heavy_tail_ratio": 1.037, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.055442, + "mean_fro": 34.760254, + "rms_per_prompt_fro": 35.710838, + "max_per_prompt_fro": 36.651428, + "heavy_tail_ratio": 1.0263, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.025238, + "mean_fro": 34.40942, + "rms_per_prompt_fro": 34.840934, + "max_per_prompt_fro": 35.473534, + "heavy_tail_ratio": 1.0182, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 152.3, + "fit_secs_per_prompt": 3.047, + "total_wall_seconds": 160.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9878, + "top_sv": 1.4919, + "stable_rank": 21.937 + }, + { + "layer": 1, + "fro_norm": 6.5833, + "top_sv": 1.4283, + "stable_rank": 21.243 + }, + { + "layer": 2, + "fro_norm": 7.1697, + "top_sv": 1.6265, + "stable_rank": 19.43 + }, + { + "layer": 3, + "fro_norm": 6.9638, + "top_sv": 1.5054, + "stable_rank": 21.398 + }, + { + "layer": 4, + "fro_norm": 7.9119, + "top_sv": 1.8098, + "stable_rank": 19.112 + }, + { + "layer": 5, + "fro_norm": 8.1684, + "top_sv": 1.8709, + "stable_rank": 19.063 + }, + { + "layer": 6, + "fro_norm": 8.4536, + "top_sv": 1.7948, + "stable_rank": 22.183 + }, + { + "layer": 7, + "fro_norm": 9.33, + "top_sv": 1.9205, + "stable_rank": 23.601 + }, + { + "layer": 8, + "fro_norm": 10.663, + "top_sv": 2.4154, + "stable_rank": 19.488 + }, + { + "layer": 9, + "fro_norm": 12.002, + "top_sv": 2.7591, + "stable_rank": 18.922 + }, + { + "layer": 10, + "fro_norm": 13.6961, + "top_sv": 3.1026, + "stable_rank": 19.487 + }, + { + "layer": 11, + "fro_norm": 16.114, + "top_sv": 3.5677, + "stable_rank": 20.4 + }, + { + "layer": 12, + "fro_norm": 18.2877, + "top_sv": 3.9656, + "stable_rank": 21.267 + }, + { + "layer": 13, + "fro_norm": 19.9352, + "top_sv": 4.2558, + "stable_rank": 21.942 + }, + { + "layer": 14, + "fro_norm": 23.3138, + "top_sv": 4.9582, + "stable_rank": 22.11 + }, + { + "layer": 15, + "fro_norm": 26.319, + "top_sv": 5.7773, + "stable_rank": 20.753 + }, + { + "layer": 16, + "fro_norm": 29.727, + "top_sv": 6.3319, + "stable_rank": 22.041 + }, + { + "layer": 17, + "fro_norm": 32.0362, + "top_sv": 6.8437, + "stable_rank": 21.913 + }, + { + "layer": 18, + "fro_norm": 33.8064, + "top_sv": 7.3384, + "stable_rank": 21.222 + }, + { + "layer": 19, + "fro_norm": 34.2844, + "top_sv": 7.2116, + "stable_rank": 22.601 + }, + { + "layer": 20, + "fro_norm": 34.7591, + "top_sv": 6.9676, + "stable_rank": 24.887 + }, + { + "layer": 21, + "fro_norm": 34.4079, + "top_sv": 5.5353, + "stable_rank": 38.64 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n50_s1.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2b96713431f4697e8e9b0d049e5b11da9ab007c --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae4513372f15515fbc7e5ef33371d5a19ea0358fc036532684dbfb25fd0caf3d +size 46145005 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n50_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..faa2aa3011cba74b0a08bf9dd9466b2aa1f53452 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n50_s1.pt", + "sha256": "ae4513372f15515fbc7e5ef33371d5a19ea0358fc036532684dbfb25fd0caf3d", + "bytes": 46145005, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.598754, + "mean_fro": 6.871367, + "rms_per_prompt_fro": 13.035245, + "max_per_prompt_fro": 17.002298, + "heavy_tail_ratio": 1.3043, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.327552, + "mean_fro": 6.463257, + "rms_per_prompt_fro": 11.790002, + "max_per_prompt_fro": 15.121107, + "heavy_tail_ratio": 1.2825, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.883331, + "mean_fro": 7.057635, + "rms_per_prompt_fro": 11.984128, + "max_per_prompt_fro": 15.58554, + "heavy_tail_ratio": 1.3005, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.58122, + "mean_fro": 6.846482, + "rms_per_prompt_fro": 10.999678, + "max_per_prompt_fro": 14.107286, + "heavy_tail_ratio": 1.2825, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.291508, + "mean_fro": 7.783177, + "rms_per_prompt_fro": 11.781961, + "max_per_prompt_fro": 14.546237, + "heavy_tail_ratio": 1.2346, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.144877, + "mean_fro": 8.03897, + "rms_per_prompt_fro": 11.773392, + "max_per_prompt_fro": 14.404887, + "heavy_tail_ratio": 1.2235, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.991692, + "mean_fro": 8.342077, + "rms_per_prompt_fro": 11.77295, + "max_per_prompt_fro": 13.863462, + "heavy_tail_ratio": 1.1776, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.88045, + "mean_fro": 9.23279, + "rms_per_prompt_fro": 12.660877, + "max_per_prompt_fro": 14.802132, + "heavy_tail_ratio": 1.1691, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.792944, + "mean_fro": 10.565343, + "rms_per_prompt_fro": 14.147085, + "max_per_prompt_fro": 16.552475, + "heavy_tail_ratio": 1.17, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.726809, + "mean_fro": 11.922649, + "rms_per_prompt_fro": 15.667327, + "max_per_prompt_fro": 18.087225, + "heavy_tail_ratio": 1.1545, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.66277, + "mean_fro": 13.629216, + "rms_per_prompt_fro": 17.574663, + "max_per_prompt_fro": 19.858463, + "heavy_tail_ratio": 1.1299, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.6056, + "mean_fro": 16.061256, + "rms_per_prompt_fro": 20.35158, + "max_per_prompt_fro": 22.907913, + "heavy_tail_ratio": 1.1256, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.504416, + "mean_fro": 18.292473, + "rms_per_prompt_fro": 22.436564, + "max_per_prompt_fro": 24.603395, + "heavy_tail_ratio": 1.0966, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.439285, + "mean_fro": 19.94001, + "rms_per_prompt_fro": 23.922071, + "max_per_prompt_fro": 26.25326, + "heavy_tail_ratio": 1.0974, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.374612, + "mean_fro": 23.305468, + "rms_per_prompt_fro": 27.324227, + "max_per_prompt_fro": 29.861521, + "heavy_tail_ratio": 1.0929, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.31888, + "mean_fro": 26.346807, + "rms_per_prompt_fro": 30.25733, + "max_per_prompt_fro": 33.088852, + "heavy_tail_ratio": 1.0936, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.241967, + "mean_fro": 29.834367, + "rms_per_prompt_fro": 33.24848, + "max_per_prompt_fro": 35.631195, + "heavy_tail_ratio": 1.0717, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.17851, + "mean_fro": 32.17863, + "rms_per_prompt_fro": 34.93287, + "max_per_prompt_fro": 37.413486, + "heavy_tail_ratio": 1.071, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.131903, + "mean_fro": 33.936748, + "rms_per_prompt_fro": 36.105629, + "max_per_prompt_fro": 38.020199, + "heavy_tail_ratio": 1.053, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.089753, + "mean_fro": 34.415505, + "rms_per_prompt_fro": 35.926767, + "max_per_prompt_fro": 37.108562, + "heavy_tail_ratio": 1.0329, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.051822, + "mean_fro": 34.834847, + "rms_per_prompt_fro": 35.726058, + "max_per_prompt_fro": 36.592823, + "heavy_tail_ratio": 1.0243, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.023727, + "mean_fro": 34.471691, + "rms_per_prompt_fro": 34.878246, + "max_per_prompt_fro": 35.386131, + "heavy_tail_ratio": 1.0146, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 154.6, + "fit_secs_per_prompt": 3.091, + "total_wall_seconds": 161.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.8713, + "top_sv": 1.3431, + "stable_rank": 26.174 + }, + { + "layer": 1, + "fro_norm": 6.4632, + "top_sv": 1.2785, + "stable_rank": 25.555 + }, + { + "layer": 2, + "fro_norm": 7.0575, + "top_sv": 1.4488, + "stable_rank": 23.728 + }, + { + "layer": 3, + "fro_norm": 6.8464, + "top_sv": 1.3658, + "stable_rank": 25.128 + }, + { + "layer": 4, + "fro_norm": 7.7831, + "top_sv": 1.6069, + "stable_rank": 23.459 + }, + { + "layer": 5, + "fro_norm": 8.0389, + "top_sv": 1.6932, + "stable_rank": 22.542 + }, + { + "layer": 6, + "fro_norm": 8.342, + "top_sv": 1.6793, + "stable_rank": 24.676 + }, + { + "layer": 7, + "fro_norm": 9.2327, + "top_sv": 1.8338, + "stable_rank": 25.347 + }, + { + "layer": 8, + "fro_norm": 10.5652, + "top_sv": 2.312, + "stable_rank": 20.883 + }, + { + "layer": 9, + "fro_norm": 11.9225, + "top_sv": 2.64, + "stable_rank": 20.394 + }, + { + "layer": 10, + "fro_norm": 13.629, + "top_sv": 2.9899, + "stable_rank": 20.779 + }, + { + "layer": 11, + "fro_norm": 16.061, + "top_sv": 3.4577, + "stable_rank": 21.576 + }, + { + "layer": 12, + "fro_norm": 18.2922, + "top_sv": 3.8672, + "stable_rank": 22.373 + }, + { + "layer": 13, + "fro_norm": 19.9397, + "top_sv": 4.1677, + "stable_rank": 22.89 + }, + { + "layer": 14, + "fro_norm": 23.3051, + "top_sv": 4.8107, + "stable_rank": 23.468 + }, + { + "layer": 15, + "fro_norm": 26.3463, + "top_sv": 5.6331, + "stable_rank": 21.875 + }, + { + "layer": 16, + "fro_norm": 29.8338, + "top_sv": 6.2902, + "stable_rank": 22.495 + }, + { + "layer": 17, + "fro_norm": 32.178, + "top_sv": 6.8276, + "stable_rank": 22.212 + }, + { + "layer": 18, + "fro_norm": 33.9358, + "top_sv": 7.4101, + "stable_rank": 20.973 + }, + { + "layer": 19, + "fro_norm": 34.4144, + "top_sv": 7.2895, + "stable_rank": 22.289 + }, + { + "layer": 20, + "fro_norm": 34.8336, + "top_sv": 7.0047, + "stable_rank": 24.73 + }, + { + "layer": 21, + "fro_norm": 34.4702, + "top_sv": 5.6333, + "stable_rank": 37.442 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n50_s2.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..a29ca93f7ded6f27aaa78edbdd07469dff748b5a --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32dea8de66bf5edaa80b24e76761715815dd2b0a6874b85d0ceb45b22f2e4dd4 +size 46145005 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n50_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b15360e3d4e9f85f6ec3c49eb9058175d6642c87 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n50_s2_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n50_s2.pt", + "sha256": "32dea8de66bf5edaa80b24e76761715815dd2b0a6874b85d0ceb45b22f2e4dd4", + "bytes": 46145005, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.621325, + "mean_fro": 6.826432, + "rms_per_prompt_fro": 12.990548, + "max_per_prompt_fro": 19.645554, + "heavy_tail_ratio": 1.5123, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.360384, + "mean_fro": 6.437847, + "rms_per_prompt_fro": 11.801443, + "max_per_prompt_fro": 18.059361, + "heavy_tail_ratio": 1.5303, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.905832, + "mean_fro": 7.03398, + "rms_per_prompt_fro": 11.990473, + "max_per_prompt_fro": 17.360987, + "heavy_tail_ratio": 1.4479, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.614396, + "mean_fro": 6.824203, + "rms_per_prompt_fro": 11.034118, + "max_per_prompt_fro": 15.802229, + "heavy_tail_ratio": 1.4321, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.310793, + "mean_fro": 7.76083, + "rms_per_prompt_fro": 11.797465, + "max_per_prompt_fro": 15.854173, + "heavy_tail_ratio": 1.3439, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.166582, + "mean_fro": 8.028244, + "rms_per_prompt_fro": 11.817024, + "max_per_prompt_fro": 15.6437, + "heavy_tail_ratio": 1.3238, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.006474, + "mean_fro": 8.342357, + "rms_per_prompt_fro": 11.816955, + "max_per_prompt_fro": 15.298518, + "heavy_tail_ratio": 1.2946, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.889605, + "mean_fro": 9.243597, + "rms_per_prompt_fro": 12.706518, + "max_per_prompt_fro": 15.987034, + "heavy_tail_ratio": 1.2582, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.799247, + "mean_fro": 10.59154, + "rms_per_prompt_fro": 14.20707, + "max_per_prompt_fro": 17.441622, + "heavy_tail_ratio": 1.2277, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.731376, + "mean_fro": 11.942278, + "rms_per_prompt_fro": 15.713858, + "max_per_prompt_fro": 19.016964, + "heavy_tail_ratio": 1.2102, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.66504, + "mean_fro": 13.671294, + "rms_per_prompt_fro": 17.640947, + "max_per_prompt_fro": 20.77474, + "heavy_tail_ratio": 1.1776, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.605137, + "mean_fro": 16.146576, + "rms_per_prompt_fro": 20.456743, + "max_per_prompt_fro": 23.907835, + "heavy_tail_ratio": 1.1687, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.501155, + "mean_fro": 18.366589, + "rms_per_prompt_fro": 22.503041, + "max_per_prompt_fro": 25.347282, + "heavy_tail_ratio": 1.1264, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.432539, + "mean_fro": 20.005013, + "rms_per_prompt_fro": 23.943745, + "max_per_prompt_fro": 27.265087, + "heavy_tail_ratio": 1.1387, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.365846, + "mean_fro": 23.385081, + "rms_per_prompt_fro": 27.330004, + "max_per_prompt_fro": 30.383566, + "heavy_tail_ratio": 1.1117, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.308883, + "mean_fro": 26.447115, + "rms_per_prompt_fro": 30.257201, + "max_per_prompt_fro": 33.130489, + "heavy_tail_ratio": 1.095, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.23418, + "mean_fro": 29.896272, + "rms_per_prompt_fro": 33.212861, + "max_per_prompt_fro": 35.40237, + "heavy_tail_ratio": 1.0659, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.173577, + "mean_fro": 32.282258, + "rms_per_prompt_fro": 34.971943, + "max_per_prompt_fro": 36.947639, + "heavy_tail_ratio": 1.0565, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.128436, + "mean_fro": 33.984116, + "rms_per_prompt_fro": 36.100601, + "max_per_prompt_fro": 38.000294, + "heavy_tail_ratio": 1.0526, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.086942, + "mean_fro": 34.444023, + "rms_per_prompt_fro": 35.910136, + "max_per_prompt_fro": 37.17416, + "heavy_tail_ratio": 1.0352, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.050424, + "mean_fro": 34.80664, + "rms_per_prompt_fro": 35.673392, + "max_per_prompt_fro": 36.726715, + "heavy_tail_ratio": 1.0295, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.023099, + "mean_fro": 34.461551, + "rms_per_prompt_fro": 34.8573, + "max_per_prompt_fro": 35.574158, + "heavy_tail_ratio": 1.0206, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 154.4, + "fit_secs_per_prompt": 3.088, + "total_wall_seconds": 162.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.8263, + "top_sv": 1.2892, + "stable_rank": 28.037 + }, + { + "layer": 1, + "fro_norm": 6.4378, + "top_sv": 1.236, + "stable_rank": 27.128 + }, + { + "layer": 2, + "fro_norm": 7.0339, + "top_sv": 1.4084, + "stable_rank": 24.942 + }, + { + "layer": 3, + "fro_norm": 6.8241, + "top_sv": 1.3449, + "stable_rank": 25.745 + }, + { + "layer": 4, + "fro_norm": 7.7607, + "top_sv": 1.5803, + "stable_rank": 24.118 + }, + { + "layer": 5, + "fro_norm": 8.0282, + "top_sv": 1.6742, + "stable_rank": 22.995 + }, + { + "layer": 6, + "fro_norm": 8.3422, + "top_sv": 1.6953, + "stable_rank": 24.214 + }, + { + "layer": 7, + "fro_norm": 9.2435, + "top_sv": 1.8949, + "stable_rank": 23.796 + }, + { + "layer": 8, + "fro_norm": 10.5914, + "top_sv": 2.3903, + "stable_rank": 19.634 + }, + { + "layer": 9, + "fro_norm": 11.9421, + "top_sv": 2.7058, + "stable_rank": 19.479 + }, + { + "layer": 10, + "fro_norm": 13.6711, + "top_sv": 3.0727, + "stable_rank": 19.796 + }, + { + "layer": 11, + "fro_norm": 16.1464, + "top_sv": 3.5691, + "stable_rank": 20.466 + }, + { + "layer": 12, + "fro_norm": 18.3663, + "top_sv": 3.9815, + "stable_rank": 21.279 + }, + { + "layer": 13, + "fro_norm": 20.0047, + "top_sv": 4.283, + "stable_rank": 21.816 + }, + { + "layer": 14, + "fro_norm": 23.3847, + "top_sv": 4.9454, + "stable_rank": 22.36 + }, + { + "layer": 15, + "fro_norm": 26.4466, + "top_sv": 5.8109, + "stable_rank": 20.713 + }, + { + "layer": 16, + "fro_norm": 29.8957, + "top_sv": 6.454, + "stable_rank": 21.456 + }, + { + "layer": 17, + "fro_norm": 32.2816, + "top_sv": 6.9662, + "stable_rank": 21.474 + }, + { + "layer": 18, + "fro_norm": 33.9832, + "top_sv": 7.4796, + "stable_rank": 20.643 + }, + { + "layer": 19, + "fro_norm": 34.443, + "top_sv": 7.3939, + "stable_rank": 21.7 + }, + { + "layer": 20, + "fro_norm": 34.8054, + "top_sv": 7.0102, + "stable_rank": 24.651 + }, + { + "layer": 21, + "fro_norm": 34.46, + "top_sv": 5.5849, + "stable_rank": 38.072 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n800_s0.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1071841c23589c5b92dd5417c051b81fabd02f7e --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6244533db3ed6740e6cc1640ef7ce71a7c85f3834047ab097b73284880f8adcd +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n800_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c6f9e19bac225df7dc6868b3e999177550628edb --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s0_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n800_s0.pt", + "sha256": "6244533db3ed6740e6cc1640ef7ce71a7c85f3834047ab097b73284880f8adcd", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.913681, + "mean_fro": 6.687707, + "rms_per_prompt_fro": 13.230309, + "max_per_prompt_fro": 24.010969, + "heavy_tail_ratio": 1.8148, + "n_prompts": 800 + }, + "1": { + "dispersion": 2.59155, + "mean_fro": 6.317776, + "rms_per_prompt_fro": 11.973061, + "max_per_prompt_fro": 21.339285, + "heavy_tail_ratio": 1.7823, + "n_prompts": 800 + }, + "2": { + "dispersion": 2.077191, + "mean_fro": 6.926495, + "rms_per_prompt_fro": 12.150406, + "max_per_prompt_fro": 21.357693, + "heavy_tail_ratio": 1.7578, + "n_prompts": 800 + }, + "3": { + "dispersion": 1.754017, + "mean_fro": 6.736807, + "rms_per_prompt_fro": 11.179887, + "max_per_prompt_fro": 18.622065, + "heavy_tail_ratio": 1.6657, + "n_prompts": 800 + }, + "4": { + "dispersion": 1.424961, + "mean_fro": 7.681076, + "rms_per_prompt_fro": 11.961192, + "max_per_prompt_fro": 18.780937, + "heavy_tail_ratio": 1.5702, + "n_prompts": 800 + }, + "5": { + "dispersion": 1.266482, + "mean_fro": 7.954712, + "rms_per_prompt_fro": 11.975691, + "max_per_prompt_fro": 18.145401, + "heavy_tail_ratio": 1.5152, + "n_prompts": 800 + }, + "6": { + "dispersion": 1.088182, + "mean_fro": 8.265727, + "rms_per_prompt_fro": 11.944425, + "max_per_prompt_fro": 17.646355, + "heavy_tail_ratio": 1.4774, + "n_prompts": 800 + }, + "7": { + "dispersion": 0.960375, + "mean_fro": 9.15569, + "rms_per_prompt_fro": 12.819191, + "max_per_prompt_fro": 18.226198, + "heavy_tail_ratio": 1.4218, + "n_prompts": 800 + }, + "8": { + "dispersion": 0.862449, + "mean_fro": 10.489577, + "rms_per_prompt_fro": 14.315291, + "max_per_prompt_fro": 19.531025, + "heavy_tail_ratio": 1.3643, + "n_prompts": 800 + }, + "9": { + "dispersion": 0.788625, + "mean_fro": 11.830669, + "rms_per_prompt_fro": 15.822277, + "max_per_prompt_fro": 20.859287, + "heavy_tail_ratio": 1.3183, + "n_prompts": 800 + }, + "10": { + "dispersion": 0.717591, + "mean_fro": 13.53175, + "rms_per_prompt_fro": 17.734289, + "max_per_prompt_fro": 22.657656, + "heavy_tail_ratio": 1.2776, + "n_prompts": 800 + }, + "11": { + "dispersion": 0.651093, + "mean_fro": 15.965302, + "rms_per_prompt_fro": 20.514593, + "max_per_prompt_fro": 25.98525, + "heavy_tail_ratio": 1.2667, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.537994, + "mean_fro": 18.170279, + "rms_per_prompt_fro": 22.53403, + "max_per_prompt_fro": 27.157608, + "heavy_tail_ratio": 1.2052, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.467341, + "mean_fro": 19.822189, + "rms_per_prompt_fro": 24.011381, + "max_per_prompt_fro": 28.444826, + "heavy_tail_ratio": 1.1846, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.397251, + "mean_fro": 23.201993, + "rms_per_prompt_fro": 27.425999, + "max_per_prompt_fro": 31.735264, + "heavy_tail_ratio": 1.1571, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.337567, + "mean_fro": 26.238825, + "rms_per_prompt_fro": 30.34605, + "max_per_prompt_fro": 34.977901, + "heavy_tail_ratio": 1.1526, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.254994, + "mean_fro": 29.705214, + "rms_per_prompt_fro": 33.277715, + "max_per_prompt_fro": 37.10759, + "heavy_tail_ratio": 1.1151, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.188187, + "mean_fro": 32.096298, + "rms_per_prompt_fro": 34.986242, + "max_per_prompt_fro": 38.78054, + "heavy_tail_ratio": 1.1085, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.140234, + "mean_fro": 33.87867, + "rms_per_prompt_fro": 36.17624, + "max_per_prompt_fro": 39.442684, + "heavy_tail_ratio": 1.0903, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.095397, + "mean_fro": 34.373359, + "rms_per_prompt_fro": 35.975575, + "max_per_prompt_fro": 38.283794, + "heavy_tail_ratio": 1.0642, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.054022, + "mean_fro": 34.804725, + "rms_per_prompt_fro": 35.732465, + "max_per_prompt_fro": 37.069771, + "heavy_tail_ratio": 1.0374, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.024719, + "mean_fro": 34.457325, + "rms_per_prompt_fro": 34.880597, + "max_per_prompt_fro": 35.838272, + "heavy_tail_ratio": 1.0275, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2431.5, + "fit_secs_per_prompt": 3.039, + "total_wall_seconds": 2439.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.6876, + "top_sv": 1.2715, + "stable_rank": 27.664 + }, + { + "layer": 1, + "fro_norm": 6.3177, + "top_sv": 1.2333, + "stable_rank": 26.239 + }, + { + "layer": 2, + "fro_norm": 6.9264, + "top_sv": 1.4344, + "stable_rank": 23.317 + }, + { + "layer": 3, + "fro_norm": 6.7367, + "top_sv": 1.3459, + "stable_rank": 25.054 + }, + { + "layer": 4, + "fro_norm": 7.681, + "top_sv": 1.6185, + "stable_rank": 22.523 + }, + { + "layer": 5, + "fro_norm": 7.9546, + "top_sv": 1.7083, + "stable_rank": 21.682 + }, + { + "layer": 6, + "fro_norm": 8.2656, + "top_sv": 1.6843, + "stable_rank": 24.084 + }, + { + "layer": 7, + "fro_norm": 9.1556, + "top_sv": 1.8415, + "stable_rank": 24.719 + }, + { + "layer": 8, + "fro_norm": 10.4894, + "top_sv": 2.3306, + "stable_rank": 20.257 + }, + { + "layer": 9, + "fro_norm": 11.8305, + "top_sv": 2.6581, + "stable_rank": 19.808 + }, + { + "layer": 10, + "fro_norm": 13.5315, + "top_sv": 3.0103, + "stable_rank": 20.205 + }, + { + "layer": 11, + "fro_norm": 15.9651, + "top_sv": 3.479, + "stable_rank": 21.058 + }, + { + "layer": 12, + "fro_norm": 18.17, + "top_sv": 3.8859, + "stable_rank": 21.864 + }, + { + "layer": 13, + "fro_norm": 19.8219, + "top_sv": 4.1799, + "stable_rank": 22.488 + }, + { + "layer": 14, + "fro_norm": 23.2016, + "top_sv": 4.8561, + "stable_rank": 22.827 + }, + { + "layer": 15, + "fro_norm": 26.2383, + "top_sv": 5.6967, + "stable_rank": 21.214 + }, + { + "layer": 16, + "fro_norm": 29.7047, + "top_sv": 6.3391, + "stable_rank": 21.958 + }, + { + "layer": 17, + "fro_norm": 32.0957, + "top_sv": 6.8637, + "stable_rank": 21.866 + }, + { + "layer": 18, + "fro_norm": 33.8778, + "top_sv": 7.3966, + "stable_rank": 20.978 + }, + { + "layer": 19, + "fro_norm": 34.3723, + "top_sv": 7.2818, + "stable_rank": 22.281 + }, + { + "layer": 20, + "fro_norm": 34.8035, + "top_sv": 6.9869, + "stable_rank": 24.813 + }, + { + "layer": 21, + "fro_norm": 34.4558, + "top_sv": 5.5739, + "stable_rank": 38.213 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n800_s1.pt b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb5da793cb821f513c98580d381358bc280b29c5 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22e1d8eb49c6f6d9fd7cbf370f128b907d9f77ae87f36680055469c0e4bebb11 +size 46145033 diff --git a/results/lenses/e28/e28_Wikipedia_en_410m_n800_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d472a6d8f898d2a3ac7a1c483b7e20d0861255e5 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_410m_n800_s1_provenance.json @@ -0,0 +1,375 @@ +{ + "artifact": "results/e28_Wikipedia_en_410m_n800_s1.pt", + "sha256": "22e1d8eb49c6f6d9fd7cbf370f128b907d9f77ae87f36680055469c0e4bebb11", + "bytes": 46145033, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 2.93132, + "mean_fro": 6.751777, + "rms_per_prompt_fro": 13.387125, + "max_per_prompt_fro": 22.19487, + "heavy_tail_ratio": 1.6579, + "n_prompts": 800 + }, + "1": { + "dispersion": 2.605859, + "mean_fro": 6.376929, + "rms_per_prompt_fro": 12.109213, + "max_per_prompt_fro": 19.273548, + "heavy_tail_ratio": 1.5916, + "n_prompts": 800 + }, + "2": { + "dispersion": 2.0851, + "mean_fro": 6.985854, + "rms_per_prompt_fro": 12.270271, + "max_per_prompt_fro": 19.291433, + "heavy_tail_ratio": 1.5722, + "n_prompts": 800 + }, + "3": { + "dispersion": 1.759446, + "mean_fro": 6.790871, + "rms_per_prompt_fro": 11.28071, + "max_per_prompt_fro": 17.139297, + "heavy_tail_ratio": 1.5193, + "n_prompts": 800 + }, + "4": { + "dispersion": 1.427759, + "mean_fro": 7.73845, + "rms_per_prompt_fro": 12.057485, + "max_per_prompt_fro": 17.873886, + "heavy_tail_ratio": 1.4824, + "n_prompts": 800 + }, + "5": { + "dispersion": 1.26736, + "mean_fro": 8.003997, + "rms_per_prompt_fro": 12.052223, + "max_per_prompt_fro": 18.013294, + "heavy_tail_ratio": 1.4946, + "n_prompts": 800 + }, + "6": { + "dispersion": 1.090064, + "mean_fro": 8.313265, + "rms_per_prompt_fro": 12.018533, + "max_per_prompt_fro": 16.853292, + "heavy_tail_ratio": 1.4023, + "n_prompts": 800 + }, + "7": { + "dispersion": 0.963344, + "mean_fro": 9.197528, + "rms_per_prompt_fro": 12.88752, + "max_per_prompt_fro": 17.873398, + "heavy_tail_ratio": 1.3869, + "n_prompts": 800 + }, + "8": { + "dispersion": 0.866726, + "mean_fro": 10.521808, + "rms_per_prompt_fro": 14.375754, + "max_per_prompt_fro": 19.793097, + "heavy_tail_ratio": 1.3768, + "n_prompts": 800 + }, + "9": { + "dispersion": 0.794524, + "mean_fro": 11.855688, + "rms_per_prompt_fro": 15.881862, + "max_per_prompt_fro": 21.319441, + "heavy_tail_ratio": 1.3424, + "n_prompts": 800 + }, + "10": { + "dispersion": 0.723199, + "mean_fro": 13.54124, + "rms_per_prompt_fro": 17.775679, + "max_per_prompt_fro": 23.136627, + "heavy_tail_ratio": 1.3016, + "n_prompts": 800 + }, + "11": { + "dispersion": 0.658209, + "mean_fro": 15.953359, + "rms_per_prompt_fro": 20.543372, + "max_per_prompt_fro": 27.136269, + "heavy_tail_ratio": 1.3209, + "n_prompts": 800 + }, + "12": { + "dispersion": 0.545157, + "mean_fro": 18.16931, + "rms_per_prompt_fro": 22.585244, + "max_per_prompt_fro": 27.989729, + "heavy_tail_ratio": 1.2393, + "n_prompts": 800 + }, + "13": { + "dispersion": 0.474187, + "mean_fro": 19.825281, + "rms_per_prompt_fro": 24.071087, + "max_per_prompt_fro": 29.441811, + "heavy_tail_ratio": 1.2231, + "n_prompts": 800 + }, + "14": { + "dispersion": 0.40369, + "mean_fro": 23.192085, + "rms_per_prompt_fro": 27.477388, + "max_per_prompt_fro": 32.696583, + "heavy_tail_ratio": 1.1899, + "n_prompts": 800 + }, + "15": { + "dispersion": 0.34369, + "mean_fro": 26.23452, + "rms_per_prompt_fro": 30.410436, + "max_per_prompt_fro": 35.626503, + "heavy_tail_ratio": 1.1715, + "n_prompts": 800 + }, + "16": { + "dispersion": 0.259421, + "mean_fro": 29.678434, + "rms_per_prompt_fro": 33.306303, + "max_per_prompt_fro": 36.620102, + "heavy_tail_ratio": 1.0995, + "n_prompts": 800 + }, + "17": { + "dispersion": 0.191891, + "mean_fro": 32.057015, + "rms_per_prompt_fro": 34.997852, + "max_per_prompt_fro": 37.982109, + "heavy_tail_ratio": 1.0853, + "n_prompts": 800 + }, + "18": { + "dispersion": 0.142519, + "mean_fro": 33.833164, + "rms_per_prompt_fro": 36.163818, + "max_per_prompt_fro": 38.831398, + "heavy_tail_ratio": 1.0738, + "n_prompts": 800 + }, + "19": { + "dispersion": 0.09684, + "mean_fro": 34.335098, + "rms_per_prompt_fro": 35.9592, + "max_per_prompt_fro": 37.941498, + "heavy_tail_ratio": 1.0551, + "n_prompts": 800 + }, + "20": { + "dispersion": 0.054901, + "mean_fro": 34.78513, + "rms_per_prompt_fro": 35.727233, + "max_per_prompt_fro": 37.254478, + "heavy_tail_ratio": 1.0427, + "n_prompts": 800 + }, + "21": { + "dispersion": 0.025136, + "mean_fro": 34.44447, + "rms_per_prompt_fro": 34.874686, + "max_per_prompt_fro": 35.857876, + "heavy_tail_ratio": 1.0282, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2473.1, + "fit_secs_per_prompt": 3.091, + "total_wall_seconds": 2479.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.7517, + "top_sv": 1.2661, + "stable_rank": 28.44 + }, + { + "layer": 1, + "fro_norm": 6.3768, + "top_sv": 1.2318, + "stable_rank": 26.802 + }, + { + "layer": 2, + "fro_norm": 6.9858, + "top_sv": 1.424, + "stable_rank": 24.067 + }, + { + "layer": 3, + "fro_norm": 6.7908, + "top_sv": 1.3407, + "stable_rank": 25.656 + }, + { + "layer": 4, + "fro_norm": 7.7384, + "top_sv": 1.6078, + "stable_rank": 23.164 + }, + { + "layer": 5, + "fro_norm": 8.0039, + "top_sv": 1.6944, + "stable_rank": 22.313 + }, + { + "layer": 6, + "fro_norm": 8.3132, + "top_sv": 1.6791, + "stable_rank": 24.512 + }, + { + "layer": 7, + "fro_norm": 9.1974, + "top_sv": 1.836, + "stable_rank": 25.095 + }, + { + "layer": 8, + "fro_norm": 10.5217, + "top_sv": 2.3178, + "stable_rank": 20.608 + }, + { + "layer": 9, + "fro_norm": 11.8555, + "top_sv": 2.6478, + "stable_rank": 20.048 + }, + { + "layer": 10, + "fro_norm": 13.541, + "top_sv": 3.0009, + "stable_rank": 20.361 + }, + { + "layer": 11, + "fro_norm": 15.9531, + "top_sv": 3.4625, + "stable_rank": 21.228 + }, + { + "layer": 12, + "fro_norm": 18.169, + "top_sv": 3.8647, + "stable_rank": 22.102 + }, + { + "layer": 13, + "fro_norm": 19.825, + "top_sv": 4.1642, + "stable_rank": 22.665 + }, + { + "layer": 14, + "fro_norm": 23.1917, + "top_sv": 4.8363, + "stable_rank": 22.996 + }, + { + "layer": 15, + "fro_norm": 26.234, + "top_sv": 5.6825, + "stable_rank": 21.313 + }, + { + "layer": 16, + "fro_norm": 29.6779, + "top_sv": 6.3179, + "stable_rank": 22.065 + }, + { + "layer": 17, + "fro_norm": 32.0563, + "top_sv": 6.8404, + "stable_rank": 21.962 + }, + { + "layer": 18, + "fro_norm": 33.8323, + "top_sv": 7.3817, + "stable_rank": 21.006 + }, + { + "layer": 19, + "fro_norm": 34.334, + "top_sv": 7.2595, + "stable_rank": 22.369 + }, + { + "layer": 20, + "fro_norm": 34.7839, + "top_sv": 6.9865, + "stable_rank": 24.788 + }, + { + "layer": 21, + "fro_norm": 34.4429, + "top_sv": 5.5746, + "stable_rank": 38.174 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n100_s0.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..899d25baa610264e3f0ff2fd872b0b64eb545e38 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791dae2001d54780df16c24e528c2d37b8850b3ae18636bc049456368ab949b0 +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n100_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..85d88acb54ab4a090ce4212a3a2258c7e9c22584 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n100_s0.pt", + "sha256": "791dae2001d54780df16c24e528c2d37b8850b3ae18636bc049456368ab949b0", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.205064, + "mean_fro": 32.493289, + "rms_per_prompt_fro": 35.669642, + "max_per_prompt_fro": 40.516212, + "heavy_tail_ratio": 1.1359, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.158405, + "mean_fro": 25.232653, + "rms_per_prompt_fro": 27.157709, + "max_per_prompt_fro": 30.508642, + "heavy_tail_ratio": 1.1234, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.110574, + "mean_fro": 23.969816, + "rms_per_prompt_fro": 25.260301, + "max_per_prompt_fro": 27.863903, + "heavy_tail_ratio": 1.1031, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.056809, + "mean_fro": 23.017101, + "rms_per_prompt_fro": 23.661863, + "max_per_prompt_fro": 24.666615, + "heavy_tail_ratio": 1.0425, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 12.1, + "fit_secs_per_prompt": 0.121, + "total_wall_seconds": 20.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.4932, + "top_sv": 5.8809, + "stable_rank": 30.528 + }, + { + "layer": 1, + "fro_norm": 25.2326, + "top_sv": 4.4367, + "stable_rank": 32.345 + }, + { + "layer": 2, + "fro_norm": 23.9698, + "top_sv": 3.6646, + "stable_rank": 42.784 + }, + { + "layer": 3, + "fro_norm": 23.0171, + "top_sv": 4.675, + "stable_rank": 24.24 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n100_s1.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..e270edb0576ae414cc4c127fb162728c0dcacfd8 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0045fd42d96e2569b8cd600dfd69658ac9905473d047bc585427308dfbbc104 +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n100_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e5f7f37a983caa2176eee660b5aac812b1c25e58 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n100_s1.pt", + "sha256": "d0045fd42d96e2569b8cd600dfd69658ac9905473d047bc585427308dfbbc104", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.201439, + "mean_fro": 32.48011, + "rms_per_prompt_fro": 35.601498, + "max_per_prompt_fro": 42.173519, + "heavy_tail_ratio": 1.1846, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.159409, + "mean_fro": 25.299687, + "rms_per_prompt_fro": 27.241658, + "max_per_prompt_fro": 31.457043, + "heavy_tail_ratio": 1.1547, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.113326, + "mean_fro": 24.016451, + "rms_per_prompt_fro": 25.340778, + "max_per_prompt_fro": 28.14131, + "heavy_tail_ratio": 1.1105, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.058891, + "mean_fro": 22.983723, + "rms_per_prompt_fro": 23.650805, + "max_per_prompt_fro": 24.916557, + "heavy_tail_ratio": 1.0535, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 12.2, + "fit_secs_per_prompt": 0.122, + "total_wall_seconds": 19.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.4801, + "top_sv": 5.8472, + "stable_rank": 30.856 + }, + { + "layer": 1, + "fro_norm": 25.2996, + "top_sv": 4.472, + "stable_rank": 32.006 + }, + { + "layer": 2, + "fro_norm": 24.0164, + "top_sv": 3.6583, + "stable_rank": 43.097 + }, + { + "layer": 3, + "fro_norm": 22.9837, + "top_sv": 4.6351, + "stable_rank": 24.588 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n100_s2.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..4830b7877dfa4e92401200035258d1b5fb558077 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ce6b3178708e6cc0a3cf26ba46743db50113b28b8ac02ec33e5d815267b209 +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n100_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..6ab6d39ae19eb5a52c911409e69ac26e7b2a89d1 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n100_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n100_s2.pt", + "sha256": "81ce6b3178708e6cc0a3cf26ba46743db50113b28b8ac02ec33e5d815267b209", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 100, + "n_prompts_used": 100, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.211007, + "mean_fro": 32.19566, + "rms_per_prompt_fro": 35.429956, + "max_per_prompt_fro": 44.987434, + "heavy_tail_ratio": 1.2698, + "n_prompts": 100 + }, + "1": { + "dispersion": 0.1657, + "mean_fro": 25.144337, + "rms_per_prompt_fro": 27.147733, + "max_per_prompt_fro": 33.336575, + "heavy_tail_ratio": 1.228, + "n_prompts": 100 + }, + "2": { + "dispersion": 0.118212, + "mean_fro": 23.936488, + "rms_per_prompt_fro": 25.311764, + "max_per_prompt_fro": 28.846191, + "heavy_tail_ratio": 1.1396, + "n_prompts": 100 + }, + "3": { + "dispersion": 0.060903, + "mean_fro": 22.953956, + "rms_per_prompt_fro": 23.642603, + "max_per_prompt_fro": 25.111322, + "heavy_tail_ratio": 1.0621, + "n_prompts": 100 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 11.2, + "fit_secs_per_prompt": 0.112, + "total_wall_seconds": 16.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.1956, + "top_sv": 5.4823, + "stable_rank": 34.489 + }, + { + "layer": 1, + "fro_norm": 25.1443, + "top_sv": 4.4445, + "stable_rank": 32.006 + }, + { + "layer": 2, + "fro_norm": 23.9364, + "top_sv": 3.6497, + "stable_rank": 43.014 + }, + { + "layer": 3, + "fro_norm": 22.9539, + "top_sv": 4.6459, + "stable_rank": 24.411 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n200_s0.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..162c6d160b1dbaa03fbd244e575431ee36db6089 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b8eca1a4c0cc2137b3b83b22ccf3542629097011045603dc45d898bb2c8e62a +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n200_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..8a8956c16ec3db1394a51ff84c3aa1485737388d --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n200_s0.pt", + "sha256": "6b8eca1a4c0cc2137b3b83b22ccf3542629097011045603dc45d898bb2c8e62a", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.20702, + "mean_fro": 32.563152, + "rms_per_prompt_fro": 35.775326, + "max_per_prompt_fro": 41.142822, + "heavy_tail_ratio": 1.15, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.160648, + "mean_fro": 25.308756, + "rms_per_prompt_fro": 27.265981, + "max_per_prompt_fro": 31.330845, + "heavy_tail_ratio": 1.1491, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.113418, + "mean_fro": 24.026826, + "rms_per_prompt_fro": 25.352777, + "max_per_prompt_fro": 28.608912, + "heavy_tail_ratio": 1.1284, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.058495, + "mean_fro": 23.034172, + "rms_per_prompt_fro": 23.698294, + "max_per_prompt_fro": 25.856049, + "heavy_tail_ratio": 1.0911, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 23.8, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 32.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.5631, + "top_sv": 5.8475, + "stable_rank": 31.01 + }, + { + "layer": 1, + "fro_norm": 25.3087, + "top_sv": 4.469, + "stable_rank": 32.072 + }, + { + "layer": 2, + "fro_norm": 24.0268, + "top_sv": 3.6598, + "stable_rank": 43.1 + }, + { + "layer": 3, + "fro_norm": 23.0341, + "top_sv": 4.6618, + "stable_rank": 24.414 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n200_s1.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3f6c074bccae4b3eaa05a503c9ba4e19f136468 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5ff2d95106e40a28ab80792cca129bdfa56976900f1c5dc59dbb1770c69857d +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n200_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..3786433a42191e1868bfdf4e0eefc2a99b0eb4db --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n200_s1.pt", + "sha256": "d5ff2d95106e40a28ab80792cca129bdfa56976900f1c5dc59dbb1770c69857d", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.205972, + "mean_fro": 32.293721, + "rms_per_prompt_fro": 35.46392, + "max_per_prompt_fro": 42.173519, + "heavy_tail_ratio": 1.1892, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.162322, + "mean_fro": 25.193632, + "rms_per_prompt_fro": 27.161519, + "max_per_prompt_fro": 31.457043, + "heavy_tail_ratio": 1.1581, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.11535, + "mean_fro": 23.962177, + "rms_per_prompt_fro": 25.306491, + "max_per_prompt_fro": 28.350622, + "heavy_tail_ratio": 1.1203, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.059538, + "mean_fro": 22.958903, + "rms_per_prompt_fro": 23.63249, + "max_per_prompt_fro": 24.916557, + "heavy_tail_ratio": 1.0543, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 23.8, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 31.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.2937, + "top_sv": 5.6534, + "stable_rank": 32.63 + }, + { + "layer": 1, + "fro_norm": 25.1936, + "top_sv": 4.4412, + "stable_rank": 32.18 + }, + { + "layer": 2, + "fro_norm": 23.9621, + "top_sv": 3.6432, + "stable_rank": 43.26 + }, + { + "layer": 3, + "fro_norm": 22.9589, + "top_sv": 4.6379, + "stable_rank": 24.505 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n200_s2.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ee0006837741b2179b0bd411a4f689aac568a98 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c10f206bb41dd5ff2eda75cebf8c1aa25a978ae3520ff6b65b7546a92221b45f +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n200_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0646f0dd162b7aa0f12db79537d4f4c364bdafeb --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n200_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n200_s2.pt", + "sha256": "c10f206bb41dd5ff2eda75cebf8c1aa25a978ae3520ff6b65b7546a92221b45f", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.20443, + "mean_fro": 32.46069, + "rms_per_prompt_fro": 35.62448, + "max_per_prompt_fro": 44.987434, + "heavy_tail_ratio": 1.2628, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.159606, + "mean_fro": 25.24869, + "rms_per_prompt_fro": 27.189057, + "max_per_prompt_fro": 33.336575, + "heavy_tail_ratio": 1.2261, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.112768, + "mean_fro": 23.970577, + "rms_per_prompt_fro": 25.286042, + "max_per_prompt_fro": 28.846191, + "heavy_tail_ratio": 1.1408, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.057791, + "mean_fro": 22.979103, + "rms_per_prompt_fro": 23.633769, + "max_per_prompt_fro": 25.111322, + "heavy_tail_ratio": 1.0625, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 22.2, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 27.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.4606, + "top_sv": 5.6629, + "stable_rank": 32.858 + }, + { + "layer": 1, + "fro_norm": 25.2486, + "top_sv": 4.4524, + "stable_rank": 32.158 + }, + { + "layer": 2, + "fro_norm": 23.9705, + "top_sv": 3.6531, + "stable_rank": 43.055 + }, + { + "layer": 3, + "fro_norm": 22.979, + "top_sv": 4.6577, + "stable_rank": 24.34 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n25_s0.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..50145737fed483cde9df5f0e7aae3ce10d1575df --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2a6fefbcb065aa660edb02278cda05fa0f108ea57727b9d62f008f016b378ba +size 2099877 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n25_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0cfa885b121272434107ee7fe8112948f02c41bc --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n25_s0.pt", + "sha256": "f2a6fefbcb065aa660edb02278cda05fa0f108ea57727b9d62f008f016b378ba", + "bytes": 2099877, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.190429, + "mean_fro": 32.950645, + "rms_per_prompt_fro": 35.951382, + "max_per_prompt_fro": 39.533691, + "heavy_tail_ratio": 1.0996, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.144662, + "mean_fro": 25.408363, + "rms_per_prompt_fro": 27.18412, + "max_per_prompt_fro": 29.456219, + "heavy_tail_ratio": 1.0836, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.10032, + "mean_fro": 24.03973, + "rms_per_prompt_fro": 25.216747, + "max_per_prompt_fro": 26.775425, + "heavy_tail_ratio": 1.0618, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.051211, + "mean_fro": 23.12874, + "rms_per_prompt_fro": 23.713568, + "max_per_prompt_fro": 24.420059, + "heavy_tail_ratio": 1.0298, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.4, + "fit_secs_per_prompt": 0.137, + "total_wall_seconds": 11.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.9506, + "top_sv": 5.8148, + "stable_rank": 32.112 + }, + { + "layer": 1, + "fro_norm": 25.4083, + "top_sv": 4.4539, + "stable_rank": 32.545 + }, + { + "layer": 2, + "fro_norm": 24.0397, + "top_sv": 3.6752, + "stable_rank": 42.785 + }, + { + "layer": 3, + "fro_norm": 23.1287, + "top_sv": 4.6739, + "stable_rank": 24.487 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n25_s1.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..e416735b369a90f3075428dfdff2d90645c72654 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e6181c3cf9fc15032ca046ae60cae559eefce70b5ef9b0b2ca03b4fc131cff4 +size 2099877 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n25_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..572bc441bf5ad2f2818ecb081b3f6fc5e41ef490 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n25_s1.pt", + "sha256": "7e6181c3cf9fc15032ca046ae60cae559eefce70b5ef9b0b2ca03b4fc131cff4", + "bytes": 2099877, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.186222, + "mean_fro": 32.756947, + "rms_per_prompt_fro": 35.676848, + "max_per_prompt_fro": 40.227047, + "heavy_tail_ratio": 1.1275, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.145748, + "mean_fro": 25.384526, + "rms_per_prompt_fro": 27.171502, + "max_per_prompt_fro": 30.344797, + "heavy_tail_ratio": 1.1168, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.104971, + "mean_fro": 24.095378, + "rms_per_prompt_fro": 25.328482, + "max_per_prompt_fro": 27.492146, + "heavy_tail_ratio": 1.0854, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.055733, + "mean_fro": 23.000352, + "rms_per_prompt_fro": 23.632599, + "max_per_prompt_fro": 24.750063, + "heavy_tail_ratio": 1.0473, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 3.3, + "fit_secs_per_prompt": 0.133, + "total_wall_seconds": 10.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.7569, + "top_sv": 6.0606, + "stable_rank": 29.213 + }, + { + "layer": 1, + "fro_norm": 25.3845, + "top_sv": 4.5186, + "stable_rank": 31.56 + }, + { + "layer": 2, + "fro_norm": 24.0953, + "top_sv": 3.6486, + "stable_rank": 43.612 + }, + { + "layer": 3, + "fro_norm": 23.0003, + "top_sv": 4.6195, + "stable_rank": 24.79 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n25_s2.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..16565238164c97bc719f3dbbcf6e4a30521c5b1f --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19a317acca9852e9b50c041e1b2b509520e66083315806138a264f319f68f3b8 +size 2099877 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n25_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..469af8c02dc3ee99d81840ec3f06bf1a520a6096 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n25_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n25_s2.pt", + "sha256": "19a317acca9852e9b50c041e1b2b509520e66083315806138a264f319f68f3b8", + "bytes": 2099877, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 25, + "n_prompts_used": 25, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.210842, + "mean_fro": 32.210326, + "rms_per_prompt_fro": 35.443677, + "max_per_prompt_fro": 44.987434, + "heavy_tail_ratio": 1.2693, + "n_prompts": 25 + }, + "1": { + "dispersion": 0.165286, + "mean_fro": 25.187609, + "rms_per_prompt_fro": 27.189626, + "max_per_prompt_fro": 33.336575, + "heavy_tail_ratio": 1.2261, + "n_prompts": 25 + }, + "2": { + "dispersion": 0.115551, + "mean_fro": 23.97226, + "rms_per_prompt_fro": 25.319413, + "max_per_prompt_fro": 28.846191, + "heavy_tail_ratio": 1.1393, + "n_prompts": 25 + }, + "3": { + "dispersion": 0.057462, + "mean_fro": 22.904483, + "rms_per_prompt_fro": 23.553364, + "max_per_prompt_fro": 24.309166, + "heavy_tail_ratio": 1.0321, + "n_prompts": 25 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 2.9, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 7.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.2103, + "top_sv": 5.5595, + "stable_rank": 33.567 + }, + { + "layer": 1, + "fro_norm": 25.1875, + "top_sv": 4.4895, + "stable_rank": 31.475 + }, + { + "layer": 2, + "fro_norm": 23.9722, + "top_sv": 3.6714, + "stable_rank": 42.635 + }, + { + "layer": 3, + "fro_norm": 22.9044, + "top_sv": 4.6078, + "stable_rank": 24.709 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n400_s0.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..92e52c5a950427d478aec7d0a0bda119547381fa --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63de5057134684e0fa7d1829e63e4a24ce8693d58d837800b13e2d3135b60d7c +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n400_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..16ddf5814efbd08b90c89f3faa27ec6f91e8f835 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n400_s0.pt", + "sha256": "63de5057134684e0fa7d1829e63e4a24ce8693d58d837800b13e2d3135b60d7c", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.209887, + "mean_fro": 32.46161, + "rms_per_prompt_fro": 35.706103, + "max_per_prompt_fro": 41.142822, + "heavy_tail_ratio": 1.1523, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.163024, + "mean_fro": 25.273657, + "rms_per_prompt_fro": 27.25602, + "max_per_prompt_fro": 31.330845, + "heavy_tail_ratio": 1.1495, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.114669, + "mean_fro": 24.012256, + "rms_per_prompt_fro": 25.351635, + "max_per_prompt_fro": 29.073936, + "heavy_tail_ratio": 1.1468, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.059259, + "mean_fro": 23.021973, + "rms_per_prompt_fro": 23.694284, + "max_per_prompt_fro": 25.856049, + "heavy_tail_ratio": 1.0912, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 47.5, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 57.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.4616, + "top_sv": 5.7551, + "stable_rank": 31.816 + }, + { + "layer": 1, + "fro_norm": 25.2736, + "top_sv": 4.4624, + "stable_rank": 32.077 + }, + { + "layer": 2, + "fro_norm": 24.0122, + "top_sv": 3.661, + "stable_rank": 43.02 + }, + { + "layer": 3, + "fro_norm": 23.0219, + "top_sv": 4.6664, + "stable_rank": 24.34 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n400_s1.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa80896a27e65de159a7358c72f41a5b1723294b --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ea7428b4c500f7da5c7c3a1a79cd080a3b197a1d10420c9e00a61a425e8d8d +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n400_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7008dc9ee734c22e52a81a815266d7763df1d598 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n400_s1.pt", + "sha256": "c6ea7428b4c500f7da5c7c3a1a79cd080a3b197a1d10420c9e00a61a425e8d8d", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.209805, + "mean_fro": 32.518223, + "rms_per_prompt_fro": 35.767156, + "max_per_prompt_fro": 42.173519, + "heavy_tail_ratio": 1.1791, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.16456, + "mean_fro": 25.328507, + "rms_per_prompt_fro": 27.333206, + "max_per_prompt_fro": 31.457043, + "heavy_tail_ratio": 1.1509, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.11601, + "mean_fro": 24.042181, + "rms_per_prompt_fro": 25.398488, + "max_per_prompt_fro": 28.350622, + "heavy_tail_ratio": 1.1162, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.060018, + "mean_fro": 23.015922, + "rms_per_prompt_fro": 23.696538, + "max_per_prompt_fro": 24.95471, + "heavy_tail_ratio": 1.0531, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 47.5, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 54.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.5182, + "top_sv": 5.7571, + "stable_rank": 31.904 + }, + { + "layer": 1, + "fro_norm": 25.3285, + "top_sv": 4.4628, + "stable_rank": 32.211 + }, + { + "layer": 2, + "fro_norm": 24.0421, + "top_sv": 3.6615, + "stable_rank": 43.116 + }, + { + "layer": 3, + "fro_norm": 23.0159, + "top_sv": 4.6508, + "stable_rank": 24.491 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n400_s2.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..2622666803e0a80a65af3142147f9cbe863b60c0 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee7c003f2a4f5303f8450d25a1a99f15ac6c2220ff674157011b065c451b385b +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n400_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f4c429871bb1c8c264c9e8a94bb82ed07eb282f8 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n400_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n400_s2.pt", + "sha256": "ee7c003f2a4f5303f8450d25a1a99f15ac6c2220ff674157011b065c451b385b", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 400, + "n_prompts_used": 400, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.20616, + "mean_fro": 32.399124, + "rms_per_prompt_fro": 35.582435, + "max_per_prompt_fro": 44.987434, + "heavy_tail_ratio": 1.2643, + "n_prompts": 400 + }, + "1": { + "dispersion": 0.161474, + "mean_fro": 25.228325, + "rms_per_prompt_fro": 27.189001, + "max_per_prompt_fro": 33.336575, + "heavy_tail_ratio": 1.2261, + "n_prompts": 400 + }, + "2": { + "dispersion": 0.114018, + "mean_fro": 23.977075, + "rms_per_prompt_fro": 25.307098, + "max_per_prompt_fro": 28.846191, + "heavy_tail_ratio": 1.1398, + "n_prompts": 400 + }, + "3": { + "dispersion": 0.058833, + "mean_fro": 22.984715, + "rms_per_prompt_fro": 23.651186, + "max_per_prompt_fro": 25.137999, + "heavy_tail_ratio": 1.0629, + "n_prompts": 400 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 44.1, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 49.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.3991, + "top_sv": 5.663, + "stable_rank": 32.732 + }, + { + "layer": 1, + "fro_norm": 25.2283, + "top_sv": 4.4452, + "stable_rank": 32.21 + }, + { + "layer": 2, + "fro_norm": 23.977, + "top_sv": 3.6455, + "stable_rank": 43.259 + }, + { + "layer": 3, + "fro_norm": 22.9847, + "top_sv": 4.6476, + "stable_rank": 24.457 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n50_s0.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b4e0b8fd0c904ab9f7e5b8f14c50231ac70854d --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47c7d8884935cfa16add972a8ebe08ee6267ea5d62ef4d02877de42692e4f2ec +size 2099877 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n50_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..87ac1fc49ebe9d3bfa4e6b856c0ea3228f3f833d --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n50_s0.pt", + "sha256": "47c7d8884935cfa16add972a8ebe08ee6267ea5d62ef4d02877de42692e4f2ec", + "bytes": 2099877, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.20375, + "mean_fro": 32.386249, + "rms_per_prompt_fro": 35.532754, + "max_per_prompt_fro": 39.533691, + "heavy_tail_ratio": 1.1126, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.156481, + "mean_fro": 25.182361, + "rms_per_prompt_fro": 27.081068, + "max_per_prompt_fro": 29.456219, + "heavy_tail_ratio": 1.0877, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.108706, + "mean_fro": 23.936355, + "rms_per_prompt_fro": 25.203808, + "max_per_prompt_fro": 26.935259, + "heavy_tail_ratio": 1.0687, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.055947, + "mean_fro": 23.058248, + "rms_per_prompt_fro": 23.694494, + "max_per_prompt_fro": 24.581585, + "heavy_tail_ratio": 1.0374, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 6.1, + "fit_secs_per_prompt": 0.122, + "total_wall_seconds": 14.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.3862, + "top_sv": 5.8489, + "stable_rank": 30.66 + }, + { + "layer": 1, + "fro_norm": 25.1823, + "top_sv": 4.4181, + "stable_rank": 32.487 + }, + { + "layer": 2, + "fro_norm": 23.9363, + "top_sv": 3.6735, + "stable_rank": 42.457 + }, + { + "layer": 3, + "fro_norm": 23.0582, + "top_sv": 4.6852, + "stable_rank": 24.221 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n50_s1.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a0364ea8e9f302f20abcfc315c68a987ad98613 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bbda35c32b666a608cb6177a52a238c86f9e9ac08db38c0372d28ce09fa0546 +size 2099877 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n50_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..6688329152a91fb2d4b3302b425c1162d4aa6fc3 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n50_s1.pt", + "sha256": "0bbda35c32b666a608cb6177a52a238c86f9e9ac08db38c0372d28ce09fa0546", + "bytes": 2099877, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.199492, + "mean_fro": 32.491016, + "rms_per_prompt_fro": 35.584583, + "max_per_prompt_fro": 42.173519, + "heavy_tail_ratio": 1.1852, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.157169, + "mean_fro": 25.273684, + "rms_per_prompt_fro": 27.187349, + "max_per_prompt_fro": 31.457043, + "heavy_tail_ratio": 1.157, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.112073, + "mean_fro": 24.028737, + "rms_per_prompt_fro": 25.339469, + "max_per_prompt_fro": 28.14131, + "heavy_tail_ratio": 1.1106, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.059065, + "mean_fro": 22.979852, + "rms_per_prompt_fro": 23.648771, + "max_per_prompt_fro": 24.916557, + "heavy_tail_ratio": 1.0536, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 6.3, + "fit_secs_per_prompt": 0.125, + "total_wall_seconds": 13.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.491, + "top_sv": 5.9206, + "stable_rank": 30.116 + }, + { + "layer": 1, + "fro_norm": 25.2736, + "top_sv": 4.4696, + "stable_rank": 31.974 + }, + { + "layer": 2, + "fro_norm": 24.0287, + "top_sv": 3.6489, + "stable_rank": 43.365 + }, + { + "layer": 3, + "fro_norm": 22.9798, + "top_sv": 4.6315, + "stable_rank": 24.618 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n50_s2.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s2.pt new file mode 100644 index 0000000000000000000000000000000000000000..67d437e13f463a96e3bc9374400d0318e968f40b --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd01517b76d37e7affe7890621a4ae1c5b18bc126641c74dd6d0faa527b24153 +size 2099877 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n50_s2_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s2_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..406f76f1ba90f5465118a8e5e6b5d558f277b4fc --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n50_s2_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n50_s2.pt", + "sha256": "cd01517b76d37e7affe7890621a4ae1c5b18bc126641c74dd6d0faa527b24153", + "bytes": 2099877, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 2, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.200563, + "mean_fro": 32.160416, + "rms_per_prompt_fro": 35.238228, + "max_per_prompt_fro": 44.987434, + "heavy_tail_ratio": 1.2767, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.157823, + "mean_fro": 25.14591, + "rms_per_prompt_fro": 27.057549, + "max_per_prompt_fro": 33.336575, + "heavy_tail_ratio": 1.2321, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.111322, + "mean_fro": 23.949499, + "rms_per_prompt_fro": 25.247382, + "max_per_prompt_fro": 28.846191, + "heavy_tail_ratio": 1.1425, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.056559, + "mean_fro": 22.943391, + "rms_per_prompt_fro": 23.583297, + "max_per_prompt_fro": 24.359318, + "heavy_tail_ratio": 1.0329, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 5.7, + "fit_secs_per_prompt": 0.114, + "total_wall_seconds": 10.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.1604, + "top_sv": 5.4991, + "stable_rank": 34.203 + }, + { + "layer": 1, + "fro_norm": 25.1459, + "top_sv": 4.4769, + "stable_rank": 31.548 + }, + { + "layer": 2, + "fro_norm": 23.9495, + "top_sv": 3.6655, + "stable_rank": 42.689 + }, + { + "layer": 3, + "fro_norm": 22.9433, + "top_sv": 4.6323, + "stable_rank": 24.532 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n800_s0.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e24ace731c2082ab4810d92731bf6a461ce4a9a --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:331d3030a6652c2b4960eb5abb29bc378c0a39e727913f4a421f402c3a8758b0 +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n800_s0_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s0_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4f6a16bb7e29815b699192205074a834a76ee55e --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s0_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n800_s0.pt", + "sha256": "331d3030a6652c2b4960eb5abb29bc378c0a39e727913f4a421f402c3a8758b0", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 0, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.214561, + "mean_fro": 32.374587, + "rms_per_prompt_fro": 35.679106, + "max_per_prompt_fro": 43.181686, + "heavy_tail_ratio": 1.2103, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.167116, + "mean_fro": 25.222986, + "rms_per_prompt_fro": 27.249183, + "max_per_prompt_fro": 32.252716, + "heavy_tail_ratio": 1.1836, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.117792, + "mean_fro": 23.976428, + "rms_per_prompt_fro": 25.349236, + "max_per_prompt_fro": 29.073936, + "heavy_tail_ratio": 1.1469, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.060924, + "mean_fro": 22.994117, + "rms_per_prompt_fro": 23.684204, + "max_per_prompt_fro": 25.856049, + "heavy_tail_ratio": 1.0917, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 95.0, + "fit_secs_per_prompt": 0.119, + "total_wall_seconds": 102.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.3745, + "top_sv": 5.7461, + "stable_rank": 31.744 + }, + { + "layer": 1, + "fro_norm": 25.223, + "top_sv": 4.454, + "stable_rank": 32.07 + }, + { + "layer": 2, + "fro_norm": 23.9764, + "top_sv": 3.6504, + "stable_rank": 43.142 + }, + { + "layer": 3, + "fro_norm": 22.9941, + "top_sv": 4.6519, + "stable_rank": 24.432 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n800_s1.pt b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s1.pt new file mode 100644 index 0000000000000000000000000000000000000000..a723a10a9e971629144f8ee38d61fe4bdf58fd4e --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a858ad5c6a133a98cd1702bdc807089ddd9225c12058d3b6208bedb342eb176 +size 2099887 diff --git a/results/lenses/e28/e28_Wikipedia_en_70m_n800_s1_provenance.json b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s1_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..cc618f818081cfb2ae1a419b8aa036367006af86 --- /dev/null +++ b/results/lenses/e28/e28_Wikipedia_en_70m_n800_s1_provenance.json @@ -0,0 +1,105 @@ +{ + "artifact": "results/e28_Wikipedia_en_70m_n800_s1.pt", + "sha256": "1a858ad5c6a133a98cd1702bdc807089ddd9225c12058d3b6208bedb342eb176", + "bytes": 2099887, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "Wikipedia_en", + "n_prompts_requested": 800, + "n_prompts_used": 800, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "corpus_file": "corpora/Wikipedia_en.jsonl", + "seed_block": 1, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": "NOT RECORDED AT FIT TIME", + "jacobian_dispersion": { + "0": { + "dispersion": 0.212081, + "mean_fro": 32.380452, + "rms_per_prompt_fro": 35.649111, + "max_per_prompt_fro": 43.695316, + "heavy_tail_ratio": 1.2257, + "n_prompts": 800 + }, + "1": { + "dispersion": 0.165226, + "mean_fro": 25.249925, + "rms_per_prompt_fro": 27.256192, + "max_per_prompt_fro": 32.948757, + "heavy_tail_ratio": 1.2089, + "n_prompts": 800 + }, + "2": { + "dispersion": 0.116508, + "mean_fro": 23.997103, + "rms_per_prompt_fro": 25.35652, + "max_per_prompt_fro": 29.788128, + "heavy_tail_ratio": 1.1748, + "n_prompts": 800 + }, + "3": { + "dispersion": 0.060325, + "mean_fro": 22.996957, + "rms_per_prompt_fro": 23.680444, + "max_per_prompt_fro": 25.833094, + "heavy_tail_ratio": 1.0909, + "n_prompts": 800 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.11.0+cu128" + }, + "fit_wall_seconds": 94.3, + "fit_secs_per_prompt": 0.118, + "total_wall_seconds": 101.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 32.3804, + "top_sv": 5.7171, + "stable_rank": 32.078 + }, + { + "layer": 1, + "fro_norm": 25.2499, + "top_sv": 4.443, + "stable_rank": 32.297 + }, + { + "layer": 2, + "fro_norm": 23.9971, + "top_sv": 3.6539, + "stable_rank": 43.132 + }, + { + "layer": 3, + "fro_norm": 22.9969, + "top_sv": 4.6529, + "stable_rank": 24.428 + } + ], + "corpus_as_originally_recorded": "wikitext", + "corpus_backfilled_from": "corpus_file", + "R4g_note": "`corpus` recorded the --corpus DEFAULT ('wikitext'), not the corpus actually fitted. Backfilled from `corpus_file`, which was correct all along. Original value preserved above.", + "anchor_agreement_gate_note": "null was ambiguous between 'ran, returned nothing' and 'never ran'. The fitter wrote no gate value for these runs, so it is the latter. Stated explicitly so that absence is never read as a passing gate." +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_1.4b_n200_db128_fin.pt b/results/lenses/ladder/lens_1.4b_n200_db128_fin.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb399726e5d0aaeb0b7473dd423d10dfb35a932a --- /dev/null +++ b/results/lenses/ladder/lens_1.4b_n200_db128_fin.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c513a1ee383df6c8393e780ef4ee2a446d809bdc738ecb78dbeb151d9ec627c8 +size 192945308 diff --git a/results/lenses/ladder/lens_1.4b_n200_db128_fin_provenance.json b/results/lenses/ladder/lens_1.4b_n200_db128_fin_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b5e9f609ac19a4465ffa95dd1882e3390ec9ec6f --- /dev/null +++ b/results/lenses/ladder/lens_1.4b_n200_db128_fin_provenance.json @@ -0,0 +1,532 @@ +{ + "artifact": "pythia/results/lens_1.4b_n200_db128_fin.pt", + "sha256": "c513a1ee383df6c8393e780ef4ee2a446d809bdc738ecb78dbeb151d9ec627c8", + "bytes": 192945308, + "model": "EleutherAI/pythia-1.4b-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": null, + "target_layer_effective": 23, + "target_layer_is_penultimate": false, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 2048, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": null, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 7.419119724412346e-08, + "ref_absmax": 0.40169620513916016, + "pass": true + }, + "1": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 8.47415471789666e-08, + "ref_absmax": 0.35168489813804626, + "pass": true + }, + "2": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 8.412475094867481e-08, + "ref_absmax": 0.35426342487335205, + "pass": true + }, + "3": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 7.370802364907197e-08, + "ref_absmax": 0.40432941913604736, + "pass": true + }, + "4": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.232405529664412e-08, + "ref_absmax": 0.4781832993030548, + "pass": true + }, + "5": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.686455227581037e-08, + "ref_absmax": 0.5240931510925293, + "pass": true + }, + "6": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.3942303096838406e-08, + "ref_absmax": 0.5524851679801941, + "pass": true + }, + "7": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 4.9029029107554e-08, + "ref_absmax": 0.6078505516052246, + "pass": true + }, + "8": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 9.276894808083775e-08, + "ref_absmax": 0.6425064206123352, + "pass": true + }, + "9": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.99035468305499e-08, + "ref_absmax": 0.7459574341773987, + "pass": true + }, + "10": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.43014916767469e-08, + "ref_absmax": 0.8021998405456543, + "pass": true + }, + "11": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.129805016231358e-08, + "ref_absmax": 0.8359926342964172, + "pass": true + }, + "12": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.907497487570476e-08, + "ref_absmax": 0.8628978133201599, + "pass": true + }, + "13": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.801179025191091e-08, + "ref_absmax": 0.8763869404792786, + "pass": true + }, + "14": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.54059666593085e-08, + "ref_absmax": 0.9113028645515442, + "pass": true + }, + "15": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.067747247502985e-08, + "ref_absmax": 0.982319176197052, + "pass": true + }, + "16": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1576542128541757e-07, + "ref_absmax": 1.0297486782073975, + "pass": true + }, + "17": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1023694439064269e-07, + "ref_absmax": 1.081391453742981, + "pass": true + }, + "18": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0563063017860767e-07, + "ref_absmax": 1.128548502922058, + "pass": true + }, + "19": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0696137806472426e-07, + "ref_absmax": 1.114507794380188, + "pass": true + }, + "20": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0470108938342471e-07, + "ref_absmax": 1.1385678052902222, + "pass": true + }, + "21": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0287323928593216e-07, + "ref_absmax": 1.1587978601455688, + "pass": true + }, + "22": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.429164954094582e-08, + "ref_absmax": 1.2642613649368286, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.1576542128541757e-07 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 3.073246, + "mean_fro": 14.451368, + "rms_per_prompt_fro": 29.166162, + "max_per_prompt_fro": 45.126831, + "heavy_tail_ratio": 1.5472, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.775817, + "mean_fro": 13.865869, + "rms_per_prompt_fro": 26.943408, + "max_per_prompt_fro": 40.843876, + "heavy_tail_ratio": 1.5159, + "n_prompts": 200 + }, + "2": { + "dispersion": 2.115892, + "mean_fro": 15.062893, + "rms_per_prompt_fro": 26.588854, + "max_per_prompt_fro": 37.730949, + "heavy_tail_ratio": 1.4191, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.713451, + "mean_fro": 16.052374, + "rms_per_prompt_fro": 26.442362, + "max_per_prompt_fro": 36.966892, + "heavy_tail_ratio": 1.398, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.428081, + "mean_fro": 16.63882, + "rms_per_prompt_fro": 25.927112, + "max_per_prompt_fro": 36.121006, + "heavy_tail_ratio": 1.3932, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.211926, + "mean_fro": 17.678509, + "rms_per_prompt_fro": 26.29244, + "max_per_prompt_fro": 36.246788, + "heavy_tail_ratio": 1.3786, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.024511, + "mean_fro": 19.616804, + "rms_per_prompt_fro": 27.911829, + "max_per_prompt_fro": 37.858646, + "heavy_tail_ratio": 1.3564, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.894743, + "mean_fro": 22.422243, + "rms_per_prompt_fro": 30.86414, + "max_per_prompt_fro": 41.511974, + "heavy_tail_ratio": 1.345, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.781544, + "mean_fro": 25.039917, + "rms_per_prompt_fro": 33.421899, + "max_per_prompt_fro": 44.494282, + "heavy_tail_ratio": 1.3313, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.674337, + "mean_fro": 29.23728, + "rms_per_prompt_fro": 37.831926, + "max_per_prompt_fro": 50.038036, + "heavy_tail_ratio": 1.3226, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.613654, + "mean_fro": 31.559141, + "rms_per_prompt_fro": 40.08947, + "max_per_prompt_fro": 52.033173, + "heavy_tail_ratio": 1.2979, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.537351, + "mean_fro": 34.072594, + "rms_per_prompt_fro": 42.2466, + "max_per_prompt_fro": 52.464481, + "heavy_tail_ratio": 1.2419, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.444817, + "mean_fro": 36.429363, + "rms_per_prompt_fro": 43.788287, + "max_per_prompt_fro": 53.04911, + "heavy_tail_ratio": 1.2115, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.374544, + "mean_fro": 39.494571, + "rms_per_prompt_fro": 46.303803, + "max_per_prompt_fro": 54.715389, + "heavy_tail_ratio": 1.1817, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.300584, + "mean_fro": 42.857578, + "rms_per_prompt_fro": 48.876135, + "max_per_prompt_fro": 57.119305, + "heavy_tail_ratio": 1.1687, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.244874, + "mean_fro": 47.019532, + "rms_per_prompt_fro": 52.461546, + "max_per_prompt_fro": 60.57988, + "heavy_tail_ratio": 1.1547, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.196416, + "mean_fro": 47.591667, + "rms_per_prompt_fro": 52.056142, + "max_per_prompt_fro": 58.190056, + "heavy_tail_ratio": 1.1178, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.168004, + "mean_fro": 49.817474, + "rms_per_prompt_fro": 53.839845, + "max_per_prompt_fro": 59.461399, + "heavy_tail_ratio": 1.1044, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.123022, + "mean_fro": 49.499748, + "rms_per_prompt_fro": 52.456233, + "max_per_prompt_fro": 56.722462, + "heavy_tail_ratio": 1.0813, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.10209, + "mean_fro": 48.83027, + "rms_per_prompt_fro": 51.262248, + "max_per_prompt_fro": 54.662098, + "heavy_tail_ratio": 1.0663, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.080021, + "mean_fro": 49.834641, + "rms_per_prompt_fro": 51.790194, + "max_per_prompt_fro": 55.797943, + "heavy_tail_ratio": 1.0774, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.055481, + "mean_fro": 49.910927, + "rms_per_prompt_fro": 51.276795, + "max_per_prompt_fro": 54.673832, + "heavy_tail_ratio": 1.0662, + "n_prompts": 200 + }, + "22": { + "dispersion": 0.02671, + "mean_fro": 48.617091, + "rms_per_prompt_fro": 49.262096, + "max_per_prompt_fro": 51.049999, + "heavy_tail_ratio": 1.0363, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 2771.5, + "fit_secs_per_prompt": 13.857, + "total_wall_seconds": 2883.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 14.4499, + "top_sv": 3.7469, + "stable_rank": 14.872 + }, + { + "layer": 1, + "fro_norm": 13.8643, + "top_sv": 3.7934, + "stable_rank": 13.358 + }, + { + "layer": 2, + "fro_norm": 15.0614, + "top_sv": 4.0997, + "stable_rank": 13.497 + }, + { + "layer": 3, + "fro_norm": 16.0509, + "top_sv": 4.3506, + "stable_rank": 13.612 + }, + { + "layer": 4, + "fro_norm": 16.6371, + "top_sv": 4.6499, + "stable_rank": 12.801 + }, + { + "layer": 5, + "fro_norm": 17.6765, + "top_sv": 4.8283, + "stable_rank": 13.403 + }, + { + "layer": 6, + "fro_norm": 19.6145, + "top_sv": 5.5008, + "stable_rank": 12.715 + }, + { + "layer": 7, + "fro_norm": 22.42, + "top_sv": 6.4968, + "stable_rank": 11.909 + }, + { + "layer": 8, + "fro_norm": 25.0368, + "top_sv": 6.9157, + "stable_rank": 13.106 + }, + { + "layer": 9, + "fro_norm": 29.2339, + "top_sv": 8.7622, + "stable_rank": 11.131 + }, + { + "layer": 10, + "fro_norm": 31.5557, + "top_sv": 8.4964, + "stable_rank": 13.794 + }, + { + "layer": 11, + "fro_norm": 34.0681, + "top_sv": 8.1878, + "stable_rank": 17.313 + }, + { + "layer": 12, + "fro_norm": 36.4241, + "top_sv": 7.3533, + "stable_rank": 24.536 + }, + { + "layer": 13, + "fro_norm": 39.4886, + "top_sv": 7.4934, + "stable_rank": 27.771 + }, + { + "layer": 14, + "fro_norm": 42.8512, + "top_sv": 7.4205, + "stable_rank": 33.347 + }, + { + "layer": 15, + "fro_norm": 47.0117, + "top_sv": 7.0694, + "stable_rank": 44.223 + }, + { + "layer": 16, + "fro_norm": 47.5824, + "top_sv": 7.1728, + "stable_rank": 44.006 + }, + { + "layer": 17, + "fro_norm": 49.8065, + "top_sv": 6.8902, + "stable_rank": 52.252 + }, + { + "layer": 18, + "fro_norm": 49.4875, + "top_sv": 6.3867, + "stable_rank": 60.039 + }, + { + "layer": 19, + "fro_norm": 48.8171, + "top_sv": 5.8645, + "stable_rank": 69.292 + }, + { + "layer": 20, + "fro_norm": 49.82, + "top_sv": 6.4479, + "stable_rank": 59.699 + }, + { + "layer": 21, + "fro_norm": 49.8943, + "top_sv": 5.7208, + "stable_rank": 76.065 + }, + { + "layer": 22, + "fro_norm": 48.5974, + "top_sv": 7.9001, + "stable_rank": 37.841 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_1.4b_n200_db128_pen.pt b/results/lenses/ladder/lens_1.4b_n200_db128_pen.pt new file mode 100644 index 0000000000000000000000000000000000000000..700cf256aedc284750a69cac5ea90616e41d8cc4 --- /dev/null +++ b/results/lenses/ladder/lens_1.4b_n200_db128_pen.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74da881b4e2d9c8b7b88b5b66907cc498150dd15c132a814ff421b6e7614da28 +size 184556430 diff --git a/results/lenses/ladder/lens_1.4b_n200_db128_pen_provenance.json b/results/lenses/ladder/lens_1.4b_n200_db128_pen_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..164505d17df12df21025a06655b6429dce5ae5a1 --- /dev/null +++ b/results/lenses/ladder/lens_1.4b_n200_db128_pen_provenance.json @@ -0,0 +1,511 @@ +{ + "artifact": "pythia/results/lens_1.4b_n200_db128_pen.pt", + "sha256": "74da881b4e2d9c8b7b88b5b66907cc498150dd15c132a814ff421b6e7614da28", + "bytes": 184556430, + "model": "EleutherAI/pythia-1.4b-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 2048, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": -2, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.183828289968322e-07, + "ref_absmax": 0.5034906268119812, + "pass": true + }, + "1": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.646853283276324e-08, + "ref_absmax": 0.4483673870563507, + "pass": true + }, + "2": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.224454837347526e-08, + "ref_absmax": 0.4787940979003906, + "pass": true + }, + "3": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0649086303070654e-07, + "ref_absmax": 0.5597160458564758, + "pass": true + }, + "4": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.862987693652957e-08, + "ref_absmax": 0.672511875629425, + "pass": true + }, + "5": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.216434693435372e-08, + "ref_absmax": 0.7254319787025452, + "pass": true + }, + "6": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 3.8227670395826884e-08, + "ref_absmax": 0.7796007990837097, + "pass": true + }, + "7": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.985606646497358e-08, + "ref_absmax": 0.8532493710517883, + "pass": true + }, + "8": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.176165867110527e-08, + "ref_absmax": 0.9650751948356628, + "pass": true + }, + "9": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 2.5878593786197683e-08, + "ref_absmax": 1.151620626449585, + "pass": true + }, + "10": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.531215732300937e-08, + "ref_absmax": 1.2507249116897583, + "pass": true + }, + "11": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 4.304448104755804e-08, + "ref_absmax": 1.384722113609314, + "pass": true + }, + "12": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 8.211013185655524e-08, + "ref_absmax": 1.4518219232559204, + "pass": true + }, + "13": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 3.97314693023146e-08, + "ref_absmax": 1.5001872777938843, + "pass": true + }, + "14": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 7.506774300798398e-08, + "ref_absmax": 1.5880228281021118, + "pass": true + }, + "15": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 6.806701851416097e-08, + "ref_absmax": 1.7513517141342163, + "pass": true + }, + "16": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 6.546721562236311e-08, + "ref_absmax": 1.820900559425354, + "pass": true + }, + "17": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 6.491590631206569e-08, + "ref_absmax": 1.8363648653030396, + "pass": true + }, + "18": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 6.176181888062509e-08, + "ref_absmax": 1.9301453828811646, + "pass": true + }, + "19": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 8.384628393688823e-08, + "ref_absmax": 1.4217599630355835, + "pass": true + }, + "20": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 7.401394733626394e-08, + "ref_absmax": 1.6106327772140503, + "pass": true + }, + "21": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0846410488912799e-07, + "ref_absmax": 1.0990667343139648, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.183828289968322e-07 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 2.897373, + "mean_fro": 12.849277, + "rms_per_prompt_fro": 25.366743, + "max_per_prompt_fro": 38.131401, + "heavy_tail_ratio": 1.5032, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.608921, + "mean_fro": 12.344032, + "rms_per_prompt_fro": 23.450158, + "max_per_prompt_fro": 34.585754, + "heavy_tail_ratio": 1.4749, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.988551, + "mean_fro": 13.457545, + "rms_per_prompt_fro": 23.264631, + "max_per_prompt_fro": 32.139359, + "heavy_tail_ratio": 1.3815, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.617573, + "mean_fro": 14.347399, + "rms_per_prompt_fro": 23.212536, + "max_per_prompt_fro": 31.553576, + "heavy_tail_ratio": 1.3593, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.354258, + "mean_fro": 14.865723, + "rms_per_prompt_fro": 22.809359, + "max_per_prompt_fro": 30.849564, + "heavy_tail_ratio": 1.3525, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.135337, + "mean_fro": 15.9777, + "rms_per_prompt_fro": 23.347881, + "max_per_prompt_fro": 31.178038, + "heavy_tail_ratio": 1.3354, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.959127, + "mean_fro": 17.785394, + "rms_per_prompt_fro": 24.894005, + "max_per_prompt_fro": 32.774372, + "heavy_tail_ratio": 1.3166, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.845753, + "mean_fro": 20.340356, + "rms_per_prompt_fro": 27.634101, + "max_per_prompt_fro": 36.014729, + "heavy_tail_ratio": 1.3033, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.732547, + "mean_fro": 22.881256, + "rms_per_prompt_fro": 30.117736, + "max_per_prompt_fro": 38.763283, + "heavy_tail_ratio": 1.2871, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.645362, + "mean_fro": 26.474791, + "rms_per_prompt_fro": 33.959657, + "max_per_prompt_fro": 43.450668, + "heavy_tail_ratio": 1.2795, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.584442, + "mean_fro": 28.739019, + "rms_per_prompt_fro": 36.175126, + "max_per_prompt_fro": 45.485264, + "heavy_tail_ratio": 1.2574, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.506679, + "mean_fro": 31.292311, + "rms_per_prompt_fro": 38.41033, + "max_per_prompt_fro": 46.393658, + "heavy_tail_ratio": 1.2078, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.414947, + "mean_fro": 33.707458, + "rms_per_prompt_fro": 40.095537, + "max_per_prompt_fro": 47.296223, + "heavy_tail_ratio": 1.1796, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.348885, + "mean_fro": 36.580459, + "rms_per_prompt_fro": 42.48509, + "max_per_prompt_fro": 49.065292, + "heavy_tail_ratio": 1.1549, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.277452, + "mean_fro": 39.858593, + "rms_per_prompt_fro": 45.049941, + "max_per_prompt_fro": 51.246361, + "heavy_tail_ratio": 1.1375, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.225428, + "mean_fro": 43.756648, + "rms_per_prompt_fro": 48.43819, + "max_per_prompt_fro": 54.530708, + "heavy_tail_ratio": 1.1258, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.176245, + "mean_fro": 44.67844, + "rms_per_prompt_fro": 48.455921, + "max_per_prompt_fro": 53.171162, + "heavy_tail_ratio": 1.0973, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.151089, + "mean_fro": 46.471508, + "rms_per_prompt_fro": 49.858732, + "max_per_prompt_fro": 53.999046, + "heavy_tail_ratio": 1.083, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.107296, + "mean_fro": 46.365823, + "rms_per_prompt_fro": 48.789886, + "max_per_prompt_fro": 51.804676, + "heavy_tail_ratio": 1.0618, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.087375, + "mean_fro": 45.662196, + "rms_per_prompt_fro": 47.615288, + "max_per_prompt_fro": 49.922176, + "heavy_tail_ratio": 1.0484, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.06569, + "mean_fro": 46.725572, + "rms_per_prompt_fro": 48.235872, + "max_per_prompt_fro": 51.097256, + "heavy_tail_ratio": 1.0593, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.040788, + "mean_fro": 46.641578, + "rms_per_prompt_fro": 47.583269, + "max_per_prompt_fro": 50.074245, + "heavy_tail_ratio": 1.0523, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 2650.9, + "fit_secs_per_prompt": 13.255, + "total_wall_seconds": 2805.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 12.8479, + "top_sv": 2.6338, + "stable_rank": 23.797 + }, + { + "layer": 1, + "fro_norm": 12.3427, + "top_sv": 2.8114, + "stable_rank": 19.274 + }, + { + "layer": 2, + "fro_norm": 13.4561, + "top_sv": 3.2566, + "stable_rank": 17.072 + }, + { + "layer": 3, + "fro_norm": 14.3459, + "top_sv": 3.447, + "stable_rank": 17.321 + }, + { + "layer": 4, + "fro_norm": 14.8642, + "top_sv": 3.5246, + "stable_rank": 17.786 + }, + { + "layer": 5, + "fro_norm": 15.9763, + "top_sv": 4.3679, + "stable_rank": 13.379 + }, + { + "layer": 6, + "fro_norm": 17.7833, + "top_sv": 4.9488, + "stable_rank": 12.913 + }, + { + "layer": 7, + "fro_norm": 20.3382, + "top_sv": 5.5993, + "stable_rank": 13.193 + }, + { + "layer": 8, + "fro_norm": 22.8789, + "top_sv": 6.1898, + "stable_rank": 13.662 + }, + { + "layer": 9, + "fro_norm": 26.4714, + "top_sv": 6.7466, + "stable_rank": 15.395 + }, + { + "layer": 10, + "fro_norm": 28.7356, + "top_sv": 6.7108, + "stable_rank": 18.336 + }, + { + "layer": 11, + "fro_norm": 31.2887, + "top_sv": 6.5106, + "stable_rank": 23.096 + }, + { + "layer": 12, + "fro_norm": 33.7029, + "top_sv": 6.0789, + "stable_rank": 30.739 + }, + { + "layer": 13, + "fro_norm": 36.5748, + "top_sv": 6.0055, + "stable_rank": 37.09 + }, + { + "layer": 14, + "fro_norm": 39.8517, + "top_sv": 6.1557, + "stable_rank": 41.912 + }, + { + "layer": 15, + "fro_norm": 43.7494, + "top_sv": 6.0209, + "stable_rank": 52.799 + }, + { + "layer": 16, + "fro_norm": 44.6699, + "top_sv": 6.5051, + "stable_rank": 47.154 + }, + { + "layer": 17, + "fro_norm": 46.4612, + "top_sv": 6.2749, + "stable_rank": 54.823 + }, + { + "layer": 18, + "fro_norm": 46.3542, + "top_sv": 6.7029, + "stable_rank": 47.825 + }, + { + "layer": 19, + "fro_norm": 45.6491, + "top_sv": 5.4122, + "stable_rank": 71.14 + }, + { + "layer": 20, + "fro_norm": 46.7089, + "top_sv": 7.0294, + "stable_rank": 44.154 + }, + { + "layer": 21, + "fro_norm": 46.6203, + "top_sv": 4.6999, + "stable_rank": 98.394 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_160m_n200.pt b/results/lenses/ladder/lens_160m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6a1a27aa8395eca6438150f37dcfe9b7f73736d --- /dev/null +++ b/results/lenses/ladder/lens_160m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af0e17b4c9305e0b54c3c538805cbb7fec71e847f676e1001ca1844d935b8143 +size 12980409 diff --git a/results/lenses/ladder/lens_160m_n200_db128.pt b/results/lenses/ladder/lens_160m_n200_db128.pt new file mode 100644 index 0000000000000000000000000000000000000000..a808b1743cde247980544847016348094176f665 --- /dev/null +++ b/results/lenses/ladder/lens_160m_n200_db128.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0022d6297697477dc152ffc6722a29be22464844de8197aa88a9daa3bd4df197 +size 12980088 diff --git a/results/lenses/ladder/lens_160m_n200_db128_pen.pt b/results/lenses/ladder/lens_160m_n200_db128_pen.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e0c0d3fb9839d1a8d93cd9d179c3a32ed04e402 --- /dev/null +++ b/results/lenses/ladder/lens_160m_n200_db128_pen.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65373e75993d2342ba95142055fcaea1c71206b3a4c8faa7de9a8e984cbd5b78 +size 11800294 diff --git a/results/lenses/ladder/lens_160m_n200_db128_pen_provenance.json b/results/lenses/ladder/lens_160m_n200_db128_pen_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..fc8f845db0ac75f578dee33140ea4b5c306a667f --- /dev/null +++ b/results/lenses/ladder/lens_160m_n200_db128_pen_provenance.json @@ -0,0 +1,259 @@ +{ + "artifact": "pythia/results/lens_160m_n200_db128_pen.pt", + "sha256": "65373e75993d2342ba95142055fcaea1c71206b3a4c8faa7de9a8e984cbd5b78", + "bytes": 11800294, + "model": "EleutherAI/pythia-160m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": -2, + "target_layer_effective": 10, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 12, + "d_model": 768, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": -2, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0765172299273942e-07, + "ref_absmax": 0.5536803603172302, + "pass": true + }, + "1": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0907743505284093e-07, + "ref_absmax": 0.5464434027671814, + "pass": true + }, + "2": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.539795453024529e-08, + "ref_absmax": 0.5379679203033447, + "pass": true + }, + "3": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.3282682105288286e-08, + "ref_absmax": 0.5593247413635254, + "pass": true + }, + "4": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 9.721576952551996e-08, + "ref_absmax": 0.6131170392036438, + "pass": true + }, + "5": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.69167046699285e-08, + "ref_absmax": 0.7749245762825012, + "pass": true + }, + "6": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.58406787558731e-08, + "ref_absmax": 0.785919189453125, + "pass": true + }, + "7": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.613739175071157e-08, + "ref_absmax": 0.7828564047813416, + "pass": true + }, + "8": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.577575165075759e-08, + "ref_absmax": 0.9061796069145203, + "pass": true + }, + "9": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.945999620645224e-08, + "ref_absmax": 1.0024327039718628, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 5.960464477539063e-08, + "worst_rel_diff": 1.0907743505284093e-07 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 0.641485, + "mean_fro": 21.017168, + "rms_per_prompt_fro": 26.927289, + "max_per_prompt_fro": 35.919056, + "heavy_tail_ratio": 1.3339, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.50993, + "mean_fro": 20.779541, + "rms_per_prompt_fro": 25.533732, + "max_per_prompt_fro": 34.364525, + "heavy_tail_ratio": 1.3458, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.438254, + "mean_fro": 19.926792, + "rms_per_prompt_fro": 23.897651, + "max_per_prompt_fro": 31.656626, + "heavy_tail_ratio": 1.3247, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.381029, + "mean_fro": 20.416662, + "rms_per_prompt_fro": 23.993088, + "max_per_prompt_fro": 31.558027, + "heavy_tail_ratio": 1.3153, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.333103, + "mean_fro": 22.77523, + "rms_per_prompt_fro": 26.296295, + "max_per_prompt_fro": 33.983177, + "heavy_tail_ratio": 1.2923, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.279857, + "mean_fro": 24.498484, + "rms_per_prompt_fro": 27.715326, + "max_per_prompt_fro": 33.405384, + "heavy_tail_ratio": 1.2053, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.235863, + "mean_fro": 26.602331, + "rms_per_prompt_fro": 29.573643, + "max_per_prompt_fro": 34.678463, + "heavy_tail_ratio": 1.1726, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.171063, + "mean_fro": 25.310974, + "rms_per_prompt_fro": 27.390442, + "max_per_prompt_fro": 29.926582, + "heavy_tail_ratio": 1.0926, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.116419, + "mean_fro": 27.832924, + "rms_per_prompt_fro": 29.408474, + "max_per_prompt_fro": 31.196423, + "heavy_tail_ratio": 1.0608, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.063732, + "mean_fro": 28.561892, + "rms_per_prompt_fro": 29.457989, + "max_per_prompt_fro": 30.291183, + "heavy_tail_ratio": 1.0283, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 89.0, + "fit_secs_per_prompt": 0.445, + "total_wall_seconds": 123.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 21.017, + "top_sv": 3.354, + "stable_rank": 39.265 + }, + { + "layer": 1, + "fro_norm": 20.7794, + "top_sv": 3.2494, + "stable_rank": 40.895 + }, + { + "layer": 2, + "fro_norm": 19.9267, + "top_sv": 3.1239, + "stable_rank": 40.69 + }, + { + "layer": 3, + "fro_norm": 20.4165, + "top_sv": 3.0023, + "stable_rank": 46.243 + }, + { + "layer": 4, + "fro_norm": 22.7751, + "top_sv": 3.2855, + "stable_rank": 48.052 + }, + { + "layer": 5, + "fro_norm": 24.4984, + "top_sv": 3.4077, + "stable_rank": 51.682 + }, + { + "layer": 6, + "fro_norm": 26.6022, + "top_sv": 3.4479, + "stable_rank": 59.528 + }, + { + "layer": 7, + "fro_norm": 25.3108, + "top_sv": 3.5701, + "stable_rank": 50.263 + }, + { + "layer": 8, + "fro_norm": 27.8327, + "top_sv": 3.8704, + "stable_rank": 51.714 + }, + { + "layer": 9, + "fro_norm": 28.5615, + "top_sv": 3.2339, + "stable_rank": 78.005 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_160m_n200_db128_provenance.json b/results/lenses/ladder/lens_160m_n200_db128_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f111e2fc076963427b7bd8b19ee89d55e2ca4614 --- /dev/null +++ b/results/lenses/ladder/lens_160m_n200_db128_provenance.json @@ -0,0 +1,184 @@ +{ + "artifact": "pythia/results/lens_160m_n200_db128.pt", + "sha256": "0022d6297697477dc152ffc6722a29be22464844de8197aa88a9daa3bd4df197", + "bytes": 12980088, + "model": "EleutherAI/pythia-160m-deduped", + "corpus": "wikitext-103-raw-v1", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "device": "cuda", + "dtype": "float32", + "n_layers": 12, + "d_model": 768, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.60956924181433e-08, + "ref_absmax": 0.45089659094810486, + "pass": true + }, + "1": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.485761062876146e-08, + "ref_absmax": 0.4595038592815399, + "pass": true + }, + "2": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.54696372772592e-08, + "ref_absmax": 0.45520830154418945, + "pass": true + }, + "3": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.8982977748553676e-08, + "ref_absmax": 0.5052698850631714, + "pass": true + }, + "4": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 9.902355839770772e-08, + "ref_absmax": 0.6019238829612732, + "pass": true + }, + "5": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.297791128001726e-08, + "ref_absmax": 0.7183194160461426, + "pass": true + }, + "6": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.652402142499243e-08, + "ref_absmax": 0.895986795425415, + "pass": true + }, + "7": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.492115325936649e-08, + "ref_absmax": 0.9181082248687744, + "pass": true + }, + "8": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1217931998915451e-07, + "ref_absmax": 1.0626672506332397, + "pass": true + }, + "9": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0759267521978225e-07, + "ref_absmax": 1.1079684495925903, + "pass": true + }, + "10": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0604503350810969e-07, + "ref_absmax": 1.124138355255127, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.1217931998915451e-07 + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 117.3, + "fit_secs_per_prompt": 0.586, + "total_wall_seconds": 135.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 26.1498, + "top_sv": 9.5296, + "stable_rank": 7.53 + }, + { + "layer": 1, + "fro_norm": 25.8198, + "top_sv": 9.3782, + "stable_rank": 7.58 + }, + { + "layer": 2, + "fro_norm": 24.9842, + "top_sv": 9.7954, + "stable_rank": 6.506 + }, + { + "layer": 3, + "fro_norm": 25.7388, + "top_sv": 10.4307, + "stable_rank": 6.089 + }, + { + "layer": 4, + "fro_norm": 28.8239, + "top_sv": 12.1963, + "stable_rank": 5.585 + }, + { + "layer": 5, + "fro_norm": 30.8919, + "top_sv": 13.4581, + "stable_rank": 5.269 + }, + { + "layer": 6, + "fro_norm": 32.5111, + "top_sv": 12.0816, + "stable_rank": 7.241 + }, + { + "layer": 7, + "fro_norm": 30.7644, + "top_sv": 12.2288, + "stable_rank": 6.329 + }, + { + "layer": 8, + "fro_norm": 33.5039, + "top_sv": 12.6192, + "stable_rank": 7.049 + }, + { + "layer": 9, + "fro_norm": 33.9332, + "top_sv": 12.59, + "stable_rank": 7.264 + }, + { + "layer": 10, + "fro_norm": 34.2446, + "top_sv": 15.3146, + "stable_rank": 5.0 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_160m_n200_provenance.json b/results/lenses/ladder/lens_160m_n200_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e963599354623ad8614c91c659e745f422401b6b --- /dev/null +++ b/results/lenses/ladder/lens_160m_n200_provenance.json @@ -0,0 +1,112 @@ +{ + "artifact": "pythia/results/lens_160m_n200.pt", + "sha256": "af0e17b4c9305e0b54c3c538805cbb7fec71e847f676e1001ca1844d935b8143", + "bytes": 12980409, + "model": "EleutherAI/pythia-160m-deduped", + "corpus": "wikitext-103-raw-v1", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 64, + "max_seq_len": 128, + "skip_first_positions": 16, + "device": "cpu", + "dtype": "float32", + "n_layers": 12, + "d_model": 768, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "centered": false, + "layout_autodetected": false, + "layout": { + "path": "gpt_neox", + "lm_head": "lm_head", + "norm": "final_layer_norm", + "embed": "embed_in" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 2470.3, + "fit_secs_per_prompt": 12.352, + "per_layer": [ + { + "layer": 0, + "fro_norm": 26.1515, + "top_sv": 9.5335, + "stable_rank": 7.525 + }, + { + "layer": 1, + "fro_norm": 25.82, + "top_sv": 9.3779, + "stable_rank": 7.58 + }, + { + "layer": 2, + "fro_norm": 24.9868, + "top_sv": 9.8036, + "stable_rank": 6.496 + }, + { + "layer": 3, + "fro_norm": 25.7402, + "top_sv": 10.4348, + "stable_rank": 6.085 + }, + { + "layer": 4, + "fro_norm": 28.8238, + "top_sv": 12.196, + "stable_rank": 5.586 + }, + { + "layer": 5, + "fro_norm": 30.8925, + "top_sv": 13.4599, + "stable_rank": 5.268 + }, + { + "layer": 6, + "fro_norm": 32.5101, + "top_sv": 12.0779, + "stable_rank": 7.245 + }, + { + "layer": 7, + "fro_norm": 30.7635, + "top_sv": 12.2269, + "stable_rank": 6.331 + }, + { + "layer": 8, + "fro_norm": 33.5032, + "top_sv": 12.618, + "stable_rank": 7.05 + }, + { + "layer": 9, + "fro_norm": 33.9327, + "top_sv": 12.5897, + "stable_rank": 7.265 + }, + { + "layer": 10, + "fro_norm": 34.244, + "top_sv": 15.3143, + "stable_rank": 5.0 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_1b_n200_db128_fin.pt b/results/lenses/ladder/lens_1b_n200_db128_fin.pt new file mode 100644 index 0000000000000000000000000000000000000000..28d42ebd68847e37fa80cc4b959810d9482cee01 --- /dev/null +++ b/results/lenses/ladder/lens_1b_n200_db128_fin.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b1f36a46322f6078200fc0caa030be5cc0f0499fa04c8caf82b6ee41ffaf1cf +size 125834246 diff --git a/results/lenses/ladder/lens_1b_n200_db128_fin_provenance.json b/results/lenses/ladder/lens_1b_n200_db128_fin_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..e8c71f8bdd496e11a479b32241ab6bb0d738eb33 --- /dev/null +++ b/results/lenses/ladder/lens_1b_n200_db128_fin_provenance.json @@ -0,0 +1,364 @@ +{ + "artifact": "pythia/results/lens_1b_n200_db128_fin.pt", + "sha256": "1b1f36a46322f6078200fc0caa030be5cc0f0499fa04c8caf82b6ee41ffaf1cf", + "bytes": 125834246, + "model": "EleutherAI/pythia-1b-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": null, + "target_layer_effective": 15, + "target_layer_is_penultimate": false, + "device": "cuda", + "dtype": "float32", + "n_layers": 16, + "d_model": 2048, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": null, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1621273391734881e-07, + "ref_absmax": 1.0257850885391235, + "pass": true + }, + "1": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.776570422016591e-08, + "ref_absmax": 0.7664644122123718, + "pass": true + }, + "2": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.742997405515519e-08, + "ref_absmax": 0.6817415356636047, + "pass": true + }, + "3": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 9.376295393810501e-08, + "ref_absmax": 0.6356950402259827, + "pass": true + }, + "4": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 9.537957494092427e-08, + "ref_absmax": 0.6249204277992249, + "pass": true + }, + "5": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.09077395291833e-08, + "ref_absmax": 0.7366989254951477, + "pass": true + }, + "6": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.768648407442613e-08, + "ref_absmax": 0.7672460079193115, + "pass": true + }, + "7": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.090316379134248e-08, + "ref_absmax": 0.8406485915184021, + "pass": true + }, + "8": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.143806229266574e-08, + "ref_absmax": 0.9701582789421082, + "pass": true + }, + "9": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1543810318573379e-07, + "ref_absmax": 1.0326684713363647, + "pass": true + }, + "10": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0594660270034582e-07, + "ref_absmax": 1.1251827478408813, + "pass": true + }, + "11": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.853656450494196e-08, + "ref_absmax": 1.2097975015640259, + "pass": true + }, + "12": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.220666501124967e-08, + "ref_absmax": 1.2928489446640015, + "pass": true + }, + "13": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 8.614813758064327e-08, + "ref_absmax": 1.3837709426879883, + "pass": true + }, + "14": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.472087416000345e-08, + "ref_absmax": 1.2585324048995972, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.1621273391734881e-07 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 1.944999, + "mean_fro": 24.804895, + "rms_per_prompt_fro": 42.567677, + "max_per_prompt_fro": 59.223969, + "heavy_tail_ratio": 1.3913, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.688314, + "mean_fro": 23.127671, + "rms_per_prompt_fro": 37.920313, + "max_per_prompt_fro": 52.651028, + "heavy_tail_ratio": 1.3885, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.306531, + "mean_fro": 26.304841, + "rms_per_prompt_fro": 39.949866, + "max_per_prompt_fro": 55.241611, + "heavy_tail_ratio": 1.3828, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.093576, + "mean_fro": 27.862741, + "rms_per_prompt_fro": 40.315147, + "max_per_prompt_fro": 55.043587, + "heavy_tail_ratio": 1.3653, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.974968, + "mean_fro": 29.589304, + "rms_per_prompt_fro": 41.582901, + "max_per_prompt_fro": 56.275574, + "heavy_tail_ratio": 1.3533, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.809343, + "mean_fro": 32.859303, + "rms_per_prompt_fro": 44.199648, + "max_per_prompt_fro": 56.971691, + "heavy_tail_ratio": 1.289, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.656803, + "mean_fro": 38.24403, + "rms_per_prompt_fro": 49.226517, + "max_per_prompt_fro": 62.204323, + "heavy_tail_ratio": 1.2636, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.557888, + "mean_fro": 41.794057, + "rms_per_prompt_fro": 52.165414, + "max_per_prompt_fro": 64.595482, + "heavy_tail_ratio": 1.2383, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.457916, + "mean_fro": 48.538612, + "rms_per_prompt_fro": 58.607547, + "max_per_prompt_fro": 71.892807, + "heavy_tail_ratio": 1.2267, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.35622, + "mean_fro": 48.123795, + "rms_per_prompt_fro": 56.04346, + "max_per_prompt_fro": 66.774361, + "heavy_tail_ratio": 1.1915, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.275558, + "mean_fro": 54.789563, + "rms_per_prompt_fro": 61.879675, + "max_per_prompt_fro": 72.793411, + "heavy_tail_ratio": 1.1764, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.178295, + "mean_fro": 54.482132, + "rms_per_prompt_fro": 59.139983, + "max_per_prompt_fro": 66.887222, + "heavy_tail_ratio": 1.131, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.113245, + "mean_fro": 56.951378, + "rms_per_prompt_fro": 60.089638, + "max_per_prompt_fro": 66.160812, + "heavy_tail_ratio": 1.101, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.062215, + "mean_fro": 54.51786, + "rms_per_prompt_fro": 56.188192, + "max_per_prompt_fro": 59.847736, + "heavy_tail_ratio": 1.0651, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.024493, + "mean_fro": 47.711497, + "rms_per_prompt_fro": 48.29227, + "max_per_prompt_fro": 49.517708, + "heavy_tail_ratio": 1.0254, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 4573.1, + "fit_secs_per_prompt": 22.865, + "total_wall_seconds": 4734.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 24.8022, + "top_sv": 5.9438, + "stable_rank": 17.412 + }, + { + "layer": 1, + "fro_norm": 23.1255, + "top_sv": 5.5836, + "stable_rank": 17.154 + }, + { + "layer": 2, + "fro_norm": 26.3021, + "top_sv": 6.1654, + "stable_rank": 18.199 + }, + { + "layer": 3, + "fro_norm": 27.8598, + "top_sv": 6.6844, + "stable_rank": 17.371 + }, + { + "layer": 4, + "fro_norm": 29.5863, + "top_sv": 7.3529, + "stable_rank": 16.191 + }, + { + "layer": 5, + "fro_norm": 32.8561, + "top_sv": 7.8007, + "stable_rank": 17.741 + }, + { + "layer": 6, + "fro_norm": 38.2396, + "top_sv": 9.4746, + "stable_rank": 16.289 + }, + { + "layer": 7, + "fro_norm": 41.7895, + "top_sv": 10.032, + "stable_rank": 17.353 + }, + { + "layer": 8, + "fro_norm": 48.5325, + "top_sv": 11.249, + "stable_rank": 18.614 + }, + { + "layer": 9, + "fro_norm": 48.1172, + "top_sv": 8.4506, + "stable_rank": 32.421 + }, + { + "layer": 10, + "fro_norm": 54.7814, + "top_sv": 9.0013, + "stable_rank": 37.039 + }, + { + "layer": 11, + "fro_norm": 54.4725, + "top_sv": 8.4661, + "stable_rank": 41.399 + }, + { + "layer": 12, + "fro_norm": 56.9405, + "top_sv": 7.4575, + "stable_rank": 58.298 + }, + { + "layer": 13, + "fro_norm": 54.5044, + "top_sv": 6.8503, + "stable_rank": 63.306 + }, + { + "layer": 14, + "fro_norm": 47.6933, + "top_sv": 5.4986, + "stable_rank": 75.233 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_1b_n200_db128_pen.pt b/results/lenses/ladder/lens_1b_n200_db128_pen.pt new file mode 100644 index 0000000000000000000000000000000000000000..c886871a03a0d75723e7f7be617d8f2ac9b36916 --- /dev/null +++ b/results/lenses/ladder/lens_1b_n200_db128_pen.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fccbb2a6520759d5e5043e56b0fc6c6ab8f10204b27d0225b546e73a414957fa +size 117445370 diff --git a/results/lenses/ladder/lens_1b_n200_db128_pen_provenance.json b/results/lenses/ladder/lens_1b_n200_db128_pen_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9fda7143023d88d155fff37f90cc935e2e7c0e5f --- /dev/null +++ b/results/lenses/ladder/lens_1b_n200_db128_pen_provenance.json @@ -0,0 +1,343 @@ +{ + "artifact": "pythia/results/lens_1b_n200_db128_pen.pt", + "sha256": "fccbb2a6520759d5e5043e56b0fc6c6ab8f10204b27d0225b546e73a414957fa", + "bytes": 117445370, + "model": "EleutherAI/pythia-1b-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": -2, + "target_layer_effective": 14, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 16, + "d_model": 2048, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": -2, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 4.979290632331072e-08, + "ref_absmax": 1.197050929069519, + "pass": true + }, + "1": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.694914295389393e-08, + "ref_absmax": 0.8902973532676697, + "pass": true + }, + "2": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.213944901764466e-08, + "ref_absmax": 0.8262420296669006, + "pass": true + }, + "3": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.316761405312802e-08, + "ref_absmax": 0.8146315217018127, + "pass": true + }, + "4": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.375875811493857e-08, + "ref_absmax": 0.8081026077270508, + "pass": true + }, + "5": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.674776074612248e-08, + "ref_absmax": 0.8929834365844727, + "pass": true + }, + "6": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.4997066456475214e-08, + "ref_absmax": 1.0837786197662354, + "pass": true + }, + "7": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 4.9696256480391844e-08, + "ref_absmax": 1.1993789672851562, + "pass": true + }, + "8": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 3.8404395674958404e-08, + "ref_absmax": 1.552026629447937, + "pass": true + }, + "9": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 7.343810552012942e-08, + "ref_absmax": 1.6232620477676392, + "pass": true + }, + "10": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 6.062755215470017e-08, + "ref_absmax": 1.966256022453308, + "pass": true + }, + "11": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 7.358402084723611e-08, + "ref_absmax": 1.620043158531189, + "pass": true + }, + "12": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 8.62041915236453e-08, + "ref_absmax": 1.382871150970459, + "pass": true + }, + "13": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.571155752470387e-08, + "ref_absmax": 1.245505690574646, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 9.571155752470387e-08 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 1.73571, + "mean_fro": 22.479978, + "rms_per_prompt_fro": 37.181841, + "max_per_prompt_fro": 49.971481, + "heavy_tail_ratio": 1.344, + "n_prompts": 200 + }, + "1": { + "dispersion": 1.505767, + "mean_fro": 20.975348, + "rms_per_prompt_fro": 33.20317, + "max_per_prompt_fro": 44.988842, + "heavy_tail_ratio": 1.355, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.166506, + "mean_fro": 23.873709, + "rms_per_prompt_fro": 35.139842, + "max_per_prompt_fro": 47.355873, + "heavy_tail_ratio": 1.3476, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.982771, + "mean_fro": 25.190741, + "rms_per_prompt_fro": 35.471305, + "max_per_prompt_fro": 47.145779, + "heavy_tail_ratio": 1.3291, + "n_prompts": 200 + }, + "4": { + "dispersion": 0.875869, + "mean_fro": 26.822136, + "rms_per_prompt_fro": 36.736237, + "max_per_prompt_fro": 47.619823, + "heavy_tail_ratio": 1.2963, + "n_prompts": 200 + }, + "5": { + "dispersion": 0.72398, + "mean_fro": 30.050229, + "rms_per_prompt_fro": 39.456074, + "max_per_prompt_fro": 49.627102, + "heavy_tail_ratio": 1.2578, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.594042, + "mean_fro": 34.862644, + "rms_per_prompt_fro": 44.015966, + "max_per_prompt_fro": 54.175922, + "heavy_tail_ratio": 1.2308, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.504802, + "mean_fro": 38.224381, + "rms_per_prompt_fro": 46.88999, + "max_per_prompt_fro": 56.599808, + "heavy_tail_ratio": 1.2071, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.41589, + "mean_fro": 44.366184, + "rms_per_prompt_fro": 52.791847, + "max_per_prompt_fro": 63.116722, + "heavy_tail_ratio": 1.1956, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.311256, + "mean_fro": 45.072667, + "rms_per_prompt_fro": 51.612752, + "max_per_prompt_fro": 59.868328, + "heavy_tail_ratio": 1.16, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.239409, + "mean_fro": 51.182945, + "rms_per_prompt_fro": 56.981335, + "max_per_prompt_fro": 65.219429, + "heavy_tail_ratio": 1.1446, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.147058, + "mean_fro": 51.338902, + "rms_per_prompt_fro": 54.984369, + "max_per_prompt_fro": 60.819519, + "heavy_tail_ratio": 1.1061, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.085984, + "mean_fro": 53.47414, + "rms_per_prompt_fro": 55.725711, + "max_per_prompt_fro": 60.089775, + "heavy_tail_ratio": 1.0783, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.037524, + "mean_fro": 51.395867, + "rms_per_prompt_fro": 52.351286, + "max_per_prompt_fro": 54.723274, + "heavy_tail_ratio": 1.0453, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 4287.5, + "fit_secs_per_prompt": 21.437, + "total_wall_seconds": 4439.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 22.478, + "top_sv": 3.523, + "stable_rank": 40.708 + }, + { + "layer": 1, + "fro_norm": 20.9733, + "top_sv": 3.538, + "stable_rank": 35.141 + }, + { + "layer": 2, + "fro_norm": 23.8712, + "top_sv": 4.3802, + "stable_rank": 29.701 + }, + { + "layer": 3, + "fro_norm": 25.188, + "top_sv": 4.7944, + "stable_rank": 27.6 + }, + { + "layer": 4, + "fro_norm": 26.8191, + "top_sv": 5.4054, + "stable_rank": 24.617 + }, + { + "layer": 5, + "fro_norm": 30.0472, + "top_sv": 6.247, + "stable_rank": 23.135 + }, + { + "layer": 6, + "fro_norm": 34.8584, + "top_sv": 6.8427, + "stable_rank": 25.951 + }, + { + "layer": 7, + "fro_norm": 38.2198, + "top_sv": 7.1358, + "stable_rank": 28.687 + }, + { + "layer": 8, + "fro_norm": 44.3614, + "top_sv": 8.0073, + "stable_rank": 30.693 + }, + { + "layer": 9, + "fro_norm": 45.0673, + "top_sv": 7.1698, + "stable_rank": 39.51 + }, + { + "layer": 10, + "fro_norm": 51.175, + "top_sv": 7.733, + "stable_rank": 43.794 + }, + { + "layer": 11, + "fro_norm": 51.3288, + "top_sv": 7.2652, + "stable_rank": 49.914 + }, + { + "layer": 12, + "fro_norm": 53.4619, + "top_sv": 5.6522, + "stable_rank": 89.466 + }, + { + "layer": 13, + "fro_norm": 51.3799, + "top_sv": 4.169, + "stable_rank": 151.886 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_2.8b_n200_db128_fin.pt b/results/lenses/ladder/lens_2.8b_n200_db128_fin.pt new file mode 100644 index 0000000000000000000000000000000000000000..9632ae779ddf236b17a7f843b9e5c9a35bda3955 --- /dev/null +++ b/results/lenses/ladder/lens_2.8b_n200_db128_fin.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:569395f9f23828eb0347712ebab3124dd25f4cebf4db9160ecfd6ab96ee4c6b6 +size 406332684 diff --git a/results/lenses/ladder/lens_2.8b_n200_db128_fin_provenance.json b/results/lenses/ladder/lens_2.8b_n200_db128_fin_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..64f15f49b4a62466a19cac95091e4ceec12c390a --- /dev/null +++ b/results/lenses/ladder/lens_2.8b_n200_db128_fin_provenance.json @@ -0,0 +1,700 @@ +{ + "artifact": "pythia/results/lens_2.8b_n200_db128_fin.pt", + "sha256": "569395f9f23828eb0347712ebab3124dd25f4cebf4db9160ecfd6ab96ee4c6b6", + "bytes": 406332684, + "model": "EleutherAI/pythia-2.8b-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": null, + "target_layer_effective": 31, + "target_layer_is_penultimate": false, + "device": "cuda", + "dtype": "float32", + "n_layers": 32, + "d_model": 2560, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": null, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0861625605196202e-07, + "ref_absmax": 0.5487635731697083, + "pass": true + }, + "1": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.1087751682261405e-07, + "ref_absmax": 0.5375719666481018, + "pass": true + }, + "2": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.82087423710167e-08, + "ref_absmax": 0.511990487575531, + "pass": true + }, + "3": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.720840675011666e-08, + "ref_absmax": 0.44343146681785583, + "pass": true + }, + "4": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.525324228678943e-08, + "ref_absmax": 0.4567178785800934, + "pass": true + }, + "5": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.39084789679752e-08, + "ref_absmax": 0.46632814407348633, + "pass": true + }, + "6": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.170305871934561e-08, + "ref_absmax": 0.482995867729187, + "pass": true + }, + "7": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.8719168179566034e-08, + "ref_absmax": 0.5075399279594421, + "pass": true + }, + "8": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0731499806511059e-07, + "ref_absmax": 0.5554176568984985, + "pass": true + }, + "9": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 9.808108284654057e-08, + "ref_absmax": 0.6077078580856323, + "pass": true + }, + "10": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.676157978607891e-08, + "ref_absmax": 0.6869935393333435, + "pass": true + }, + "11": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.490082942931486e-08, + "ref_absmax": 0.7957808375358582, + "pass": true + }, + "12": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.394715337105477e-08, + "ref_absmax": 0.806043803691864, + "pass": true + }, + "13": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.797745785924437e-08, + "ref_absmax": 0.8768295645713806, + "pass": true + }, + "14": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.629194921400287e-08, + "ref_absmax": 0.8991234302520752, + "pass": true + }, + "15": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.25535771388194e-08, + "ref_absmax": 0.952857494354248, + "pass": true + }, + "16": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.911370817335686e-08, + "ref_absmax": 1.0083049535751343, + "pass": true + }, + "17": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.899934050537183e-08, + "ref_absmax": 1.0102595090866089, + "pass": true + }, + "18": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.9110682151458594e-08, + "ref_absmax": 1.0083565711975098, + "pass": true + }, + "19": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.7551631583000214e-08, + "ref_absmax": 1.0356725454330444, + "pass": true + }, + "20": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.153837855089273e-07, + "ref_absmax": 1.0331546068191528, + "pass": true + }, + "21": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.445664466908657e-08, + "ref_absmax": 1.094533920288086, + "pass": true + }, + "22": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.4433031531204446e-08, + "ref_absmax": 1.0950087308883667, + "pass": true + }, + "23": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1415383073429794e-07, + "ref_absmax": 1.0442863702774048, + "pass": true + }, + "24": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.060438977198123e-07, + "ref_absmax": 1.1241503953933716, + "pass": true + }, + "25": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0549761802203149e-07, + "ref_absmax": 1.1299713850021362, + "pass": true + }, + "26": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0820085671274328e-07, + "ref_absmax": 1.1017407178878784, + "pass": true + }, + "27": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0019285119998974e-07, + "ref_absmax": 1.189798355102539, + "pass": true + }, + "28": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0588981381288148e-07, + "ref_absmax": 1.1257861852645874, + "pass": true + }, + "29": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0093245438660521e-07, + "ref_absmax": 1.1810798645019531, + "pass": true + }, + "30": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0149828726715151e-07, + "ref_absmax": 1.1744955778121948, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.153837855089273e-07 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 4.354734, + "mean_fro": 12.800961, + "rms_per_prompt_fro": 29.621806, + "max_per_prompt_fro": 49.337948, + "heavy_tail_ratio": 1.6656, + "n_prompts": 200 + }, + "1": { + "dispersion": 3.98429, + "mean_fro": 13.101943, + "rms_per_prompt_fro": 29.250772, + "max_per_prompt_fro": 49.149296, + "heavy_tail_ratio": 1.6803, + "n_prompts": 200 + }, + "2": { + "dispersion": 3.157497, + "mean_fro": 14.23256, + "rms_per_prompt_fro": 29.020105, + "max_per_prompt_fro": 45.208035, + "heavy_tail_ratio": 1.5578, + "n_prompts": 200 + }, + "3": { + "dispersion": 2.535749, + "mean_fro": 14.591548, + "rms_per_prompt_fro": 27.437346, + "max_per_prompt_fro": 43.706936, + "heavy_tail_ratio": 1.593, + "n_prompts": 200 + }, + "4": { + "dispersion": 2.077953, + "mean_fro": 16.427964, + "rms_per_prompt_fro": 28.821377, + "max_per_prompt_fro": 47.786591, + "heavy_tail_ratio": 1.658, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.626043, + "mean_fro": 17.184721, + "rms_per_prompt_fro": 27.84796, + "max_per_prompt_fro": 40.95554, + "heavy_tail_ratio": 1.4707, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.379366, + "mean_fro": 17.492298, + "rms_per_prompt_fro": 26.982208, + "max_per_prompt_fro": 35.258102, + "heavy_tail_ratio": 1.3067, + "n_prompts": 200 + }, + "7": { + "dispersion": 1.098862, + "mean_fro": 19.849886, + "rms_per_prompt_fro": 28.75742, + "max_per_prompt_fro": 37.049168, + "heavy_tail_ratio": 1.2883, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.981051, + "mean_fro": 20.645256, + "rms_per_prompt_fro": 29.058158, + "max_per_prompt_fro": 36.322208, + "heavy_tail_ratio": 1.25, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.843625, + "mean_fro": 23.228242, + "rms_per_prompt_fro": 31.539345, + "max_per_prompt_fro": 39.124928, + "heavy_tail_ratio": 1.2405, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.73697, + "mean_fro": 26.49205, + "rms_per_prompt_fro": 34.914975, + "max_per_prompt_fro": 42.931221, + "heavy_tail_ratio": 1.2296, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.677661, + "mean_fro": 28.310585, + "rms_per_prompt_fro": 36.669156, + "max_per_prompt_fro": 44.921524, + "heavy_tail_ratio": 1.225, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.58037, + "mean_fro": 31.194172, + "rms_per_prompt_fro": 39.21506, + "max_per_prompt_fro": 47.656384, + "heavy_tail_ratio": 1.2153, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.529322, + "mean_fro": 33.85759, + "rms_per_prompt_fro": 41.870253, + "max_per_prompt_fro": 49.946491, + "heavy_tail_ratio": 1.1929, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.449156, + "mean_fro": 36.39196, + "rms_per_prompt_fro": 43.808971, + "max_per_prompt_fro": 51.781006, + "heavy_tail_ratio": 1.182, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.397351, + "mean_fro": 39.317448, + "rms_per_prompt_fro": 46.476991, + "max_per_prompt_fro": 54.85675, + "heavy_tail_ratio": 1.1803, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.343031, + "mean_fro": 42.002159, + "rms_per_prompt_fro": 48.675967, + "max_per_prompt_fro": 57.067669, + "heavy_tail_ratio": 1.1724, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.291247, + "mean_fro": 43.531403, + "rms_per_prompt_fro": 49.466062, + "max_per_prompt_fro": 56.598057, + "heavy_tail_ratio": 1.1442, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.247792, + "mean_fro": 45.056633, + "rms_per_prompt_fro": 50.33034, + "max_per_prompt_fro": 56.896435, + "heavy_tail_ratio": 1.1305, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.211488, + "mean_fro": 47.239236, + "rms_per_prompt_fro": 51.995105, + "max_per_prompt_fro": 58.215702, + "heavy_tail_ratio": 1.1196, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.185012, + "mean_fro": 48.289795, + "rms_per_prompt_fro": 52.567433, + "max_per_prompt_fro": 58.002899, + "heavy_tail_ratio": 1.1034, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.155788, + "mean_fro": 48.991703, + "rms_per_prompt_fro": 52.669806, + "max_per_prompt_fro": 57.449986, + "heavy_tail_ratio": 1.0908, + "n_prompts": 200 + }, + "22": { + "dispersion": 0.133959, + "mean_fro": 49.421272, + "rms_per_prompt_fro": 52.627479, + "max_per_prompt_fro": 56.579224, + "heavy_tail_ratio": 1.0751, + "n_prompts": 200 + }, + "23": { + "dispersion": 0.113206, + "mean_fro": 49.83834, + "rms_per_prompt_fro": 52.583732, + "max_per_prompt_fro": 56.293888, + "heavy_tail_ratio": 1.0706, + "n_prompts": 200 + }, + "24": { + "dispersion": 0.09436, + "mean_fro": 50.595002, + "rms_per_prompt_fro": 52.928274, + "max_per_prompt_fro": 56.065018, + "heavy_tail_ratio": 1.0593, + "n_prompts": 200 + }, + "25": { + "dispersion": 0.076181, + "mean_fro": 51.340379, + "rms_per_prompt_fro": 53.26006, + "max_per_prompt_fro": 55.957291, + "heavy_tail_ratio": 1.0506, + "n_prompts": 200 + }, + "26": { + "dispersion": 0.062567, + "mean_fro": 52.41668, + "rms_per_prompt_fro": 54.031582, + "max_per_prompt_fro": 56.623928, + "heavy_tail_ratio": 1.048, + "n_prompts": 200 + }, + "27": { + "dispersion": 0.048262, + "mean_fro": 52.514663, + "rms_per_prompt_fro": 53.76697, + "max_per_prompt_fro": 55.862991, + "heavy_tail_ratio": 1.039, + "n_prompts": 200 + }, + "28": { + "dispersion": 0.032667, + "mean_fro": 51.972468, + "rms_per_prompt_fro": 52.814527, + "max_per_prompt_fro": 54.141281, + "heavy_tail_ratio": 1.0251, + "n_prompts": 200 + }, + "29": { + "dispersion": 0.019604, + "mean_fro": 52.205788, + "rms_per_prompt_fro": 52.715038, + "max_per_prompt_fro": 53.621696, + "heavy_tail_ratio": 1.0172, + "n_prompts": 200 + }, + "30": { + "dispersion": 0.007015, + "mean_fro": 51.074112, + "rms_per_prompt_fro": 51.252939, + "max_per_prompt_fro": 51.642681, + "heavy_tail_ratio": 1.0076, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 7275.6, + "fit_secs_per_prompt": 36.378, + "total_wall_seconds": 7536.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 12.798, + "top_sv": 4.8203, + "stable_rank": 7.049 + }, + { + "layer": 1, + "fro_norm": 13.0989, + "top_sv": 5.199, + "stable_rank": 6.348 + }, + { + "layer": 2, + "fro_norm": 14.2296, + "top_sv": 5.5815, + "stable_rank": 6.5 + }, + { + "layer": 3, + "fro_norm": 14.5885, + "top_sv": 5.5671, + "stable_rank": 6.867 + }, + { + "layer": 4, + "fro_norm": 16.4245, + "top_sv": 6.3833, + "stable_rank": 6.621 + }, + { + "layer": 5, + "fro_norm": 17.1806, + "top_sv": 7.0917, + "stable_rank": 5.869 + }, + { + "layer": 6, + "fro_norm": 17.488, + "top_sv": 6.34, + "stable_rank": 7.609 + }, + { + "layer": 7, + "fro_norm": 19.8455, + "top_sv": 7.5812, + "stable_rank": 6.852 + }, + { + "layer": 8, + "fro_norm": 20.6407, + "top_sv": 6.5107, + "stable_rank": 10.051 + }, + { + "layer": 9, + "fro_norm": 23.223, + "top_sv": 7.3651, + "stable_rank": 9.942 + }, + { + "layer": 10, + "fro_norm": 26.4852, + "top_sv": 8.2911, + "stable_rank": 10.204 + }, + { + "layer": 11, + "fro_norm": 28.3034, + "top_sv": 7.6533, + "stable_rank": 13.677 + }, + { + "layer": 12, + "fro_norm": 31.1872, + "top_sv": 7.9606, + "stable_rank": 15.348 + }, + { + "layer": 13, + "fro_norm": 33.8482, + "top_sv": 7.6723, + "stable_rank": 19.463 + }, + { + "layer": 14, + "fro_norm": 36.3806, + "top_sv": 7.4022, + "stable_rank": 24.156 + }, + { + "layer": 15, + "fro_norm": 39.3052, + "top_sv": 7.7855, + "stable_rank": 25.488 + }, + { + "layer": 16, + "fro_norm": 41.989, + "top_sv": 7.5272, + "stable_rank": 31.118 + }, + { + "layer": 17, + "fro_norm": 43.5174, + "top_sv": 7.3597, + "stable_rank": 34.963 + }, + { + "layer": 18, + "fro_norm": 45.042, + "top_sv": 7.0737, + "stable_rank": 40.546 + }, + { + "layer": 19, + "fro_norm": 47.2211, + "top_sv": 7.1535, + "stable_rank": 43.574 + }, + { + "layer": 20, + "fro_norm": 48.2694, + "top_sv": 6.7795, + "stable_rank": 50.693 + }, + { + "layer": 21, + "fro_norm": 48.9689, + "top_sv": 6.4525, + "stable_rank": 57.595 + }, + { + "layer": 22, + "fro_norm": 49.3964, + "top_sv": 6.6047, + "stable_rank": 55.936 + }, + { + "layer": 23, + "fro_norm": 49.8114, + "top_sv": 5.9084, + "stable_rank": 71.074 + }, + { + "layer": 24, + "fro_norm": 50.5659, + "top_sv": 5.7232, + "stable_rank": 78.062 + }, + { + "layer": 25, + "fro_norm": 51.3093, + "top_sv": 5.4582, + "stable_rank": 88.367 + }, + { + "layer": 26, + "fro_norm": 52.3837, + "top_sv": 5.434, + "stable_rank": 92.929 + }, + { + "layer": 27, + "fro_norm": 52.4786, + "top_sv": 5.6466, + "stable_rank": 86.376 + }, + { + "layer": 28, + "fro_norm": 51.9318, + "top_sv": 5.6217, + "stable_rank": 85.336 + }, + { + "layer": 29, + "fro_norm": 52.1604, + "top_sv": 5.8078, + "stable_rank": 80.66 + }, + { + "layer": 30, + "fro_norm": 51.0154, + "top_sv": 4.3612, + "stable_rank": 136.831 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_2.8b_n200_db128_pen.pt b/results/lenses/ladder/lens_2.8b_n200_db128_pen.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7ec74d7bd62dbe1b6f4415a34200fa1a02e7b49 --- /dev/null +++ b/results/lenses/ladder/lens_2.8b_n200_db128_pen.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3437c2bcf657a06647e243021a9a3b76b89415edee7f51c743684b149e7541f0 +size 393225214 diff --git a/results/lenses/ladder/lens_2.8b_n200_db128_pen_provenance.json b/results/lenses/ladder/lens_2.8b_n200_db128_pen_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a54d717415541317b2f6f8b56d93fbdbfddf4d31 --- /dev/null +++ b/results/lenses/ladder/lens_2.8b_n200_db128_pen_provenance.json @@ -0,0 +1,679 @@ +{ + "artifact": "pythia/results/lens_2.8b_n200_db128_pen.pt", + "sha256": "3437c2bcf657a06647e243021a9a3b76b89415edee7f51c743684b149e7541f0", + "bytes": 393225214, + "model": "EleutherAI/pythia-2.8b-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": -2, + "target_layer_effective": 30, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 32, + "d_model": 2560, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": -2, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.053971274996015e-08, + "ref_absmax": 0.49227723479270935, + "pass": true + }, + "1": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.116727218188308e-08, + "ref_absmax": 0.48722660541534424, + "pass": true + }, + "2": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.018917578327033e-08, + "ref_absmax": 0.49514421820640564, + "pass": true + }, + "3": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.895048795676901e-08, + "ref_absmax": 0.505548357963562, + "pass": true + }, + "4": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.1792988242980371e-07, + "ref_absmax": 0.505424439907074, + "pass": true + }, + "5": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.1542408598263352e-07, + "ref_absmax": 0.5163969397544861, + "pass": true + }, + "6": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.1203338236287591e-07, + "ref_absmax": 0.5320257544517517, + "pass": true + }, + "7": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0681280937662656e-07, + "ref_absmax": 0.5580289959907532, + "pass": true + }, + "8": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 4.890394043500055e-08, + "ref_absmax": 0.6094053387641907, + "pass": true + }, + "9": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 4.6072100072287124e-08, + "ref_absmax": 0.6468626856803894, + "pass": true + }, + "10": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.675762799092584e-08, + "ref_absmax": 0.6870248317718506, + "pass": true + }, + "11": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.178540817461511e-08, + "ref_absmax": 0.7287931442260742, + "pass": true + }, + "12": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.486332203296442e-08, + "ref_absmax": 0.7961795330047607, + "pass": true + }, + "13": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.708871731471237e-08, + "ref_absmax": 0.8884451389312744, + "pass": true + }, + "14": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.020408341420247e-08, + "ref_absmax": 0.9900432229042053, + "pass": true + }, + "15": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1225938743869094e-07, + "ref_absmax": 1.0619093179702759, + "pass": true + }, + "16": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.672430953185087e-08, + "ref_absmax": 1.050777792930603, + "pass": true + }, + "17": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.6613055989746246e-08, + "ref_absmax": 1.0528427362442017, + "pass": true + }, + "18": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.5392246902299375e-08, + "ref_absmax": 1.0760467052459717, + "pass": true + }, + "19": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.4846475938412234e-08, + "ref_absmax": 1.086754322052002, + "pass": true + }, + "20": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0826167583228058e-07, + "ref_absmax": 1.1011217832565308, + "pass": true + }, + "21": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.689102819382819e-08, + "ref_absmax": 1.0476984977722168, + "pass": true + }, + "22": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1275694912620694e-07, + "ref_absmax": 1.0572234392166138, + "pass": true + }, + "23": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.600607464288006e-08, + "ref_absmax": 1.0642532110214233, + "pass": true + }, + "24": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 5.8083666501244384e-08, + "ref_absmax": 1.0261859893798828, + "pass": true + }, + "25": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1386910950826422e-07, + "ref_absmax": 1.046897530555725, + "pass": true + }, + "26": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1467981967745154e-07, + "ref_absmax": 1.039496660232544, + "pass": true + }, + "27": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1351437438198518e-07, + "ref_absmax": 1.0501691102981567, + "pass": true + }, + "28": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1283968978112486e-07, + "ref_absmax": 1.056448221206665, + "pass": true + }, + "29": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1201045550395856e-07, + "ref_absmax": 1.0642693042755127, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.1792988242980371e-07 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 4.418416, + "mean_fro": 11.4343, + "rms_per_prompt_fro": 26.616182, + "max_per_prompt_fro": 45.299362, + "heavy_tail_ratio": 1.7019, + "n_prompts": 200 + }, + "1": { + "dispersion": 4.123117, + "mean_fro": 11.570233, + "rms_per_prompt_fro": 26.188417, + "max_per_prompt_fro": 45.069904, + "heavy_tail_ratio": 1.721, + "n_prompts": 200 + }, + "2": { + "dispersion": 3.288672, + "mean_fro": 12.551829, + "rms_per_prompt_fro": 25.99372, + "max_per_prompt_fro": 41.419498, + "heavy_tail_ratio": 1.5934, + "n_prompts": 200 + }, + "3": { + "dispersion": 2.626305, + "mean_fro": 12.94799, + "rms_per_prompt_fro": 24.656676, + "max_per_prompt_fro": 39.208958, + "heavy_tail_ratio": 1.5902, + "n_prompts": 200 + }, + "4": { + "dispersion": 2.1716, + "mean_fro": 14.543183, + "rms_per_prompt_fro": 25.899936, + "max_per_prompt_fro": 42.961643, + "heavy_tail_ratio": 1.6588, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.724826, + "mean_fro": 15.113483, + "rms_per_prompt_fro": 24.947896, + "max_per_prompt_fro": 36.650139, + "heavy_tail_ratio": 1.4691, + "n_prompts": 200 + }, + "6": { + "dispersion": 1.427161, + "mean_fro": 15.678135, + "rms_per_prompt_fro": 24.425511, + "max_per_prompt_fro": 31.682831, + "heavy_tail_ratio": 1.2971, + "n_prompts": 200 + }, + "7": { + "dispersion": 1.165801, + "mean_fro": 17.65267, + "rms_per_prompt_fro": 25.978837, + "max_per_prompt_fro": 33.127178, + "heavy_tail_ratio": 1.2752, + "n_prompts": 200 + }, + "8": { + "dispersion": 1.006128, + "mean_fro": 18.82354, + "rms_per_prompt_fro": 26.661257, + "max_per_prompt_fro": 32.922745, + "heavy_tail_ratio": 1.2349, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.874959, + "mean_fro": 21.135389, + "rms_per_prompt_fro": 28.94051, + "max_per_prompt_fro": 35.44994, + "heavy_tail_ratio": 1.2249, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.767417, + "mean_fro": 24.167644, + "rms_per_prompt_fro": 32.129493, + "max_per_prompt_fro": 38.931671, + "heavy_tail_ratio": 1.2117, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.688895, + "mean_fro": 26.302559, + "rms_per_prompt_fro": 34.18215, + "max_per_prompt_fro": 41.352184, + "heavy_tail_ratio": 1.2098, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.588333, + "mean_fro": 29.089727, + "rms_per_prompt_fro": 36.66152, + "max_per_prompt_fro": 43.946098, + "heavy_tail_ratio": 1.1987, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.528614, + "mean_fro": 31.969438, + "rms_per_prompt_fro": 39.526101, + "max_per_prompt_fro": 46.478096, + "heavy_tail_ratio": 1.1759, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.44279, + "mean_fro": 34.641159, + "rms_per_prompt_fro": 41.609647, + "max_per_prompt_fro": 48.517254, + "heavy_tail_ratio": 1.166, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.390658, + "mean_fro": 37.501499, + "rms_per_prompt_fro": 44.224078, + "max_per_prompt_fro": 51.565746, + "heavy_tail_ratio": 1.166, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.333914, + "mean_fro": 40.306972, + "rms_per_prompt_fro": 46.552608, + "max_per_prompt_fro": 53.97427, + "heavy_tail_ratio": 1.1594, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.281205, + "mean_fro": 41.936535, + "rms_per_prompt_fro": 47.468095, + "max_per_prompt_fro": 53.72374, + "heavy_tail_ratio": 1.1318, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.237206, + "mean_fro": 43.537667, + "rms_per_prompt_fro": 48.426844, + "max_per_prompt_fro": 54.208069, + "heavy_tail_ratio": 1.1194, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.201721, + "mean_fro": 45.646148, + "rms_per_prompt_fro": 50.03869, + "max_per_prompt_fro": 55.324219, + "heavy_tail_ratio": 1.1056, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.175299, + "mean_fro": 46.751172, + "rms_per_prompt_fro": 50.683521, + "max_per_prompt_fro": 55.286209, + "heavy_tail_ratio": 1.0908, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.146459, + "mean_fro": 47.522427, + "rms_per_prompt_fro": 50.883607, + "max_per_prompt_fro": 54.924599, + "heavy_tail_ratio": 1.0794, + "n_prompts": 200 + }, + "22": { + "dispersion": 0.124612, + "mean_fro": 48.004989, + "rms_per_prompt_fro": 50.908203, + "max_per_prompt_fro": 54.219257, + "heavy_tail_ratio": 1.065, + "n_prompts": 200 + }, + "23": { + "dispersion": 0.104266, + "mean_fro": 48.57571, + "rms_per_prompt_fro": 51.045332, + "max_per_prompt_fro": 54.14946, + "heavy_tail_ratio": 1.0608, + "n_prompts": 200 + }, + "24": { + "dispersion": 0.085761, + "mean_fro": 49.342293, + "rms_per_prompt_fro": 51.414603, + "max_per_prompt_fro": 54.006844, + "heavy_tail_ratio": 1.0504, + "n_prompts": 200 + }, + "25": { + "dispersion": 0.067878, + "mean_fro": 50.124864, + "rms_per_prompt_fro": 51.798114, + "max_per_prompt_fro": 53.986523, + "heavy_tail_ratio": 1.0422, + "n_prompts": 200 + }, + "26": { + "dispersion": 0.054365, + "mean_fro": 51.196263, + "rms_per_prompt_fro": 52.56948, + "max_per_prompt_fro": 54.660439, + "heavy_tail_ratio": 1.0398, + "n_prompts": 200 + }, + "27": { + "dispersion": 0.040424, + "mean_fro": 51.328769, + "rms_per_prompt_fro": 52.355954, + "max_per_prompt_fro": 54.006607, + "heavy_tail_ratio": 1.0315, + "n_prompts": 200 + }, + "28": { + "dispersion": 0.025223, + "mean_fro": 50.954759, + "rms_per_prompt_fro": 51.593376, + "max_per_prompt_fro": 52.563293, + "heavy_tail_ratio": 1.0188, + "n_prompts": 200 + }, + "29": { + "dispersion": 0.012407, + "mean_fro": 51.257648, + "rms_per_prompt_fro": 51.574653, + "max_per_prompt_fro": 52.077423, + "heavy_tail_ratio": 1.0097, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 7050.0, + "fit_secs_per_prompt": 35.25, + "total_wall_seconds": 7359.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 11.4322, + "top_sv": 2.7345, + "stable_rank": 17.479 + }, + { + "layer": 1, + "fro_norm": 11.568, + "top_sv": 2.8983, + "stable_rank": 15.93 + }, + { + "layer": 2, + "fro_norm": 12.5491, + "top_sv": 3.1124, + "stable_rank": 16.257 + }, + { + "layer": 3, + "fro_norm": 12.9451, + "top_sv": 3.1869, + "stable_rank": 16.499 + }, + { + "layer": 4, + "fro_norm": 14.5402, + "top_sv": 3.8186, + "stable_rank": 14.499 + }, + { + "layer": 5, + "fro_norm": 15.1105, + "top_sv": 3.9004, + "stable_rank": 15.009 + }, + { + "layer": 6, + "fro_norm": 15.6751, + "top_sv": 3.3217, + "stable_rank": 22.269 + }, + { + "layer": 7, + "fro_norm": 17.6485, + "top_sv": 3.9743, + "stable_rank": 19.72 + }, + { + "layer": 8, + "fro_norm": 18.8191, + "top_sv": 3.4065, + "stable_rank": 30.52 + }, + { + "layer": 9, + "fro_norm": 21.1308, + "top_sv": 3.8755, + "stable_rank": 29.729 + }, + { + "layer": 10, + "fro_norm": 24.1618, + "top_sv": 4.3063, + "stable_rank": 31.481 + }, + { + "layer": 11, + "fro_norm": 26.2955, + "top_sv": 4.059, + "stable_rank": 41.97 + }, + { + "layer": 12, + "fro_norm": 29.0824, + "top_sv": 4.2604, + "stable_rank": 46.597 + }, + { + "layer": 13, + "fro_norm": 31.962, + "top_sv": 4.4691, + "stable_rank": 51.149 + }, + { + "layer": 14, + "fro_norm": 34.6306, + "top_sv": 3.9089, + "stable_rank": 78.489 + }, + { + "layer": 15, + "fro_norm": 37.489, + "top_sv": 4.2835, + "stable_rank": 76.596 + }, + { + "layer": 16, + "fro_norm": 40.2931, + "top_sv": 4.0614, + "stable_rank": 98.424 + }, + { + "layer": 17, + "fro_norm": 41.9215, + "top_sv": 4.0418, + "stable_rank": 107.577 + }, + { + "layer": 18, + "fro_norm": 43.5216, + "top_sv": 3.8251, + "stable_rank": 129.459 + }, + { + "layer": 19, + "fro_norm": 45.6291, + "top_sv": 3.9826, + "stable_rank": 131.263 + }, + { + "layer": 20, + "fro_norm": 46.7307, + "top_sv": 3.9047, + "stable_rank": 143.229 + }, + { + "layer": 21, + "fro_norm": 47.499, + "top_sv": 3.6286, + "stable_rank": 171.352 + }, + { + "layer": 22, + "fro_norm": 47.9789, + "top_sv": 3.6514, + "stable_rank": 172.657 + }, + { + "layer": 23, + "fro_norm": 48.5463, + "top_sv": 3.1689, + "stable_rank": 234.687 + }, + { + "layer": 24, + "fro_norm": 49.31, + "top_sv": 2.8063, + "stable_rank": 308.747 + }, + { + "layer": 25, + "fro_norm": 50.0893, + "top_sv": 2.8865, + "stable_rank": 301.124 + }, + { + "layer": 26, + "fro_norm": 51.1574, + "top_sv": 2.8153, + "stable_rank": 330.198 + }, + { + "layer": 27, + "fro_norm": 51.2848, + "top_sv": 2.8989, + "stable_rank": 312.979 + }, + { + "layer": 28, + "fro_norm": 50.9028, + "top_sv": 2.6931, + "stable_rank": 357.25 + }, + { + "layer": 29, + "fro_norm": 51.1964, + "top_sv": 2.6043, + "stable_rank": 386.443 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_410m_n200_ckpt.pt b/results/lenses/ladder/lens_410m_n200_ckpt.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f00e82d31db985ead3b60496884cf99e402585b --- /dev/null +++ b/results/lenses/ladder/lens_410m_n200_ckpt.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a93a2b4ba77cca11c9a314d1e8381d2bda8c60ebf338ef63b6f01d915eec1ec +size 96476869 diff --git a/results/lenses/ladder/lens_410m_n200_db128.pt b/results/lenses/ladder/lens_410m_n200_db128.pt new file mode 100644 index 0000000000000000000000000000000000000000..adbee970bb2584773f2a63aa246ed3888804b612 --- /dev/null +++ b/results/lenses/ladder/lens_410m_n200_db128.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74f66f5ac1244b030a339508d66b295496a19a4d39470a4c44b9a21bd5147de +size 48241648 diff --git a/results/lenses/ladder/lens_410m_n200_db128_pen.pt b/results/lenses/ladder/lens_410m_n200_db128_pen.pt new file mode 100644 index 0000000000000000000000000000000000000000..14c71bb86160cd7d234bab732f601017b2929d70 --- /dev/null +++ b/results/lenses/ladder/lens_410m_n200_db128_pen.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bad084ffea3a88708029f5b916bc9bc56a46600532ede1bd0604947ea45872a +size 46144398 diff --git a/results/lenses/ladder/lens_410m_n200_db128_pen_provenance.json b/results/lenses/ladder/lens_410m_n200_db128_pen_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..dedf0c3169080d8aae7216e5da90c8ba09cb7060 --- /dev/null +++ b/results/lenses/ladder/lens_410m_n200_db128_pen_provenance.json @@ -0,0 +1,511 @@ +{ + "artifact": "pythia/results/lens_410m_n200_db128_pen.pt", + "sha256": "4bad084ffea3a88708029f5b916bc9bc56a46600532ede1bd0604947ea45872a", + "bytes": 46144398, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": -2, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 8.473323942630852e-08, + "ref_absmax": 0.17585968971252441, + "pass": true + }, + "1": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 8.186736177514637e-08, + "ref_absmax": 0.18201589584350586, + "pass": true + }, + "2": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 8.624282858539251e-08, + "ref_absmax": 0.17278145253658295, + "pass": true + }, + "3": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 8.441938164322495e-08, + "ref_absmax": 0.17651350796222687, + "pass": true + }, + "4": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 7.274153535849648e-08, + "ref_absmax": 0.20485079288482666, + "pass": true + }, + "5": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 5.945881537585939e-08, + "ref_absmax": 0.25061315298080444, + "pass": true + }, + "6": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 1.0854825072865727e-07, + "ref_absmax": 0.27455368638038635, + "pass": true + }, + "7": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 9.87980525323077e-08, + "ref_absmax": 0.301648885011673, + "pass": true + }, + "8": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 8.461076551843935e-08, + "ref_absmax": 0.3522284924983978, + "pass": true + }, + "9": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 7.417675668362248e-08, + "ref_absmax": 0.40177440643310547, + "pass": true + }, + "10": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.529210250233289e-08, + "ref_absmax": 0.4564460515975952, + "pass": true + }, + "11": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.7757048901276104e-08, + "ref_absmax": 0.5159945487976074, + "pass": true + }, + "12": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0281289923533934e-07, + "ref_absmax": 0.579738974571228, + "pass": true + }, + "13": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 9.64524127426369e-08, + "ref_absmax": 0.6179694533348083, + "pass": true + }, + "14": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.030236732181889e-08, + "ref_absmax": 0.7422526478767395, + "pass": true + }, + "15": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.316003963518307e-08, + "ref_absmax": 0.8147158622741699, + "pass": true + }, + "16": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.316616219011234e-08, + "ref_absmax": 0.9436166882514954, + "pass": true + }, + "17": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1615566113072658e-07, + "ref_absmax": 1.0262891054153442, + "pass": true + }, + "18": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.096203354491885e-07, + "ref_absmax": 1.0874742269515991, + "pass": true + }, + "19": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.048218468854704e-07, + "ref_absmax": 1.137256145477295, + "pass": true + }, + "20": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0000627039315365e-07, + "ref_absmax": 1.1920181512832642, + "pass": true + }, + "21": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0124782886686974e-07, + "ref_absmax": 1.1774009466171265, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.1615566113072658e-07 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 2.364437, + "mean_fro": 8.312783, + "rms_per_prompt_fro": 15.247641, + "max_per_prompt_fro": 21.704403, + "heavy_tail_ratio": 1.4235, + "n_prompts": 200 + }, + "1": { + "dispersion": 2.077954, + "mean_fro": 7.823097, + "rms_per_prompt_fro": 13.724919, + "max_per_prompt_fro": 19.592503, + "heavy_tail_ratio": 1.4275, + "n_prompts": 200 + }, + "2": { + "dispersion": 1.651819, + "mean_fro": 8.444405, + "rms_per_prompt_fro": 13.751212, + "max_per_prompt_fro": 18.946533, + "heavy_tail_ratio": 1.3778, + "n_prompts": 200 + }, + "3": { + "dispersion": 1.420183, + "mean_fro": 8.183704, + "rms_per_prompt_fro": 12.731338, + "max_per_prompt_fro": 17.235622, + "heavy_tail_ratio": 1.3538, + "n_prompts": 200 + }, + "4": { + "dispersion": 1.16357, + "mean_fro": 9.352962, + "rms_per_prompt_fro": 13.757346, + "max_per_prompt_fro": 18.681889, + "heavy_tail_ratio": 1.358, + "n_prompts": 200 + }, + "5": { + "dispersion": 1.051403, + "mean_fro": 9.543461, + "rms_per_prompt_fro": 13.668831, + "max_per_prompt_fro": 18.275652, + "heavy_tail_ratio": 1.337, + "n_prompts": 200 + }, + "6": { + "dispersion": 0.924073, + "mean_fro": 9.896236, + "rms_per_prompt_fro": 13.727164, + "max_per_prompt_fro": 18.085121, + "heavy_tail_ratio": 1.3175, + "n_prompts": 200 + }, + "7": { + "dispersion": 0.810342, + "mean_fro": 10.974287, + "rms_per_prompt_fro": 14.765786, + "max_per_prompt_fro": 18.988771, + "heavy_tail_ratio": 1.286, + "n_prompts": 200 + }, + "8": { + "dispersion": 0.730484, + "mean_fro": 12.621095, + "rms_per_prompt_fro": 16.602782, + "max_per_prompt_fro": 21.087879, + "heavy_tail_ratio": 1.2701, + "n_prompts": 200 + }, + "9": { + "dispersion": 0.671794, + "mean_fro": 14.061591, + "rms_per_prompt_fro": 18.18134, + "max_per_prompt_fro": 22.651909, + "heavy_tail_ratio": 1.2459, + "n_prompts": 200 + }, + "10": { + "dispersion": 0.612544, + "mean_fro": 15.919509, + "rms_per_prompt_fro": 20.215545, + "max_per_prompt_fro": 24.773321, + "heavy_tail_ratio": 1.2255, + "n_prompts": 200 + }, + "11": { + "dispersion": 0.562712, + "mean_fro": 18.746285, + "rms_per_prompt_fro": 23.434443, + "max_per_prompt_fro": 28.19455, + "heavy_tail_ratio": 1.2031, + "n_prompts": 200 + }, + "12": { + "dispersion": 0.466847, + "mean_fro": 20.247509, + "rms_per_prompt_fro": 24.522458, + "max_per_prompt_fro": 28.458569, + "heavy_tail_ratio": 1.1605, + "n_prompts": 200 + }, + "13": { + "dispersion": 0.402353, + "mean_fro": 21.92836, + "rms_per_prompt_fro": 25.967783, + "max_per_prompt_fro": 29.87108, + "heavy_tail_ratio": 1.1503, + "n_prompts": 200 + }, + "14": { + "dispersion": 0.345839, + "mean_fro": 25.251056, + "rms_per_prompt_fro": 29.293832, + "max_per_prompt_fro": 33.469563, + "heavy_tail_ratio": 1.1425, + "n_prompts": 200 + }, + "15": { + "dispersion": 0.296264, + "mean_fro": 28.221912, + "rms_per_prompt_fro": 32.131663, + "max_per_prompt_fro": 36.422871, + "heavy_tail_ratio": 1.1336, + "n_prompts": 200 + }, + "16": { + "dispersion": 0.222337, + "mean_fro": 31.387956, + "rms_per_prompt_fro": 34.702314, + "max_per_prompt_fro": 38.504932, + "heavy_tail_ratio": 1.1096, + "n_prompts": 200 + }, + "17": { + "dispersion": 0.166069, + "mean_fro": 33.490116, + "rms_per_prompt_fro": 36.164189, + "max_per_prompt_fro": 39.699699, + "heavy_tail_ratio": 1.0978, + "n_prompts": 200 + }, + "18": { + "dispersion": 0.12263, + "mean_fro": 35.019765, + "rms_per_prompt_fro": 37.104921, + "max_per_prompt_fro": 39.946423, + "heavy_tail_ratio": 1.0766, + "n_prompts": 200 + }, + "19": { + "dispersion": 0.082939, + "mean_fro": 35.156215, + "rms_per_prompt_fro": 36.585095, + "max_per_prompt_fro": 38.673088, + "heavy_tail_ratio": 1.0571, + "n_prompts": 200 + }, + "20": { + "dispersion": 0.047779, + "mean_fro": 35.197197, + "rms_per_prompt_fro": 36.028229, + "max_per_prompt_fro": 37.543106, + "heavy_tail_ratio": 1.042, + "n_prompts": 200 + }, + "21": { + "dispersion": 0.022615, + "mean_fro": 34.62461, + "rms_per_prompt_fro": 35.013939, + "max_per_prompt_fro": 35.945892, + "heavy_tail_ratio": 1.0266, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 398.3, + "fit_secs_per_prompt": 1.992, + "total_wall_seconds": 451.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.3127, + "top_sv": 1.8411, + "stable_rank": 20.386 + }, + { + "layer": 1, + "fro_norm": 7.823, + "top_sv": 1.7355, + "stable_rank": 20.32 + }, + { + "layer": 2, + "fro_norm": 8.4443, + "top_sv": 1.9097, + "stable_rank": 19.552 + }, + { + "layer": 3, + "fro_norm": 8.1836, + "top_sv": 1.8435, + "stable_rank": 19.707 + }, + { + "layer": 4, + "fro_norm": 9.3528, + "top_sv": 2.0805, + "stable_rank": 20.209 + }, + { + "layer": 5, + "fro_norm": 9.5433, + "top_sv": 2.2055, + "stable_rank": 18.724 + }, + { + "layer": 6, + "fro_norm": 9.8961, + "top_sv": 2.2297, + "stable_rank": 19.699 + }, + { + "layer": 7, + "fro_norm": 10.9742, + "top_sv": 2.5256, + "stable_rank": 18.88 + }, + { + "layer": 8, + "fro_norm": 12.6209, + "top_sv": 3.0091, + "stable_rank": 17.592 + }, + { + "layer": 9, + "fro_norm": 14.0614, + "top_sv": 3.3816, + "stable_rank": 17.29 + }, + { + "layer": 10, + "fro_norm": 15.9193, + "top_sv": 3.8076, + "stable_rank": 17.48 + }, + { + "layer": 11, + "fro_norm": 18.746, + "top_sv": 4.4176, + "stable_rank": 18.007 + }, + { + "layer": 12, + "fro_norm": 20.2472, + "top_sv": 4.4389, + "stable_rank": 20.805 + }, + { + "layer": 13, + "fro_norm": 21.928, + "top_sv": 4.7785, + "stable_rank": 21.058 + }, + { + "layer": 14, + "fro_norm": 25.2506, + "top_sv": 5.4258, + "stable_rank": 21.658 + }, + { + "layer": 15, + "fro_norm": 28.2214, + "top_sv": 6.2892, + "stable_rank": 20.136 + }, + { + "layer": 16, + "fro_norm": 31.3874, + "top_sv": 6.7078, + "stable_rank": 21.896 + }, + { + "layer": 17, + "fro_norm": 33.4893, + "top_sv": 7.2021, + "stable_rank": 21.622 + }, + { + "layer": 18, + "fro_norm": 35.0188, + "top_sv": 7.7284, + "stable_rank": 20.532 + }, + { + "layer": 19, + "fro_norm": 35.1551, + "top_sv": 7.7563, + "stable_rank": 20.543 + }, + { + "layer": 20, + "fro_norm": 35.1959, + "top_sv": 7.0684, + "stable_rank": 24.794 + }, + { + "layer": 21, + "fro_norm": 34.623, + "top_sv": 5.6003, + "stable_rank": 38.222 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_410m_n200_db128_provenance.json b/results/lenses/ladder/lens_410m_n200_db128_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..285beeff41b0ed9ab3f9b3b7174ff2a4c5e13ff2 --- /dev/null +++ b/results/lenses/ladder/lens_410m_n200_db128_provenance.json @@ -0,0 +1,340 @@ +{ + "artifact": "pythia/results/lens_410m_n200_db128.pt", + "sha256": "f74f66f5ac1244b030a339508d66b295496a19a4d39470a4c44b9a21bd5147de", + "bytes": 48241648, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext-103-raw-v1", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 8.428828447633164e-08, + "ref_absmax": 0.17678804695606232, + "pass": true + }, + "1": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 9.131576150222463e-08, + "ref_absmax": 0.16318279504776, + "pass": true + }, + "2": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 8.385960124927325e-08, + "ref_absmax": 0.1776917725801468, + "pass": true + }, + "3": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 8.648239516798643e-08, + "ref_absmax": 0.17230282723903656, + "pass": true + }, + "4": { + "max_abs_diff": 1.4901161193847656e-08, + "max_rel_diff": 7.077900720126853e-08, + "ref_absmax": 0.21053080260753632, + "pass": true + }, + "5": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 1.1795366897836199e-07, + "ref_absmax": 0.25266125798225403, + "pass": true + }, + "6": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 1.0794644517783259e-07, + "ref_absmax": 0.2760843336582184, + "pass": true + }, + "7": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 9.893793099217827e-08, + "ref_absmax": 0.30122241377830505, + "pass": true + }, + "8": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 9.164470630391859e-08, + "ref_absmax": 0.3251941502094269, + "pass": true + }, + "9": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 8.224087299015642e-08, + "ref_absmax": 0.36237847805023193, + "pass": true + }, + "10": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.604537779772348e-08, + "ref_absmax": 0.45124009251594543, + "pass": true + }, + "11": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 5.9435307520658524e-08, + "ref_absmax": 0.5014245510101318, + "pass": true + }, + "12": { + "max_abs_diff": 2.9802322387695312e-08, + "max_rel_diff": 6.010326221274247e-08, + "ref_absmax": 0.495851993560791, + "pass": true + }, + "13": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 1.0550160262209463e-07, + "ref_absmax": 0.5649643540382385, + "pass": true + }, + "14": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.81037008752486e-08, + "ref_absmax": 0.67652827501297, + "pass": true + }, + "15": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.436985123575547e-08, + "ref_absmax": 0.8014624714851379, + "pass": true + }, + "16": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.313901816427471e-08, + "ref_absmax": 0.9440223574638367, + "pass": true + }, + "17": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.1164765251320541e-07, + "ref_absmax": 1.0677276849746704, + "pass": true + }, + "18": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.055348378942676e-07, + "ref_absmax": 1.129572868347168, + "pass": true + }, + "19": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.94655220174393e-08, + "ref_absmax": 1.1984986066818237, + "pass": true + }, + "20": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.685206573349702e-08, + "ref_absmax": 1.2308388948440552, + "pass": true + }, + "21": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.558317781950302e-08, + "ref_absmax": 1.247178554534912, + "pass": true + }, + "22": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 1.0237387615238434e-07, + "ref_absmax": 1.1644502878189087, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 1.1795366897836199e-07 + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 488.9, + "fit_secs_per_prompt": 2.444, + "total_wall_seconds": 516.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.0599, + "top_sv": 2.2539, + "stable_rank": 16.157 + }, + { + "layer": 1, + "fro_norm": 8.5297, + "top_sv": 2.1498, + "stable_rank": 15.743 + }, + { + "layer": 2, + "fro_norm": 9.2171, + "top_sv": 2.3361, + "stable_rank": 15.567 + }, + { + "layer": 3, + "fro_norm": 8.9814, + "top_sv": 2.3763, + "stable_rank": 14.286 + }, + { + "layer": 4, + "fro_norm": 10.2779, + "top_sv": 2.7413, + "stable_rank": 14.057 + }, + { + "layer": 5, + "fro_norm": 10.4907, + "top_sv": 3.0239, + "stable_rank": 12.036 + }, + { + "layer": 6, + "fro_norm": 10.8763, + "top_sv": 3.0619, + "stable_rank": 12.618 + }, + { + "layer": 7, + "fro_norm": 12.0564, + "top_sv": 3.4409, + "stable_rank": 12.277 + }, + { + "layer": 8, + "fro_norm": 13.844, + "top_sv": 4.2028, + "stable_rank": 10.85 + }, + { + "layer": 9, + "fro_norm": 15.236, + "top_sv": 4.2581, + "stable_rank": 12.803 + }, + { + "layer": 10, + "fro_norm": 17.309, + "top_sv": 4.9105, + "stable_rank": 12.425 + }, + { + "layer": 11, + "fro_norm": 20.2151, + "top_sv": 5.4771, + "stable_rank": 13.622 + }, + { + "layer": 12, + "fro_norm": 21.5511, + "top_sv": 4.4612, + "stable_rank": 23.337 + }, + { + "layer": 13, + "fro_norm": 23.2674, + "top_sv": 4.6155, + "stable_rank": 25.413 + }, + { + "layer": 14, + "fro_norm": 26.6938, + "top_sv": 4.9122, + "stable_rank": 29.53 + }, + { + "layer": 15, + "fro_norm": 29.7047, + "top_sv": 5.4847, + "stable_rank": 29.332 + }, + { + "layer": 16, + "fro_norm": 32.7868, + "top_sv": 5.248, + "stable_rank": 39.031 + }, + { + "layer": 17, + "fro_norm": 34.884, + "top_sv": 5.2549, + "stable_rank": 44.068 + }, + { + "layer": 18, + "fro_norm": 36.4595, + "top_sv": 5.0823, + "stable_rank": 51.464 + }, + { + "layer": 19, + "fro_norm": 36.4515, + "top_sv": 4.6253, + "stable_rank": 62.108 + }, + { + "layer": 20, + "fro_norm": 36.6396, + "top_sv": 4.6789, + "stable_rank": 61.321 + }, + { + "layer": 21, + "fro_norm": 36.5158, + "top_sv": 4.4661, + "stable_rank": 66.849 + }, + { + "layer": 22, + "fro_norm": 34.4489, + "top_sv": 5.4493, + "stable_rank": 39.963 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_70m_n200.pt b/results/lenses/ladder/lens_70m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..90e317df458295ee79475ad684a211020d27ac6d --- /dev/null +++ b/results/lenses/ladder/lens_70m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e03a9982b3d6070f27065d19140e5fe93b97d15f1d541dfefcf38853ce0c3f +size 2624155 diff --git a/results/lenses/ladder/lens_70m_n200_db128.pt b/results/lenses/ladder/lens_70m_n200_db128.pt new file mode 100644 index 0000000000000000000000000000000000000000..65177a09a696561c23ff7c3988b483a7807c0648 --- /dev/null +++ b/results/lenses/ladder/lens_70m_n200_db128.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf8de535f1420effec69636094169eb709461a688003af93206b0b220e4309ea +size 2623800 diff --git a/results/lenses/ladder/lens_70m_n200_db128_pen.pt b/results/lenses/ladder/lens_70m_n200_db128_pen.pt new file mode 100644 index 0000000000000000000000000000000000000000..d60df14fce78598818bc5183352816d08e5d07c4 --- /dev/null +++ b/results/lenses/ladder/lens_70m_n200_db128_pen.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b53b45cd4f89a5174109f70fcffb7d6bce639093afc9b6dcecb34a88ab71a390 +size 2099408 diff --git a/results/lenses/ladder/lens_70m_n200_db128_pen_provenance.json b/results/lenses/ladder/lens_70m_n200_db128_pen_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..bc51645917d11f5e651445a4e7fba0188e704466 --- /dev/null +++ b/results/lenses/ladder/lens_70m_n200_db128_pen_provenance.json @@ -0,0 +1,133 @@ +{ + "artifact": "pythia/results/lens_70m_n200_db128_pen.pt", + "sha256": "b53b45cd4f89a5174109f70fcffb7d6bce639093afc9b6dcecb34a88ab71a390", + "bytes": 2099408, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": true, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "target_layer_arg": -2, + "skip_first_arg": null, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.767996661625755e-08, + "ref_absmax": 0.767310380935669, + "pass": true + }, + "1": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 8.392756145029176e-08, + "ref_absmax": 0.710191547870636, + "pass": true + }, + "2": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.2480181200453e-08, + "ref_absmax": 0.8223578333854675, + "pass": true + }, + "3": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.3601769477549e-08, + "ref_absmax": 0.9371538758277893, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 5.960464477539063e-08, + "worst_rel_diff": 8.392756145029176e-08 + }, + "jacobian_dispersion": { + "0": { + "dispersion": 0.229049, + "mean_fro": 34.68082, + "rms_per_prompt_fro": 38.448025, + "max_per_prompt_fro": 44.867264, + "heavy_tail_ratio": 1.167, + "n_prompts": 200 + }, + "1": { + "dispersion": 0.174573, + "mean_fro": 27.030013, + "rms_per_prompt_fro": 29.294505, + "max_per_prompt_fro": 33.658653, + "heavy_tail_ratio": 1.149, + "n_prompts": 200 + }, + "2": { + "dispersion": 0.121218, + "mean_fro": 25.570791, + "rms_per_prompt_fro": 27.076292, + "max_per_prompt_fro": 30.517342, + "heavy_tail_ratio": 1.1271, + "n_prompts": 200 + }, + "3": { + "dispersion": 0.057114, + "mean_fro": 23.192881, + "rms_per_prompt_fro": 23.846008, + "max_per_prompt_fro": 25.37808, + "heavy_tail_ratio": 1.0642, + "n_prompts": 200 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 14.4, + "fit_secs_per_prompt": 0.072, + "total_wall_seconds": 41.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 34.6808, + "top_sv": 5.1937, + "stable_rank": 44.589 + }, + { + "layer": 1, + "fro_norm": 27.03, + "top_sv": 4.3976, + "stable_rank": 37.779 + }, + { + "layer": 2, + "fro_norm": 25.5707, + "top_sv": 3.5877, + "stable_rank": 50.798 + }, + { + "layer": 3, + "fro_norm": 23.1928, + "top_sv": 4.4564, + "stable_rank": 27.086 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_70m_n200_db128_provenance.json b/results/lenses/ladder/lens_70m_n200_db128_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b9f3f706be6cf8457d8766e12fc228407c262ce1 --- /dev/null +++ b/results/lenses/ladder/lens_70m_n200_db128_provenance.json @@ -0,0 +1,106 @@ +{ + "artifact": "pythia/results/lens_70m_n200_db128.pt", + "sha256": "bf8de535f1420effec69636094169eb709461a688003af93206b0b220e4309ea", + "bytes": 2623800, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext-103-raw-v1", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": { + "dim_batch_ref": 128, + "dim_batch_fast": 128, + "n_prompts": 3, + "layers": { + "0": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.397999729364611e-08, + "ref_absmax": 0.9316137433052063, + "pass": true + }, + "1": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 7.441136702089702e-08, + "ref_absmax": 0.801015317440033, + "pass": true + }, + "2": { + "max_abs_diff": 5.960464477539063e-08, + "max_rel_diff": 6.096124669163314e-08, + "ref_absmax": 0.9777464866638184, + "pass": true + }, + "3": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 9.751302774050613e-08, + "ref_absmax": 1.2224960327148438, + "pass": true + }, + "4": { + "max_abs_diff": 1.1920928955078125e-07, + "max_rel_diff": 8.964788373171508e-08, + "ref_absmax": 1.3297501802444458, + "pass": true + } + }, + "ALL_LAYERS_AGREE": true, + "worst_max_abs_diff": 1.1920928955078125e-07, + "worst_rel_diff": 9.751302774050613e-08 + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 20.1, + "fit_secs_per_prompt": 0.1, + "total_wall_seconds": 32.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 50.5122, + "top_sv": 17.3095, + "stable_rank": 8.516 + }, + { + "layer": 1, + "fro_norm": 40.2718, + "top_sv": 17.8208, + "stable_rank": 5.107 + }, + { + "layer": 2, + "fro_norm": 37.1094, + "top_sv": 15.9076, + "stable_rank": 5.442 + }, + { + "layer": 3, + "fro_norm": 34.2093, + "top_sv": 15.5109, + "stable_rank": 4.864 + }, + { + "layer": 4, + "fro_norm": 34.1635, + "top_sv": 18.6978, + "stable_rank": 3.338 + } + ] +} \ No newline at end of file diff --git a/results/lenses/ladder/lens_70m_n200_provenance.json b/results/lenses/ladder/lens_70m_n200_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..333efcf1252caf42be8ab9207eac161396669c7a --- /dev/null +++ b/results/lenses/ladder/lens_70m_n200_provenance.json @@ -0,0 +1,70 @@ +{ + "artifact": "pythia/results/lens_70m_n200.pt", + "sha256": "28e03a9982b3d6070f27065d19140e5fe93b97d15f1d541dfefcf38853ce0c3f", + "bytes": 2624155, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext-103-raw-v1", + "n_prompts_requested": 200, + "n_prompts_used": 200, + "dim_batch": 64, + "max_seq_len": 128, + "skip_first_positions": 16, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "centered": false, + "layout_autodetected": false, + "layout": { + "path": "gpt_neox", + "lm_head": "lm_head", + "norm": "final_layer_norm", + "embed": "embed_in" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 574.0, + "fit_secs_per_prompt": 2.87, + "per_layer": [ + { + "layer": 0, + "fro_norm": 50.4707, + "top_sv": 17.2394, + "stable_rank": 8.571 + }, + { + "layer": 1, + "fro_norm": 40.248, + "top_sv": 17.7945, + "stable_rank": 5.116 + }, + { + "layer": 2, + "fro_norm": 37.1045, + "top_sv": 15.9163, + "stable_rank": 5.435 + }, + { + "layer": 3, + "fro_norm": 34.2007, + "top_sv": 15.4959, + "stable_rank": 4.871 + }, + { + "layer": 4, + "fro_norm": 34.1625, + "top_sv": 18.6961, + "stable_rank": 3.339 + } + ] +} \ No newline at end of file diff --git a/results/lenses/misc/e66_D1_refit_410m_pilecc_s0.pt b/results/lenses/misc/e66_D1_refit_410m_pilecc_s0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6f0646ca91e6756b4b706f27aa986f4299a3b8d --- /dev/null +++ b/results/lenses/misc/e66_D1_refit_410m_pilecc_s0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00eebd6ddffff67532bb954e8d51674f0cd8d5021c0919d7139a942c607fff21 +size 54531448 diff --git a/results/lenses/misc/jeval_410m_skip0.pt b/results/lenses/misc/jeval_410m_skip0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e2e3dd4752fa224ff9720be60865a9ac433d6b5 --- /dev/null +++ b/results/lenses/misc/jeval_410m_skip0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b48d7ff83ccf0c6ecfb8594d9cdbfeeb4e24dd9de95b7c7f92fa71c6d0e293a9 +size 27268007 diff --git a/results/lenses/misc/t0_smoke_160m_lens.pt b/results/lenses/misc/t0_smoke_160m_lens.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f3767a2db1f8445c5abd28927248a7219f56f9c --- /dev/null +++ b/results/lenses/misc/t0_smoke_160m_lens.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9350644d564378205650c306a9a232879af5a79a99380eff15b8f25f142afec +size 12980477 diff --git a/results/lenses/misc/t0_smoke_1b_lens.pt b/results/lenses/misc/t0_smoke_1b_lens.pt new file mode 100644 index 0000000000000000000000000000000000000000..89154aceb4fa866d0cc01e8b397f4ce65eef932c --- /dev/null +++ b/results/lenses/misc/t0_smoke_1b_lens.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f82cdddf6a0d14e504e2ce3f7f955646fb2f5da34d52c5f764e2244574bfe69d +size 125834483 diff --git a/results/lenses/misc/t0_smoke_410m_lens.pt b/results/lenses/misc/t0_smoke_410m_lens.pt new file mode 100644 index 0000000000000000000000000000000000000000..34480e3998feb1f9586ed5868f4347560bac62e1 --- /dev/null +++ b/results/lenses/misc/t0_smoke_410m_lens.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bef7f86cd5ba2d1a1956a433596bfa850082b0f8d6e797e46dccba2afab5948 +size 48242013 diff --git a/results/lenses/misc/t0_smoke_70m_lens.pt b/results/lenses/misc/t0_smoke_70m_lens.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ff62a38e8515e6170cfdd522ff319af822ed84e --- /dev/null +++ b/results/lenses/misc/t0_smoke_70m_lens.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c4eddb4511300814c4fe1dc1650eaa6ea88c20e3e07b0c530dfd8c250e5b49f +size 2624199 diff --git a/results/lenses/misc/t16_robust_aggregation_1b_median.pt b/results/lenses/misc/t16_robust_aggregation_1b_median.pt new file mode 100644 index 0000000000000000000000000000000000000000..da31203a2dc0cb7bc4106c306e9f25f87496d25e --- /dev/null +++ b/results/lenses/misc/t16_robust_aggregation_1b_median.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6b60b393d31338719af4f33370285dec800ad1ec8656014e9513b3e06e44117 +size 117445614 diff --git a/results/lenses/misc/t16_robust_aggregation_1b_trimmed.pt b/results/lenses/misc/t16_robust_aggregation_1b_trimmed.pt new file mode 100644 index 0000000000000000000000000000000000000000..122395d2cbd1e5944e2bd7730c8a02f7bab9b923 --- /dev/null +++ b/results/lenses/misc/t16_robust_aggregation_1b_trimmed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:366538a394f1ee21392b755e9df0a2e3a54a59cc806b724f29135f141bbfc234 +size 117445632 diff --git a/results/lenses/plad/plad_160m_pile_A.pt b/results/lenses/plad/plad_160m_pile_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d10ac2da32c394eeede2530ee0a0639bdc599e5 --- /dev/null +++ b/results/lenses/plad/plad_160m_pile_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a755a1ff2da0849c6a75e8e2545088c32c60647986f9fd3aec5d21a109786dea +size 11800118 diff --git a/results/lenses/plad/plad_160m_pile_A_provenance.json b/results/lenses/plad/plad_160m_pile_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4020a94f31707a4a3ed774e321e503391ea906e6 --- /dev/null +++ b/results/lenses/plad/plad_160m_pile_A_provenance.json @@ -0,0 +1,189 @@ +{ + "artifact": "results/plad_160m_pile_A.pt", + "sha256": "a755a1ff2da0849c6a75e8e2545088c32c60647986f9fd3aec5d21a109786dea", + "bytes": 11800118, + "model": "EleutherAI/pythia-160m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 10, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 12, + "d_model": 768, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.913092, + "mean_fro": 16.555662, + "rms_per_prompt_fro": 22.898894, + "max_per_prompt_fro": 28.508936, + "heavy_tail_ratio": 1.245, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.722271, + "mean_fro": 16.329058, + "rms_per_prompt_fro": 21.42949, + "max_per_prompt_fro": 26.815962, + "heavy_tail_ratio": 1.2514, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.617868, + "mean_fro": 15.726634, + "rms_per_prompt_fro": 20.003564, + "max_per_prompt_fro": 24.988049, + "heavy_tail_ratio": 1.2492, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.541216, + "mean_fro": 16.843685, + "rms_per_prompt_fro": 20.910711, + "max_per_prompt_fro": 25.562738, + "heavy_tail_ratio": 1.2225, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.461721, + "mean_fro": 19.005217, + "rms_per_prompt_fro": 22.977625, + "max_per_prompt_fro": 27.709206, + "heavy_tail_ratio": 1.2059, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.390002, + "mean_fro": 21.006542, + "rms_per_prompt_fro": 24.76637, + "max_per_prompt_fro": 28.17878, + "heavy_tail_ratio": 1.1378, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.326927, + "mean_fro": 23.420942, + "rms_per_prompt_fro": 26.979129, + "max_per_prompt_fro": 30.056028, + "heavy_tail_ratio": 1.114, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.237008, + "mean_fro": 23.578515, + "rms_per_prompt_fro": 26.224222, + "max_per_prompt_fro": 27.385286, + "heavy_tail_ratio": 1.0443, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.163247, + "mean_fro": 26.512197, + "rms_per_prompt_fro": 28.594449, + "max_per_prompt_fro": 29.573009, + "heavy_tail_ratio": 1.0342, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.086573, + "mean_fro": 27.795761, + "rms_per_prompt_fro": 28.973966, + "max_per_prompt_fro": 29.670923, + "heavy_tail_ratio": 1.0241, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 34.4, + "fit_secs_per_prompt": 0.688, + "total_wall_seconds": 44.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 16.5556, + "top_sv": 2.2268, + "stable_rank": 55.276 + }, + { + "layer": 1, + "fro_norm": 16.329, + "top_sv": 2.1924, + "stable_rank": 55.473 + }, + { + "layer": 2, + "fro_norm": 15.7265, + "top_sv": 2.1535, + "stable_rank": 53.328 + }, + { + "layer": 3, + "fro_norm": 16.8436, + "top_sv": 2.045, + "stable_rank": 67.838 + }, + { + "layer": 4, + "fro_norm": 19.0051, + "top_sv": 2.3507, + "stable_rank": 65.363 + }, + { + "layer": 5, + "fro_norm": 21.0064, + "top_sv": 2.7328, + "stable_rank": 59.088 + }, + { + "layer": 6, + "fro_norm": 23.4208, + "top_sv": 3.2242, + "stable_rank": 52.768 + }, + { + "layer": 7, + "fro_norm": 23.5783, + "top_sv": 3.5597, + "stable_rank": 43.874 + }, + { + "layer": 8, + "fro_norm": 26.5119, + "top_sv": 3.8087, + "stable_rank": 48.455 + }, + { + "layer": 9, + "fro_norm": 27.7954, + "top_sv": 3.1857, + "stable_rank": 76.127 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_160m_pile_B.pt b/results/lenses/plad/plad_160m_pile_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bd668ba9880b3213997ff1285cbbd41d6bc8b75 --- /dev/null +++ b/results/lenses/plad/plad_160m_pile_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a335c68b6dbb16f5481138b6e7eb8c8dc3c76673a86d7860e6867832a2f27cfd +size 11800118 diff --git a/results/lenses/plad/plad_160m_pile_B_provenance.json b/results/lenses/plad/plad_160m_pile_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ca2e5588be27829831ecca9e42d0f1d36494a483 --- /dev/null +++ b/results/lenses/plad/plad_160m_pile_B_provenance.json @@ -0,0 +1,189 @@ +{ + "artifact": "results/plad_160m_pile_B.pt", + "sha256": "a335c68b6dbb16f5481138b6e7eb8c8dc3c76673a86d7860e6867832a2f27cfd", + "bytes": 11800118, + "model": "EleutherAI/pythia-160m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 10, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 12, + "d_model": 768, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.981523, + "mean_fro": 16.574175, + "rms_per_prompt_fro": 23.3309, + "max_per_prompt_fro": 31.589676, + "heavy_tail_ratio": 1.354, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.792089, + "mean_fro": 16.333607, + "rms_per_prompt_fro": 21.865624, + "max_per_prompt_fro": 29.533768, + "heavy_tail_ratio": 1.3507, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.67665, + "mean_fro": 15.719687, + "rms_per_prompt_fro": 20.354717, + "max_per_prompt_fro": 26.514057, + "heavy_tail_ratio": 1.3026, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.591056, + "mean_fro": 16.865954, + "rms_per_prompt_fro": 21.274218, + "max_per_prompt_fro": 26.903215, + "heavy_tail_ratio": 1.2646, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.499856, + "mean_fro": 18.899742, + "rms_per_prompt_fro": 23.146255, + "max_per_prompt_fro": 27.503872, + "heavy_tail_ratio": 1.1883, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.420263, + "mean_fro": 20.81806, + "rms_per_prompt_fro": 24.809878, + "max_per_prompt_fro": 27.887552, + "heavy_tail_ratio": 1.1241, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.346133, + "mean_fro": 23.115863, + "rms_per_prompt_fro": 26.819714, + "max_per_prompt_fro": 29.310322, + "heavy_tail_ratio": 1.0929, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.248599, + "mean_fro": 23.510693, + "rms_per_prompt_fro": 26.27102, + "max_per_prompt_fro": 27.730881, + "heavy_tail_ratio": 1.0556, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.168599, + "mean_fro": 26.415726, + "rms_per_prompt_fro": 28.555868, + "max_per_prompt_fro": 30.046568, + "heavy_tail_ratio": 1.0522, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.088742, + "mean_fro": 27.77104, + "rms_per_prompt_fro": 28.977081, + "max_per_prompt_fro": 29.877253, + "heavy_tail_ratio": 1.0311, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 34.8, + "fit_secs_per_prompt": 0.696, + "total_wall_seconds": 44.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 16.5741, + "top_sv": 2.2526, + "stable_rank": 54.139 + }, + { + "layer": 1, + "fro_norm": 16.3335, + "top_sv": 2.2174, + "stable_rank": 54.259 + }, + { + "layer": 2, + "fro_norm": 15.7196, + "top_sv": 2.1709, + "stable_rank": 52.431 + }, + { + "layer": 3, + "fro_norm": 16.8659, + "top_sv": 2.0048, + "stable_rank": 70.772 + }, + { + "layer": 4, + "fro_norm": 18.8996, + "top_sv": 2.2828, + "stable_rank": 68.544 + }, + { + "layer": 5, + "fro_norm": 20.8179, + "top_sv": 2.7313, + "stable_rank": 58.095 + }, + { + "layer": 6, + "fro_norm": 23.1157, + "top_sv": 3.249, + "stable_rank": 50.619 + }, + { + "layer": 7, + "fro_norm": 23.5105, + "top_sv": 3.611, + "stable_rank": 42.39 + }, + { + "layer": 8, + "fro_norm": 26.4155, + "top_sv": 3.8472, + "stable_rank": 47.143 + }, + { + "layer": 9, + "fro_norm": 27.7707, + "top_sv": 3.1921, + "stable_rank": 75.687 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_160m_wikitext_A.pt b/results/lenses/plad/plad_160m_wikitext_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec3cd54ffc46f3671f3edb5e96744d5057ac6044 --- /dev/null +++ b/results/lenses/plad/plad_160m_wikitext_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aea350f39ba5fdd4278211c0adff1acd01b1377955e6df8efcec988709e0901 +size 11800174 diff --git a/results/lenses/plad/plad_160m_wikitext_A_provenance.json b/results/lenses/plad/plad_160m_wikitext_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..17b137001d10aafba0198499a7e98a465fd0f874 --- /dev/null +++ b/results/lenses/plad/plad_160m_wikitext_A_provenance.json @@ -0,0 +1,189 @@ +{ + "artifact": "results/plad_160m_wikitext_A.pt", + "sha256": "9aea350f39ba5fdd4278211c0adff1acd01b1377955e6df8efcec988709e0901", + "bytes": 11800174, + "model": "EleutherAI/pythia-160m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 10, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 12, + "d_model": 768, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.564511, + "mean_fro": 21.171055, + "rms_per_prompt_fro": 26.48084, + "max_per_prompt_fro": 33.444771, + "heavy_tail_ratio": 1.263, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.442722, + "mean_fro": 20.936891, + "rms_per_prompt_fro": 25.148005, + "max_per_prompt_fro": 31.710749, + "heavy_tail_ratio": 1.261, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.376064, + "mean_fro": 20.060703, + "rms_per_prompt_fro": 23.532364, + "max_per_prompt_fro": 29.54355, + "heavy_tail_ratio": 1.2554, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.322647, + "mean_fro": 20.595564, + "rms_per_prompt_fro": 23.686215, + "max_per_prompt_fro": 29.382389, + "heavy_tail_ratio": 1.2405, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.278291, + "mean_fro": 23.019446, + "rms_per_prompt_fro": 26.026137, + "max_per_prompt_fro": 31.867363, + "heavy_tail_ratio": 1.2244, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.232338, + "mean_fro": 24.740453, + "rms_per_prompt_fro": 27.464551, + "max_per_prompt_fro": 32.486626, + "heavy_tail_ratio": 1.1829, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.191191, + "mean_fro": 26.91205, + "rms_per_prompt_fro": 29.372267, + "max_per_prompt_fro": 34.668541, + "heavy_tail_ratio": 1.1803, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.134272, + "mean_fro": 25.47846, + "rms_per_prompt_fro": 27.135117, + "max_per_prompt_fro": 29.613083, + "heavy_tail_ratio": 1.0913, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.087811, + "mean_fro": 27.97196, + "rms_per_prompt_fro": 29.174238, + "max_per_prompt_fro": 30.818113, + "heavy_tail_ratio": 1.0563, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.04593, + "mean_fro": 28.66619, + "rms_per_prompt_fro": 29.317118, + "max_per_prompt_fro": 30.097876, + "heavy_tail_ratio": 1.0266, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 31.1, + "fit_secs_per_prompt": 0.621, + "total_wall_seconds": 52.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 21.171, + "top_sv": 3.3082, + "stable_rank": 40.955 + }, + { + "layer": 1, + "fro_norm": 20.9368, + "top_sv": 3.1791, + "stable_rank": 43.371 + }, + { + "layer": 2, + "fro_norm": 20.0606, + "top_sv": 3.0538, + "stable_rank": 43.152 + }, + { + "layer": 3, + "fro_norm": 20.5954, + "top_sv": 2.997, + "stable_rank": 47.223 + }, + { + "layer": 4, + "fro_norm": 23.0193, + "top_sv": 3.2705, + "stable_rank": 49.541 + }, + { + "layer": 5, + "fro_norm": 24.7403, + "top_sv": 3.356, + "stable_rank": 54.347 + }, + { + "layer": 6, + "fro_norm": 26.9119, + "top_sv": 3.3706, + "stable_rank": 63.75 + }, + { + "layer": 7, + "fro_norm": 25.4783, + "top_sv": 3.4993, + "stable_rank": 53.013 + }, + { + "layer": 8, + "fro_norm": 27.9717, + "top_sv": 3.8269, + "stable_rank": 53.423 + }, + { + "layer": 9, + "fro_norm": 28.6658, + "top_sv": 3.2496, + "stable_rank": 77.818 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_160m_wikitext_B.pt b/results/lenses/plad/plad_160m_wikitext_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b21aedc815274ba65f6e00fa1d274e5acafa72e --- /dev/null +++ b/results/lenses/plad/plad_160m_wikitext_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c5d5e19b75fc00bd7ac95929a25d425a43c3a1124395bac3f0c03a0a8fa04ca +size 11800174 diff --git a/results/lenses/plad/plad_160m_wikitext_B_provenance.json b/results/lenses/plad/plad_160m_wikitext_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4cb7e797692f3a65b22c033dcd7d655f85cea1b4 --- /dev/null +++ b/results/lenses/plad/plad_160m_wikitext_B_provenance.json @@ -0,0 +1,189 @@ +{ + "artifact": "results/plad_160m_wikitext_B.pt", + "sha256": "7c5d5e19b75fc00bd7ac95929a25d425a43c3a1124395bac3f0c03a0a8fa04ca", + "bytes": 11800174, + "model": "EleutherAI/pythia-160m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 10, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 12, + "d_model": 768, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.664669, + "mean_fro": 20.649408, + "rms_per_prompt_fro": 26.642288, + "max_per_prompt_fro": 31.675861, + "heavy_tail_ratio": 1.1889, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.532237, + "mean_fro": 20.436283, + "rms_per_prompt_fro": 25.296763, + "max_per_prompt_fro": 29.905817, + "heavy_tail_ratio": 1.1822, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.459038, + "mean_fro": 19.677716, + "rms_per_prompt_fro": 23.768841, + "max_per_prompt_fro": 28.106764, + "heavy_tail_ratio": 1.1825, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.399667, + "mean_fro": 20.156563, + "rms_per_prompt_fro": 23.846728, + "max_per_prompt_fro": 27.555391, + "heavy_tail_ratio": 1.1555, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.348845, + "mean_fro": 22.440498, + "rms_per_prompt_fro": 26.062348, + "max_per_prompt_fro": 29.692545, + "heavy_tail_ratio": 1.1393, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.293911, + "mean_fro": 24.096505, + "rms_per_prompt_fro": 27.409829, + "max_per_prompt_fro": 30.097103, + "heavy_tail_ratio": 1.098, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.248821, + "mean_fro": 26.197045, + "rms_per_prompt_fro": 29.275374, + "max_per_prompt_fro": 31.748158, + "heavy_tail_ratio": 1.0845, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.183538, + "mean_fro": 25.101656, + "rms_per_prompt_fro": 27.308223, + "max_per_prompt_fro": 28.921268, + "heavy_tail_ratio": 1.0591, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.127372, + "mean_fro": 27.66322, + "rms_per_prompt_fro": 29.372189, + "max_per_prompt_fro": 30.649035, + "heavy_tail_ratio": 1.0435, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.070973, + "mean_fro": 28.46978, + "rms_per_prompt_fro": 29.462758, + "max_per_prompt_fro": 30.293047, + "heavy_tail_ratio": 1.0282, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 32.2, + "fit_secs_per_prompt": 0.643, + "total_wall_seconds": 42.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 20.6493, + "top_sv": 3.2853, + "stable_rank": 39.505 + }, + { + "layer": 1, + "fro_norm": 20.4362, + "top_sv": 3.2456, + "stable_rank": 39.646 + }, + { + "layer": 2, + "fro_norm": 19.6776, + "top_sv": 3.2427, + "stable_rank": 36.825 + }, + { + "layer": 3, + "fro_norm": 20.1565, + "top_sv": 3.1229, + "stable_rank": 41.658 + }, + { + "layer": 4, + "fro_norm": 22.4404, + "top_sv": 3.4116, + "stable_rank": 43.265 + }, + { + "layer": 5, + "fro_norm": 24.0963, + "top_sv": 3.5708, + "stable_rank": 45.539 + }, + { + "layer": 6, + "fro_norm": 26.1969, + "top_sv": 3.701, + "stable_rank": 50.101 + }, + { + "layer": 7, + "fro_norm": 25.1015, + "top_sv": 3.6939, + "stable_rank": 46.178 + }, + { + "layer": 8, + "fro_norm": 27.6629, + "top_sv": 3.9619, + "stable_rank": 48.753 + }, + { + "layer": 9, + "fro_norm": 28.4694, + "top_sv": 3.266, + "stable_rank": 75.986 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410mLM_code_A.pt b/results/lenses/plad/plad_410mLM_code_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b3e8457c6380f97b2302454b7060e756b386070 --- /dev/null +++ b/results/lenses/plad/plad_410mLM_code_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30afa8ead88116edf5eb29f691b5583aadd9cd7177b6df62c1353ccdee643c6 +size 46144204 diff --git a/results/lenses/plad/plad_410mLM_code_B.pt b/results/lenses/plad/plad_410mLM_code_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..eda2d6bfe9b8e5e5f969d2ea5c46cbbb07ce886c --- /dev/null +++ b/results/lenses/plad/plad_410mLM_code_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f834d586ad588383c753fb73c8be8105530b419d7bded06d5511dc7f118e5e6 +size 46144204 diff --git a/results/lenses/plad/plad_410mLM_pile_A.pt b/results/lenses/plad/plad_410mLM_pile_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..99c8cd7db3f64531eb9e113986c7794e1b8a7b28 --- /dev/null +++ b/results/lenses/plad/plad_410mLM_pile_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22543312f585298839b191357f1b934cda2a11b5767677e1d369da8a08b9e19a +size 46144204 diff --git a/results/lenses/plad/plad_410mLM_pile_B.pt b/results/lenses/plad/plad_410mLM_pile_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c61743892372c0123e1cd3ff091eebfd028b3b3 --- /dev/null +++ b/results/lenses/plad/plad_410mLM_pile_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88c415a2d9b914e44758483b1ae450a176a87832cbffbd470b69598093888bfa +size 46144204 diff --git a/results/lenses/plad/plad_410mLM_wikitext_A.pt b/results/lenses/plad/plad_410mLM_wikitext_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..096bdb101bb7c1a00d580d0d28ac425d8bfc380b --- /dev/null +++ b/results/lenses/plad/plad_410mLM_wikitext_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d895cafeba61e2bcb3e831054cc23288f7f6af704ba289fa9304fa9d1fa013d4 +size 46144372 diff --git a/results/lenses/plad/plad_410mLM_wikitext_B.pt b/results/lenses/plad/plad_410mLM_wikitext_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..e32e3726d71cebed8f5832765f67106b4a43d377 --- /dev/null +++ b/results/lenses/plad/plad_410mLM_wikitext_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab6958198f5c001301c699364ce3018947eccd7a5311ff8d1e607da626a3ec2 +size 46144372 diff --git a/results/lenses/plad/plad_410m_code_A.pt b/results/lenses/plad/plad_410m_code_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..09144b5a0d3c50a3fdca41b0a973da96db6bbc46 --- /dev/null +++ b/results/lenses/plad/plad_410m_code_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:587398c25f85452ac2013dd2c167bd6517edb95413c4e8518d797c62f552e50c +size 46144126 diff --git a/results/lenses/plad/plad_410m_code_A_provenance.json b/results/lenses/plad/plad_410m_code_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..70801a19e0a682f8e6e7bbf7d76f661436cb0bf3 --- /dev/null +++ b/results/lenses/plad/plad_410m_code_A_provenance.json @@ -0,0 +1,368 @@ +{ + "artifact": "results/plad_410m_code_A.pt", + "sha256": "587398c25f85452ac2013dd2c167bd6517edb95413c4e8518d797c62f552e50c", + "bytes": 46144126, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.713946, + "mean_fro": 6.919973, + "rms_per_prompt_fro": 15.024384, + "max_per_prompt_fro": 29.136526, + "heavy_tail_ratio": 1.9393, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.298644, + "mean_fro": 6.485599, + "rms_per_prompt_fro": 13.446702, + "max_per_prompt_fro": 25.729727, + "heavy_tail_ratio": 1.9135, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.672838, + "mean_fro": 7.000559, + "rms_per_prompt_fro": 13.416327, + "max_per_prompt_fro": 24.512585, + "heavy_tail_ratio": 1.8271, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.240896, + "mean_fro": 6.729721, + "rms_per_prompt_fro": 12.115173, + "max_per_prompt_fro": 19.907906, + "heavy_tail_ratio": 1.6432, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.858944, + "mean_fro": 7.669265, + "rms_per_prompt_fro": 12.967509, + "max_per_prompt_fro": 20.365643, + "heavy_tail_ratio": 1.5705, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.658169, + "mean_fro": 7.92138, + "rms_per_prompt_fro": 12.914932, + "max_per_prompt_fro": 19.429527, + "heavy_tail_ratio": 1.5044, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.447723, + "mean_fro": 8.323853, + "rms_per_prompt_fro": 13.022833, + "max_per_prompt_fro": 18.995975, + "heavy_tail_ratio": 1.4587, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.265118, + "mean_fro": 9.242006, + "rms_per_prompt_fro": 13.909505, + "max_per_prompt_fro": 19.020033, + "heavy_tail_ratio": 1.3674, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.131778, + "mean_fro": 10.565512, + "rms_per_prompt_fro": 15.426291, + "max_per_prompt_fro": 19.865644, + "heavy_tail_ratio": 1.2878, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.030573, + "mean_fro": 11.918001, + "rms_per_prompt_fro": 16.982936, + "max_per_prompt_fro": 21.44342, + "heavy_tail_ratio": 1.2626, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.926377, + "mean_fro": 13.586765, + "rms_per_prompt_fro": 18.857615, + "max_per_prompt_fro": 23.286633, + "heavy_tail_ratio": 1.2349, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.823931, + "mean_fro": 15.978941, + "rms_per_prompt_fro": 21.580038, + "max_per_prompt_fro": 26.242798, + "heavy_tail_ratio": 1.2161, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.664069, + "mean_fro": 18.043156, + "rms_per_prompt_fro": 23.275454, + "max_per_prompt_fro": 26.8624, + "heavy_tail_ratio": 1.1541, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.573991, + "mean_fro": 19.741478, + "rms_per_prompt_fro": 24.76742, + "max_per_prompt_fro": 28.332306, + "heavy_tail_ratio": 1.1439, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.479433, + "mean_fro": 22.760923, + "rms_per_prompt_fro": 27.684551, + "max_per_prompt_fro": 30.923445, + "heavy_tail_ratio": 1.117, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.402042, + "mean_fro": 25.585373, + "rms_per_prompt_fro": 30.295094, + "max_per_prompt_fro": 33.897038, + "heavy_tail_ratio": 1.1189, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.291863, + "mean_fro": 29.210134, + "rms_per_prompt_fro": 33.200284, + "max_per_prompt_fro": 35.748421, + "heavy_tail_ratio": 1.0768, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.215227, + "mean_fro": 31.922094, + "rms_per_prompt_fro": 35.190071, + "max_per_prompt_fro": 37.325512, + "heavy_tail_ratio": 1.0607, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.154743, + "mean_fro": 33.706997, + "rms_per_prompt_fro": 36.221189, + "max_per_prompt_fro": 38.310303, + "heavy_tail_ratio": 1.0577, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.104522, + "mean_fro": 34.390274, + "rms_per_prompt_fro": 36.14289, + "max_per_prompt_fro": 38.000111, + "heavy_tail_ratio": 1.0514, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.058425, + "mean_fro": 34.85598, + "rms_per_prompt_fro": 35.859762, + "max_per_prompt_fro": 37.382595, + "heavy_tail_ratio": 1.0425, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.027866, + "mean_fro": 34.49477, + "rms_per_prompt_fro": 34.972086, + "max_per_prompt_fro": 35.899288, + "heavy_tail_ratio": 1.0265, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.7, + "fit_secs_per_prompt": 3.034, + "total_wall_seconds": 162.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9199, + "top_sv": 1.258, + "stable_rank": 30.259 + }, + { + "layer": 1, + "fro_norm": 6.4855, + "top_sv": 1.1725, + "stable_rank": 30.596 + }, + { + "layer": 2, + "fro_norm": 7.0005, + "top_sv": 1.2878, + "stable_rank": 29.55 + }, + { + "layer": 3, + "fro_norm": 6.7296, + "top_sv": 1.2481, + "stable_rank": 29.071 + }, + { + "layer": 4, + "fro_norm": 7.6692, + "top_sv": 1.4243, + "stable_rank": 28.995 + }, + { + "layer": 5, + "fro_norm": 7.9213, + "top_sv": 1.5538, + "stable_rank": 25.989 + }, + { + "layer": 6, + "fro_norm": 8.3237, + "top_sv": 1.6261, + "stable_rank": 26.202 + }, + { + "layer": 7, + "fro_norm": 9.2419, + "top_sv": 1.8228, + "stable_rank": 25.706 + }, + { + "layer": 8, + "fro_norm": 10.5654, + "top_sv": 2.2923, + "stable_rank": 21.244 + }, + { + "layer": 9, + "fro_norm": 11.9178, + "top_sv": 2.5639, + "stable_rank": 21.607 + }, + { + "layer": 10, + "fro_norm": 13.5865, + "top_sv": 2.9252, + "stable_rank": 21.573 + }, + { + "layer": 11, + "fro_norm": 15.9787, + "top_sv": 3.4353, + "stable_rank": 21.635 + }, + { + "layer": 12, + "fro_norm": 18.0428, + "top_sv": 3.8541, + "stable_rank": 21.916 + }, + { + "layer": 13, + "fro_norm": 19.7411, + "top_sv": 4.2392, + "stable_rank": 21.685 + }, + { + "layer": 14, + "fro_norm": 22.7606, + "top_sv": 4.8035, + "stable_rank": 22.452 + }, + { + "layer": 15, + "fro_norm": 25.5849, + "top_sv": 5.5627, + "stable_rank": 21.154 + }, + { + "layer": 16, + "fro_norm": 29.2096, + "top_sv": 6.4689, + "stable_rank": 20.389 + }, + { + "layer": 17, + "fro_norm": 31.9214, + "top_sv": 6.9701, + "stable_rank": 20.974 + }, + { + "layer": 18, + "fro_norm": 33.7061, + "top_sv": 7.2823, + "stable_rank": 21.423 + }, + { + "layer": 19, + "fro_norm": 34.3892, + "top_sv": 7.2556, + "stable_rank": 22.464 + }, + { + "layer": 20, + "fro_norm": 34.8546, + "top_sv": 7.0893, + "stable_rank": 24.172 + }, + { + "layer": 21, + "fro_norm": 34.4931, + "top_sv": 5.5645, + "stable_rank": 38.425 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_code_B.pt b/results/lenses/plad/plad_410m_code_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..e30a9621a99061918d144f0b84b2ac71c04669a7 --- /dev/null +++ b/results/lenses/plad/plad_410m_code_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78897e2eac3bea90605659ea3205b89f636baeaed6b4d40dbb8d1cc6b7d8f2ae +size 46144126 diff --git a/results/lenses/plad/plad_410m_code_B_provenance.json b/results/lenses/plad/plad_410m_code_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9b28e519e28770908727cb9d7ba8f28487437488 --- /dev/null +++ b/results/lenses/plad/plad_410m_code_B_provenance.json @@ -0,0 +1,368 @@ +{ + "artifact": "results/plad_410m_code_B.pt", + "sha256": "78897e2eac3bea90605659ea3205b89f636baeaed6b4d40dbb8d1cc6b7d8f2ae", + "bytes": 46144126, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.519694, + "mean_fro": 7.288487, + "rms_per_prompt_fro": 15.49501, + "max_per_prompt_fro": 23.124804, + "heavy_tail_ratio": 1.4924, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.102217, + "mean_fro": 6.829753, + "rms_per_prompt_fro": 13.832933, + "max_per_prompt_fro": 19.959238, + "heavy_tail_ratio": 1.4429, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.557518, + "mean_fro": 7.38134, + "rms_per_prompt_fro": 13.922229, + "max_per_prompt_fro": 19.482925, + "heavy_tail_ratio": 1.3994, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.174648, + "mean_fro": 7.085949, + "rms_per_prompt_fro": 12.625419, + "max_per_prompt_fro": 16.869507, + "heavy_tail_ratio": 1.3362, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.818431, + "mean_fro": 8.075409, + "rms_per_prompt_fro": 13.557144, + "max_per_prompt_fro": 17.634926, + "heavy_tail_ratio": 1.3008, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.625788, + "mean_fro": 8.301267, + "rms_per_prompt_fro": 13.451607, + "max_per_prompt_fro": 17.452738, + "heavy_tail_ratio": 1.2974, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.420289, + "mean_fro": 8.690286, + "rms_per_prompt_fro": 13.51972, + "max_per_prompt_fro": 17.744358, + "heavy_tail_ratio": 1.3125, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.240336, + "mean_fro": 9.567411, + "rms_per_prompt_fro": 14.320266, + "max_per_prompt_fro": 18.768539, + "heavy_tail_ratio": 1.3106, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.120063, + "mean_fro": 10.91246, + "rms_per_prompt_fro": 15.88902, + "max_per_prompt_fro": 20.823996, + "heavy_tail_ratio": 1.3106, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.026704, + "mean_fro": 12.253334, + "rms_per_prompt_fro": 17.444135, + "max_per_prompt_fro": 22.276936, + "heavy_tail_ratio": 1.277, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.931251, + "mean_fro": 13.938882, + "rms_per_prompt_fro": 19.370787, + "max_per_prompt_fro": 24.22246, + "heavy_tail_ratio": 1.2505, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.844047, + "mean_fro": 16.330183, + "rms_per_prompt_fro": 22.175683, + "max_per_prompt_fro": 27.460121, + "heavy_tail_ratio": 1.2383, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.681601, + "mean_fro": 18.387573, + "rms_per_prompt_fro": 23.844374, + "max_per_prompt_fro": 28.70298, + "heavy_tail_ratio": 1.2038, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.59539, + "mean_fro": 20.138516, + "rms_per_prompt_fro": 25.436705, + "max_per_prompt_fro": 30.662514, + "heavy_tail_ratio": 1.2054, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.497646, + "mean_fro": 23.202481, + "rms_per_prompt_fro": 28.394816, + "max_per_prompt_fro": 33.231274, + "heavy_tail_ratio": 1.1703, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.423483, + "mean_fro": 26.04353, + "rms_per_prompt_fro": 31.072486, + "max_per_prompt_fro": 35.899979, + "heavy_tail_ratio": 1.1554, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.304653, + "mean_fro": 29.568385, + "rms_per_prompt_fro": 33.77343, + "max_per_prompt_fro": 37.873398, + "heavy_tail_ratio": 1.1214, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.225983, + "mean_fro": 32.105995, + "rms_per_prompt_fro": 35.549077, + "max_per_prompt_fro": 39.419685, + "heavy_tail_ratio": 1.1089, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.163385, + "mean_fro": 33.891205, + "rms_per_prompt_fro": 36.555162, + "max_per_prompt_fro": 39.626827, + "heavy_tail_ratio": 1.084, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.108844, + "mean_fro": 34.441105, + "rms_per_prompt_fro": 36.267061, + "max_per_prompt_fro": 38.304001, + "heavy_tail_ratio": 1.0562, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.061743, + "mean_fro": 34.851189, + "rms_per_prompt_fro": 35.910988, + "max_per_prompt_fro": 37.125557, + "heavy_tail_ratio": 1.0338, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.029874, + "mean_fro": 34.485287, + "rms_per_prompt_fro": 34.996609, + "max_per_prompt_fro": 35.79924, + "heavy_tail_ratio": 1.0229, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.8, + "fit_secs_per_prompt": 3.036, + "total_wall_seconds": 162.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.2884, + "top_sv": 1.2668, + "stable_rank": 33.103 + }, + { + "layer": 1, + "fro_norm": 6.8297, + "top_sv": 1.1856, + "stable_rank": 33.181 + }, + { + "layer": 2, + "fro_norm": 7.3812, + "top_sv": 1.2934, + "stable_rank": 32.566 + }, + { + "layer": 3, + "fro_norm": 7.0859, + "top_sv": 1.2841, + "stable_rank": 30.451 + }, + { + "layer": 4, + "fro_norm": 8.0753, + "top_sv": 1.4589, + "stable_rank": 30.638 + }, + { + "layer": 5, + "fro_norm": 8.3012, + "top_sv": 1.5744, + "stable_rank": 27.801 + }, + { + "layer": 6, + "fro_norm": 8.6902, + "top_sv": 1.6532, + "stable_rank": 27.632 + }, + { + "layer": 7, + "fro_norm": 9.5673, + "top_sv": 1.8502, + "stable_rank": 26.739 + }, + { + "layer": 8, + "fro_norm": 10.9123, + "top_sv": 2.2391, + "stable_rank": 23.752 + }, + { + "layer": 9, + "fro_norm": 12.2531, + "top_sv": 2.5014, + "stable_rank": 23.995 + }, + { + "layer": 10, + "fro_norm": 13.9387, + "top_sv": 2.8597, + "stable_rank": 23.758 + }, + { + "layer": 11, + "fro_norm": 16.3299, + "top_sv": 3.3672, + "stable_rank": 23.52 + }, + { + "layer": 12, + "fro_norm": 18.3873, + "top_sv": 3.7578, + "stable_rank": 23.942 + }, + { + "layer": 13, + "fro_norm": 20.1382, + "top_sv": 4.1617, + "stable_rank": 23.415 + }, + { + "layer": 14, + "fro_norm": 23.2021, + "top_sv": 4.7037, + "stable_rank": 24.332 + }, + { + "layer": 15, + "fro_norm": 26.043, + "top_sv": 5.4936, + "stable_rank": 22.473 + }, + { + "layer": 16, + "fro_norm": 29.5678, + "top_sv": 6.4461, + "stable_rank": 21.04 + }, + { + "layer": 17, + "fro_norm": 32.1053, + "top_sv": 6.9664, + "stable_rank": 21.239 + }, + { + "layer": 18, + "fro_norm": 33.8903, + "top_sv": 7.4139, + "stable_rank": 20.896 + }, + { + "layer": 19, + "fro_norm": 34.44, + "top_sv": 7.3577, + "stable_rank": 21.91 + }, + { + "layer": 20, + "fro_norm": 34.8499, + "top_sv": 7.0653, + "stable_rank": 24.33 + }, + { + "layer": 21, + "fro_norm": 34.4837, + "top_sv": 5.5792, + "stable_rank": 38.202 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_code_LM_A.pt b/results/lenses/plad/plad_410m_code_LM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b3e8457c6380f97b2302454b7060e756b386070 --- /dev/null +++ b/results/lenses/plad/plad_410m_code_LM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30afa8ead88116edf5eb29f691b5583aadd9cd7177b6df62c1353ccdee643c6 +size 46144204 diff --git a/results/lenses/plad/plad_410m_code_LM_A_provenance.json b/results/lenses/plad/plad_410m_code_LM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..60a03313daec060767db4ed9f123915628e83a8c --- /dev/null +++ b/results/lenses/plad/plad_410m_code_LM_A_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_code_LM_A.pt", + "sha256": "f30afa8ead88116edf5eb29f691b5583aadd9cd7177b6df62c1353ccdee643c6", + "bytes": 46144204, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.67179, + "mean_fro": 6.94841, + "rms_per_prompt_fro": 15.018519, + "max_per_prompt_fro": 29.136526, + "heavy_tail_ratio": 1.94, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.263688, + "mean_fro": 6.511862, + "rms_per_prompt_fro": 13.446148, + "max_per_prompt_fro": 25.729727, + "heavy_tail_ratio": 1.9135, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.643464, + "mean_fro": 7.022424, + "rms_per_prompt_fro": 13.404303, + "max_per_prompt_fro": 24.512585, + "heavy_tail_ratio": 1.8287, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.218595, + "mean_fro": 6.750026, + "rms_per_prompt_fro": 12.109847, + "max_per_prompt_fro": 19.907906, + "heavy_tail_ratio": 1.6439, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.838572, + "mean_fro": 7.688786, + "rms_per_prompt_fro": 12.954114, + "max_per_prompt_fro": 20.365643, + "heavy_tail_ratio": 1.5721, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.641687, + "mean_fro": 7.940871, + "rms_per_prompt_fro": 12.906512, + "max_per_prompt_fro": 19.429527, + "heavy_tail_ratio": 1.5054, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.437646, + "mean_fro": 8.345395, + "rms_per_prompt_fro": 13.029634, + "max_per_prompt_fro": 18.995975, + "heavy_tail_ratio": 1.4579, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.25676, + "mean_fro": 9.267051, + "rms_per_prompt_fro": 13.921441, + "max_per_prompt_fro": 19.020033, + "heavy_tail_ratio": 1.3662, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.123909, + "mean_fro": 10.599702, + "rms_per_prompt_fro": 15.447622, + "max_per_prompt_fro": 19.865644, + "heavy_tail_ratio": 1.286, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.023438, + "mean_fro": 11.962677, + "rms_per_prompt_fro": 17.01662, + "max_per_prompt_fro": 21.44342, + "heavy_tail_ratio": 1.2601, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.919937, + "mean_fro": 13.644495, + "rms_per_prompt_fro": 18.906055, + "max_per_prompt_fro": 23.286633, + "heavy_tail_ratio": 1.2317, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.818007, + "mean_fro": 16.058741, + "rms_per_prompt_fro": 21.652563, + "max_per_prompt_fro": 26.242798, + "heavy_tail_ratio": 1.212, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.659406, + "mean_fro": 18.12166, + "rms_per_prompt_fro": 23.343946, + "max_per_prompt_fro": 26.8624, + "heavy_tail_ratio": 1.1507, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.569658, + "mean_fro": 19.81872, + "rms_per_prompt_fro": 24.83008, + "max_per_prompt_fro": 28.332306, + "heavy_tail_ratio": 1.141, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.475545, + "mean_fro": 22.845421, + "rms_per_prompt_fro": 27.750796, + "max_per_prompt_fro": 30.923445, + "heavy_tail_ratio": 1.1143, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.39893, + "mean_fro": 25.681362, + "rms_per_prompt_fro": 30.37498, + "max_per_prompt_fro": 33.897038, + "heavy_tail_ratio": 1.116, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.289129, + "mean_fro": 29.317659, + "rms_per_prompt_fro": 33.287212, + "max_per_prompt_fro": 36.836906, + "heavy_tail_ratio": 1.1066, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.213709, + "mean_fro": 32.023621, + "rms_per_prompt_fro": 35.279931, + "max_per_prompt_fro": 38.391636, + "heavy_tail_ratio": 1.0882, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.153674, + "mean_fro": 33.795819, + "rms_per_prompt_fro": 36.299827, + "max_per_prompt_fro": 38.980076, + "heavy_tail_ratio": 1.0738, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.103846, + "mean_fro": 34.458515, + "rms_per_prompt_fro": 36.203516, + "max_per_prompt_fro": 38.162895, + "heavy_tail_ratio": 1.0541, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.058147, + "mean_fro": 34.895539, + "rms_per_prompt_fro": 35.895737, + "max_per_prompt_fro": 37.382595, + "heavy_tail_ratio": 1.0414, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.027807, + "mean_fro": 34.518605, + "rms_per_prompt_fro": 34.995239, + "max_per_prompt_fro": 35.899288, + "heavy_tail_ratio": 1.0258, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.5, + "fit_secs_per_prompt": 3.031, + "total_wall_seconds": 162.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9483, + "top_sv": 1.2603, + "stable_rank": 30.395 + }, + { + "layer": 1, + "fro_norm": 6.5118, + "top_sv": 1.1783, + "stable_rank": 30.54 + }, + { + "layer": 2, + "fro_norm": 7.0223, + "top_sv": 1.2924, + "stable_rank": 29.523 + }, + { + "layer": 3, + "fro_norm": 6.7499, + "top_sv": 1.2528, + "stable_rank": 29.029 + }, + { + "layer": 4, + "fro_norm": 7.6887, + "top_sv": 1.423, + "stable_rank": 29.194 + }, + { + "layer": 5, + "fro_norm": 7.9408, + "top_sv": 1.5505, + "stable_rank": 26.228 + }, + { + "layer": 6, + "fro_norm": 8.3453, + "top_sv": 1.6238, + "stable_rank": 26.411 + }, + { + "layer": 7, + "fro_norm": 9.2669, + "top_sv": 1.8232, + "stable_rank": 25.835 + }, + { + "layer": 8, + "fro_norm": 10.5996, + "top_sv": 2.2892, + "stable_rank": 21.439 + }, + { + "layer": 9, + "fro_norm": 11.9625, + "top_sv": 2.5616, + "stable_rank": 21.808 + }, + { + "layer": 10, + "fro_norm": 13.6443, + "top_sv": 2.9283, + "stable_rank": 21.711 + }, + { + "layer": 11, + "fro_norm": 16.0585, + "top_sv": 3.4438, + "stable_rank": 21.743 + }, + { + "layer": 12, + "fro_norm": 18.1214, + "top_sv": 3.8607, + "stable_rank": 22.032 + }, + { + "layer": 13, + "fro_norm": 19.8184, + "top_sv": 4.2517, + "stable_rank": 21.728 + }, + { + "layer": 14, + "fro_norm": 22.845, + "top_sv": 4.8175, + "stable_rank": 22.487 + }, + { + "layer": 15, + "fro_norm": 25.6808, + "top_sv": 5.5863, + "stable_rank": 21.133 + }, + { + "layer": 16, + "fro_norm": 29.317, + "top_sv": 6.4991, + "stable_rank": 20.348 + }, + { + "layer": 17, + "fro_norm": 32.0229, + "top_sv": 7.0032, + "stable_rank": 20.909 + }, + { + "layer": 18, + "fro_norm": 33.7948, + "top_sv": 7.3096, + "stable_rank": 21.375 + }, + { + "layer": 19, + "fro_norm": 34.4574, + "top_sv": 7.2782, + "stable_rank": 22.414 + }, + { + "layer": 20, + "fro_norm": 34.8942, + "top_sv": 7.0951, + "stable_rank": 24.187 + }, + { + "layer": 21, + "fro_norm": 34.5169, + "top_sv": 5.5718, + "stable_rank": 38.378 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_code_LM_B.pt b/results/lenses/plad/plad_410m_code_LM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..eda2d6bfe9b8e5e5f969d2ea5c46cbbb07ce886c --- /dev/null +++ b/results/lenses/plad/plad_410m_code_LM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f834d586ad588383c753fb73c8be8105530b419d7bded06d5511dc7f118e5e6 +size 46144204 diff --git a/results/lenses/plad/plad_410m_code_LM_B_provenance.json b/results/lenses/plad/plad_410m_code_LM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d14ef225142eac5b39cb16e1e204b529fa43871b --- /dev/null +++ b/results/lenses/plad/plad_410m_code_LM_B_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_code_LM_B.pt", + "sha256": "0f834d586ad588383c753fb73c8be8105530b419d7bded06d5511dc7f118e5e6", + "bytes": 46144204, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.499124, + "mean_fro": 7.284318, + "rms_per_prompt_fro": 15.450868, + "max_per_prompt_fro": 23.124804, + "heavy_tail_ratio": 1.4967, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.083871, + "mean_fro": 6.8245, + "rms_per_prompt_fro": 13.791353, + "max_per_prompt_fro": 19.959238, + "heavy_tail_ratio": 1.4472, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.546972, + "mean_fro": 7.377098, + "rms_per_prompt_fro": 13.893589, + "max_per_prompt_fro": 19.482925, + "heavy_tail_ratio": 1.4023, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.166598, + "mean_fro": 7.081293, + "rms_per_prompt_fro": 12.601117, + "max_per_prompt_fro": 16.869507, + "heavy_tail_ratio": 1.3387, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.813919, + "mean_fro": 8.072085, + "rms_per_prompt_fro": 13.540713, + "max_per_prompt_fro": 17.634926, + "heavy_tail_ratio": 1.3024, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.621887, + "mean_fro": 8.29917, + "rms_per_prompt_fro": 13.438218, + "max_per_prompt_fro": 17.452738, + "heavy_tail_ratio": 1.2987, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.416691, + "mean_fro": 8.688404, + "rms_per_prompt_fro": 13.50674, + "max_per_prompt_fro": 17.744358, + "heavy_tail_ratio": 1.3137, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.237567, + "mean_fro": 9.565654, + "rms_per_prompt_fro": 14.308784, + "max_per_prompt_fro": 18.768539, + "heavy_tail_ratio": 1.3117, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.117716, + "mean_fro": 10.913303, + "rms_per_prompt_fro": 15.881446, + "max_per_prompt_fro": 20.823996, + "heavy_tail_ratio": 1.3112, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.024573, + "mean_fro": 12.25664, + "rms_per_prompt_fro": 17.439667, + "max_per_prompt_fro": 22.276936, + "heavy_tail_ratio": 1.2774, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.929223, + "mean_fro": 13.942428, + "rms_per_prompt_fro": 19.365542, + "max_per_prompt_fro": 24.22246, + "heavy_tail_ratio": 1.2508, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.842386, + "mean_fro": 16.333607, + "rms_per_prompt_fro": 22.170344, + "max_per_prompt_fro": 27.460121, + "heavy_tail_ratio": 1.2386, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.679831, + "mean_fro": 18.39317, + "rms_per_prompt_fro": 23.839071, + "max_per_prompt_fro": 28.70298, + "heavy_tail_ratio": 1.204, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.593925, + "mean_fro": 20.146427, + "rms_per_prompt_fro": 25.435012, + "max_per_prompt_fro": 30.662514, + "heavy_tail_ratio": 1.2055, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.496345, + "mean_fro": 23.210366, + "rms_per_prompt_fro": 28.392119, + "max_per_prompt_fro": 33.231274, + "heavy_tail_ratio": 1.1704, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.422279, + "mean_fro": 26.048267, + "rms_per_prompt_fro": 31.064988, + "max_per_prompt_fro": 35.899979, + "heavy_tail_ratio": 1.1556, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.303996, + "mean_fro": 29.568106, + "rms_per_prompt_fro": 33.764607, + "max_per_prompt_fro": 37.873398, + "heavy_tail_ratio": 1.1217, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.225652, + "mean_fro": 32.108253, + "rms_per_prompt_fro": 35.546784, + "max_per_prompt_fro": 39.419685, + "heavy_tail_ratio": 1.109, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.163248, + "mean_fro": 33.893654, + "rms_per_prompt_fro": 36.555657, + "max_per_prompt_fro": 39.626827, + "heavy_tail_ratio": 1.084, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.108641, + "mean_fro": 34.439641, + "rms_per_prompt_fro": 36.262194, + "max_per_prompt_fro": 38.304001, + "heavy_tail_ratio": 1.0563, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.061545, + "mean_fro": 34.857006, + "rms_per_prompt_fro": 35.91363, + "max_per_prompt_fro": 37.211964, + "heavy_tail_ratio": 1.0362, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.029781, + "mean_fro": 34.490624, + "rms_per_prompt_fro": 35.000443, + "max_per_prompt_fro": 35.912029, + "heavy_tail_ratio": 1.026, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.6, + "fit_secs_per_prompt": 3.031, + "total_wall_seconds": 164.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.2842, + "top_sv": 1.2687, + "stable_rank": 32.965 + }, + { + "layer": 1, + "fro_norm": 6.8244, + "top_sv": 1.1843, + "stable_rank": 33.206 + }, + { + "layer": 2, + "fro_norm": 7.377, + "top_sv": 1.2921, + "stable_rank": 32.599 + }, + { + "layer": 3, + "fro_norm": 7.0812, + "top_sv": 1.2815, + "stable_rank": 30.535 + }, + { + "layer": 4, + "fro_norm": 8.072, + "top_sv": 1.4592, + "stable_rank": 30.601 + }, + { + "layer": 5, + "fro_norm": 8.2991, + "top_sv": 1.5754, + "stable_rank": 27.749 + }, + { + "layer": 6, + "fro_norm": 8.6883, + "top_sv": 1.6537, + "stable_rank": 27.604 + }, + { + "layer": 7, + "fro_norm": 9.5655, + "top_sv": 1.8487, + "stable_rank": 26.773 + }, + { + "layer": 8, + "fro_norm": 10.9132, + "top_sv": 2.2377, + "stable_rank": 23.785 + }, + { + "layer": 9, + "fro_norm": 12.2565, + "top_sv": 2.5025, + "stable_rank": 23.988 + }, + { + "layer": 10, + "fro_norm": 13.9422, + "top_sv": 2.8606, + "stable_rank": 23.754 + }, + { + "layer": 11, + "fro_norm": 16.3334, + "top_sv": 3.3683, + "stable_rank": 23.515 + }, + { + "layer": 12, + "fro_norm": 18.3929, + "top_sv": 3.7614, + "stable_rank": 23.912 + }, + { + "layer": 13, + "fro_norm": 20.1461, + "top_sv": 4.1631, + "stable_rank": 23.417 + }, + { + "layer": 14, + "fro_norm": 23.21, + "top_sv": 4.7035, + "stable_rank": 24.35 + }, + { + "layer": 15, + "fro_norm": 26.0478, + "top_sv": 5.4897, + "stable_rank": 22.513 + }, + { + "layer": 16, + "fro_norm": 29.5675, + "top_sv": 6.4425, + "stable_rank": 21.063 + }, + { + "layer": 17, + "fro_norm": 32.1076, + "top_sv": 6.9629, + "stable_rank": 21.264 + }, + { + "layer": 18, + "fro_norm": 33.8927, + "top_sv": 7.4045, + "stable_rank": 20.952 + }, + { + "layer": 19, + "fro_norm": 34.4385, + "top_sv": 7.35, + "stable_rank": 21.954 + }, + { + "layer": 20, + "fro_norm": 34.8557, + "top_sv": 7.0749, + "stable_rank": 24.272 + }, + { + "layer": 21, + "fro_norm": 34.489, + "top_sv": 5.5836, + "stable_rank": 38.154 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_pile_A.pt b/results/lenses/plad/plad_410m_pile_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec2812fd4c736f682a685d3be4536463968403cf --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2343666bc27c555b2d59354ced6f70d0de73bb090e24c56a4ffd807a3b806e4 +size 46144126 diff --git a/results/lenses/plad/plad_410m_pile_A_provenance.json b/results/lenses/plad/plad_410m_pile_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..708f42b535429f8e49bb9f2a868402654def003f --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_A_provenance.json @@ -0,0 +1,368 @@ +{ + "artifact": "results/plad_410m_pile_A.pt", + "sha256": "f2343666bc27c555b2d59354ced6f70d0de73bb090e24c56a4ffd807a3b806e4", + "bytes": 46144126, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.201029, + "mean_fro": 7.093643, + "rms_per_prompt_fro": 14.539424, + "max_per_prompt_fro": 29.136526, + "heavy_tail_ratio": 2.004, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.839526, + "mean_fro": 6.644515, + "rms_per_prompt_fro": 13.019733, + "max_per_prompt_fro": 25.729727, + "heavy_tail_ratio": 1.9762, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.288991, + "mean_fro": 7.232116, + "rms_per_prompt_fro": 13.115858, + "max_per_prompt_fro": 24.512585, + "heavy_tail_ratio": 1.8689, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.948297, + "mean_fro": 6.940479, + "rms_per_prompt_fro": 11.917223, + "max_per_prompt_fro": 19.907906, + "heavy_tail_ratio": 1.6705, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.59922, + "mean_fro": 7.980213, + "rms_per_prompt_fro": 12.865777, + "max_per_prompt_fro": 20.365643, + "heavy_tail_ratio": 1.5829, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.434924, + "mean_fro": 8.207283, + "rms_per_prompt_fro": 12.806843, + "max_per_prompt_fro": 19.429527, + "heavy_tail_ratio": 1.5171, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.263661, + "mean_fro": 8.602848, + "rms_per_prompt_fro": 12.943388, + "max_per_prompt_fro": 18.995975, + "heavy_tail_ratio": 1.4676, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.113545, + "mean_fro": 9.524809, + "rms_per_prompt_fro": 13.847201, + "max_per_prompt_fro": 19.020033, + "heavy_tail_ratio": 1.3736, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.004463, + "mean_fro": 10.965172, + "rms_per_prompt_fro": 15.524388, + "max_per_prompt_fro": 19.865644, + "heavy_tail_ratio": 1.2796, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.917912, + "mean_fro": 12.389916, + "rms_per_prompt_fro": 17.158632, + "max_per_prompt_fro": 21.44342, + "heavy_tail_ratio": 1.2497, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.832758, + "mean_fro": 14.160503, + "rms_per_prompt_fro": 19.170403, + "max_per_prompt_fro": 23.286633, + "heavy_tail_ratio": 1.2147, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.751321, + "mean_fro": 16.726421, + "rms_per_prompt_fro": 22.135324, + "max_per_prompt_fro": 26.242798, + "heavy_tail_ratio": 1.1856, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.607594, + "mean_fro": 18.687527, + "rms_per_prompt_fro": 23.694092, + "max_per_prompt_fro": 27.498405, + "heavy_tail_ratio": 1.1606, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.5295, + "mean_fro": 20.487957, + "rms_per_prompt_fro": 25.33806, + "max_per_prompt_fro": 28.693178, + "heavy_tail_ratio": 1.1324, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.449528, + "mean_fro": 23.64857, + "rms_per_prompt_fro": 28.472015, + "max_per_prompt_fro": 32.119041, + "heavy_tail_ratio": 1.1281, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.380558, + "mean_fro": 26.555256, + "rms_per_prompt_fro": 31.201669, + "max_per_prompt_fro": 34.809654, + "heavy_tail_ratio": 1.1156, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.277695, + "mean_fro": 29.969359, + "rms_per_prompt_fro": 33.875914, + "max_per_prompt_fro": 37.032349, + "heavy_tail_ratio": 1.0932, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.207957, + "mean_fro": 32.467368, + "rms_per_prompt_fro": 35.683935, + "max_per_prompt_fro": 38.70195, + "heavy_tail_ratio": 1.0846, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.150806, + "mean_fro": 34.174217, + "rms_per_prompt_fro": 36.66061, + "max_per_prompt_fro": 39.161572, + "heavy_tail_ratio": 1.0682, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.10307, + "mean_fro": 34.575645, + "rms_per_prompt_fro": 36.313811, + "max_per_prompt_fro": 37.935268, + "heavy_tail_ratio": 1.0447, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.059468, + "mean_fro": 34.918888, + "rms_per_prompt_fro": 35.942164, + "max_per_prompt_fro": 37.094536, + "heavy_tail_ratio": 1.0321, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.028654, + "mean_fro": 34.534214, + "rms_per_prompt_fro": 35.025499, + "max_per_prompt_fro": 35.834873, + "heavy_tail_ratio": 1.0231, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.0, + "fit_secs_per_prompt": 3.021, + "total_wall_seconds": 162.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.0935, + "top_sv": 1.2784, + "stable_rank": 30.789 + }, + { + "layer": 1, + "fro_norm": 6.6444, + "top_sv": 1.2137, + "stable_rank": 29.972 + }, + { + "layer": 2, + "fro_norm": 7.232, + "top_sv": 1.3926, + "stable_rank": 26.968 + }, + { + "layer": 3, + "fro_norm": 6.9404, + "top_sv": 1.3076, + "stable_rank": 28.171 + }, + { + "layer": 4, + "fro_norm": 7.9801, + "top_sv": 1.5376, + "stable_rank": 26.935 + }, + { + "layer": 5, + "fro_norm": 8.2072, + "top_sv": 1.6413, + "stable_rank": 25.005 + }, + { + "layer": 6, + "fro_norm": 8.6027, + "top_sv": 1.7002, + "stable_rank": 25.602 + }, + { + "layer": 7, + "fro_norm": 9.5247, + "top_sv": 1.8842, + "stable_rank": 25.552 + }, + { + "layer": 8, + "fro_norm": 10.965, + "top_sv": 2.377, + "stable_rank": 21.28 + }, + { + "layer": 9, + "fro_norm": 12.3897, + "top_sv": 2.6576, + "stable_rank": 21.734 + }, + { + "layer": 10, + "fro_norm": 14.1603, + "top_sv": 3.048, + "stable_rank": 21.583 + }, + { + "layer": 11, + "fro_norm": 16.7262, + "top_sv": 3.6082, + "stable_rank": 21.489 + }, + { + "layer": 12, + "fro_norm": 18.6872, + "top_sv": 3.9262, + "stable_rank": 22.654 + }, + { + "layer": 13, + "fro_norm": 20.4876, + "top_sv": 4.3322, + "stable_rank": 22.365 + }, + { + "layer": 14, + "fro_norm": 23.6482, + "top_sv": 4.8956, + "stable_rank": 23.334 + }, + { + "layer": 15, + "fro_norm": 26.5548, + "top_sv": 5.7471, + "stable_rank": 21.349 + }, + { + "layer": 16, + "fro_norm": 29.9687, + "top_sv": 6.5473, + "stable_rank": 20.951 + }, + { + "layer": 17, + "fro_norm": 32.4666, + "top_sv": 7.0495, + "stable_rank": 21.211 + }, + { + "layer": 18, + "fro_norm": 34.1733, + "top_sv": 7.4704, + "stable_rank": 20.926 + }, + { + "layer": 19, + "fro_norm": 34.5745, + "top_sv": 7.4272, + "stable_rank": 21.67 + }, + { + "layer": 20, + "fro_norm": 34.9175, + "top_sv": 7.171, + "stable_rank": 23.709 + }, + { + "layer": 21, + "fro_norm": 34.5326, + "top_sv": 5.6492, + "stable_rank": 37.367 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_pile_B.pt b/results/lenses/plad/plad_410m_pile_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..040d390720b39e0a0373d2dab6fac25338c06c0f --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e64f302f14e7cfd2765720c1baa93f45833c80d870a6d152046924c8e05c8d +size 46144126 diff --git a/results/lenses/plad/plad_410m_pile_B_provenance.json b/results/lenses/plad/plad_410m_pile_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..2e8a1c654c2aa100b8a17bea0f7c3f8cc5f6d1c8 --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_B_provenance.json @@ -0,0 +1,368 @@ +{ + "artifact": "results/plad_410m_pile_B.pt", + "sha256": "a8e64f302f14e7cfd2765720c1baa93f45833c80d870a6d152046924c8e05c8d", + "bytes": 46144126, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.45472, + "mean_fro": 6.91474, + "rms_per_prompt_fro": 14.594393, + "max_per_prompt_fro": 23.070173, + "heavy_tail_ratio": 1.5808, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.109921, + "mean_fro": 6.499671, + "rms_per_prompt_fro": 13.176745, + "max_per_prompt_fro": 20.843794, + "heavy_tail_ratio": 1.5819, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.546637, + "mean_fro": 7.057953, + "rms_per_prompt_fro": 13.291903, + "max_per_prompt_fro": 19.886864, + "heavy_tail_ratio": 1.4962, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.181035, + "mean_fro": 6.794089, + "rms_per_prompt_fro": 12.117569, + "max_per_prompt_fro": 17.225756, + "heavy_tail_ratio": 1.4216, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.818744, + "mean_fro": 7.76867, + "rms_per_prompt_fro": 13.042909, + "max_per_prompt_fro": 17.280107, + "heavy_tail_ratio": 1.3249, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.653319, + "mean_fro": 7.995348, + "rms_per_prompt_fro": 13.023632, + "max_per_prompt_fro": 16.794432, + "heavy_tail_ratio": 1.2895, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.448457, + "mean_fro": 8.38339, + "rms_per_prompt_fro": 13.117949, + "max_per_prompt_fro": 16.535435, + "heavy_tail_ratio": 1.2605, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.255267, + "mean_fro": 9.282065, + "rms_per_prompt_fro": 13.939384, + "max_per_prompt_fro": 17.095018, + "heavy_tail_ratio": 1.2264, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.127775, + "mean_fro": 10.64199, + "rms_per_prompt_fro": 15.52336, + "max_per_prompt_fro": 18.73192, + "heavy_tail_ratio": 1.2067, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.029301, + "mean_fro": 12.007204, + "rms_per_prompt_fro": 17.104688, + "max_per_prompt_fro": 20.550196, + "heavy_tail_ratio": 1.2014, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.928763, + "mean_fro": 13.700489, + "rms_per_prompt_fro": 19.027227, + "max_per_prompt_fro": 22.878252, + "heavy_tail_ratio": 1.2024, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.833785, + "mean_fro": 16.123034, + "rms_per_prompt_fro": 21.83338, + "max_per_prompt_fro": 26.577879, + "heavy_tail_ratio": 1.2173, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.660233, + "mean_fro": 18.157368, + "rms_per_prompt_fro": 23.395775, + "max_per_prompt_fro": 26.625357, + "heavy_tail_ratio": 1.138, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.569386, + "mean_fro": 19.904557, + "rms_per_prompt_fro": 24.935463, + "max_per_prompt_fro": 28.42799, + "heavy_tail_ratio": 1.1401, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.477286, + "mean_fro": 23.043419, + "rms_per_prompt_fro": 28.007814, + "max_per_prompt_fro": 32.271133, + "heavy_tail_ratio": 1.1522, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.400346, + "mean_fro": 25.959019, + "rms_per_prompt_fro": 30.718924, + "max_per_prompt_fro": 35.262959, + "heavy_tail_ratio": 1.1479, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.289789, + "mean_fro": 29.559813, + "rms_per_prompt_fro": 33.570742, + "max_per_prompt_fro": 37.725597, + "heavy_tail_ratio": 1.1238, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.213481, + "mean_fro": 32.194546, + "rms_per_prompt_fro": 35.464911, + "max_per_prompt_fro": 38.964504, + "heavy_tail_ratio": 1.0987, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.154117, + "mean_fro": 33.942017, + "rms_per_prompt_fro": 36.463861, + "max_per_prompt_fro": 39.75565, + "heavy_tail_ratio": 1.0903, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.103905, + "mean_fro": 34.479514, + "rms_per_prompt_fro": 36.226559, + "max_per_prompt_fro": 38.119759, + "heavy_tail_ratio": 1.0523, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.059492, + "mean_fro": 34.884453, + "rms_per_prompt_fro": 35.907144, + "max_per_prompt_fro": 37.382595, + "heavy_tail_ratio": 1.0411, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.028536, + "mean_fro": 34.519257, + "rms_per_prompt_fro": 35.008307, + "max_per_prompt_fro": 35.899288, + "heavy_tail_ratio": 1.0255, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.0, + "fit_secs_per_prompt": 3.02, + "total_wall_seconds": 162.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9147, + "top_sv": 1.2, + "stable_rank": 33.203 + }, + { + "layer": 1, + "fro_norm": 6.4996, + "top_sv": 1.1297, + "stable_rank": 33.1 + }, + { + "layer": 2, + "fro_norm": 7.0579, + "top_sv": 1.281, + "stable_rank": 30.357 + }, + { + "layer": 3, + "fro_norm": 6.794, + "top_sv": 1.2424, + "stable_rank": 29.903 + }, + { + "layer": 4, + "fro_norm": 7.7686, + "top_sv": 1.4731, + "stable_rank": 27.81 + }, + { + "layer": 5, + "fro_norm": 7.9953, + "top_sv": 1.5728, + "stable_rank": 25.843 + }, + { + "layer": 6, + "fro_norm": 8.3833, + "top_sv": 1.6455, + "stable_rank": 25.955 + }, + { + "layer": 7, + "fro_norm": 9.2819, + "top_sv": 1.8446, + "stable_rank": 25.32 + }, + { + "layer": 8, + "fro_norm": 10.6419, + "top_sv": 2.3164, + "stable_rank": 21.106 + }, + { + "layer": 9, + "fro_norm": 12.007, + "top_sv": 2.5957, + "stable_rank": 21.397 + }, + { + "layer": 10, + "fro_norm": 13.7003, + "top_sv": 2.9509, + "stable_rank": 21.555 + }, + { + "layer": 11, + "fro_norm": 16.1228, + "top_sv": 3.4623, + "stable_rank": 21.684 + }, + { + "layer": 12, + "fro_norm": 18.157, + "top_sv": 3.8145, + "stable_rank": 22.658 + }, + { + "layer": 13, + "fro_norm": 19.9042, + "top_sv": 4.2121, + "stable_rank": 22.331 + }, + { + "layer": 14, + "fro_norm": 23.0431, + "top_sv": 4.7642, + "stable_rank": 23.394 + }, + { + "layer": 15, + "fro_norm": 25.9585, + "top_sv": 5.5678, + "stable_rank": 21.736 + }, + { + "layer": 16, + "fro_norm": 29.5592, + "top_sv": 6.3971, + "stable_rank": 21.351 + }, + { + "layer": 17, + "fro_norm": 32.1938, + "top_sv": 6.907, + "stable_rank": 21.725 + }, + { + "layer": 18, + "fro_norm": 33.9411, + "top_sv": 7.3365, + "stable_rank": 21.403 + }, + { + "layer": 19, + "fro_norm": 34.4784, + "top_sv": 7.2877, + "stable_rank": 22.383 + }, + { + "layer": 20, + "fro_norm": 34.8832, + "top_sv": 7.064, + "stable_rank": 24.385 + }, + { + "layer": 21, + "fro_norm": 34.5176, + "top_sv": 5.5734, + "stable_rank": 38.357 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_pile_LM_A.pt b/results/lenses/plad/plad_410m_pile_LM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..99c8cd7db3f64531eb9e113986c7794e1b8a7b28 --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_LM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22543312f585298839b191357f1b934cda2a11b5767677e1d369da8a08b9e19a +size 46144204 diff --git a/results/lenses/plad/plad_410m_pile_LM_A_provenance.json b/results/lenses/plad/plad_410m_pile_LM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..ea8f0bcfd0f5410afcacc897cc545696b681d264 --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_LM_A_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_pile_LM_A.pt", + "sha256": "22543312f585298839b191357f1b934cda2a11b5767677e1d369da8a08b9e19a", + "bytes": 46144204, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.184438, + "mean_fro": 7.103298, + "rms_per_prompt_fro": 14.530435, + "max_per_prompt_fro": 29.136526, + "heavy_tail_ratio": 2.0052, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.826391, + "mean_fro": 6.655066, + "rms_per_prompt_fro": 13.018083, + "max_per_prompt_fro": 25.729727, + "heavy_tail_ratio": 1.9765, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.282526, + "mean_fro": 7.240881, + "rms_per_prompt_fro": 13.11884, + "max_per_prompt_fro": 24.512585, + "heavy_tail_ratio": 1.8685, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.943383, + "mean_fro": 6.947722, + "rms_per_prompt_fro": 11.919714, + "max_per_prompt_fro": 19.907906, + "heavy_tail_ratio": 1.6702, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.595587, + "mean_fro": 7.990179, + "rms_per_prompt_fro": 12.872839, + "max_per_prompt_fro": 20.365643, + "heavy_tail_ratio": 1.5821, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.432591, + "mean_fro": 8.214525, + "rms_per_prompt_fro": 12.812004, + "max_per_prompt_fro": 19.429527, + "heavy_tail_ratio": 1.5165, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.261357, + "mean_fro": 8.611459, + "rms_per_prompt_fro": 12.949747, + "max_per_prompt_fro": 18.995975, + "heavy_tail_ratio": 1.4669, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.111101, + "mean_fro": 9.536723, + "rms_per_prompt_fro": 13.856503, + "max_per_prompt_fro": 19.020033, + "heavy_tail_ratio": 1.3726, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.002027, + "mean_fro": 10.978649, + "rms_per_prompt_fro": 15.53402, + "max_per_prompt_fro": 19.865644, + "heavy_tail_ratio": 1.2788, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.91596, + "mean_fro": 12.407503, + "rms_per_prompt_fro": 17.174245, + "max_per_prompt_fro": 21.44342, + "heavy_tail_ratio": 1.2486, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.831525, + "mean_fro": 14.176548, + "rms_per_prompt_fro": 19.185668, + "max_per_prompt_fro": 23.286633, + "heavy_tail_ratio": 1.2138, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.750355, + "mean_fro": 16.74308, + "rms_per_prompt_fro": 22.151262, + "max_per_prompt_fro": 26.242798, + "heavy_tail_ratio": 1.1847, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.606057, + "mean_fro": 18.704136, + "rms_per_prompt_fro": 23.70381, + "max_per_prompt_fro": 27.498405, + "heavy_tail_ratio": 1.1601, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.527964, + "mean_fro": 20.505404, + "rms_per_prompt_fro": 25.346899, + "max_per_prompt_fro": 28.693178, + "heavy_tail_ratio": 1.132, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.448027, + "mean_fro": 23.662876, + "rms_per_prompt_fro": 28.474487, + "max_per_prompt_fro": 32.119041, + "heavy_tail_ratio": 1.128, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.37907, + "mean_fro": 26.567596, + "rms_per_prompt_fro": 31.199342, + "max_per_prompt_fro": 34.809654, + "heavy_tail_ratio": 1.1157, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.276675, + "mean_fro": 29.983657, + "rms_per_prompt_fro": 33.878554, + "max_per_prompt_fro": 37.032349, + "heavy_tail_ratio": 1.0931, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.207022, + "mean_fro": 32.483669, + "rms_per_prompt_fro": 35.688041, + "max_per_prompt_fro": 38.70195, + "heavy_tail_ratio": 1.0845, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.150086, + "mean_fro": 34.194428, + "rms_per_prompt_fro": 36.670811, + "max_per_prompt_fro": 39.161572, + "heavy_tail_ratio": 1.0679, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.10267, + "mean_fro": 34.591777, + "rms_per_prompt_fro": 36.324164, + "max_per_prompt_fro": 37.935268, + "heavy_tail_ratio": 1.0444, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.059271, + "mean_fro": 34.932362, + "rms_per_prompt_fro": 35.952706, + "max_per_prompt_fro": 37.094536, + "heavy_tail_ratio": 1.0318, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.028586, + "mean_fro": 34.543166, + "rms_per_prompt_fro": 35.033416, + "max_per_prompt_fro": 35.834873, + "heavy_tail_ratio": 1.0229, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.4, + "fit_secs_per_prompt": 3.027, + "total_wall_seconds": 163.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 7.1032, + "top_sv": 1.2759, + "stable_rank": 30.995 + }, + { + "layer": 1, + "fro_norm": 6.655, + "top_sv": 1.2144, + "stable_rank": 30.03 + }, + { + "layer": 2, + "fro_norm": 7.2408, + "top_sv": 1.391, + "stable_rank": 27.097 + }, + { + "layer": 3, + "fro_norm": 6.9476, + "top_sv": 1.3059, + "stable_rank": 28.306 + }, + { + "layer": 4, + "fro_norm": 7.9901, + "top_sv": 1.5375, + "stable_rank": 27.006 + }, + { + "layer": 5, + "fro_norm": 8.2144, + "top_sv": 1.6429, + "stable_rank": 24.999 + }, + { + "layer": 6, + "fro_norm": 8.6113, + "top_sv": 1.7025, + "stable_rank": 25.583 + }, + { + "layer": 7, + "fro_norm": 9.5366, + "top_sv": 1.8846, + "stable_rank": 25.607 + }, + { + "layer": 8, + "fro_norm": 10.9785, + "top_sv": 2.3824, + "stable_rank": 21.236 + }, + { + "layer": 9, + "fro_norm": 12.4073, + "top_sv": 2.6655, + "stable_rank": 21.667 + }, + { + "layer": 10, + "fro_norm": 14.1763, + "top_sv": 3.0538, + "stable_rank": 21.55 + }, + { + "layer": 11, + "fro_norm": 16.7429, + "top_sv": 3.6124, + "stable_rank": 21.481 + }, + { + "layer": 12, + "fro_norm": 18.7038, + "top_sv": 3.9263, + "stable_rank": 22.694 + }, + { + "layer": 13, + "fro_norm": 20.5051, + "top_sv": 4.331, + "stable_rank": 22.415 + }, + { + "layer": 14, + "fro_norm": 23.6624, + "top_sv": 4.8952, + "stable_rank": 23.366 + }, + { + "layer": 15, + "fro_norm": 26.5671, + "top_sv": 5.7425, + "stable_rank": 21.403 + }, + { + "layer": 16, + "fro_norm": 29.9831, + "top_sv": 6.5481, + "stable_rank": 20.966 + }, + { + "layer": 17, + "fro_norm": 32.4829, + "top_sv": 7.047, + "stable_rank": 21.247 + }, + { + "layer": 18, + "fro_norm": 34.1935, + "top_sv": 7.4676, + "stable_rank": 20.966 + }, + { + "layer": 19, + "fro_norm": 34.5906, + "top_sv": 7.4291, + "stable_rank": 21.679 + }, + { + "layer": 20, + "fro_norm": 34.9311, + "top_sv": 7.1763, + "stable_rank": 23.693 + }, + { + "layer": 21, + "fro_norm": 34.5415, + "top_sv": 5.6538, + "stable_rank": 37.326 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_pile_LM_B.pt b/results/lenses/plad/plad_410m_pile_LM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c61743892372c0123e1cd3ff091eebfd028b3b3 --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_LM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88c415a2d9b914e44758483b1ae450a176a87832cbffbd470b69598093888bfa +size 46144204 diff --git a/results/lenses/plad/plad_410m_pile_LM_B_provenance.json b/results/lenses/plad/plad_410m_pile_LM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..91cc07f1b4766505c312c958f7aebf15d4c999a9 --- /dev/null +++ b/results/lenses/plad/plad_410m_pile_LM_B_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_pile_LM_B.pt", + "sha256": "88c415a2d9b914e44758483b1ae450a176a87832cbffbd470b69598093888bfa", + "bytes": 46144204, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 3.533212, + "mean_fro": 6.957493, + "rms_per_prompt_fro": 14.813436, + "max_per_prompt_fro": 23.070173, + "heavy_tail_ratio": 1.5574, + "n_prompts": 50 + }, + "1": { + "dispersion": 3.169602, + "mean_fro": 6.533674, + "rms_per_prompt_fro": 13.341504, + "max_per_prompt_fro": 20.843794, + "heavy_tail_ratio": 1.5623, + "n_prompts": 50 + }, + "2": { + "dispersion": 2.58947, + "mean_fro": 7.087787, + "rms_per_prompt_fro": 13.428448, + "max_per_prompt_fro": 19.886864, + "heavy_tail_ratio": 1.481, + "n_prompts": 50 + }, + "3": { + "dispersion": 2.212823, + "mean_fro": 6.821863, + "rms_per_prompt_fro": 12.227745, + "max_per_prompt_fro": 17.225756, + "heavy_tail_ratio": 1.4087, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.840221, + "mean_fro": 7.793947, + "rms_per_prompt_fro": 13.135103, + "max_per_prompt_fro": 17.280107, + "heavy_tail_ratio": 1.3156, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.671147, + "mean_fro": 8.023529, + "rms_per_prompt_fro": 13.113369, + "max_per_prompt_fro": 16.794432, + "heavy_tail_ratio": 1.2807, + "n_prompts": 50 + }, + "6": { + "dispersion": 1.459947, + "mean_fro": 8.41053, + "rms_per_prompt_fro": 13.191259, + "max_per_prompt_fro": 16.535435, + "heavy_tail_ratio": 1.2535, + "n_prompts": 50 + }, + "7": { + "dispersion": 1.265384, + "mean_fro": 9.308252, + "rms_per_prompt_fro": 14.010029, + "max_per_prompt_fro": 17.361248, + "heavy_tail_ratio": 1.2392, + "n_prompts": 50 + }, + "8": { + "dispersion": 1.137897, + "mean_fro": 10.66486, + "rms_per_prompt_fro": 15.593676, + "max_per_prompt_fro": 19.416708, + "heavy_tail_ratio": 1.2452, + "n_prompts": 50 + }, + "9": { + "dispersion": 1.037473, + "mean_fro": 12.01798, + "rms_per_prompt_fro": 17.154476, + "max_per_prompt_fro": 20.950607, + "heavy_tail_ratio": 1.2213, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.935399, + "mean_fro": 13.70229, + "rms_per_prompt_fro": 19.062437, + "max_per_prompt_fro": 22.979149, + "heavy_tail_ratio": 1.2055, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.839583, + "mean_fro": 16.115347, + "rms_per_prompt_fro": 21.857444, + "max_per_prompt_fro": 26.577879, + "heavy_tail_ratio": 1.216, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.664575, + "mean_fro": 18.15056, + "rms_per_prompt_fro": 23.417562, + "max_per_prompt_fro": 27.496004, + "heavy_tail_ratio": 1.1742, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.573734, + "mean_fro": 19.89879, + "rms_per_prompt_fro": 24.962745, + "max_per_prompt_fro": 29.23694, + "heavy_tail_ratio": 1.1712, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.480603, + "mean_fro": 23.03133, + "rms_per_prompt_fro": 28.024529, + "max_per_prompt_fro": 32.271133, + "heavy_tail_ratio": 1.1515, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.403003, + "mean_fro": 25.954689, + "rms_per_prompt_fro": 30.742916, + "max_per_prompt_fro": 35.262959, + "heavy_tail_ratio": 1.147, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.291217, + "mean_fro": 29.553386, + "rms_per_prompt_fro": 33.582022, + "max_per_prompt_fro": 37.725597, + "heavy_tail_ratio": 1.1234, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.215114, + "mean_fro": 32.197956, + "rms_per_prompt_fro": 35.492513, + "max_per_prompt_fro": 38.964504, + "heavy_tail_ratio": 1.0978, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.155696, + "mean_fro": 33.938869, + "rms_per_prompt_fro": 36.485408, + "max_per_prompt_fro": 39.75565, + "heavy_tail_ratio": 1.0896, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.10478, + "mean_fro": 34.478081, + "rms_per_prompt_fro": 36.239406, + "max_per_prompt_fro": 38.119759, + "heavy_tail_ratio": 1.0519, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.059939, + "mean_fro": 34.871239, + "rms_per_prompt_fro": 35.901113, + "max_per_prompt_fro": 37.382595, + "heavy_tail_ratio": 1.0413, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.028784, + "mean_fro": 34.513466, + "rms_per_prompt_fro": 35.006668, + "max_per_prompt_fro": 35.899288, + "heavy_tail_ratio": 1.0255, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.5, + "fit_secs_per_prompt": 3.03, + "total_wall_seconds": 162.6, + "per_layer": [ + { + "layer": 0, + "fro_norm": 6.9574, + "top_sv": 1.2116, + "stable_rank": 32.975 + }, + { + "layer": 1, + "fro_norm": 6.5336, + "top_sv": 1.139, + "stable_rank": 32.905 + }, + { + "layer": 2, + "fro_norm": 7.0877, + "top_sv": 1.2912, + "stable_rank": 30.132 + }, + { + "layer": 3, + "fro_norm": 6.8218, + "top_sv": 1.2516, + "stable_rank": 29.705 + }, + { + "layer": 4, + "fro_norm": 7.7939, + "top_sv": 1.4708, + "stable_rank": 28.079 + }, + { + "layer": 5, + "fro_norm": 8.0234, + "top_sv": 1.574, + "stable_rank": 25.984 + }, + { + "layer": 6, + "fro_norm": 8.4104, + "top_sv": 1.6469, + "stable_rank": 26.08 + }, + { + "layer": 7, + "fro_norm": 9.3081, + "top_sv": 1.8456, + "stable_rank": 25.435 + }, + { + "layer": 8, + "fro_norm": 10.6647, + "top_sv": 2.3104, + "stable_rank": 21.308 + }, + { + "layer": 9, + "fro_norm": 12.0178, + "top_sv": 2.5902, + "stable_rank": 21.527 + }, + { + "layer": 10, + "fro_norm": 13.7021, + "top_sv": 2.9522, + "stable_rank": 21.542 + }, + { + "layer": 11, + "fro_norm": 16.1151, + "top_sv": 3.4637, + "stable_rank": 21.647 + }, + { + "layer": 12, + "fro_norm": 18.1503, + "top_sv": 3.8187, + "stable_rank": 22.591 + }, + { + "layer": 13, + "fro_norm": 19.8985, + "top_sv": 4.2128, + "stable_rank": 22.31 + }, + { + "layer": 14, + "fro_norm": 23.0309, + "top_sv": 4.7713, + "stable_rank": 23.3 + }, + { + "layer": 15, + "fro_norm": 25.9541, + "top_sv": 5.563, + "stable_rank": 21.767 + }, + { + "layer": 16, + "fro_norm": 29.5528, + "top_sv": 6.4252, + "stable_rank": 21.156 + }, + { + "layer": 17, + "fro_norm": 32.1973, + "top_sv": 6.927, + "stable_rank": 21.604 + }, + { + "layer": 18, + "fro_norm": 33.938, + "top_sv": 7.3557, + "stable_rank": 21.288 + }, + { + "layer": 19, + "fro_norm": 34.4769, + "top_sv": 7.3199, + "stable_rank": 22.184 + }, + { + "layer": 20, + "fro_norm": 34.87, + "top_sv": 7.0471, + "stable_rank": 24.484 + }, + { + "layer": 21, + "fro_norm": 34.5118, + "top_sv": 5.5702, + "stable_rank": 38.387 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_A.pt b/results/lenses/plad/plad_410m_wikitext_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f903e77a9bf9e7c9abce51ddb5f38d30b93ded3 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e9e97dfc8c551a0fa170d72902bce9481a038d250ba80be326dea9e87bf2d5d +size 46144230 diff --git a/results/lenses/plad/plad_410m_wikitext_A_provenance.json b/results/lenses/plad/plad_410m_wikitext_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d2aa3706b97e0ffe5de7519be40ec11b11abb17f --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_A_provenance.json @@ -0,0 +1,368 @@ +{ + "artifact": "results/plad_410m_wikitext_A.pt", + "sha256": "3e9e97dfc8c551a0fa170d72902bce9481a038d250ba80be326dea9e87bf2d5d", + "bytes": 46144230, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 1.992096, + "mean_fro": 8.992118, + "rms_per_prompt_fro": 15.554275, + "max_per_prompt_fro": 21.35177, + "heavy_tail_ratio": 1.3727, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.743749, + "mean_fro": 8.42288, + "rms_per_prompt_fro": 13.951881, + "max_per_prompt_fro": 19.380335, + "heavy_tail_ratio": 1.3891, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.380619, + "mean_fro": 9.067018, + "rms_per_prompt_fro": 13.989733, + "max_per_prompt_fro": 18.982204, + "heavy_tail_ratio": 1.3569, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.192038, + "mean_fro": 8.702469, + "rms_per_prompt_fro": 12.88447, + "max_per_prompt_fro": 17.260042, + "heavy_tail_ratio": 1.3396, + "n_prompts": 50 + }, + "4": { + "dispersion": 0.983, + "mean_fro": 9.935296, + "rms_per_prompt_fro": 13.990787, + "max_per_prompt_fro": 18.704479, + "heavy_tail_ratio": 1.3369, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.902433, + "mean_fro": 10.088699, + "rms_per_prompt_fro": 13.915214, + "max_per_prompt_fro": 18.306854, + "heavy_tail_ratio": 1.3156, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.78874, + "mean_fro": 10.418619, + "rms_per_prompt_fro": 13.934254, + "max_per_prompt_fro": 18.107841, + "heavy_tail_ratio": 1.2995, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.687824, + "mean_fro": 11.512639, + "rms_per_prompt_fro": 14.956791, + "max_per_prompt_fro": 19.004868, + "heavy_tail_ratio": 1.2707, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.618719, + "mean_fro": 13.248595, + "rms_per_prompt_fro": 16.856038, + "max_per_prompt_fro": 21.10013, + "heavy_tail_ratio": 1.2518, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.56565, + "mean_fro": 14.692585, + "rms_per_prompt_fro": 18.384234, + "max_per_prompt_fro": 22.659031, + "heavy_tail_ratio": 1.2325, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.515231, + "mean_fro": 16.609268, + "rms_per_prompt_fro": 20.445129, + "max_per_prompt_fro": 24.784271, + "heavy_tail_ratio": 1.2122, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.47091, + "mean_fro": 19.536711, + "rms_per_prompt_fro": 23.694335, + "max_per_prompt_fro": 28.20462, + "heavy_tail_ratio": 1.1904, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.382845, + "mean_fro": 20.956253, + "rms_per_prompt_fro": 24.643386, + "max_per_prompt_fro": 28.459875, + "heavy_tail_ratio": 1.1549, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.324671, + "mean_fro": 22.635711, + "rms_per_prompt_fro": 26.052427, + "max_per_prompt_fro": 29.533745, + "heavy_tail_ratio": 1.1336, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.274433, + "mean_fro": 26.028427, + "rms_per_prompt_fro": 29.383699, + "max_per_prompt_fro": 33.452335, + "heavy_tail_ratio": 1.1385, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.232881, + "mean_fro": 28.988655, + "rms_per_prompt_fro": 32.187597, + "max_per_prompt_fro": 36.410156, + "heavy_tail_ratio": 1.1312, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.170039, + "mean_fro": 32.103816, + "rms_per_prompt_fro": 34.726171, + "max_per_prompt_fro": 38.488338, + "heavy_tail_ratio": 1.1083, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.126159, + "mean_fro": 34.192063, + "rms_per_prompt_fro": 36.284831, + "max_per_prompt_fro": 39.684338, + "heavy_tail_ratio": 1.0937, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.09196, + "mean_fro": 35.547691, + "rms_per_prompt_fro": 37.146229, + "max_per_prompt_fro": 39.655983, + "heavy_tail_ratio": 1.0676, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.062283, + "mean_fro": 35.494678, + "rms_per_prompt_fro": 36.58334, + "max_per_prompt_fro": 37.943092, + "heavy_tail_ratio": 1.0372, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.037196, + "mean_fro": 35.325703, + "rms_per_prompt_fro": 35.976692, + "max_per_prompt_fro": 37.134029, + "heavy_tail_ratio": 1.0322, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.017904, + "mean_fro": 34.652645, + "rms_per_prompt_fro": 34.961484, + "max_per_prompt_fro": 35.695461, + "heavy_tail_ratio": 1.021, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 137.0, + "fit_secs_per_prompt": 2.74, + "total_wall_seconds": 167.2, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.992, + "top_sv": 2.3465, + "stable_rank": 14.685 + }, + { + "layer": 1, + "fro_norm": 8.4228, + "top_sv": 2.1424, + "stable_rank": 15.457 + }, + { + "layer": 2, + "fro_norm": 9.0669, + "top_sv": 2.383, + "stable_rank": 14.476 + }, + { + "layer": 3, + "fro_norm": 8.7024, + "top_sv": 2.1715, + "stable_rank": 16.06 + }, + { + "layer": 4, + "fro_norm": 9.9352, + "top_sv": 2.4866, + "stable_rank": 15.964 + }, + { + "layer": 5, + "fro_norm": 10.0886, + "top_sv": 2.5724, + "stable_rank": 15.381 + }, + { + "layer": 6, + "fro_norm": 10.4185, + "top_sv": 2.5544, + "stable_rank": 16.635 + }, + { + "layer": 7, + "fro_norm": 11.5125, + "top_sv": 2.7953, + "stable_rank": 16.962 + }, + { + "layer": 8, + "fro_norm": 13.2484, + "top_sv": 3.4061, + "stable_rank": 15.129 + }, + { + "layer": 9, + "fro_norm": 14.6924, + "top_sv": 3.7212, + "stable_rank": 15.589 + }, + { + "layer": 10, + "fro_norm": 16.609, + "top_sv": 4.1939, + "stable_rank": 15.684 + }, + { + "layer": 11, + "fro_norm": 19.5364, + "top_sv": 4.8694, + "stable_rank": 16.096 + }, + { + "layer": 12, + "fro_norm": 20.9559, + "top_sv": 4.7098, + "stable_rank": 19.798 + }, + { + "layer": 13, + "fro_norm": 22.6354, + "top_sv": 5.0857, + "stable_rank": 19.81 + }, + { + "layer": 14, + "fro_norm": 26.028, + "top_sv": 5.7581, + "stable_rank": 20.432 + }, + { + "layer": 15, + "fro_norm": 28.9881, + "top_sv": 6.6243, + "stable_rank": 19.15 + }, + { + "layer": 16, + "fro_norm": 32.1033, + "top_sv": 7.0411, + "stable_rank": 20.788 + }, + { + "layer": 17, + "fro_norm": 34.1913, + "top_sv": 7.5229, + "stable_rank": 20.657 + }, + { + "layer": 18, + "fro_norm": 35.5467, + "top_sv": 8.0049, + "stable_rank": 19.719 + }, + { + "layer": 19, + "fro_norm": 35.4936, + "top_sv": 7.9549, + "stable_rank": 19.908 + }, + { + "layer": 20, + "fro_norm": 35.3245, + "top_sv": 7.0341, + "stable_rank": 25.219 + }, + { + "layer": 21, + "fro_norm": 34.651, + "top_sv": 5.5548, + "stable_rank": 38.913 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_B.pt b/results/lenses/plad/plad_410m_wikitext_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b912bc07bdbdfefa60c5c7b798b9e46bc080fb5 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:861ad01659c21a1eb8a99a381238e0f35dba27470ea5dd082192f947ecc4ab94 +size 46144230 diff --git a/results/lenses/plad/plad_410m_wikitext_B_provenance.json b/results/lenses/plad/plad_410m_wikitext_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..2228bd5bd44a685372741a6fe06703fc47434cf7 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_B_provenance.json @@ -0,0 +1,368 @@ +{ + "artifact": "results/plad_410m_wikitext_B.pt", + "sha256": "861ad01659c21a1eb8a99a381238e0f35dba27470ea5dd082192f947ecc4ab94", + "bytes": 46144230, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 2.262787, + "mean_fro": 8.448131, + "rms_per_prompt_fro": 15.260016, + "max_per_prompt_fro": 20.631985, + "heavy_tail_ratio": 1.352, + "n_prompts": 50 + }, + "1": { + "dispersion": 2.008111, + "mean_fro": 7.942075, + "rms_per_prompt_fro": 13.77466, + "max_per_prompt_fro": 17.883482, + "heavy_tail_ratio": 1.2983, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.63156, + "mean_fro": 8.475485, + "rms_per_prompt_fro": 13.749002, + "max_per_prompt_fro": 18.007618, + "heavy_tail_ratio": 1.3097, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.416819, + "mean_fro": 8.238171, + "rms_per_prompt_fro": 12.80716, + "max_per_prompt_fro": 16.417717, + "heavy_tail_ratio": 1.2819, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.166425, + "mean_fro": 9.305072, + "rms_per_prompt_fro": 13.695929, + "max_per_prompt_fro": 17.213795, + "heavy_tail_ratio": 1.2569, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.050421, + "mean_fro": 9.497901, + "rms_per_prompt_fro": 13.600322, + "max_per_prompt_fro": 17.267988, + "heavy_tail_ratio": 1.2697, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.921921, + "mean_fro": 9.824585, + "rms_per_prompt_fro": 13.620153, + "max_per_prompt_fro": 17.088648, + "heavy_tail_ratio": 1.2547, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.812383, + "mean_fro": 10.877182, + "rms_per_prompt_fro": 14.643381, + "max_per_prompt_fro": 18.057991, + "heavy_tail_ratio": 1.2332, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.733071, + "mean_fro": 12.444842, + "rms_per_prompt_fro": 16.383157, + "max_per_prompt_fro": 19.9904, + "heavy_tail_ratio": 1.2202, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.675077, + "mean_fro": 13.84548, + "rms_per_prompt_fro": 17.919481, + "max_per_prompt_fro": 21.632399, + "heavy_tail_ratio": 1.2072, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.61469, + "mean_fro": 15.645626, + "rms_per_prompt_fro": 19.88097, + "max_per_prompt_fro": 23.683722, + "heavy_tail_ratio": 1.1913, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.567233, + "mean_fro": 18.380819, + "rms_per_prompt_fro": 23.010797, + "max_per_prompt_fro": 27.367882, + "heavy_tail_ratio": 1.1893, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.47531, + "mean_fro": 19.965965, + "rms_per_prompt_fro": 24.25113, + "max_per_prompt_fro": 27.571602, + "heavy_tail_ratio": 1.1369, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.41368, + "mean_fro": 21.60673, + "rms_per_prompt_fro": 25.690027, + "max_per_prompt_fro": 28.954369, + "heavy_tail_ratio": 1.1271, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.359611, + "mean_fro": 24.913553, + "rms_per_prompt_fro": 29.049786, + "max_per_prompt_fro": 32.46352, + "heavy_tail_ratio": 1.1175, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.310808, + "mean_fro": 27.845825, + "rms_per_prompt_fro": 31.880835, + "max_per_prompt_fro": 35.452213, + "heavy_tail_ratio": 1.112, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.236268, + "mean_fro": 31.035448, + "rms_per_prompt_fro": 34.50756, + "max_per_prompt_fro": 37.286232, + "heavy_tail_ratio": 1.0805, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.178283, + "mean_fro": 33.169373, + "rms_per_prompt_fro": 36.004939, + "max_per_prompt_fro": 38.919029, + "heavy_tail_ratio": 1.0809, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.133239, + "mean_fro": 34.800434, + "rms_per_prompt_fro": 37.046352, + "max_per_prompt_fro": 39.91909, + "heavy_tail_ratio": 1.0775, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.089953, + "mean_fro": 35.034473, + "rms_per_prompt_fro": 36.576269, + "max_per_prompt_fro": 38.651585, + "heavy_tail_ratio": 1.0567, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.050832, + "mean_fro": 35.134808, + "rms_per_prompt_fro": 36.016722, + "max_per_prompt_fro": 37.532619, + "heavy_tail_ratio": 1.0421, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.02391, + "mean_fro": 34.61731, + "rms_per_prompt_fro": 35.02872, + "max_per_prompt_fro": 35.946171, + "heavy_tail_ratio": 1.0262, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 142.3, + "fit_secs_per_prompt": 2.845, + "total_wall_seconds": 155.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.448, + "top_sv": 1.8571, + "stable_rank": 20.695 + }, + { + "layer": 1, + "fro_norm": 7.942, + "top_sv": 1.7633, + "stable_rank": 20.287 + }, + { + "layer": 2, + "fro_norm": 8.4754, + "top_sv": 1.8735, + "stable_rank": 20.465 + }, + { + "layer": 3, + "fro_norm": 8.2381, + "top_sv": 1.8502, + "stable_rank": 19.825 + }, + { + "layer": 4, + "fro_norm": 9.305, + "top_sv": 2.0497, + "stable_rank": 20.608 + }, + { + "layer": 5, + "fro_norm": 9.4978, + "top_sv": 2.1957, + "stable_rank": 18.711 + }, + { + "layer": 6, + "fro_norm": 9.8245, + "top_sv": 2.2125, + "stable_rank": 19.717 + }, + { + "layer": 7, + "fro_norm": 10.877, + "top_sv": 2.5127, + "stable_rank": 18.739 + }, + { + "layer": 8, + "fro_norm": 12.4447, + "top_sv": 2.9702, + "stable_rank": 17.555 + }, + { + "layer": 9, + "fro_norm": 13.8453, + "top_sv": 3.312, + "stable_rank": 17.475 + }, + { + "layer": 10, + "fro_norm": 15.6454, + "top_sv": 3.7184, + "stable_rank": 17.704 + }, + { + "layer": 11, + "fro_norm": 18.3805, + "top_sv": 4.2708, + "stable_rank": 18.523 + }, + { + "layer": 12, + "fro_norm": 19.9656, + "top_sv": 4.3598, + "stable_rank": 20.972 + }, + { + "layer": 13, + "fro_norm": 21.6064, + "top_sv": 4.7198, + "stable_rank": 20.957 + }, + { + "layer": 14, + "fro_norm": 24.9131, + "top_sv": 5.367, + "stable_rank": 21.547 + }, + { + "layer": 15, + "fro_norm": 27.8453, + "top_sv": 6.2011, + "stable_rank": 20.164 + }, + { + "layer": 16, + "fro_norm": 31.0349, + "top_sv": 6.6296, + "stable_rank": 21.914 + }, + { + "layer": 17, + "fro_norm": 33.1687, + "top_sv": 7.1543, + "stable_rank": 21.494 + }, + { + "layer": 18, + "fro_norm": 34.7995, + "top_sv": 7.6124, + "stable_rank": 20.898 + }, + { + "layer": 19, + "fro_norm": 35.0334, + "top_sv": 7.6001, + "stable_rank": 21.248 + }, + { + "layer": 20, + "fro_norm": 35.1335, + "top_sv": 7.0494, + "stable_rank": 24.839 + }, + { + "layer": 21, + "fro_norm": 34.6157, + "top_sv": 5.5394, + "stable_rank": 39.05 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_DOCLM_A.pt b/results/lenses/plad/plad_410m_wikitext_DOCLM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..4210bef628afa67124ab672190f2beee7ec033ad --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOCLM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a760b1dca294f1405a990f7a69437f70f9ad10613f9f1f677033d4e0c4e01535 +size 46144450 diff --git a/results/lenses/plad/plad_410m_wikitext_DOCLM_A_provenance.json b/results/lenses/plad/plad_410m_wikitext_DOCLM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..5be09b5ee33ed65486c52dfc311a6ebfe3b37cb6 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOCLM_A_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_wikitext_DOCLM_A.pt", + "sha256": "a760b1dca294f1405a990f7a69437f70f9ad10613f9f1f677033d4e0c4e01535", + "bytes": 46144450, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 2.187799, + "mean_fro": 9.048888, + "rms_per_prompt_fro": 16.156254, + "max_per_prompt_fro": 20.631985, + "heavy_tail_ratio": 1.277, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.919584, + "mean_fro": 8.486632, + "rms_per_prompt_fro": 14.500931, + "max_per_prompt_fro": 17.883482, + "heavy_tail_ratio": 1.2333, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.538186, + "mean_fro": 9.155201, + "rms_per_prompt_fro": 14.585778, + "max_per_prompt_fro": 18.007618, + "heavy_tail_ratio": 1.2346, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.333748, + "mean_fro": 8.769715, + "rms_per_prompt_fro": 13.397151, + "max_per_prompt_fro": 16.417717, + "heavy_tail_ratio": 1.2255, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.089493, + "mean_fro": 9.964831, + "rms_per_prompt_fro": 14.404243, + "max_per_prompt_fro": 17.213795, + "heavy_tail_ratio": 1.1951, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.991723, + "mean_fro": 10.109323, + "rms_per_prompt_fro": 14.267126, + "max_per_prompt_fro": 17.267988, + "heavy_tail_ratio": 1.2103, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.872927, + "mean_fro": 10.409865, + "rms_per_prompt_fro": 14.246412, + "max_per_prompt_fro": 17.088648, + "heavy_tail_ratio": 1.1995, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.769267, + "mean_fro": 11.474767, + "rms_per_prompt_fro": 15.263024, + "max_per_prompt_fro": 18.057991, + "heavy_tail_ratio": 1.1831, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.695729, + "mean_fro": 13.167287, + "rms_per_prompt_fro": 17.146463, + "max_per_prompt_fro": 19.9904, + "heavy_tail_ratio": 1.1659, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.640831, + "mean_fro": 14.576009, + "rms_per_prompt_fro": 18.671128, + "max_per_prompt_fro": 21.632399, + "heavy_tail_ratio": 1.1586, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.584885, + "mean_fro": 16.410717, + "rms_per_prompt_fro": 20.659816, + "max_per_prompt_fro": 23.683722, + "heavy_tail_ratio": 1.1464, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.539176, + "mean_fro": 19.215919, + "rms_per_prompt_fro": 23.839946, + "max_per_prompt_fro": 27.367882, + "heavy_tail_ratio": 1.148, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.445737, + "mean_fro": 20.651506, + "rms_per_prompt_fro": 24.831121, + "max_per_prompt_fro": 27.804264, + "heavy_tail_ratio": 1.1197, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.384259, + "mean_fro": 22.271194, + "rms_per_prompt_fro": 26.203067, + "max_per_prompt_fro": 29.512173, + "heavy_tail_ratio": 1.1263, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.331909, + "mean_fro": 25.655001, + "rms_per_prompt_fro": 29.608013, + "max_per_prompt_fro": 33.452335, + "heavy_tail_ratio": 1.1298, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.284609, + "mean_fro": 28.648777, + "rms_per_prompt_fro": 32.470689, + "max_per_prompt_fro": 36.410156, + "heavy_tail_ratio": 1.1213, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.210025, + "mean_fro": 31.585375, + "rms_per_prompt_fro": 34.744269, + "max_per_prompt_fro": 38.488338, + "heavy_tail_ratio": 1.1078, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.157195, + "mean_fro": 33.706993, + "rms_per_prompt_fro": 36.25963, + "max_per_prompt_fro": 39.684338, + "heavy_tail_ratio": 1.0944, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.115279, + "mean_fro": 35.217308, + "rms_per_prompt_fro": 37.191864, + "max_per_prompt_fro": 39.91909, + "heavy_tail_ratio": 1.0733, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.077264, + "mean_fro": 35.308891, + "rms_per_prompt_fro": 36.647569, + "max_per_prompt_fro": 38.651585, + "heavy_tail_ratio": 1.0547, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.045232, + "mean_fro": 35.316131, + "rms_per_prompt_fro": 36.106006, + "max_per_prompt_fro": 37.532619, + "heavy_tail_ratio": 1.0395, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.021651, + "mean_fro": 34.688195, + "rms_per_prompt_fro": 35.061703, + "max_per_prompt_fro": 35.946171, + "heavy_tail_ratio": 1.0252, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.1, + "fit_secs_per_prompt": 3.022, + "total_wall_seconds": 164.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 9.0488, + "top_sv": 2.2239, + "stable_rank": 16.555 + }, + { + "layer": 1, + "fro_norm": 8.4865, + "top_sv": 2.1034, + "stable_rank": 16.278 + }, + { + "layer": 2, + "fro_norm": 9.1551, + "top_sv": 2.3213, + "stable_rank": 15.554 + }, + { + "layer": 3, + "fro_norm": 8.7696, + "top_sv": 2.1704, + "stable_rank": 16.326 + }, + { + "layer": 4, + "fro_norm": 9.9647, + "top_sv": 2.4152, + "stable_rank": 17.022 + }, + { + "layer": 5, + "fro_norm": 10.1092, + "top_sv": 2.5156, + "stable_rank": 16.149 + }, + { + "layer": 6, + "fro_norm": 10.4097, + "top_sv": 2.4666, + "stable_rank": 17.811 + }, + { + "layer": 7, + "fro_norm": 11.4746, + "top_sv": 2.7233, + "stable_rank": 17.753 + }, + { + "layer": 8, + "fro_norm": 13.1671, + "top_sv": 3.2713, + "stable_rank": 16.201 + }, + { + "layer": 9, + "fro_norm": 14.5758, + "top_sv": 3.6297, + "stable_rank": 16.126 + }, + { + "layer": 10, + "fro_norm": 16.4105, + "top_sv": 4.0658, + "stable_rank": 16.291 + }, + { + "layer": 11, + "fro_norm": 19.2156, + "top_sv": 4.6948, + "stable_rank": 16.752 + }, + { + "layer": 12, + "fro_norm": 20.6512, + "top_sv": 4.6239, + "stable_rank": 19.947 + }, + { + "layer": 13, + "fro_norm": 22.2709, + "top_sv": 4.9683, + "stable_rank": 20.094 + }, + { + "layer": 14, + "fro_norm": 25.6545, + "top_sv": 5.6549, + "stable_rank": 20.581 + }, + { + "layer": 15, + "fro_norm": 28.6483, + "top_sv": 6.5354, + "stable_rank": 19.215 + }, + { + "layer": 16, + "fro_norm": 31.5848, + "top_sv": 6.8618, + "stable_rank": 21.187 + }, + { + "layer": 17, + "fro_norm": 33.7062, + "top_sv": 7.3782, + "stable_rank": 20.87 + }, + { + "layer": 18, + "fro_norm": 35.2164, + "top_sv": 7.9418, + "stable_rank": 19.663 + }, + { + "layer": 19, + "fro_norm": 35.3078, + "top_sv": 7.8537, + "stable_rank": 20.211 + }, + { + "layer": 20, + "fro_norm": 35.3149, + "top_sv": 7.1331, + "stable_rank": 24.511 + }, + { + "layer": 21, + "fro_norm": 34.6866, + "top_sv": 5.6477, + "stable_rank": 37.72 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_DOCLM_B.pt b/results/lenses/plad/plad_410m_wikitext_DOCLM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..f96ccb36d9e2f78305f0962dc15236651645b8c6 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOCLM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba3687fc3a4c388be6e57f1448e190c27f0b17f967b598e1d8f13c3358f1bea +size 46144450 diff --git a/results/lenses/plad/plad_410m_wikitext_DOCLM_B_provenance.json b/results/lenses/plad/plad_410m_wikitext_DOCLM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..50b82a6522336b5d278d3bf184524c0227e54a93 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOCLM_B_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_wikitext_DOCLM_B.pt", + "sha256": "bba3687fc3a4c388be6e57f1448e190c27f0b17f967b598e1d8f13c3358f1bea", + "bytes": 46144450, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 2.037904, + "mean_fro": 8.559522, + "rms_per_prompt_fro": 14.91889, + "max_per_prompt_fro": 21.35177, + "heavy_tail_ratio": 1.4312, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.804103, + "mean_fro": 8.036702, + "rms_per_prompt_fro": 13.457824, + "max_per_prompt_fro": 17.323643, + "heavy_tail_ratio": 1.2873, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.465344, + "mean_fro": 8.610519, + "rms_per_prompt_fro": 13.519732, + "max_per_prompt_fro": 16.628773, + "heavy_tail_ratio": 1.23, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.266097, + "mean_fro": 8.335649, + "rms_per_prompt_fro": 12.548121, + "max_per_prompt_fro": 14.986617, + "heavy_tail_ratio": 1.1943, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.058076, + "mean_fro": 9.475047, + "rms_per_prompt_fro": 13.592897, + "max_per_prompt_fro": 16.355455, + "heavy_tail_ratio": 1.2032, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.959563, + "mean_fro": 9.662471, + "rms_per_prompt_fro": 13.525951, + "max_per_prompt_fro": 16.492722, + "heavy_tail_ratio": 1.2193, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.844811, + "mean_fro": 9.9968, + "rms_per_prompt_fro": 13.578035, + "max_per_prompt_fro": 16.538963, + "heavy_tail_ratio": 1.2181, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.743869, + "mean_fro": 11.073485, + "rms_per_prompt_fro": 14.62316, + "max_per_prompt_fro": 17.367277, + "heavy_tail_ratio": 1.1877, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.675322, + "mean_fro": 12.699221, + "rms_per_prompt_fro": 16.437139, + "max_per_prompt_fro": 20.027235, + "heavy_tail_ratio": 1.2184, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.623858, + "mean_fro": 14.093328, + "rms_per_prompt_fro": 17.959225, + "max_per_prompt_fro": 21.595469, + "heavy_tail_ratio": 1.2025, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.571877, + "mean_fro": 15.932646, + "rms_per_prompt_fro": 19.97548, + "max_per_prompt_fro": 24.116192, + "heavy_tail_ratio": 1.2073, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.529095, + "mean_fro": 18.714033, + "rms_per_prompt_fro": 23.141132, + "max_per_prompt_fro": 27.891216, + "heavy_tail_ratio": 1.2053, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.437, + "mean_fro": 20.215404, + "rms_per_prompt_fro": 24.2332, + "max_per_prompt_fro": 26.56078, + "heavy_tail_ratio": 1.096, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.379118, + "mean_fro": 21.831562, + "rms_per_prompt_fro": 25.638079, + "max_per_prompt_fro": 27.830687, + "heavy_tail_ratio": 1.0855, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.329493, + "mean_fro": 25.16424, + "rms_per_prompt_fro": 29.01529, + "max_per_prompt_fro": 31.442699, + "heavy_tail_ratio": 1.0837, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.283382, + "mean_fro": 28.109302, + "rms_per_prompt_fro": 31.844035, + "max_per_prompt_fro": 34.580444, + "heavy_tail_ratio": 1.0859, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.216106, + "mean_fro": 31.264731, + "rms_per_prompt_fro": 34.477865, + "max_per_prompt_fro": 36.817612, + "heavy_tail_ratio": 1.0679, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.161947, + "mean_fro": 33.325828, + "rms_per_prompt_fro": 35.923126, + "max_per_prompt_fro": 37.795784, + "heavy_tail_ratio": 1.0521, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.120646, + "mean_fro": 34.831428, + "rms_per_prompt_fro": 36.872755, + "max_per_prompt_fro": 38.917923, + "heavy_tail_ratio": 1.0555, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.083058, + "mean_fro": 34.982487, + "rms_per_prompt_fro": 36.406307, + "max_per_prompt_fro": 37.953892, + "heavy_tail_ratio": 1.0425, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.047142, + "mean_fro": 35.101165, + "rms_per_prompt_fro": 35.919004, + "max_per_prompt_fro": 37.10667, + "heavy_tail_ratio": 1.0331, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.021983, + "mean_fro": 34.5812, + "rms_per_prompt_fro": 34.959231, + "max_per_prompt_fro": 35.733311, + "heavy_tail_ratio": 1.0221, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.7, + "fit_secs_per_prompt": 3.034, + "total_wall_seconds": 164.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.5594, + "top_sv": 1.9139, + "stable_rank": 20.001 + }, + { + "layer": 1, + "fro_norm": 8.0366, + "top_sv": 1.8229, + "stable_rank": 19.437 + }, + { + "layer": 2, + "fro_norm": 8.6104, + "top_sv": 1.9473, + "stable_rank": 19.552 + }, + { + "layer": 3, + "fro_norm": 8.3355, + "top_sv": 1.9266, + "stable_rank": 18.719 + }, + { + "layer": 4, + "fro_norm": 9.4749, + "top_sv": 2.1502, + "stable_rank": 19.417 + }, + { + "layer": 5, + "fro_norm": 9.6623, + "top_sv": 2.3067, + "stable_rank": 17.546 + }, + { + "layer": 6, + "fro_norm": 9.9967, + "top_sv": 2.3459, + "stable_rank": 18.16 + }, + { + "layer": 7, + "fro_norm": 11.0733, + "top_sv": 2.6433, + "stable_rank": 17.55 + }, + { + "layer": 8, + "fro_norm": 12.6991, + "top_sv": 3.1131, + "stable_rank": 16.641 + }, + { + "layer": 9, + "fro_norm": 14.0931, + "top_sv": 3.4107, + "stable_rank": 17.073 + }, + { + "layer": 10, + "fro_norm": 15.9325, + "top_sv": 3.8702, + "stable_rank": 16.948 + }, + { + "layer": 11, + "fro_norm": 18.7137, + "top_sv": 4.4315, + "stable_rank": 17.833 + }, + { + "layer": 12, + "fro_norm": 20.2151, + "top_sv": 4.4225, + "stable_rank": 20.894 + }, + { + "layer": 13, + "fro_norm": 21.8312, + "top_sv": 4.7408, + "stable_rank": 21.206 + }, + { + "layer": 14, + "fro_norm": 25.1638, + "top_sv": 5.3724, + "stable_rank": 21.939 + }, + { + "layer": 15, + "fro_norm": 28.1088, + "top_sv": 6.2178, + "stable_rank": 20.437 + }, + { + "layer": 16, + "fro_norm": 31.2642, + "top_sv": 6.5695, + "stable_rank": 22.648 + }, + { + "layer": 17, + "fro_norm": 33.3251, + "top_sv": 7.0379, + "stable_rank": 22.421 + }, + { + "layer": 18, + "fro_norm": 34.8305, + "top_sv": 7.4483, + "stable_rank": 21.868 + }, + { + "layer": 19, + "fro_norm": 34.9814, + "top_sv": 7.4879, + "stable_rank": 21.825 + }, + { + "layer": 20, + "fro_norm": 35.0999, + "top_sv": 6.9266, + "stable_rank": 25.679 + }, + { + "layer": 21, + "fro_norm": 34.5796, + "top_sv": 5.4549, + "stable_rank": 40.185 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_DOC_A.pt b/results/lenses/plad/plad_410m_wikitext_DOC_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..31d0ec23a7bc6eb4f9e177b3e6c5f421bbda92e9 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOC_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2024766c135a89bebf8fa59a97d1b36e0244360e029b65efbe5fc3f553afa6 +size 46144398 diff --git a/results/lenses/plad/plad_410m_wikitext_DOC_A_provenance.json b/results/lenses/plad/plad_410m_wikitext_DOC_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..2574c3c330f607a4e48aaa074b0dd89002dd102f --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOC_A_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_wikitext_DOC_A.pt", + "sha256": "3c2024766c135a89bebf8fa59a97d1b36e0244360e029b65efbe5fc3f553afa6", + "bytes": 46144398, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 2.136649, + "mean_fro": 8.807482, + "rms_per_prompt_fro": 15.598567, + "max_per_prompt_fro": 20.631985, + "heavy_tail_ratio": 1.3227, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.88796, + "mean_fro": 8.276729, + "rms_per_prompt_fro": 14.065473, + "max_per_prompt_fro": 17.883482, + "heavy_tail_ratio": 1.2714, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.498044, + "mean_fro": 8.924908, + "rms_per_prompt_fro": 14.105997, + "max_per_prompt_fro": 18.007618, + "heavy_tail_ratio": 1.2766, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.292482, + "mean_fro": 8.585239, + "rms_per_prompt_fro": 12.998861, + "max_per_prompt_fro": 16.417717, + "heavy_tail_ratio": 1.263, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.05476, + "mean_fro": 9.749324, + "rms_per_prompt_fro": 13.975103, + "max_per_prompt_fro": 17.213795, + "heavy_tail_ratio": 1.2317, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.960862, + "mean_fro": 9.919084, + "rms_per_prompt_fro": 13.889772, + "max_per_prompt_fro": 17.267988, + "heavy_tail_ratio": 1.2432, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.840183, + "mean_fro": 10.244503, + "rms_per_prompt_fro": 13.897013, + "max_per_prompt_fro": 17.088648, + "heavy_tail_ratio": 1.2297, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.735404, + "mean_fro": 11.311346, + "rms_per_prompt_fro": 14.900972, + "max_per_prompt_fro": 18.057991, + "heavy_tail_ratio": 1.2119, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.664197, + "mean_fro": 12.967704, + "rms_per_prompt_fro": 16.728827, + "max_per_prompt_fro": 19.9904, + "heavy_tail_ratio": 1.195, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.608022, + "mean_fro": 14.387385, + "rms_per_prompt_fro": 18.24433, + "max_per_prompt_fro": 21.632399, + "heavy_tail_ratio": 1.1857, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.551963, + "mean_fro": 16.250801, + "rms_per_prompt_fro": 20.244893, + "max_per_prompt_fro": 23.683722, + "heavy_tail_ratio": 1.1699, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.506125, + "mean_fro": 19.070685, + "rms_per_prompt_fro": 23.404357, + "max_per_prompt_fro": 27.367882, + "heavy_tail_ratio": 1.1693, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.41586, + "mean_fro": 20.564282, + "rms_per_prompt_fro": 24.469424, + "max_per_prompt_fro": 27.804264, + "heavy_tail_ratio": 1.1363, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.357718, + "mean_fro": 22.234224, + "rms_per_prompt_fro": 25.907573, + "max_per_prompt_fro": 29.512173, + "heavy_tail_ratio": 1.1391, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.307269, + "mean_fro": 25.563423, + "rms_per_prompt_fro": 29.22816, + "max_per_prompt_fro": 33.452335, + "heavy_tail_ratio": 1.1445, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.263003, + "mean_fro": 28.510458, + "rms_per_prompt_fro": 32.041028, + "max_per_prompt_fro": 36.410156, + "heavy_tail_ratio": 1.1364, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.193662, + "mean_fro": 31.578199, + "rms_per_prompt_fro": 34.500711, + "max_per_prompt_fro": 38.488338, + "heavy_tail_ratio": 1.1156, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.145914, + "mean_fro": 33.737575, + "rms_per_prompt_fro": 36.115191, + "max_per_prompt_fro": 39.684338, + "heavy_tail_ratio": 1.0988, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.106424, + "mean_fro": 35.174858, + "rms_per_prompt_fro": 36.999276, + "max_per_prompt_fro": 39.91909, + "heavy_tail_ratio": 1.0789, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.070909, + "mean_fro": 35.267247, + "rms_per_prompt_fro": 36.496219, + "max_per_prompt_fro": 38.651585, + "heavy_tail_ratio": 1.0591, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.042023, + "mean_fro": 35.22718, + "rms_per_prompt_fro": 35.959735, + "max_per_prompt_fro": 37.532619, + "heavy_tail_ratio": 1.0437, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.020069, + "mean_fro": 34.628391, + "rms_per_prompt_fro": 34.974152, + "max_per_prompt_fro": 35.946171, + "heavy_tail_ratio": 1.0278, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 139.3, + "fit_secs_per_prompt": 2.786, + "total_wall_seconds": 151.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.8074, + "top_sv": 2.0232, + "stable_rank": 18.951 + }, + { + "layer": 1, + "fro_norm": 8.2766, + "top_sv": 1.9022, + "stable_rank": 18.932 + }, + { + "layer": 2, + "fro_norm": 8.9248, + "top_sv": 2.1423, + "stable_rank": 17.356 + }, + { + "layer": 3, + "fro_norm": 8.5851, + "top_sv": 2.0068, + "stable_rank": 18.301 + }, + { + "layer": 4, + "fro_norm": 9.7492, + "top_sv": 2.2414, + "stable_rank": 18.919 + }, + { + "layer": 5, + "fro_norm": 9.9189, + "top_sv": 2.3382, + "stable_rank": 17.996 + }, + { + "layer": 6, + "fro_norm": 10.2444, + "top_sv": 2.3273, + "stable_rank": 19.375 + }, + { + "layer": 7, + "fro_norm": 11.3112, + "top_sv": 2.582, + "stable_rank": 19.191 + }, + { + "layer": 8, + "fro_norm": 12.9675, + "top_sv": 3.1063, + "stable_rank": 17.427 + }, + { + "layer": 9, + "fro_norm": 14.3872, + "top_sv": 3.4664, + "stable_rank": 17.226 + }, + { + "layer": 10, + "fro_norm": 16.2506, + "top_sv": 3.8744, + "stable_rank": 17.593 + }, + { + "layer": 11, + "fro_norm": 19.0704, + "top_sv": 4.4731, + "stable_rank": 18.176 + }, + { + "layer": 12, + "fro_norm": 20.5639, + "top_sv": 4.4925, + "stable_rank": 20.953 + }, + { + "layer": 13, + "fro_norm": 22.2339, + "top_sv": 4.858, + "stable_rank": 20.947 + }, + { + "layer": 14, + "fro_norm": 25.5629, + "top_sv": 5.5285, + "stable_rank": 21.38 + }, + { + "layer": 15, + "fro_norm": 28.51, + "top_sv": 6.3643, + "stable_rank": 20.067 + }, + { + "layer": 16, + "fro_norm": 31.5777, + "top_sv": 6.8174, + "stable_rank": 21.455 + }, + { + "layer": 17, + "fro_norm": 33.7368, + "top_sv": 7.325, + "stable_rank": 21.213 + }, + { + "layer": 18, + "fro_norm": 35.1739, + "top_sv": 7.8687, + "stable_rank": 19.982 + }, + { + "layer": 19, + "fro_norm": 35.2661, + "top_sv": 7.7603, + "stable_rank": 20.652 + }, + { + "layer": 20, + "fro_norm": 35.2259, + "top_sv": 7.036, + "stable_rank": 25.065 + }, + { + "layer": 21, + "fro_norm": 34.6268, + "top_sv": 5.5831, + "stable_rank": 38.465 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_DOC_B.pt b/results/lenses/plad/plad_410m_wikitext_DOC_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..373d2d0c3f505be7fc0685ac7fc16433a6e57121 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOC_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a2ff99b4360f9638c47c981653b54e3a1bcab11455bdf84d0319862280b1c77 +size 46144398 diff --git a/results/lenses/plad/plad_410m_wikitext_DOC_B_provenance.json b/results/lenses/plad/plad_410m_wikitext_DOC_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b3049ad14427d64363f9a9e9a4c3a2fca038575d --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_DOC_B_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_wikitext_DOC_B.pt", + "sha256": "7a2ff99b4360f9638c47c981653b54e3a1bcab11455bdf84d0319862280b1c77", + "bytes": 46144398, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 2.073975, + "mean_fro": 8.698249, + "rms_per_prompt_fro": 15.250427, + "max_per_prompt_fro": 21.35177, + "heavy_tail_ratio": 1.4001, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.822675, + "mean_fro": 8.152827, + "rms_per_prompt_fro": 13.697416, + "max_per_prompt_fro": 19.380335, + "heavy_tail_ratio": 1.4149, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.472058, + "mean_fro": 8.688536, + "rms_per_prompt_fro": 13.660794, + "max_per_prompt_fro": 18.982204, + "heavy_tail_ratio": 1.3895, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.283329, + "mean_fro": 8.423296, + "rms_per_prompt_fro": 12.72818, + "max_per_prompt_fro": 17.260042, + "heavy_tail_ratio": 1.356, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.066174, + "mean_fro": 9.567679, + "rms_per_prompt_fro": 13.752766, + "max_per_prompt_fro": 18.704479, + "heavy_tail_ratio": 1.3601, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.966482, + "mean_fro": 9.744887, + "rms_per_prompt_fro": 13.665381, + "max_per_prompt_fro": 18.306854, + "heavy_tail_ratio": 1.3397, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.847748, + "mean_fro": 10.074415, + "rms_per_prompt_fro": 13.694344, + "max_per_prompt_fro": 18.107841, + "heavy_tail_ratio": 1.3223, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.743635, + "mean_fro": 11.159088, + "rms_per_prompt_fro": 14.735217, + "max_per_prompt_fro": 19.004868, + "heavy_tail_ratio": 1.2898, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.668704, + "mean_fro": 12.806542, + "rms_per_prompt_fro": 16.543279, + "max_per_prompt_fro": 21.10013, + "heavy_tail_ratio": 1.2755, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.614744, + "mean_fro": 14.233424, + "rms_per_prompt_fro": 18.086777, + "max_per_prompt_fro": 22.659031, + "heavy_tail_ratio": 1.2528, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.560868, + "mean_fro": 16.095503, + "rms_per_prompt_fro": 20.108871, + "max_per_prompt_fro": 24.784271, + "heavy_tail_ratio": 1.2325, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.51569, + "mean_fro": 18.947119, + "rms_per_prompt_fro": 23.326436, + "max_per_prompt_fro": 28.20462, + "heavy_tail_ratio": 1.2091, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.429423, + "mean_fro": 20.458923, + "rms_per_prompt_fro": 24.46038, + "max_per_prompt_fro": 28.459875, + "heavy_tail_ratio": 1.1635, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.370133, + "mean_fro": 22.098311, + "rms_per_prompt_fro": 25.866661, + "max_per_prompt_fro": 29.533745, + "heavy_tail_ratio": 1.1418, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.317998, + "mean_fro": 25.470777, + "rms_per_prompt_fro": 29.241495, + "max_per_prompt_fro": 32.671421, + "heavy_tail_ratio": 1.1173, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.273211, + "mean_fro": 28.417516, + "rms_per_prompt_fro": 32.065375, + "max_per_prompt_fro": 35.509422, + "heavy_tail_ratio": 1.1074, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.207122, + "mean_fro": 31.652412, + "rms_per_prompt_fro": 34.776215, + "max_per_prompt_fro": 37.117622, + "heavy_tail_ratio": 1.0673, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.155625, + "mean_fro": 33.693096, + "rms_per_prompt_fro": 36.220083, + "max_per_prompt_fro": 38.161434, + "heavy_tail_ratio": 1.0536, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.117645, + "mean_fro": 35.207352, + "rms_per_prompt_fro": 37.220766, + "max_per_prompt_fro": 38.917923, + "heavy_tail_ratio": 1.0456, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.081022, + "mean_fro": 35.28263, + "rms_per_prompt_fro": 36.684124, + "max_per_prompt_fro": 37.953892, + "heavy_tail_ratio": 1.0346, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.045164, + "mean_fro": 35.264107, + "rms_per_prompt_fro": 36.051651, + "max_per_prompt_fro": 37.10667, + "heavy_tail_ratio": 1.0293, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.021283, + "mean_fro": 34.658383, + "rms_per_prompt_fro": 35.025253, + "max_per_prompt_fro": 35.733311, + "heavy_tail_ratio": 1.0202, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 140.3, + "fit_secs_per_prompt": 2.806, + "total_wall_seconds": 151.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.6981, + "top_sv": 2.1419, + "stable_rank": 16.492 + }, + { + "layer": 1, + "fro_norm": 8.1527, + "top_sv": 1.9669, + "stable_rank": 17.181 + }, + { + "layer": 2, + "fro_norm": 8.6884, + "top_sv": 2.0737, + "stable_rank": 17.554 + }, + { + "layer": 3, + "fro_norm": 8.4232, + "top_sv": 1.9861, + "stable_rank": 17.987 + }, + { + "layer": 4, + "fro_norm": 9.5676, + "top_sv": 2.2455, + "stable_rank": 18.155 + }, + { + "layer": 5, + "fro_norm": 9.7448, + "top_sv": 2.3841, + "stable_rank": 16.707 + }, + { + "layer": 6, + "fro_norm": 10.0743, + "top_sv": 2.3956, + "stable_rank": 17.685 + }, + { + "layer": 7, + "fro_norm": 11.159, + "top_sv": 2.6986, + "stable_rank": 17.099 + }, + { + "layer": 8, + "fro_norm": 12.8064, + "top_sv": 3.2141, + "stable_rank": 15.876 + }, + { + "layer": 9, + "fro_norm": 14.2332, + "top_sv": 3.5189, + "stable_rank": 16.36 + }, + { + "layer": 10, + "fro_norm": 16.0953, + "top_sv": 3.9777, + "stable_rank": 16.373 + }, + { + "layer": 11, + "fro_norm": 18.9468, + "top_sv": 4.6009, + "stable_rank": 16.958 + }, + { + "layer": 12, + "fro_norm": 20.4586, + "top_sv": 4.5673, + "stable_rank": 20.065 + }, + { + "layer": 13, + "fro_norm": 22.098, + "top_sv": 4.9329, + "stable_rank": 20.068 + }, + { + "layer": 14, + "fro_norm": 25.4703, + "top_sv": 5.5719, + "stable_rank": 20.896 + }, + { + "layer": 15, + "fro_norm": 28.417, + "top_sv": 6.4294, + "stable_rank": 19.535 + }, + { + "layer": 16, + "fro_norm": 31.6519, + "top_sv": 6.8331, + "stable_rank": 21.457 + }, + { + "layer": 17, + "fro_norm": 33.6923, + "top_sv": 7.3118, + "stable_rank": 21.233 + }, + { + "layer": 18, + "fro_norm": 35.2064, + "top_sv": 7.7042, + "stable_rank": 20.882 + }, + { + "layer": 19, + "fro_norm": 35.2816, + "top_sv": 7.7494, + "stable_rank": 20.728 + }, + { + "layer": 20, + "fro_norm": 35.2628, + "top_sv": 7.048, + "stable_rank": 25.032 + }, + { + "layer": 21, + "fro_norm": 34.6568, + "top_sv": 5.5112, + "stable_rank": 39.544 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_LM_A.pt b/results/lenses/plad/plad_410m_wikitext_LM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..096bdb101bb7c1a00d580d0d28ac425d8bfc380b --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_LM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d895cafeba61e2bcb3e831054cc23288f7f6af704ba289fa9304fa9d1fa013d4 +size 46144372 diff --git a/results/lenses/plad/plad_410m_wikitext_LM_A_provenance.json b/results/lenses/plad/plad_410m_wikitext_LM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..40875762638c918167751c0f9ab9243de24056e1 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_LM_A_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_wikitext_LM_A.pt", + "sha256": "d895cafeba61e2bcb3e831054cc23288f7f6af704ba289fa9304fa9d1fa013d4", + "bytes": 46144372, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 2.211806, + "mean_fro": 8.858653, + "rms_per_prompt_fro": 15.876045, + "max_per_prompt_fro": 21.35177, + "heavy_tail_ratio": 1.3449, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.9308, + "mean_fro": 8.307793, + "rms_per_prompt_fro": 14.222591, + "max_per_prompt_fro": 17.883482, + "heavy_tail_ratio": 1.2574, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.549866, + "mean_fro": 8.954554, + "rms_per_prompt_fro": 14.298902, + "max_per_prompt_fro": 18.007618, + "heavy_tail_ratio": 1.2594, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.342848, + "mean_fro": 8.609469, + "rms_per_prompt_fro": 13.177967, + "max_per_prompt_fro": 16.417717, + "heavy_tail_ratio": 1.2458, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.100248, + "mean_fro": 9.812578, + "rms_per_prompt_fro": 14.220615, + "max_per_prompt_fro": 17.213795, + "heavy_tail_ratio": 1.2105, + "n_prompts": 50 + }, + "5": { + "dispersion": 1.004372, + "mean_fro": 9.978725, + "rms_per_prompt_fro": 14.127466, + "max_per_prompt_fro": 17.267988, + "heavy_tail_ratio": 1.2223, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.879511, + "mean_fro": 10.319172, + "rms_per_prompt_fro": 14.147094, + "max_per_prompt_fro": 17.088648, + "heavy_tail_ratio": 1.2079, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.770458, + "mean_fro": 11.403305, + "rms_per_prompt_fro": 15.173072, + "max_per_prompt_fro": 18.057991, + "heavy_tail_ratio": 1.1901, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.697508, + "mean_fro": 13.098725, + "rms_per_prompt_fro": 17.066127, + "max_per_prompt_fro": 20.027235, + "heavy_tail_ratio": 1.1735, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.63995, + "mean_fro": 14.520549, + "rms_per_prompt_fro": 18.595091, + "max_per_prompt_fro": 21.632399, + "heavy_tail_ratio": 1.1633, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.584646, + "mean_fro": 16.38174, + "rms_per_prompt_fro": 20.621781, + "max_per_prompt_fro": 24.116192, + "heavy_tail_ratio": 1.1695, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.539552, + "mean_fro": 19.208283, + "rms_per_prompt_fro": 23.833386, + "max_per_prompt_fro": 27.891216, + "heavy_tail_ratio": 1.1703, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.443655, + "mean_fro": 20.624953, + "rms_per_prompt_fro": 24.781329, + "max_per_prompt_fro": 27.804264, + "heavy_tail_ratio": 1.122, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.383726, + "mean_fro": 22.266817, + "rms_per_prompt_fro": 26.192877, + "max_per_prompt_fro": 29.512173, + "heavy_tail_ratio": 1.1267, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.331581, + "mean_fro": 25.590187, + "rms_per_prompt_fro": 29.529574, + "max_per_prompt_fro": 33.452335, + "heavy_tail_ratio": 1.1328, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.285007, + "mean_fro": 28.546339, + "rms_per_prompt_fro": 32.359604, + "max_per_prompt_fro": 36.410156, + "heavy_tail_ratio": 1.1252, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.212229, + "mean_fro": 31.652396, + "rms_per_prompt_fro": 34.849692, + "max_per_prompt_fro": 38.488338, + "heavy_tail_ratio": 1.1044, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.159045, + "mean_fro": 33.766834, + "rms_per_prompt_fro": 36.353013, + "max_per_prompt_fro": 39.684338, + "heavy_tail_ratio": 1.0916, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.117488, + "mean_fro": 35.288406, + "rms_per_prompt_fro": 37.303841, + "max_per_prompt_fro": 39.91909, + "heavy_tail_ratio": 1.0701, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.079607, + "mean_fro": 35.353721, + "rms_per_prompt_fro": 36.733975, + "max_per_prompt_fro": 38.651585, + "heavy_tail_ratio": 1.0522, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.046195, + "mean_fro": 35.313565, + "rms_per_prompt_fro": 36.120011, + "max_per_prompt_fro": 37.532619, + "heavy_tail_ratio": 1.0391, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.022056, + "mean_fro": 34.682613, + "rms_per_prompt_fro": 35.063005, + "max_per_prompt_fro": 35.946171, + "heavy_tail_ratio": 1.0252, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 150.6, + "fit_secs_per_prompt": 3.012, + "total_wall_seconds": 161.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.8585, + "top_sv": 2.0814, + "stable_rank": 18.114 + }, + { + "layer": 1, + "fro_norm": 8.3077, + "top_sv": 1.9739, + "stable_rank": 17.715 + }, + { + "layer": 2, + "fro_norm": 8.9544, + "top_sv": 2.1975, + "stable_rank": 16.604 + }, + { + "layer": 3, + "fro_norm": 8.6094, + "top_sv": 2.0547, + "stable_rank": 17.557 + }, + { + "layer": 4, + "fro_norm": 9.8124, + "top_sv": 2.3122, + "stable_rank": 18.01 + }, + { + "layer": 5, + "fro_norm": 9.9786, + "top_sv": 2.4473, + "stable_rank": 16.625 + }, + { + "layer": 6, + "fro_norm": 10.319, + "top_sv": 2.4551, + "stable_rank": 17.665 + }, + { + "layer": 7, + "fro_norm": 11.4032, + "top_sv": 2.7251, + "stable_rank": 17.511 + }, + { + "layer": 8, + "fro_norm": 13.0985, + "top_sv": 3.2652, + "stable_rank": 16.092 + }, + { + "layer": 9, + "fro_norm": 14.5204, + "top_sv": 3.6034, + "stable_rank": 16.238 + }, + { + "layer": 10, + "fro_norm": 16.3815, + "top_sv": 4.0565, + "stable_rank": 16.308 + }, + { + "layer": 11, + "fro_norm": 19.208, + "top_sv": 4.6747, + "stable_rank": 16.883 + }, + { + "layer": 12, + "fro_norm": 20.6247, + "top_sv": 4.5813, + "stable_rank": 20.267 + }, + { + "layer": 13, + "fro_norm": 22.2665, + "top_sv": 4.9471, + "stable_rank": 20.258 + }, + { + "layer": 14, + "fro_norm": 25.5897, + "top_sv": 5.5981, + "stable_rank": 20.895 + }, + { + "layer": 15, + "fro_norm": 28.5459, + "top_sv": 6.4469, + "stable_rank": 19.606 + }, + { + "layer": 16, + "fro_norm": 31.6519, + "top_sv": 6.8563, + "stable_rank": 21.312 + }, + { + "layer": 17, + "fro_norm": 33.7661, + "top_sv": 7.3573, + "stable_rank": 21.063 + }, + { + "layer": 18, + "fro_norm": 35.2874, + "top_sv": 7.9146, + "stable_rank": 19.879 + }, + { + "layer": 19, + "fro_norm": 35.3526, + "top_sv": 7.8494, + "stable_rank": 20.285 + }, + { + "layer": 20, + "fro_norm": 35.3123, + "top_sv": 7.0595, + "stable_rank": 25.021 + }, + { + "layer": 21, + "fro_norm": 34.681, + "top_sv": 5.5843, + "stable_rank": 38.57 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_410m_wikitext_LM_B.pt b/results/lenses/plad/plad_410m_wikitext_LM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..e32e3726d71cebed8f5832765f67106b4a43d377 --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_LM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab6958198f5c001301c699364ce3018947eccd7a5311ff8d1e607da626a3ec2 +size 46144372 diff --git a/results/lenses/plad/plad_410m_wikitext_LM_B_provenance.json b/results/lenses/plad/plad_410m_wikitext_LM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d5f30cfe3e63b52e54e6aebecadc178a31fafb7b --- /dev/null +++ b/results/lenses/plad/plad_410m_wikitext_LM_B_provenance.json @@ -0,0 +1,369 @@ +{ + "artifact": "results/plad_410m_wikitext_LM_B.pt", + "sha256": "8ab6958198f5c001301c699364ce3018947eccd7a5311ff8d1e607da626a3ec2", + "bytes": 46144372, + "model": "EleutherAI/pythia-410m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 22, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 24, + "d_model": 1024, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 2.062986, + "mean_fro": 8.694588, + "rms_per_prompt_fro": 15.216735, + "max_per_prompt_fro": 19.283012, + "heavy_tail_ratio": 1.2672, + "n_prompts": 50 + }, + "1": { + "dispersion": 1.83745, + "mean_fro": 8.16377, + "rms_per_prompt_fro": 13.751652, + "max_per_prompt_fro": 17.181873, + "heavy_tail_ratio": 1.2494, + "n_prompts": 50 + }, + "2": { + "dispersion": 1.49072, + "mean_fro": 8.75857, + "rms_per_prompt_fro": 13.822789, + "max_per_prompt_fro": 16.670076, + "heavy_tail_ratio": 1.206, + "n_prompts": 50 + }, + "3": { + "dispersion": 1.288729, + "mean_fro": 8.446359, + "rms_per_prompt_fro": 12.778113, + "max_per_prompt_fro": 15.261189, + "heavy_tail_ratio": 1.1943, + "n_prompts": 50 + }, + "4": { + "dispersion": 1.070952, + "mean_fro": 9.578958, + "rms_per_prompt_fro": 13.78489, + "max_per_prompt_fro": 16.358732, + "heavy_tail_ratio": 1.1867, + "n_prompts": 50 + }, + "5": { + "dispersion": 0.966613, + "mean_fro": 9.749108, + "rms_per_prompt_fro": 13.671757, + "max_per_prompt_fro": 16.238359, + "heavy_tail_ratio": 1.1877, + "n_prompts": 50 + }, + "6": { + "dispersion": 0.854998, + "mean_fro": 10.045268, + "rms_per_prompt_fro": 13.681485, + "max_per_prompt_fro": 16.175442, + "heavy_tail_ratio": 1.1823, + "n_prompts": 50 + }, + "7": { + "dispersion": 0.756658, + "mean_fro": 11.103506, + "rms_per_prompt_fro": 14.716474, + "max_per_prompt_fro": 17.325182, + "heavy_tail_ratio": 1.1773, + "n_prompts": 50 + }, + "8": { + "dispersion": 0.686314, + "mean_fro": 12.721985, + "rms_per_prompt_fro": 16.520534, + "max_per_prompt_fro": 19.262226, + "heavy_tail_ratio": 1.166, + "n_prompts": 50 + }, + "9": { + "dispersion": 0.635931, + "mean_fro": 14.102774, + "rms_per_prompt_fro": 18.037943, + "max_per_prompt_fro": 20.69235, + "heavy_tail_ratio": 1.1472, + "n_prompts": 50 + }, + "10": { + "dispersion": 0.582179, + "mean_fro": 15.911907, + "rms_per_prompt_fro": 20.014743, + "max_per_prompt_fro": 22.584604, + "heavy_tail_ratio": 1.1284, + "n_prompts": 50 + }, + "11": { + "dispersion": 0.538103, + "mean_fro": 18.6646, + "rms_per_prompt_fro": 23.147888, + "max_per_prompt_fro": 25.793814, + "heavy_tail_ratio": 1.1143, + "n_prompts": 50 + }, + "12": { + "dispersion": 0.446373, + "mean_fro": 20.192129, + "rms_per_prompt_fro": 24.284116, + "max_per_prompt_fro": 26.504976, + "heavy_tail_ratio": 1.0915, + "n_prompts": 50 + }, + "13": { + "dispersion": 0.385911, + "mean_fro": 21.786835, + "rms_per_prompt_fro": 25.64849, + "max_per_prompt_fro": 27.483522, + "heavy_tail_ratio": 1.0715, + "n_prompts": 50 + }, + "14": { + "dispersion": 0.33649, + "mean_fro": 25.167338, + "rms_per_prompt_fro": 29.095116, + "max_per_prompt_fro": 31.115555, + "heavy_tail_ratio": 1.0694, + "n_prompts": 50 + }, + "15": { + "dispersion": 0.288814, + "mean_fro": 28.149432, + "rms_per_prompt_fro": 31.956913, + "max_per_prompt_fro": 34.168682, + "heavy_tail_ratio": 1.0692, + "n_prompts": 50 + }, + "16": { + "dispersion": 0.218136, + "mean_fro": 31.142115, + "rms_per_prompt_fro": 34.371302, + "max_per_prompt_fro": 36.42894, + "heavy_tail_ratio": 1.0599, + "n_prompts": 50 + }, + "17": { + "dispersion": 0.163096, + "mean_fro": 33.22173, + "rms_per_prompt_fro": 35.828622, + "max_per_prompt_fro": 37.851063, + "heavy_tail_ratio": 1.0564, + "n_prompts": 50 + }, + "18": { + "dispersion": 0.120524, + "mean_fro": 34.726314, + "rms_per_prompt_fro": 36.759465, + "max_per_prompt_fro": 38.68256, + "heavy_tail_ratio": 1.0523, + "n_prompts": 50 + }, + "19": { + "dispersion": 0.081985, + "mean_fro": 34.916026, + "rms_per_prompt_fro": 36.319122, + "max_per_prompt_fro": 37.774918, + "heavy_tail_ratio": 1.0401, + "n_prompts": 50 + }, + "20": { + "dispersion": 0.046998, + "mean_fro": 35.089817, + "rms_per_prompt_fro": 35.904921, + "max_per_prompt_fro": 37.218494, + "heavy_tail_ratio": 1.0366, + "n_prompts": 50 + }, + "21": { + "dispersion": 0.021889, + "mean_fro": 34.581505, + "rms_per_prompt_fro": 34.957925, + "max_per_prompt_fro": 35.78508, + "heavy_tail_ratio": 1.0237, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 151.3, + "fit_secs_per_prompt": 3.027, + "total_wall_seconds": 162.9, + "per_layer": [ + { + "layer": 0, + "fro_norm": 8.6945, + "top_sv": 2.0627, + "stable_rank": 17.767 + }, + { + "layer": 1, + "fro_norm": 8.1637, + "top_sv": 1.9597, + "stable_rank": 17.353 + }, + { + "layer": 2, + "fro_norm": 8.7585, + "top_sv": 2.1034, + "stable_rank": 17.338 + }, + { + "layer": 3, + "fro_norm": 8.4462, + "top_sv": 2.0442, + "stable_rank": 17.072 + }, + { + "layer": 4, + "fro_norm": 9.5788, + "top_sv": 2.2721, + "stable_rank": 17.774 + }, + { + "layer": 5, + "fro_norm": 9.749, + "top_sv": 2.3826, + "stable_rank": 16.742 + }, + { + "layer": 6, + "fro_norm": 10.0451, + "top_sv": 2.3482, + "stable_rank": 18.299 + }, + { + "layer": 7, + "fro_norm": 11.1034, + "top_sv": 2.6296, + "stable_rank": 17.829 + }, + { + "layer": 8, + "fro_norm": 12.7218, + "top_sv": 3.0976, + "stable_rank": 16.867 + }, + { + "layer": 9, + "fro_norm": 14.1026, + "top_sv": 3.4264, + "stable_rank": 16.94 + }, + { + "layer": 10, + "fro_norm": 15.9117, + "top_sv": 3.8577, + "stable_rank": 17.013 + }, + { + "layer": 11, + "fro_norm": 18.6643, + "top_sv": 4.4263, + "stable_rank": 17.781 + }, + { + "layer": 12, + "fro_norm": 20.1918, + "top_sv": 4.4394, + "stable_rank": 20.687 + }, + { + "layer": 13, + "fro_norm": 21.7865, + "top_sv": 4.7422, + "stable_rank": 21.106 + }, + { + "layer": 14, + "fro_norm": 25.1669, + "top_sv": 5.4119, + "stable_rank": 21.625 + }, + { + "layer": 15, + "fro_norm": 28.1489, + "top_sv": 6.3025, + "stable_rank": 19.948 + }, + { + "layer": 16, + "fro_norm": 31.1416, + "top_sv": 6.5553, + "stable_rank": 22.568 + }, + { + "layer": 17, + "fro_norm": 33.221, + "top_sv": 7.0554, + "stable_rank": 22.17 + }, + { + "layer": 18, + "fro_norm": 34.7254, + "top_sv": 7.4771, + "stable_rank": 21.569 + }, + { + "layer": 19, + "fro_norm": 34.9149, + "top_sv": 7.4959, + "stable_rank": 21.696 + }, + { + "layer": 20, + "fro_norm": 35.0886, + "top_sv": 7.0002, + "stable_rank": 25.125 + }, + { + "layer": 21, + "fro_norm": 34.58, + "top_sv": 5.518, + "stable_rank": 39.272 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70mLM_code_A.pt b/results/lenses/plad/plad_70mLM_code_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..892622ec4fc0fe3745003dccacd0382c844fe78c --- /dev/null +++ b/results/lenses/plad/plad_70mLM_code_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb0194df77fdb77c4ae72cc11f723c752ef82947e6c3b42547e47be9d4d23cb +size 2099240 diff --git a/results/lenses/plad/plad_70mLM_code_B.pt b/results/lenses/plad/plad_70mLM_code_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ca0acf656526f58e3df2d3b9ccd659fa0f096b3 --- /dev/null +++ b/results/lenses/plad/plad_70mLM_code_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5205c2cacc78d7653ff4eadd7bae088393e207b03dd100c0ee509247f8c9f9e +size 2099240 diff --git a/results/lenses/plad/plad_70mLM_pile_A.pt b/results/lenses/plad/plad_70mLM_pile_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae4b40723947cf5a04f58a76bd2f7edb608b8b53 --- /dev/null +++ b/results/lenses/plad/plad_70mLM_pile_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e3a8a210ebd4ed42b5050a7b55cf7928181c3fe9ae95068fc7a6975c99434d8 +size 2099240 diff --git a/results/lenses/plad/plad_70mLM_pile_B.pt b/results/lenses/plad/plad_70mLM_pile_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..f461bcdc637d3aeb9486c87c0c385e67ef7caa89 --- /dev/null +++ b/results/lenses/plad/plad_70mLM_pile_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12cbd2c37ac458b26008143fc5c47105c50581e4591dbef20bb4d93e3537832a +size 2099240 diff --git a/results/lenses/plad/plad_70mLM_wikitext_A.pt b/results/lenses/plad/plad_70mLM_wikitext_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..5becedbbadcae96e82fc3e6414accfff74cfe722 --- /dev/null +++ b/results/lenses/plad/plad_70mLM_wikitext_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d49641422dfb1efbc05811bd7769537586150da7817ac7313d50e0c49f173d0 +size 2099336 diff --git a/results/lenses/plad/plad_70mLM_wikitext_B.pt b/results/lenses/plad/plad_70mLM_wikitext_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d8c31475b3ccbab78c6c096d98e3969bbd874f2 --- /dev/null +++ b/results/lenses/plad/plad_70mLM_wikitext_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a81cb137cef05dc9e38539c91270826da99d2789d207886a1b3a85a8dac8147e +size 2099336 diff --git a/results/lenses/plad/plad_70m_code_A.pt b/results/lenses/plad/plad_70m_code_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..f765a2d3dab09b7abb8d1793b897f9d2acfae913 --- /dev/null +++ b/results/lenses/plad/plad_70m_code_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d182821d8e428041d8ef9497740330c231bd497c6d3d1253691f445aefad07cd +size 2099629 diff --git a/results/lenses/plad/plad_70m_code_A_provenance.json b/results/lenses/plad/plad_70m_code_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4ba820ce1920bdf27131dcac41563cc09181a3da --- /dev/null +++ b/results/lenses/plad/plad_70m_code_A_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_code_A.pt", + "sha256": "d182821d8e428041d8ef9497740330c231bd497c6d3d1253691f445aefad07cd", + "bytes": 2099629, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.285214, + "mean_fro": 30.09614, + "rms_per_prompt_fro": 34.119169, + "max_per_prompt_fro": 40.390011, + "heavy_tail_ratio": 1.1838, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.229884, + "mean_fro": 23.874693, + "rms_per_prompt_fro": 26.47707, + "max_per_prompt_fro": 31.163086, + "heavy_tail_ratio": 1.177, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.177364, + "mean_fro": 23.28725, + "rms_per_prompt_fro": 25.268159, + "max_per_prompt_fro": 29.477167, + "heavy_tail_ratio": 1.1666, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.109582, + "mean_fro": 22.12643, + "rms_per_prompt_fro": 23.307247, + "max_per_prompt_fro": 24.680311, + "heavy_tail_ratio": 1.0589, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 104.0, + "fit_secs_per_prompt": 2.081, + "total_wall_seconds": 128.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.0961, + "top_sv": 4.6824, + "stable_rank": 41.312 + }, + { + "layer": 1, + "fro_norm": 23.8747, + "top_sv": 4.4265, + "stable_rank": 29.09 + }, + { + "layer": 2, + "fro_norm": 23.2873, + "top_sv": 3.5892, + "stable_rank": 42.096 + }, + { + "layer": 3, + "fro_norm": 22.1264, + "top_sv": 4.3295, + "stable_rank": 26.119 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_code_B.pt b/results/lenses/plad/plad_70m_code_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..df70f80f9127421e76cf04ae1ac6b2fb0ae9e9fe --- /dev/null +++ b/results/lenses/plad/plad_70m_code_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cf1dedecd51891129b250e61be609735e685b0fc5148c19e86661ced90de009 +size 2099629 diff --git a/results/lenses/plad/plad_70m_code_B_provenance.json b/results/lenses/plad/plad_70m_code_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d72b29f3a13fbcd3d56f0f04704c5edd4e150665 --- /dev/null +++ b/results/lenses/plad/plad_70m_code_B_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_code_B.pt", + "sha256": "5cf1dedecd51891129b250e61be609735e685b0fc5148c19e86661ced90de009", + "bytes": 2099629, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.302729, + "mean_fro": 30.249835, + "rms_per_prompt_fro": 34.526296, + "max_per_prompt_fro": 41.213936, + "heavy_tail_ratio": 1.1937, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.244277, + "mean_fro": 24.046129, + "rms_per_prompt_fro": 26.82278, + "max_per_prompt_fro": 30.84178, + "heavy_tail_ratio": 1.1498, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.187472, + "mean_fro": 23.43053, + "rms_per_prompt_fro": 25.532525, + "max_per_prompt_fro": 28.383266, + "heavy_tail_ratio": 1.1117, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.115602, + "mean_fro": 22.200564, + "rms_per_prompt_fro": 23.448695, + "max_per_prompt_fro": 24.721634, + "heavy_tail_ratio": 1.0543, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 137.8, + "fit_secs_per_prompt": 2.755, + "total_wall_seconds": 162.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.2498, + "top_sv": 4.6726, + "stable_rank": 41.91 + }, + { + "layer": 1, + "fro_norm": 24.0461, + "top_sv": 4.3961, + "stable_rank": 29.92 + }, + { + "layer": 2, + "fro_norm": 23.4305, + "top_sv": 3.5711, + "stable_rank": 43.049 + }, + { + "layer": 3, + "fro_norm": 22.2006, + "top_sv": 4.2949, + "stable_rank": 26.719 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_code_LM_A.pt b/results/lenses/plad/plad_70m_code_LM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..892622ec4fc0fe3745003dccacd0382c844fe78c --- /dev/null +++ b/results/lenses/plad/plad_70m_code_LM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb0194df77fdb77c4ae72cc11f723c752ef82947e6c3b42547e47be9d4d23cb +size 2099240 diff --git a/results/lenses/plad/plad_70m_code_LM_A_provenance.json b/results/lenses/plad/plad_70m_code_LM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..57cb84a194a3973d63542b2994aed70d1575784b --- /dev/null +++ b/results/lenses/plad/plad_70m_code_LM_A_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_code_LM_A.pt", + "sha256": "bdb0194df77fdb77c4ae72cc11f723c752ef82947e6c3b42547e47be9d4d23cb", + "bytes": 2099240, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.352725, + "mean_fro": 30.109678, + "rms_per_prompt_fro": 35.019576, + "max_per_prompt_fro": 41.320232, + "heavy_tail_ratio": 1.1799, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.274377, + "mean_fro": 23.895314, + "rms_per_prompt_fro": 26.975013, + "max_per_prompt_fro": 31.744646, + "heavy_tail_ratio": 1.1768, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.198455, + "mean_fro": 23.318144, + "rms_per_prompt_fro": 25.527298, + "max_per_prompt_fro": 29.785957, + "heavy_tail_ratio": 1.1668, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.110795, + "mean_fro": 22.123314, + "rms_per_prompt_fro": 23.316699, + "max_per_prompt_fro": 24.685787, + "heavy_tail_ratio": 1.0587, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.6, + "fit_secs_per_prompt": 0.112, + "total_wall_seconds": 14.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.1096, + "top_sv": 4.6874, + "stable_rank": 41.261 + }, + { + "layer": 1, + "fro_norm": 23.8953, + "top_sv": 4.4333, + "stable_rank": 29.051 + }, + { + "layer": 2, + "fro_norm": 23.3181, + "top_sv": 3.5929, + "stable_rank": 42.122 + }, + { + "layer": 3, + "fro_norm": 22.1232, + "top_sv": 4.3274, + "stable_rank": 26.136 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_code_LM_B.pt b/results/lenses/plad/plad_70m_code_LM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ca0acf656526f58e3df2d3b9ccd659fa0f096b3 --- /dev/null +++ b/results/lenses/plad/plad_70m_code_LM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5205c2cacc78d7653ff4eadd7bae088393e207b03dd100c0ee509247f8c9f9e +size 2099240 diff --git a/results/lenses/plad/plad_70m_code_LM_B_provenance.json b/results/lenses/plad/plad_70m_code_LM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4ffbe0e7bf046d2e37a5ce23446474ae1e85eae3 --- /dev/null +++ b/results/lenses/plad/plad_70m_code_LM_B_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_code_LM_B.pt", + "sha256": "c5205c2cacc78d7653ff4eadd7bae088393e207b03dd100c0ee509247f8c9f9e", + "bytes": 2099240, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.367317, + "mean_fro": 30.260959, + "rms_per_prompt_fro": 35.384841, + "max_per_prompt_fro": 42.703888, + "heavy_tail_ratio": 1.2068, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.28722, + "mean_fro": 24.055669, + "rms_per_prompt_fro": 27.292538, + "max_per_prompt_fro": 31.637482, + "heavy_tail_ratio": 1.1592, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.20809, + "mean_fro": 23.432062, + "rms_per_prompt_fro": 25.75492, + "max_per_prompt_fro": 28.795031, + "heavy_tail_ratio": 1.118, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.116728, + "mean_fro": 22.200594, + "rms_per_prompt_fro": 23.460551, + "max_per_prompt_fro": 24.729326, + "heavy_tail_ratio": 1.0541, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.7, + "fit_secs_per_prompt": 0.113, + "total_wall_seconds": 15.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.2609, + "top_sv": 4.6741, + "stable_rank": 41.915 + }, + { + "layer": 1, + "fro_norm": 24.0556, + "top_sv": 4.4005, + "stable_rank": 29.884 + }, + { + "layer": 2, + "fro_norm": 23.432, + "top_sv": 3.5767, + "stable_rank": 42.92 + }, + { + "layer": 3, + "fro_norm": 22.2005, + "top_sv": 4.2934, + "stable_rank": 26.738 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_code_OE_A.pt b/results/lenses/plad/plad_70m_code_OE_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..46a2369ea1963e679473ba035364f02f7f24ead1 --- /dev/null +++ b/results/lenses/plad/plad_70m_code_OE_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4f9fb1112e3b119c13c2019995b313383c93ecabf8f847b72430774780f738c +size 2099723 diff --git a/results/lenses/plad/plad_70m_code_OE_A_provenance.json b/results/lenses/plad/plad_70m_code_OE_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9a662238dc981ec34372ac06d4538e1b4bd181e4 --- /dev/null +++ b/results/lenses/plad/plad_70m_code_OE_A_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_code_OE_A.pt", + "sha256": "a4f9fb1112e3b119c13c2019995b313383c93ecabf8f847b72430774780f738c", + "bytes": 2099723, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "code", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.303931, + "mean_fro": 29.969554, + "rms_per_prompt_fro": 34.222168, + "max_per_prompt_fro": 41.213936, + "heavy_tail_ratio": 1.2043, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.247286, + "mean_fro": 23.788961, + "rms_per_prompt_fro": 26.567977, + "max_per_prompt_fro": 30.84178, + "heavy_tail_ratio": 1.1609, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.189785, + "mean_fro": 23.262571, + "rms_per_prompt_fro": 25.374177, + "max_per_prompt_fro": 28.383266, + "heavy_tail_ratio": 1.1186, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.116115, + "mean_fro": 22.150322, + "rms_per_prompt_fro": 23.401005, + "max_per_prompt_fro": 24.721634, + "heavy_tail_ratio": 1.0564, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 174.7, + "fit_secs_per_prompt": 3.495, + "total_wall_seconds": 199.7, + "per_layer": [ + { + "layer": 0, + "fro_norm": 29.9696, + "top_sv": 4.6325, + "stable_rank": 41.854 + }, + { + "layer": 1, + "fro_norm": 23.789, + "top_sv": 4.3738, + "stable_rank": 29.582 + }, + { + "layer": 2, + "fro_norm": 23.2626, + "top_sv": 3.5616, + "stable_rank": 42.661 + }, + { + "layer": 3, + "fro_norm": 22.1503, + "top_sv": 4.3228, + "stable_rank": 26.256 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_pile_A.pt b/results/lenses/plad/plad_70m_pile_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..5788d35658319037d31509ce41bc8a947d2ef65f --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49584ea7e7778d96e2073fa5149ee39294e7702d5c06d5e9f8b6d8d77247486 +size 2099629 diff --git a/results/lenses/plad/plad_70m_pile_A_provenance.json b/results/lenses/plad/plad_70m_pile_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..200e7cb5648bebd3832cb046656443634ae77e79 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_A_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_pile_A.pt", + "sha256": "d49584ea7e7778d96e2073fa5149ee39294e7702d5c06d5e9f8b6d8d77247486", + "bytes": 2099629, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.25222, + "mean_fro": 30.718534, + "rms_per_prompt_fro": 34.37485, + "max_per_prompt_fro": 40.390011, + "heavy_tail_ratio": 1.175, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.201514, + "mean_fro": 24.237076, + "rms_per_prompt_fro": 26.567133, + "max_per_prompt_fro": 30.718653, + "heavy_tail_ratio": 1.1563, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.152733, + "mean_fro": 23.592215, + "rms_per_prompt_fro": 25.329876, + "max_per_prompt_fro": 28.47057, + "heavy_tail_ratio": 1.124, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.094159, + "mean_fro": 22.130921, + "rms_per_prompt_fro": 23.149394, + "max_per_prompt_fro": 24.10297, + "heavy_tail_ratio": 1.0412, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 108.6, + "fit_secs_per_prompt": 2.171, + "total_wall_seconds": 134.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.7185, + "top_sv": 4.7461, + "stable_rank": 41.891 + }, + { + "layer": 1, + "fro_norm": 24.2371, + "top_sv": 4.4587, + "stable_rank": 29.55 + }, + { + "layer": 2, + "fro_norm": 23.5922, + "top_sv": 3.6484, + "stable_rank": 41.815 + }, + { + "layer": 3, + "fro_norm": 22.1309, + "top_sv": 4.2528, + "stable_rank": 27.08 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_pile_B.pt b/results/lenses/plad/plad_70m_pile_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..135d194a1aba1f18b0720abaf34722492f85baec --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:039158ee86ff4b0d1ed29318434652dd6256d0c371b345f2eafaff81dcdbc89f +size 2099629 diff --git a/results/lenses/plad/plad_70m_pile_B_provenance.json b/results/lenses/plad/plad_70m_pile_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..09da5f9d8d95d412c892142792b39f8aab864c32 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_B_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_pile_B.pt", + "sha256": "039158ee86ff4b0d1ed29318434652dd6256d0c371b345f2eafaff81dcdbc89f", + "bytes": 2099629, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.277661, + "mean_fro": 30.718994, + "rms_per_prompt_fro": 34.722804, + "max_per_prompt_fro": 41.480824, + "heavy_tail_ratio": 1.1946, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.226276, + "mean_fro": 24.421208, + "rms_per_prompt_fro": 27.043393, + "max_per_prompt_fro": 31.163086, + "heavy_tail_ratio": 1.1523, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.175872, + "mean_fro": 23.649248, + "rms_per_prompt_fro": 25.644686, + "max_per_prompt_fro": 29.477167, + "heavy_tail_ratio": 1.1494, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.107054, + "mean_fro": 22.228555, + "rms_per_prompt_fro": 23.388133, + "max_per_prompt_fro": 24.448627, + "heavy_tail_ratio": 1.0453, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 106.4, + "fit_secs_per_prompt": 2.128, + "total_wall_seconds": 131.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.719, + "top_sv": 4.7575, + "stable_rank": 41.692 + }, + { + "layer": 1, + "fro_norm": 24.4212, + "top_sv": 4.5153, + "stable_rank": 29.252 + }, + { + "layer": 2, + "fro_norm": 23.6492, + "top_sv": 3.6795, + "stable_rank": 41.311 + }, + { + "layer": 3, + "fro_norm": 22.2286, + "top_sv": 4.2994, + "stable_rank": 26.731 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_pile_LM_A.pt b/results/lenses/plad/plad_70m_pile_LM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae4b40723947cf5a04f58a76bd2f7edb608b8b53 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_LM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e3a8a210ebd4ed42b5050a7b55cf7928181c3fe9ae95068fc7a6975c99434d8 +size 2099240 diff --git a/results/lenses/plad/plad_70m_pile_LM_A_provenance.json b/results/lenses/plad/plad_70m_pile_LM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..97de7779b389312bc32b2d9294dc26c32603a721 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_LM_A_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_pile_LM_A.pt", + "sha256": "8e3a8a210ebd4ed42b5050a7b55cf7928181c3fe9ae95068fc7a6975c99434d8", + "bytes": 2099240, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.320606, + "mean_fro": 30.79607, + "rms_per_prompt_fro": 35.39011, + "max_per_prompt_fro": 41.320232, + "heavy_tail_ratio": 1.1676, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.248418, + "mean_fro": 24.295447, + "rms_per_prompt_fro": 27.145936, + "max_per_prompt_fro": 31.72485, + "heavy_tail_ratio": 1.1687, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.175277, + "mean_fro": 23.641974, + "rms_per_prompt_fro": 25.630314, + "max_per_prompt_fro": 29.011332, + "heavy_tail_ratio": 1.1319, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.095634, + "mean_fro": 22.146499, + "rms_per_prompt_fro": 23.181298, + "max_per_prompt_fro": 24.128771, + "heavy_tail_ratio": 1.0409, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.5, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 15.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.796, + "top_sv": 4.7537, + "stable_rank": 41.969 + }, + { + "layer": 1, + "fro_norm": 24.2954, + "top_sv": 4.47, + "stable_rank": 29.542 + }, + { + "layer": 2, + "fro_norm": 23.6419, + "top_sv": 3.6576, + "stable_rank": 41.781 + }, + { + "layer": 3, + "fro_norm": 22.1465, + "top_sv": 4.2549, + "stable_rank": 27.091 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_pile_LM_B.pt b/results/lenses/plad/plad_70m_pile_LM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..f461bcdc637d3aeb9486c87c0c385e67ef7caa89 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_LM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12cbd2c37ac458b26008143fc5c47105c50581e4591dbef20bb4d93e3537832a +size 2099240 diff --git a/results/lenses/plad/plad_70m_pile_LM_B_provenance.json b/results/lenses/plad/plad_70m_pile_LM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9558faf066d0311d19ec770020d90de6b91df670 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_LM_B_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_pile_LM_B.pt", + "sha256": "12cbd2c37ac458b26008143fc5c47105c50581e4591dbef20bb4d93e3537832a", + "bytes": 2099240, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.341088, + "mean_fro": 30.71718, + "rms_per_prompt_fro": 35.572145, + "max_per_prompt_fro": 42.328762, + "heavy_tail_ratio": 1.1899, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.267571, + "mean_fro": 24.395742, + "rms_per_prompt_fro": 27.466302, + "max_per_prompt_fro": 31.744646, + "heavy_tail_ratio": 1.1558, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.196004, + "mean_fro": 23.625072, + "rms_per_prompt_fro": 25.83684, + "max_per_prompt_fro": 29.785957, + "heavy_tail_ratio": 1.1528, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.107708, + "mean_fro": 22.225845, + "rms_per_prompt_fro": 23.392195, + "max_per_prompt_fro": 24.466993, + "heavy_tail_ratio": 1.0459, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.6, + "fit_secs_per_prompt": 0.112, + "total_wall_seconds": 15.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.7171, + "top_sv": 4.7546, + "stable_rank": 41.738 + }, + { + "layer": 1, + "fro_norm": 24.3957, + "top_sv": 4.505, + "stable_rank": 29.325 + }, + { + "layer": 2, + "fro_norm": 23.625, + "top_sv": 3.6639, + "stable_rank": 41.577 + }, + { + "layer": 3, + "fro_norm": 22.2258, + "top_sv": 4.3003, + "stable_rank": 26.713 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_pile_OE_A.pt b/results/lenses/plad/plad_70m_pile_OE_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..2465f60fbd5e22e0851f1e86475a7db8dceb2b70 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_OE_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7167868d8b325e24c1b941bded26037fc8cc7b8eb8ece0982aed3268ce056141 +size 2099723 diff --git a/results/lenses/plad/plad_70m_pile_OE_A_provenance.json b/results/lenses/plad/plad_70m_pile_OE_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..686a22950960689945d7b8e012121a93715375db --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_OE_A_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_pile_OE_A.pt", + "sha256": "7167868d8b325e24c1b941bded26037fc8cc7b8eb8ece0982aed3268ce056141", + "bytes": 2099723, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.280097, + "mean_fro": 30.463385, + "rms_per_prompt_fro": 34.466694, + "max_per_prompt_fro": 40.213627, + "heavy_tail_ratio": 1.1667, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.226034, + "mean_fro": 24.218931, + "rms_per_prompt_fro": 26.816752, + "max_per_prompt_fro": 31.163086, + "heavy_tail_ratio": 1.1621, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.174609, + "mean_fro": 23.531109, + "rms_per_prompt_fro": 25.502874, + "max_per_prompt_fro": 29.477167, + "heavy_tail_ratio": 1.1558, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.106843, + "mean_fro": 22.09798, + "rms_per_prompt_fro": 23.248532, + "max_per_prompt_fro": 24.448627, + "heavy_tail_ratio": 1.0516, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 227.5, + "fit_secs_per_prompt": 4.55, + "total_wall_seconds": 253.3, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.4634, + "top_sv": 4.7172, + "stable_rank": 41.704 + }, + { + "layer": 1, + "fro_norm": 24.2189, + "top_sv": 4.4502, + "stable_rank": 29.617 + }, + { + "layer": 2, + "fro_norm": 23.5311, + "top_sv": 3.6261, + "stable_rank": 42.112 + }, + { + "layer": 3, + "fro_norm": 22.098, + "top_sv": 4.2579, + "stable_rank": 26.935 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_pile_OE_B.pt b/results/lenses/plad/plad_70m_pile_OE_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f219f60a52d7cdd8d3acd10c0e17451cbb14d01 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_OE_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb07baa36f7712c308baaf05d9560873586e9072f9434aecfeaed324dd9f24d5 +size 2099723 diff --git a/results/lenses/plad/plad_70m_pile_OE_B_provenance.json b/results/lenses/plad/plad_70m_pile_OE_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..b38769517e622c7701558919b28bf7c24d5980e4 --- /dev/null +++ b/results/lenses/plad/plad_70m_pile_OE_B_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_pile_OE_B.pt", + "sha256": "fb07baa36f7712c308baaf05d9560873586e9072f9434aecfeaed324dd9f24d5", + "bytes": 2099723, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "pile", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.252314, + "mean_fro": 30.946852, + "rms_per_prompt_fro": 34.631639, + "max_per_prompt_fro": 41.480824, + "heavy_tail_ratio": 1.1978, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.204606, + "mean_fro": 24.414371, + "rms_per_prompt_fro": 26.795885, + "max_per_prompt_fro": 30.98089, + "heavy_tail_ratio": 1.1562, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.156409, + "mean_fro": 23.687456, + "rms_per_prompt_fro": 25.472651, + "max_per_prompt_fro": 28.47057, + "heavy_tail_ratio": 1.1177, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.095866, + "mean_fro": 22.247595, + "rms_per_prompt_fro": 23.289589, + "max_per_prompt_fro": 24.415501, + "heavy_tail_ratio": 1.0483, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 211.6, + "fit_secs_per_prompt": 4.231, + "total_wall_seconds": 235.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 30.9469, + "top_sv": 4.7857, + "stable_rank": 41.816 + }, + { + "layer": 1, + "fro_norm": 24.4144, + "top_sv": 4.5229, + "stable_rank": 29.138 + }, + { + "layer": 2, + "fro_norm": 23.6875, + "top_sv": 3.7011, + "stable_rank": 40.961 + }, + { + "layer": 3, + "fro_norm": 22.2476, + "top_sv": 4.2902, + "stable_rank": 26.891 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_A.pt b/results/lenses/plad/plad_70m_wikitext_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..e437c5994450e049c1ba72143a85411503f509df --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac0cb01cd47db95dbf1ea68d3d2e0120f26ed58c251b4b1ad2f84b61f76ec6f4 +size 2099733 diff --git a/results/lenses/plad/plad_70m_wikitext_A_provenance.json b/results/lenses/plad/plad_70m_wikitext_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..928e3b7bfdcd022b7291f9243662cc6a603a6663 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_A_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_wikitext_A.pt", + "sha256": "ac0cb01cd47db95dbf1ea68d3d2e0120f26ed58c251b4b1ad2f84b61f76ec6f4", + "bytes": 2099733, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.126055, + "mean_fro": 34.505839, + "rms_per_prompt_fro": 36.61613, + "max_per_prompt_fro": 42.70137, + "heavy_tail_ratio": 1.1662, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.101933, + "mean_fro": 26.83785, + "rms_per_prompt_fro": 28.17249, + "max_per_prompt_fro": 32.573502, + "heavy_tail_ratio": 1.1562, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.078737, + "mean_fro": 25.42897, + "rms_per_prompt_fro": 26.411101, + "max_per_prompt_fro": 29.641071, + "heavy_tail_ratio": 1.1223, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.046108, + "mean_fro": 23.200948, + "rms_per_prompt_fro": 23.729795, + "max_per_prompt_fro": 25.256557, + "heavy_tail_ratio": 1.0643, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 97.0, + "fit_secs_per_prompt": 1.94, + "total_wall_seconds": 122.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 34.5058, + "top_sv": 5.069, + "stable_rank": 46.339 + }, + { + "layer": 1, + "fro_norm": 26.8379, + "top_sv": 4.34, + "stable_rank": 38.24 + }, + { + "layer": 2, + "fro_norm": 25.429, + "top_sv": 3.5506, + "stable_rank": 51.292 + }, + { + "layer": 3, + "fro_norm": 23.2009, + "top_sv": 4.4315, + "stable_rank": 27.41 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_B.pt b/results/lenses/plad/plad_70m_wikitext_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ff9360c55a7c183696a65c44b835a8a8f64c8b6 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d654950f5d2a30c7ba052252d9cd8dd1ead5eb8aee84d97e7953abdc47d878 +size 2099733 diff --git a/results/lenses/plad/plad_70m_wikitext_B_provenance.json b/results/lenses/plad/plad_70m_wikitext_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d4a6365f8014607fd9b8f63189979da04e3a38bc --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_B_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_wikitext_B.pt", + "sha256": "80d654950f5d2a30c7ba052252d9cd8dd1ead5eb8aee84d97e7953abdc47d878", + "bytes": 2099733, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.155054, + "mean_fro": 34.266471, + "rms_per_prompt_fro": 36.82735, + "max_per_prompt_fro": 43.189819, + "heavy_tail_ratio": 1.1728, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.123074, + "mean_fro": 26.804993, + "rms_per_prompt_fro": 28.406645, + "max_per_prompt_fro": 31.975203, + "heavy_tail_ratio": 1.1256, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.09406, + "mean_fro": 25.394476, + "rms_per_prompt_fro": 26.561939, + "max_per_prompt_fro": 29.189302, + "heavy_tail_ratio": 1.0989, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.057256, + "mean_fro": 23.228408, + "rms_per_prompt_fro": 23.884135, + "max_per_prompt_fro": 24.817972, + "heavy_tail_ratio": 1.0391, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 99.6, + "fit_secs_per_prompt": 1.992, + "total_wall_seconds": 126.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 34.2665, + "top_sv": 5.2983, + "stable_rank": 41.828 + }, + { + "layer": 1, + "fro_norm": 26.805, + "top_sv": 4.3369, + "stable_rank": 38.201 + }, + { + "layer": 2, + "fro_norm": 25.3945, + "top_sv": 3.535, + "stable_rank": 51.605 + }, + { + "layer": 3, + "fro_norm": 23.2284, + "top_sv": 4.492, + "stable_rank": 26.74 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_DOCLM_A.pt b/results/lenses/plad/plad_70m_wikitext_DOCLM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..853ee06424bb59aed99c4a465e0c972259eeba69 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOCLM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:764fbc7efedebb346ec35b483259f079f1c25a16baa826a8ba28e2a5f6566218 +size 2099424 diff --git a/results/lenses/plad/plad_70m_wikitext_DOCLM_A_provenance.json b/results/lenses/plad/plad_70m_wikitext_DOCLM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..345062e92d534c332a8761ca720b7f989a32c3f1 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOCLM_A_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_wikitext_DOCLM_A.pt", + "sha256": "764fbc7efedebb346ec35b483259f079f1c25a16baa826a8ba28e2a5f6566218", + "bytes": 2099424, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.213226, + "mean_fro": 35.375152, + "rms_per_prompt_fro": 38.964506, + "max_per_prompt_fro": 44.659336, + "heavy_tail_ratio": 1.1462, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.165291, + "mean_fro": 27.399687, + "rms_per_prompt_fro": 29.577594, + "max_per_prompt_fro": 33.443409, + "heavy_tail_ratio": 1.1307, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.117451, + "mean_fro": 25.662929, + "rms_per_prompt_fro": 27.128163, + "max_per_prompt_fro": 30.094536, + "heavy_tail_ratio": 1.1093, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.055325, + "mean_fro": 23.215393, + "rms_per_prompt_fro": 23.848947, + "max_per_prompt_fro": 24.931845, + "heavy_tail_ratio": 1.0454, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.6, + "fit_secs_per_prompt": 0.111, + "total_wall_seconds": 16.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 35.3751, + "top_sv": 5.2271, + "stable_rank": 45.801 + }, + { + "layer": 1, + "fro_norm": 27.3997, + "top_sv": 4.4185, + "stable_rank": 38.453 + }, + { + "layer": 2, + "fro_norm": 25.6629, + "top_sv": 3.5898, + "stable_rank": 51.106 + }, + { + "layer": 3, + "fro_norm": 23.2153, + "top_sv": 4.3957, + "stable_rank": 27.892 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_DOCLM_B.pt b/results/lenses/plad/plad_70m_wikitext_DOCLM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..b80f88d5eda25d051157e6e7d14d8bd2257e04bc --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOCLM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88dfd6794419e02fd4503c5f4af3d5a0df021e001b0601cc51b1f2c18f0016e7 +size 2099424 diff --git a/results/lenses/plad/plad_70m_wikitext_DOCLM_B_provenance.json b/results/lenses/plad/plad_70m_wikitext_DOCLM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..3db164878ed9c1dc0593a5fac8279bd9bea6fb53 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOCLM_B_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_wikitext_DOCLM_B.pt", + "sha256": "88dfd6794419e02fd4503c5f4af3d5a0df021e001b0601cc51b1f2c18f0016e7", + "bytes": 2099424, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.201385, + "mean_fro": 35.338954, + "rms_per_prompt_fro": 38.734214, + "max_per_prompt_fro": 42.690891, + "heavy_tail_ratio": 1.1021, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.152924, + "mean_fro": 27.400656, + "rms_per_prompt_fro": 29.421258, + "max_per_prompt_fro": 32.624393, + "heavy_tail_ratio": 1.1089, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.106061, + "mean_fro": 25.830659, + "rms_per_prompt_fro": 27.165957, + "max_per_prompt_fro": 29.589869, + "heavy_tail_ratio": 1.0892, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.049982, + "mean_fro": 23.476839, + "rms_per_prompt_fro": 24.056392, + "max_per_prompt_fro": 25.369312, + "heavy_tail_ratio": 1.0546, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.5, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 17.4, + "per_layer": [ + { + "layer": 0, + "fro_norm": 35.3389, + "top_sv": 5.5072, + "stable_rank": 41.175 + }, + { + "layer": 1, + "fro_norm": 27.4006, + "top_sv": 4.4665, + "stable_rank": 37.634 + }, + { + "layer": 2, + "fro_norm": 25.8306, + "top_sv": 3.6206, + "stable_rank": 50.898 + }, + { + "layer": 3, + "fro_norm": 23.4768, + "top_sv": 4.532, + "stable_rank": 26.834 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_DOC_A.pt b/results/lenses/plad/plad_70m_wikitext_DOC_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..41d4b4071f96e64dc116d54974d23c5960759e49 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOC_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:199defda6edf274467cf8429f452dd4d223fa5a74830822bd597a443fe46cd23 +size 2099408 diff --git a/results/lenses/plad/plad_70m_wikitext_DOC_A_provenance.json b/results/lenses/plad/plad_70m_wikitext_DOC_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..60d3627eda973e66f6864bafa087f65848e1da67 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOC_A_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_wikitext_DOC_A.pt", + "sha256": "199defda6edf274467cf8429f452dd4d223fa5a74830822bd597a443fe46cd23", + "bytes": 2099408, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.219228, + "mean_fro": 34.098044, + "rms_per_prompt_fro": 37.650597, + "max_per_prompt_fro": 44.659336, + "heavy_tail_ratio": 1.1862, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.168785, + "mean_fro": 26.62007, + "rms_per_prompt_fro": 28.779058, + "max_per_prompt_fro": 33.443409, + "heavy_tail_ratio": 1.1621, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.118253, + "mean_fro": 25.205952, + "rms_per_prompt_fro": 26.654659, + "max_per_prompt_fro": 30.094536, + "heavy_tail_ratio": 1.1291, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.054177, + "mean_fro": 23.035789, + "rms_per_prompt_fro": 23.651566, + "max_per_prompt_fro": 24.840597, + "heavy_tail_ratio": 1.0503, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.2, + "fit_secs_per_prompt": 0.104, + "total_wall_seconds": 15.8, + "per_layer": [ + { + "layer": 0, + "fro_norm": 34.098, + "top_sv": 5.059, + "stable_rank": 45.428 + }, + { + "layer": 1, + "fro_norm": 26.62, + "top_sv": 4.288, + "stable_rank": 38.539 + }, + { + "layer": 2, + "fro_norm": 25.2059, + "top_sv": 3.5081, + "stable_rank": 51.626 + }, + { + "layer": 3, + "fro_norm": 23.0358, + "top_sv": 4.3902, + "stable_rank": 27.532 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_DOC_B.pt b/results/lenses/plad/plad_70m_wikitext_DOC_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..a53a9bccb61fc291172a671ced4b36e3fd5c8b7e --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOC_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53548198e4a986a0a1dd3087aedc933e4b3eada353fa75bdb0b55660bff2831e +size 2099408 diff --git a/results/lenses/plad/plad_70m_wikitext_DOC_B_provenance.json b/results/lenses/plad/plad_70m_wikitext_DOC_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..221a89b920817cbbdfa7293ce3456db2e5cdfafa --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_DOC_B_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_wikitext_DOC_B.pt", + "sha256": "53548198e4a986a0a1dd3087aedc933e4b3eada353fa75bdb0b55660bff2831e", + "bytes": 2099408, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": false, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.20872, + "mean_fro": 34.746311, + "rms_per_prompt_fro": 38.200723, + "max_per_prompt_fro": 42.690891, + "heavy_tail_ratio": 1.1175, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.159849, + "mean_fro": 27.068966, + "rms_per_prompt_fro": 29.152275, + "max_per_prompt_fro": 32.469372, + "heavy_tail_ratio": 1.1138, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.109226, + "mean_fro": 25.63451, + "rms_per_prompt_fro": 26.998216, + "max_per_prompt_fro": 29.377508, + "heavy_tail_ratio": 1.0881, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.05073, + "mean_fro": 23.401419, + "rms_per_prompt_fro": 23.987658, + "max_per_prompt_fro": 25.273874, + "heavy_tail_ratio": 1.0536, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.2, + "fit_secs_per_prompt": 0.103, + "total_wall_seconds": 15.1, + "per_layer": [ + { + "layer": 0, + "fro_norm": 34.7462, + "top_sv": 5.312, + "stable_rank": 42.786 + }, + { + "layer": 1, + "fro_norm": 27.0689, + "top_sv": 4.391, + "stable_rank": 38.003 + }, + { + "layer": 2, + "fro_norm": 25.6345, + "top_sv": 3.5718, + "stable_rank": 51.507 + }, + { + "layer": 3, + "fro_norm": 23.4014, + "top_sv": 4.5381, + "stable_rank": 26.591 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_LM_A.pt b/results/lenses/plad/plad_70m_wikitext_LM_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..5becedbbadcae96e82fc3e6414accfff74cfe722 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_LM_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d49641422dfb1efbc05811bd7769537586150da7817ac7313d50e0c49f173d0 +size 2099336 diff --git a/results/lenses/plad/plad_70m_wikitext_LM_A_provenance.json b/results/lenses/plad/plad_70m_wikitext_LM_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..9a93b3f4686cae51c03245441f046f02b1f86d77 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_LM_A_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_wikitext_LM_A.pt", + "sha256": "2d49641422dfb1efbc05811bd7769537586150da7817ac7313d50e0c49f173d0", + "bytes": 2099336, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.214674, + "mean_fro": 35.312465, + "rms_per_prompt_fro": 38.918669, + "max_per_prompt_fro": 44.659336, + "heavy_tail_ratio": 1.1475, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.165466, + "mean_fro": 27.371633, + "rms_per_prompt_fro": 29.549531, + "max_per_prompt_fro": 33.443409, + "heavy_tail_ratio": 1.1318, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.115917, + "mean_fro": 25.744574, + "rms_per_prompt_fro": 27.19579, + "max_per_prompt_fro": 30.094536, + "heavy_tail_ratio": 1.1066, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.05407, + "mean_fro": 23.254882, + "rms_per_prompt_fro": 23.875301, + "max_per_prompt_fro": 25.273874, + "heavy_tail_ratio": 1.0586, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.5, + "fit_secs_per_prompt": 0.11, + "total_wall_seconds": 15.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 35.3124, + "top_sv": 5.278, + "stable_rank": 44.762 + }, + { + "layer": 1, + "fro_norm": 27.3716, + "top_sv": 4.475, + "stable_rank": 37.412 + }, + { + "layer": 2, + "fro_norm": 25.7445, + "top_sv": 3.6284, + "stable_rank": 50.344 + }, + { + "layer": 3, + "fro_norm": 23.2548, + "top_sv": 4.3888, + "stable_rank": 28.077 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_LM_B.pt b/results/lenses/plad/plad_70m_wikitext_LM_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d8c31475b3ccbab78c6c096d98e3969bbd874f2 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_LM_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a81cb137cef05dc9e38539c91270826da99d2789d207886a1b3a85a8dac8147e +size 2099336 diff --git a/results/lenses/plad/plad_70m_wikitext_LM_B_provenance.json b/results/lenses/plad/plad_70m_wikitext_LM_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..557f36ffb30a4e7fc76ef62c607b4ca4c01247f4 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_LM_B_provenance.json @@ -0,0 +1,99 @@ +{ + "artifact": "results/plad_70m_wikitext_LM_B.pt", + "sha256": "a81cb137cef05dc9e38539c91270826da99d2789d207886a1b3a85a8dac8147e", + "bytes": 2099336, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "require_full_window": true, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cuda", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.200823, + "mean_fro": 35.389244, + "rms_per_prompt_fro": 38.780269, + "max_per_prompt_fro": 42.373501, + "heavy_tail_ratio": 1.0927, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.153537, + "mean_fro": 27.419615, + "rms_per_prompt_fro": 29.449444, + "max_per_prompt_fro": 32.624393, + "heavy_tail_ratio": 1.1078, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.107541, + "mean_fro": 25.749065, + "rms_per_prompt_fro": 27.098255, + "max_per_prompt_fro": 29.589869, + "heavy_tail_ratio": 1.0919, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.051253, + "mean_fro": 23.437135, + "rms_per_prompt_fro": 24.030237, + "max_per_prompt_fro": 25.369312, + "heavy_tail_ratio": 1.0557, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.15.0", + "torch": "2.6.0+cu124" + }, + "fit_wall_seconds": 5.5, + "fit_secs_per_prompt": 0.109, + "total_wall_seconds": 15.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 35.3892, + "top_sv": 5.4477, + "stable_rank": 42.201 + }, + { + "layer": 1, + "fro_norm": 27.4196, + "top_sv": 4.4223, + "stable_rank": 38.444 + }, + { + "layer": 2, + "fro_norm": 25.749, + "top_sv": 3.5988, + "stable_rank": 51.192 + }, + { + "layer": 3, + "fro_norm": 23.4371, + "top_sv": 4.539, + "stable_rank": 26.662 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_OE_A.pt b/results/lenses/plad/plad_70m_wikitext_OE_A.pt new file mode 100644 index 0000000000000000000000000000000000000000..dfabbfd9676ee2ad650eb2bec96a8de5e9d4163e --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_OE_A.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f859489597475161578c659b8dedc37f020d486223d4fba5af8b5e1b886cc3c +size 2099827 diff --git a/results/lenses/plad/plad_70m_wikitext_OE_A_provenance.json b/results/lenses/plad/plad_70m_wikitext_OE_A_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..4847c691a6d47d81448377f38e1eacb46602e5cc --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_OE_A_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_wikitext_OE_A.pt", + "sha256": "1f859489597475161578c659b8dedc37f020d486223d4fba5af8b5e1b886cc3c", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.145383, + "mean_fro": 34.39482, + "rms_per_prompt_fro": 36.810223, + "max_per_prompt_fro": 42.70137, + "heavy_tail_ratio": 1.16, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.115952, + "mean_fro": 26.929411, + "rms_per_prompt_fro": 28.447855, + "max_per_prompt_fro": 32.573502, + "heavy_tail_ratio": 1.145, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.089366, + "mean_fro": 25.539599, + "rms_per_prompt_fro": 26.656367, + "max_per_prompt_fro": 29.641071, + "heavy_tail_ratio": 1.112, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.053847, + "mean_fro": 23.223505, + "rms_per_prompt_fro": 23.840564, + "max_per_prompt_fro": 25.256557, + "heavy_tail_ratio": 1.0594, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 205.3, + "fit_secs_per_prompt": 4.106, + "total_wall_seconds": 231.0, + "per_layer": [ + { + "layer": 0, + "fro_norm": 34.3948, + "top_sv": 5.1912, + "stable_rank": 43.899 + }, + { + "layer": 1, + "fro_norm": 26.9294, + "top_sv": 4.3962, + "stable_rank": 37.522 + }, + { + "layer": 2, + "fro_norm": 25.5396, + "top_sv": 3.5846, + "stable_rank": 50.763 + }, + { + "layer": 3, + "fro_norm": 23.2235, + "top_sv": 4.4418, + "stable_rank": 27.336 + } + ] +} \ No newline at end of file diff --git a/results/lenses/plad/plad_70m_wikitext_OE_B.pt b/results/lenses/plad/plad_70m_wikitext_OE_B.pt new file mode 100644 index 0000000000000000000000000000000000000000..84eb52b8e9897e5d07c31214b3542c406762a511 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_OE_B.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff0f8c04896771c3476c78b3d8ca042bfc334b522e5c06ab7c782651e60c0fb +size 2099827 diff --git a/results/lenses/plad/plad_70m_wikitext_OE_B_provenance.json b/results/lenses/plad/plad_70m_wikitext_OE_B_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..3cf0cb80b790b05da3e582cb6628dbcde190d612 --- /dev/null +++ b/results/lenses/plad/plad_70m_wikitext_OE_B_provenance.json @@ -0,0 +1,98 @@ +{ + "artifact": "results/plad_70m_wikitext_OE_B.pt", + "sha256": "8ff0f8c04896771c3476c78b3d8ca042bfc334b522e5c06ab7c782651e60c0fb", + "bytes": 2099827, + "model": "EleutherAI/pythia-70m-deduped", + "corpus": "wikitext", + "n_prompts_requested": 50, + "n_prompts_used": 50, + "dim_batch": 128, + "max_seq_len": 128, + "skip_first_positions": 16, + "skip_first_is_library_default": false, + "target_layer_arg": -2, + "target_layer_effective": 4, + "target_layer_is_penultimate": true, + "device": "cpu", + "dtype": "float32", + "n_layers": 6, + "d_model": 512, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "centered": false, + "fitter": "pythia/fastfit.py:fast_fit (device-resident accumulator)", + "anchor_agreement_gate": null, + "jacobian_dispersion": { + "0": { + "dispersion": 0.146547, + "mean_fro": 34.212182, + "rms_per_prompt_fro": 36.633348, + "max_per_prompt_fro": 43.189819, + "heavy_tail_ratio": 1.179, + "n_prompts": 50 + }, + "1": { + "dispersion": 0.11798, + "mean_fro": 26.605178, + "rms_per_prompt_fro": 28.130876, + "max_per_prompt_fro": 31.975203, + "heavy_tail_ratio": 1.1367, + "n_prompts": 50 + }, + "2": { + "dispersion": 0.089618, + "mean_fro": 25.210371, + "rms_per_prompt_fro": 26.315793, + "max_per_prompt_fro": 28.621368, + "heavy_tail_ratio": 1.0876, + "n_prompts": 50 + }, + "3": { + "dispersion": 0.053314, + "mean_fro": 23.164106, + "rms_per_prompt_fro": 23.773569, + "max_per_prompt_fro": 24.817972, + "heavy_tail_ratio": 1.0439, + "n_prompts": 50 + } + }, + "layout_autodetected": false, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "fit_wall_seconds": 207.1, + "fit_secs_per_prompt": 4.142, + "total_wall_seconds": 233.5, + "per_layer": [ + { + "layer": 0, + "fro_norm": 34.2122, + "top_sv": 5.1448, + "stable_rank": 44.221 + }, + { + "layer": 1, + "fro_norm": 26.6052, + "top_sv": 4.27, + "stable_rank": 38.823 + }, + { + "layer": 2, + "fro_norm": 25.2104, + "top_sv": 3.4899, + "stable_rank": 52.182 + }, + { + "layer": 3, + "fro_norm": 23.1641, + "top_sv": 4.4673, + "stable_rank": 26.887 + } + ] +} \ No newline at end of file diff --git a/results/m1/m1_5shard.json b/results/m1/m1_5shard.json new file mode 100644 index 0000000000000000000000000000000000000000..990af0535199dd9e1ae798e7cc93347f10d370c4 --- /dev/null +++ b/results/m1/m1_5shard.json @@ -0,0 +1,288 @@ +{ + "experiment": "E35 / M1 \u2014 n-gram containment against the Pythia stream (inverted query)", + "prereg": "pythia/specs/PREREG_E36_QLADDER.md, section E35", + "mode": "STREAM", + "gates": { + "G1_tokenizer_identical": true, + "G1_mismatched_probes": [], + "G2_vocab_fits_uint16": true, + "vocab_size": 50277 + }, + "k_sweep": [ + 8, + 13, + 20, + 32 + ], + "window": 128, + "tokens_streamed": 0, + "kgrams_probed": 0, + "runtime_s": 0.9, + "containment": { + "8": { + "CONTROL_PubMed_2023": { + "containment": 0.34787169014475844, + "n_distinct_kgrams": 287928, + "n_found": 100162 + }, + "Github": { + "containment": 0.7798426536253455, + "n_distinct_kgrams": 211635, + "n_found": 165042 + }, + "OOD_CommonPile": { + "containment": 0.09085169147963497, + "n_distinct_kgrams": 289835, + "n_found": 26332 + }, + "OOD_News_2024": { + "containment": 0.053965862122024075, + "n_distinct_kgrams": 188471, + "n_found": 10171 + }, + "OOD_Wikipedia_2023": { + "containment": 0.32380599161880463, + "n_distinct_kgrams": 288503, + "n_found": 93419 + }, + "OOD_arXiv_2023": { + "containment": 0.0401850428385403, + "n_distinct_kgrams": 284907, + "n_found": 11449 + }, + "Pile-CC": { + "containment": 0.433747803378771, + "n_distinct_kgrams": 285666, + "n_found": 123907 + }, + "StackExchange": { + "containment": 0.44562262670421904, + "n_distinct_kgrams": 285468, + "n_found": 127211 + }, + "TRAP_FineWeb": { + "containment": 0.10673636753377864, + "n_distinct_kgrams": 289236, + "n_found": 30872 + }, + "USPTO_Backgrounds": { + "containment": 0.5111694995891182, + "n_distinct_kgrams": 277452, + "n_found": 141825 + }, + "Wikipedia_en": { + "containment": 0.44922148871627204, + "n_distinct_kgrams": 288114, + "n_found": 129427 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 24200, + "n_found": 0 + } + }, + "13": { + "CONTROL_PubMed_2023": { + "containment": 0.3011107354305898, + "n_distinct_kgrams": 278014, + "n_found": 83713 + }, + "Github": { + "containment": 0.7156167293808401, + "n_distinct_kgrams": 222172, + "n_found": 158990 + }, + "OOD_CommonPile": { + "containment": 0.0183007663041721, + "n_distinct_kgrams": 278349, + "n_found": 5094 + }, + "OOD_News_2024": { + "containment": 0.002778172749844138, + "n_distinct_kgrams": 182854, + "n_found": 508 + }, + "OOD_Wikipedia_2023": { + "containment": 0.20917897342890865, + "n_distinct_kgrams": 277896, + "n_found": 58130 + }, + "OOD_arXiv_2023": { + "containment": 0.002104322509044263, + "n_distinct_kgrams": 277524, + "n_found": 584 + }, + "Pile-CC": { + "containment": 0.3854749619092294, + "n_distinct_kgrams": 275001, + "n_found": 106006 + }, + "StackExchange": { + "containment": 0.3721316686896711, + "n_distinct_kgrams": 276816, + "n_found": 103012 + }, + "TRAP_FineWeb": { + "containment": 0.052700149358478346, + "n_distinct_kgrams": 277855, + "n_found": 14643 + }, + "USPTO_Backgrounds": { + "containment": 0.4553867772284655, + "n_distinct_kgrams": 273271, + "n_found": 124444 + }, + "Wikipedia_en": { + "containment": 0.37257131343369504, + "n_distinct_kgrams": 277928, + "n_found": 103548 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 23200, + "n_found": 0 + } + }, + "20": { + "CONTROL_PubMed_2023": { + "containment": 0.29342643116891065, + "n_distinct_kgrams": 261517, + "n_found": 76736 + }, + "Github": { + "containment": 0.6629392023013173, + "n_distinct_kgrams": 219005, + "n_found": 145187 + }, + "OOD_CommonPile": { + "containment": 0.0018845997996895954, + "n_distinct_kgrams": 261594, + "n_found": 493 + }, + "OOD_News_2024": { + "containment": 0.0009513595793260915, + "n_distinct_kgrams": 173436, + "n_found": 165 + }, + "OOD_Wikipedia_2023": { + "containment": 0.15561650962704524, + "n_distinct_kgrams": 261399, + "n_found": 40678 + }, + "OOD_arXiv_2023": { + "containment": 0.00016463426307023756, + "n_distinct_kgrams": 261185, + "n_found": 43 + }, + "Pile-CC": { + "containment": 0.37259608890466944, + "n_distinct_kgrams": 259109, + "n_found": 96543 + }, + "StackExchange": { + "containment": 0.3528183796040939, + "n_distinct_kgrams": 261072, + "n_found": 92111 + }, + "TRAP_FineWeb": { + "containment": 0.03867481544696468, + "n_distinct_kgrams": 261307, + "n_found": 10106 + }, + "USPTO_Backgrounds": { + "containment": 0.4278531301406003, + "n_distinct_kgrams": 257752, + "n_found": 110280 + }, + "Wikipedia_en": { + "containment": 0.34699295723461476, + "n_distinct_kgrams": 261403, + "n_found": 90705 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 21800, + "n_found": 0 + } + }, + "32": { + "CONTROL_PubMed_2023": { + "containment": 0.2897504188324241, + "n_distinct_kgrams": 232790, + "n_found": 67451 + }, + "Github": { + "containment": 0.6077905034597948, + "n_distinct_kgrams": 201168, + "n_found": 122268 + }, + "OOD_CommonPile": { + "containment": 0.00019329896907216494, + "n_distinct_kgrams": 232800, + "n_found": 45 + }, + "OOD_News_2024": { + "containment": 0.0005486844483462006, + "n_distinct_kgrams": 154916, + "n_found": 85 + }, + "OOD_Wikipedia_2023": { + "containment": 0.11009801099132467, + "n_distinct_kgrams": 232729, + "n_found": 25623 + }, + "OOD_arXiv_2023": { + "containment": 0.0, + "n_distinct_kgrams": 232587, + "n_found": 0 + }, + "Pile-CC": { + "containment": 0.36025645462725825, + "n_distinct_kgrams": 231152, + "n_found": 83274 + }, + "StackExchange": { + "containment": 0.34294262795436375, + "n_distinct_kgrams": 232622, + "n_found": 79776 + }, + "TRAP_FineWeb": { + "containment": 0.02606299990974225, + "n_distinct_kgrams": 232667, + "n_found": 6064 + }, + "USPTO_Backgrounds": { + "containment": 0.4081210357566042, + "n_distinct_kgrams": 229552, + "n_found": 93685 + }, + "Wikipedia_en": { + "containment": 0.3318978670001762, + "n_distinct_kgrams": 232677, + "n_found": 77225 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 19400, + "n_found": 0 + } + } + }, + "k_ranges": { + "8": 0.7396576107868053, + "13": 0.7135124068717958, + "20": 0.6627745680382471, + "32": 0.6077905034597948 + }, + "k_selected": 32, + "k_selection_rule": "largest k in the sweep whose containment range across real Q corpora is >= 0.30; fixed before running", + "controls": { + "M1_C2_random_containment": { + "8": 0.0, + "13": 0.0, + "20": 0.0, + "32": 0.0 + }, + "M1_C2_fires": true + } +} \ No newline at end of file diff --git a/results/m1/m1_partial.json b/results/m1/m1_partial.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe97903ce057230b3eefdf13e2a2c21bc3f5671 --- /dev/null +++ b/results/m1/m1_partial.json @@ -0,0 +1,288 @@ +{ + "experiment": "E35 / M1 \u2014 n-gram containment against the Pythia stream (inverted query)", + "prereg": "pythia/specs/PREREG_E36_QLADDER.md, section E35", + "mode": "STREAM", + "gates": { + "G1_tokenizer_identical": true, + "G1_mismatched_probes": [], + "G2_vocab_fits_uint16": true, + "vocab_size": 50277 + }, + "k_sweep": [ + 8, + 13, + 20, + 32 + ], + "window": 128, + "tokens_streamed": 0, + "kgrams_probed": 0, + "runtime_s": 0.5, + "containment": { + "8": { + "CONTROL_PubMed_2023": { + "containment": 0.2401190575421633, + "n_distinct_kgrams": 287928, + "n_found": 69137 + }, + "Github": { + "containment": 0.7083374678101448, + "n_distinct_kgrams": 211635, + "n_found": 149909 + }, + "OOD_CommonPile": { + "containment": 0.06627563958804147, + "n_distinct_kgrams": 289835, + "n_found": 19209 + }, + "OOD_News_2024": { + "containment": 0.04525364644958641, + "n_distinct_kgrams": 188471, + "n_found": 8529 + }, + "OOD_Wikipedia_2023": { + "containment": 0.2335608295234365, + "n_distinct_kgrams": 288503, + "n_found": 67383 + }, + "OOD_arXiv_2023": { + "containment": 0.034386659506435435, + "n_distinct_kgrams": 284907, + "n_found": 9797 + }, + "Pile-CC": { + "containment": 0.30552813425468905, + "n_distinct_kgrams": 285666, + "n_found": 87279 + }, + "StackExchange": { + "containment": 0.3301490885142993, + "n_distinct_kgrams": 285468, + "n_found": 94247 + }, + "TRAP_FineWeb": { + "containment": 0.08378625067418993, + "n_distinct_kgrams": 289236, + "n_found": 24234 + }, + "USPTO_Backgrounds": { + "containment": 0.38791214336173463, + "n_distinct_kgrams": 277452, + "n_found": 107627 + }, + "Wikipedia_en": { + "containment": 0.3255759872828117, + "n_distinct_kgrams": 288114, + "n_found": 93803 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 24200, + "n_found": 0 + } + }, + "13": { + "CONTROL_PubMed_2023": { + "containment": 0.19475637917514946, + "n_distinct_kgrams": 278014, + "n_found": 54145 + }, + "Github": { + "containment": 0.6349855067245197, + "n_distinct_kgrams": 222172, + "n_found": 141076 + }, + "OOD_CommonPile": { + "containment": 0.011891546224344258, + "n_distinct_kgrams": 278349, + "n_found": 3310 + }, + "OOD_News_2024": { + "containment": 0.002220350662277008, + "n_distinct_kgrams": 182854, + "n_found": 406 + }, + "OOD_Wikipedia_2023": { + "containment": 0.13488139447850994, + "n_distinct_kgrams": 277896, + "n_found": 37483 + }, + "OOD_arXiv_2023": { + "containment": 0.0017656130640953576, + "n_distinct_kgrams": 277524, + "n_found": 490 + }, + "Pile-CC": { + "containment": 0.25664997581826976, + "n_distinct_kgrams": 275001, + "n_found": 70579 + }, + "StackExchange": { + "containment": 0.25172316629096586, + "n_distinct_kgrams": 276816, + "n_found": 69681 + }, + "TRAP_FineWeb": { + "containment": 0.03806301848086232, + "n_distinct_kgrams": 277855, + "n_found": 10576 + }, + "USPTO_Backgrounds": { + "containment": 0.3295958956493737, + "n_distinct_kgrams": 273271, + "n_found": 90069 + }, + "Wikipedia_en": { + "containment": 0.2451282346507009, + "n_distinct_kgrams": 277928, + "n_found": 68128 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 23200, + "n_found": 0 + } + }, + "20": { + "CONTROL_PubMed_2023": { + "containment": 0.18819044268632631, + "n_distinct_kgrams": 261517, + "n_found": 49215 + }, + "Github": { + "containment": 0.5773521152485103, + "n_distinct_kgrams": 219005, + "n_found": 126443 + }, + "OOD_CommonPile": { + "containment": 0.0012882558468466402, + "n_distinct_kgrams": 261594, + "n_found": 337 + }, + "OOD_News_2024": { + "containment": 0.0007553218478285939, + "n_distinct_kgrams": 173436, + "n_found": 131 + }, + "OOD_Wikipedia_2023": { + "containment": 0.09550151301267411, + "n_distinct_kgrams": 261399, + "n_found": 24964 + }, + "OOD_arXiv_2023": { + "containment": 0.00016080555927790646, + "n_distinct_kgrams": 261185, + "n_found": 42 + }, + "Pile-CC": { + "containment": 0.24451099730229364, + "n_distinct_kgrams": 259109, + "n_found": 63355 + }, + "StackExchange": { + "containment": 0.23242630385487528, + "n_distinct_kgrams": 261072, + "n_found": 60680 + }, + "TRAP_FineWeb": { + "containment": 0.027106047675722427, + "n_distinct_kgrams": 261307, + "n_found": 7083 + }, + "USPTO_Backgrounds": { + "containment": 0.3017706943108104, + "n_distinct_kgrams": 257752, + "n_found": 77782 + }, + "Wikipedia_en": { + "containment": 0.21928210464302245, + "n_distinct_kgrams": 261403, + "n_found": 57321 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 21800, + "n_found": 0 + } + }, + "32": { + "CONTROL_PubMed_2023": { + "containment": 0.1853258301473431, + "n_distinct_kgrams": 232790, + "n_found": 43142 + }, + "Github": { + "containment": 0.5195011134971765, + "n_distinct_kgrams": 201168, + "n_found": 104507 + }, + "OOD_CommonPile": { + "containment": 0.00012457044673539519, + "n_distinct_kgrams": 232800, + "n_found": 29 + }, + "OOD_News_2024": { + "containment": 0.0004324924475199463, + "n_distinct_kgrams": 154916, + "n_found": 67 + }, + "OOD_Wikipedia_2023": { + "containment": 0.06429366344546661, + "n_distinct_kgrams": 232729, + "n_found": 14963 + }, + "OOD_arXiv_2023": { + "containment": 0.0, + "n_distinct_kgrams": 232587, + "n_found": 0 + }, + "Pile-CC": { + "containment": 0.23348705613622206, + "n_distinct_kgrams": 231152, + "n_found": 53971 + }, + "StackExchange": { + "containment": 0.2230184591311226, + "n_distinct_kgrams": 232622, + "n_found": 51879 + }, + "TRAP_FineWeb": { + "containment": 0.01772490297291838, + "n_distinct_kgrams": 232667, + "n_found": 4124 + }, + "USPTO_Backgrounds": { + "containment": 0.2827638182198369, + "n_distinct_kgrams": 229552, + "n_found": 64909 + }, + "Wikipedia_en": { + "containment": 0.2045797392952462, + "n_distinct_kgrams": 232677, + "n_found": 47601 + }, + "_CONTROL_random": { + "containment": 0.0, + "n_distinct_kgrams": 19400, + "n_found": 0 + } + } + }, + "k_ranges": { + "8": 0.6739508083037093, + "13": 0.6332198936604243, + "20": 0.5771913096892324, + "32": 0.5195011134971765 + }, + "k_selected": 32, + "k_selection_rule": "largest k in the sweep whose containment range across real Q corpora is >= 0.30; fixed before running", + "controls": { + "M1_C2_random_containment": { + "8": 0.0, + "13": 0.0, + "20": 0.0, + "32": 0.0 + }, + "M1_C2_fires": true + } +} \ No newline at end of file diff --git a/results/m1/merged/shard00.npz b/results/m1/merged/shard00.npz new file mode 100644 index 0000000000000000000000000000000000000000..a574bce91077ac6dea83ca1572880c5fef26ba81 --- /dev/null +++ b/results/m1/merged/shard00.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:271dacb8f530278fba62a1a0599a0294781f4cb2b81f0c8d5383ef341903abb6 +size 1039631 diff --git a/results/m1/merged/shard01.npz b/results/m1/merged/shard01.npz new file mode 100644 index 0000000000000000000000000000000000000000..6e0ac33380e075f09d929bb5fb74c346b40b385e --- /dev/null +++ b/results/m1/merged/shard01.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc499f9d5558cfd18fb25daefc6041313953e788557f8bfda08f7be5ff9baf87 +size 1052626 diff --git a/results/m1/merged/shard02.npz b/results/m1/merged/shard02.npz new file mode 100644 index 0000000000000000000000000000000000000000..4ae2b58ce3b33c2d304b1760dacbceb695e3247f --- /dev/null +++ b/results/m1/merged/shard02.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1bb6cbeb70b055f16dc6ad1f6d9653eb747b55c6ecec451b7861b3e422f2f23 +size 1048923 diff --git a/results/m1/merged/shard03.npz b/results/m1/merged/shard03.npz new file mode 100644 index 0000000000000000000000000000000000000000..e82b9b6cc81893334490c28147b7c0b40cfd36d7 --- /dev/null +++ b/results/m1/merged/shard03.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5303578e3918fa2cab600d501238fd844e8fa26832007dfdbbe0483fa051834 +size 1046557 diff --git a/results/m1/merged/shard04.npz b/results/m1/merged/shard04.npz new file mode 100644 index 0000000000000000000000000000000000000000..fa670289b9e60abf2e5f94205564e20139bf3e32 --- /dev/null +++ b/results/m1/merged/shard04.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f3c825f329c831d753ef2636640e6270a09205632ddf544b49020cb08c30e29 +size 1047396 diff --git a/results/m1/merged/shard05.npz b/results/m1/merged/shard05.npz new file mode 100644 index 0000000000000000000000000000000000000000..26cc20db03c64a645684541e09db3405d470a132 --- /dev/null +++ b/results/m1/merged/shard05.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a573dffbbc1225482769693b643325350b7270ddd36aececf883949022762208 +size 1050274 diff --git a/results/m1/merged/shard06.npz b/results/m1/merged/shard06.npz new file mode 100644 index 0000000000000000000000000000000000000000..6701eb63a7b50c03b0fad0f0a9e069f867981a3a --- /dev/null +++ b/results/m1/merged/shard06.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efd6470fd6ce2369bf099a1b140f2f086c2e845a3600dd97822cb4a8c48400cc +size 1049424 diff --git a/results/m1/merged/shard07.npz b/results/m1/merged/shard07.npz new file mode 100644 index 0000000000000000000000000000000000000000..05d15a559390c367f63e6a20ada862c0dbeb24ba --- /dev/null +++ b/results/m1/merged/shard07.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37db3648354e22ed6b5619794399de0fccd29611e5a9ad92ff0ca694c14f05d0 +size 1037538 diff --git a/results/m1/merged/shard08.npz b/results/m1/merged/shard08.npz new file mode 100644 index 0000000000000000000000000000000000000000..0f24ff5ef13f3b5747b857d616015dd432f5199f --- /dev/null +++ b/results/m1/merged/shard08.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33a51b8473d3ce54db3d1a317bf34c7360114c6204462b7b91f62ebaa4a9cad7 +size 1042641 diff --git a/results/m1/merged/shard09.npz b/results/m1/merged/shard09.npz new file mode 100644 index 0000000000000000000000000000000000000000..1a17e94c79c7304d01ad4656e58834b9b950636b --- /dev/null +++ b/results/m1/merged/shard09.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f1527d8af10e27a1e12767ee503213f00652fa99d4c22ec31a2e9e9caf63020 +size 1048836 diff --git a/results/m1/merged/shard10.npz b/results/m1/merged/shard10.npz new file mode 100644 index 0000000000000000000000000000000000000000..15d0cf6d02ff564c9237e397f48ada73483d1f41 --- /dev/null +++ b/results/m1/merged/shard10.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a931f59851c54b1cb23c9b2025b60b16c7a09a2c32946d725560b4d8ba54e2f +size 1031060 diff --git a/results/m1/merged/shard11.npz b/results/m1/merged/shard11.npz new file mode 100644 index 0000000000000000000000000000000000000000..c26cbdb9f2180ab7552c8227108a00b5cc5d77d9 --- /dev/null +++ b/results/m1/merged/shard11.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b7c65ef6da707ba0dbe1fb592479c13fd0d1534dda5ba3d558053f280a76250 +size 1050414 diff --git a/results/m1/merged/shard12.npz b/results/m1/merged/shard12.npz new file mode 100644 index 0000000000000000000000000000000000000000..2e0da3b68d65e798bc5392e22038ee997186170d --- /dev/null +++ b/results/m1/merged/shard12.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6583e9d941f58bd395ce589cdedf5602b071915bda715db14a800d99d66a85 +size 1028856 diff --git a/results/m1/merged/shard13.npz b/results/m1/merged/shard13.npz new file mode 100644 index 0000000000000000000000000000000000000000..d647ac74c4f36e71844f4b983d47e94ba7a796c7 --- /dev/null +++ b/results/m1/merged/shard13.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4c1165f15b052a3a762158f05e2ca1db3eee20b7e9c04af1ee48131d0958ff0 +size 1049628 diff --git a/results/m1/merged/shard14.npz b/results/m1/merged/shard14.npz new file mode 100644 index 0000000000000000000000000000000000000000..c9ca3b638449e9ca177bdb4e4a62e424442b2058 --- /dev/null +++ b/results/m1/merged/shard14.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c275d4ccc84c7d4aea84681e16f1338c5f91e81b5c60cb0e6304c8c9c5c4ea4 +size 1041328 diff --git a/results/m1/merged/shard15.npz b/results/m1/merged/shard15.npz new file mode 100644 index 0000000000000000000000000000000000000000..1259e431e26fd3263481cd496c8fa715839d0f59 --- /dev/null +++ b/results/m1/merged/shard15.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:debd79b3c5163291183a84b26f3392ef71d762c2f048656d50fbd73d6704c8c9 +size 1057596 diff --git a/results/m1/merged/shard16.npz b/results/m1/merged/shard16.npz new file mode 100644 index 0000000000000000000000000000000000000000..91d76a8f352513b2e61b775b674a8177c340f45f --- /dev/null +++ b/results/m1/merged/shard16.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:889d11318360d394fac55174b9a39c9a8ecb199d66ea10ee9bfb04bad10e1ffe +size 1051198 diff --git a/results/m1/merged/shard17.npz b/results/m1/merged/shard17.npz new file mode 100644 index 0000000000000000000000000000000000000000..ba6466c6d5319adaa1e49361c0a0cdde0d412ff6 --- /dev/null +++ b/results/m1/merged/shard17.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8605129b53db7ddc2ad5b38e154d50a2fb669089393b150f0c45296df469daca +size 1049840 diff --git a/results/m1/merged/shard18.npz b/results/m1/merged/shard18.npz new file mode 100644 index 0000000000000000000000000000000000000000..45693fca543a5fc02ad3b470942e97dbdde2a9d2 --- /dev/null +++ b/results/m1/merged/shard18.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ec716f947bcc7479cec5e0f84b7e800f28441db2026ba92f513e2bb2d628e31 +size 1052755 diff --git a/results/m1/merged/shard19.npz b/results/m1/merged/shard19.npz new file mode 100644 index 0000000000000000000000000000000000000000..bf99a3ea7bb96bf5a5695a7f36c27430b387518f --- /dev/null +++ b/results/m1/merged/shard19.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13678ac5fb4cf617ef89b98d59e920285d0845b98645cb457f9a9e684fb48633 +size 1044727 diff --git a/results/os1_operator_space_410m.json b/results/os1_operator_space_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..97b1b895f2eadbccf490d45e1ee0e3a761bc1a20 --- /dev/null +++ b/results/os1_operator_space_410m.json @@ -0,0 +1,408 @@ +{ + "experiment": "OS1 \u2014 corpus displacement of the operator itself", + "prereg": "docs/experiments/preregs/OS1_operator_space.md", + "status": "PRE-REGISTERED", + "recomputes_not_remeasures": true, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "N": 200, + "k_subspace": 32, + "scope": "Displacement of the estimated operator. Says nothing about whether the displacement matters for a readout, nor about estimation error at N=200.", + "C4_band_matches": { + "required": "source_layers == [9..21] on all operators", + "n_checked": 32, + "fires": true + }, + "contrasts": { + "BETWEEN_corpus": { + "n": 84, + "median_d_rel": 0.46083697529132706, + "median_theta": 0.805525650198643, + "min_d_rel": 0.12955496973039707, + "max_d_rel": 0.6170678345791398 + }, + "WITHIN_seed_block": { + "n": 24, + "median_d_rel": 0.05122771490488408, + "median_theta": 0.1048034425251759 + }, + "WITHIN_source_disjoint_docs": { + "n": 12, + "median_d_rel": 0.049502175875797305, + "median_theta": 0.09503095897917564, + "note": "R6's four disjoint document blocks per source" + } + }, + "PRIMARY": { + "SEP_d_rel": 9.309428669309066, + "SEP_d_rel_ci": [ + 8.078577572182624, + 10.442952702479129 + ], + "SEP_theta": 8.476455029514748, + "SEP_theta_ci": [ + 7.519699999258505, + 9.258069789472367 + ], + "n_boot": 10000, + "bootstrap_unit": "fitting corpus, resampled with replacement; within-source pairs resampled" + }, + "C1_self_distance": { + "required": "exactly 0 on both metrics", + "d_rel": 0.0, + "theta": 0.00037359047564677894, + "fires": false + }, + "C2_metric_can_separate": { + "required": "a norm-matched random operator sits above every BETWEEN pair", + "random_vs_real_d_rel": 1.4143145449163792, + "max_between_d_rel": 0.6170678345791398, + "random_vs_real_theta": 1.4193653693565955, + "max_between_theta": 0.9929391650053171, + "fires": true + }, + "C3_fp16_roundtrip": { + "required": "at least 10x below the within-source median d_rel", + "roundtrip_d_rel": 0.0, + "within_source_median": 0.049502175875797305, + "ratio": Infinity, + "fires": true + }, + "C5_per_layer": { + "reported_not_gated": true, + "by_layer": { + "9": { + "between": 0.6749385048982096, + "within_source": 0.0825266583928897, + "SEP": 8.178430073891866 + }, + "10": { + "between": 0.6596151156484106, + "within_source": 0.0782536860811751, + "SEP": 8.429189073140533 + }, + "11": { + "between": 0.6440691114071331, + "within_source": 0.0740760720659662, + "SEP": 8.694698482845808 + }, + "12": { + "between": 0.591508225341446, + "within_source": 0.06664058522687494, + "SEP": 8.87609590053392 + }, + "13": { + "between": 0.5739258433169966, + "within_source": 0.061570112271851093, + "SEP": 9.321500678493754 + }, + "14": { + "between": 0.5409255037678098, + "within_source": 0.05592392194894912, + "SEP": 9.67252447461751 + }, + "15": { + "between": 0.5084862463669395, + "within_source": 0.0508886435485831, + "SEP": 9.99213598376798 + }, + "16": { + "between": 0.44518740575913474, + "within_source": 0.04351667381159293, + "SEP": 10.230271911097578 + }, + "17": { + "between": 0.3869656987890778, + "within_source": 0.03701472938394206, + "SEP": 10.454370604069672 + }, + "18": { + "between": 0.32884959876742936, + "within_source": 0.03183492441525846, + "SEP": 10.329837585849958 + }, + "19": { + "between": 0.2725807195908455, + "within_source": 0.02630828822731425, + "SEP": 10.361020726078332 + }, + "20": { + "between": 0.21046841786214693, + "within_source": 0.01981154579013223, + "SEP": 10.623523277369776 + }, + "21": { + "between": 0.14667093464265588, + "within_source": 0.013365412093153357, + "SEP": 10.973917872520397 + } + }, + "min_SEP": 8.178430073891866, + "max_SEP": 10.973917872520397, + "carried_by_few_layers": false + }, + "controls_fired": { + "C1_self_distance": false, + "C2_metric_can_separate": true, + "C3_fp16_roundtrip": true, + "C4_band_matches": true + }, + "VERDICT": "UNCLEAR \u2014 a control did not fire: C1_self_distance", + "provenance": { + "schema_version": 1, + "experiment": "OS1", + "utc": "2026-08-31T20:17:22+00:00", + "script": { + "path": "experiments/os1_operator_space.py", + "sha256": "c9a106f07f8cde303ce4e48b481ecc1ce260b58e1b98085714a7487501360d78", + "n_lines": 283 + }, + "git": { + "commit": "7275c33b11aeef0e42658e8d0ee66134a17bfe84", + "commit_short": "7275c33b11ae", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "research-paper-writing/", + "xperiments/os1_operator_space.py" + ], + "dirty_diff_sha256": "09af1a172a575483069d6dfa9fb171fa78010211b86b916c6ed1de9e50e586e4", + "describe": "7275c33-dirty" + }, + "argv": [ + "experiments/os1_operator_space.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/r6/lens_R6_Pile-CC_b0_410m_n200.pt", + "sha256": "33dae3b0d750d08e1cc7236e477e442e503d99f669a0f3ad0cfa4ef490e272e8", + "bytes": 27268171, + "exists": true + }, + { + "path": "results/r6/lens_R6_Pile-CC_b1_410m_n200.pt", + "sha256": "10e7a7af7110bb0515adc8cc8b27c638c7cb040aa47c3224280a1b53db0e46d9", + "bytes": 27268171, + "exists": true + }, + { + "path": "results/r6/lens_R6_Pile-CC_b2_410m_n200.pt", + "sha256": "8fc8cc67463f391d6e0e5fbe83230263309c13c7babeb27b5331c79588432e75", + "bytes": 27268171, + "exists": true + }, + { + "path": "results/r6/lens_R6_Pile-CC_b3_410m_n200.pt", + "sha256": "4ebb1fce54b6c476f46cb5b38067b98af5f7bc85331f042471bbc7e5efab9b66", + "bytes": 27268171, + "exists": true + }, + { + "path": "results/r6/lens_R6_Wikipedia_en_b0_410m_n200.pt", + "sha256": "b9bd821d39b93b9353433600c62f0b7c0711dc02d8c3622db934e90f66829886", + "bytes": 27268266, + "exists": true + }, + { + "path": "results/r6/lens_R6_Wikipedia_en_b1_410m_n200.pt", + "sha256": "b24b4903c5ef156ae1a08afda45f6dc3ff89451d1ffe03c0350b9382ce7d0d27", + "bytes": 27268266, + "exists": true + }, + { + "path": "results/r6/lens_R6_Wikipedia_en_b2_410m_n200.pt", + "sha256": "51c9fde7e588d29c445bfe2b9067da68ad018609f362de42790d3ded25a4a529", + "bytes": 27268266, + "exists": true + }, + { + "path": "results/r6/lens_R6_Wikipedia_en_b3_410m_n200.pt", + "sha256": "49ad6f5c39d632a4c2b0bc96c10362b3a6f3168ad1ee9b3889a3f0b694bbc55f", + "bytes": 27268266, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "c1e05ebc9894276d6a779fc2dbc0fcdcfd3b84d753fc3b752b0c836adb52d785" + } +} \ No newline at end of file diff --git a/results/os1b_selfdistance_floor.json b/results/os1b_selfdistance_floor.json new file mode 100644 index 0000000000000000000000000000000000000000..cacc727f6bcbbb923c67c8f2c3399f037a336309 --- /dev/null +++ b/results/os1b_selfdistance_floor.json @@ -0,0 +1,138 @@ +{ + "experiment": "OS1b \u2014 the numerical floor of OS1's theta self-distance control", + "status": "MEASUREMENT ONLY \u2014 no registration, no decision rule, adjudicates nothing", + "about": "results/os1_operator_space_410m.json:C1_self_distance", + "metric": "mean principal angle over the top-32 left singular subspaces, radians", + "operators": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "os1_c1_required": "exactly 0 on both metrics", + "os1_c1_measured_theta": 0.0003735905, + "os1_c1_measured_d_rel": 0.0, + "float32_eps": 1.1920928955078125e-07, + "float64_eps": 2.220446049250313e-16, + "predicted_floor_sqrt_2eps": { + "float32": 0.00048828125, + "float64": 2.1073424255447017e-08 + }, + "theta_self_distance_by_layer": { + "9": { + "float32": 0.0005157937412150204, + "float64": 1.392044176064328e-08 + }, + "10": { + "float32": 0.00045887965825386345, + "float64": 1.1371627301464958e-08 + }, + "11": { + "float32": 0.00035494580515660346, + "float64": 2.0748647508995134e-08 + }, + "12": { + "float32": 0.00028916774317622185, + "float64": 1.7980508363294786e-08 + }, + "13": { + "float32": 0.00020395449246279895, + "float64": 1.847799301645098e-09 + }, + "14": { + "float32": 0.0002822724636644125, + "float64": 8.722442911679253e-09 + }, + "15": { + "float32": 0.0003526768123265356, + "float64": 1.8639207378106388e-08 + }, + "16": { + "float32": 0.00048097758553922176, + "float64": 1.6911174239957414e-08 + }, + "17": { + "float32": 0.0006052301614545286, + "float64": 1.626573765603435e-08 + }, + "18": { + "float32": 0.0003004071768373251, + "float64": 4.134287674683542e-09 + }, + "19": { + "float32": 0.00037312990752980113, + "float64": 1.3100549204247785e-08 + }, + "20": { + "float32": 0.0002451541367918253, + "float64": 2.019068672127607e-08 + }, + "21": { + "float32": 0.00039408649899996817, + "float64": 1.6297269894172045e-08 + } + }, + "summary": { + "float32_min": 0.00020395449246279895, + "float32_max": 0.0006052301614545286, + "float32_mean": 0.00037359047564677894, + "float64_mean": 1.3856183070476932e-08, + "ratio_f32_over_f64": 26962.004885947274, + "random_matrix_control": { + "float32": 0.00028591742739081383, + "float64": 1.9200554738297506e-08 + } + }, + "READING": "theta(U,U) on the stored operators is 2.040e-04..6.052e-04 in float32 and ~1.39e-08 in float64, a factor of 26962. A random matrix gives the same float32 value (2.859e-04), so the floor is a property of the arithmetic, not of these operators. OS1's measured 3.736e-04 lies inside that float32 distribution and below the predicted floor sqrt(2*eps32) = 4.883e-04. C1 required exact zero from a quantity whose float32 floor is ~4e-04, so it could not fire. In float64 the self-distance is ~1e-08, which is zero. OS1's PRIMARY and its other three controls are unaffected.", + "provenance": { + "schema_version": 1, + "experiment": "OS1b", + "utc": "2026-08-31T03:40:04+00:00", + "script": { + "path": "experiments/os1b_selfdistance_floor.py", + "sha256": "9e61663b49e8ab0de68fd90594d682b3bee3a673253f4debc9ef4bdebf844e8e", + "n_lines": 104 + }, + "git": { + "commit": "99a6dd48ac17cd06f63d77e1e2fa51296cd189f7", + "commit_short": "99a6dd48ac17", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "PAPER_REWRITE_SCOPE.md", + "xperiments/e66b_determinism_floor.py" + ], + "dirty_diff_sha256": "e28f1f06f2d548944180fd4fdef14a82157efc3f05383027d0039cdb4af76894", + "describe": "99a6dd4-dirty" + }, + "argv": [ + "experiments/os1b_selfdistance_floor.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "8393fd879df3ba723b64b331687da0f38b5673e47323a5e49918f00df1f3b70d" + } +} \ No newline at end of file diff --git a/results/os1c_selfdistance_adjudication.json b/results/os1c_selfdistance_adjudication.json new file mode 100644 index 0000000000000000000000000000000000000000..d69c874e2e8f0fb32ebbaac18a7b91633d472aef --- /dev/null +++ b/results/os1c_selfdistance_adjudication.json @@ -0,0 +1,733 @@ +{ + "experiment": "OS1c \u2014 is OS1's C1 a failed control, or an unfireable one?", + "prereg": "docs/experiments/preregs/OS1c_selfdistance_adjudication.md", + "status": "PRE-REGISTERED", + "adjudicates": "results/os1_operator_space_410m.json:C1_self_distance \u2014 and nothing else", + "recomputes_not_remeasures": true, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "k_subspace": 32, + "N": 200, + "device": "cpu", + "n_operators": 24, + "torch_num_threads": 10, + "float32_eps": 1.1920928955078125e-07, + "float64_eps": 2.220446049250313e-16, + "PRED_theta_sqrt_2eps32": 0.00048828125, + "PRED_d_rel": 0.0, + "rule_threshold_10x_PRED_theta": 0.0048828125, + "os1_as_stored": { + "C1_registered_requirement": "exactly 0 on both metrics", + "C1_implemented_test": "self_dr == 0.0 and self_th < 1e-6 (os1_operator_space.py:194)", + "C1_implemented_scope": "ONE operator, Pile-CC|s0 \u2014 not the 24 the registration describes; recorded in the 2026-08-30 prereg amendment", + "C1_theta_measured": 0.0003735905, + "C1_d_rel_measured": 0.0, + "C1_fires": false, + "VERDICT": "UNCLEAR \u2014 a control did not fire: C1_self_distance", + "PRIMARY_untouched_by_this_file": { + "SEP_d_rel": 9.309428669309066, + "SEP_theta": 8.476455029514748 + } + }, + "PRIMARY": { + "FLOOR": { + "theta_32": 0.0004431792622199282, + "d_rel": 0.0 + }, + "FLOOR_argmax_operator": { + "theta_32": "Github|s0", + "d_rel": "Wikipedia_en|s0" + }, + "F64": { + "theta_32": 1.732492597787057e-08, + "d_rel": 0.0 + }, + "RAND": { + "theta_32": 0.00035187862410496635, + "d_rel": 0.0 + }, + "RAND_float64": { + "theta_32": 1.5621062404991232e-08, + "d_rel": 0.0 + }, + "conditions_per_metric": { + "theta_32": { + "FLOOR_le_10xPRED": true, + "F64_le_FLOOR_over_100": true, + "RAND_within_10x_of_FLOOR": true, + "RAND_over_FLOOR_ratio": 0.7939871156027742, + "all_three_hold": true + }, + "d_rel": { + "FLOOR_le_10xPRED": true, + "F64_le_FLOOR_over_100": true, + "RAND_within_10x_of_FLOOR": null, + "RAND_over_FLOOR_ratio": null, + "all_three_hold": false + } + }, + "operators_exceeding_10xPRED": { + "theta_32": [], + "d_rel": [] + }, + "d_rel_half_of_C1_was_fireable": true, + "theta_half_of_C1_was_fireable": false + }, + "per_operator": { + "Wikipedia_en|s0": { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "theta_32": { + "float32": 0.00029680083389393985, + "float64": 1.5080104083714718e-08, + "float32_max_layer": 0.0004284140304662287, + "float32_min_layer": 0.00014700791507493705 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Wikipedia_en|s1": { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "theta_32": { + "float32": 0.00036992424037844804, + "float64": 1.2937307501666824e-08, + "float32_max_layer": 0.0005573734524659812, + "float32_min_layer": 0.00011448153236415237 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Wikipedia_en|s2": { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "theta_32": { + "float32": 0.0003744769760837349, + "float64": 1.5421186112439368e-08, + "float32_max_layer": 0.0006743366830050945, + "float32_min_layer": 4.57763671875e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "USPTO_Backgrounds|s0": { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "theta_32": { + "float32": 0.0003635708198999055, + "float64": 1.3388054352500245e-08, + "float32_max_layer": 0.0007409093668684363, + "float32_min_layer": 9.780222171684727e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "USPTO_Backgrounds|s1": { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "theta_32": { + "float32": 0.0003918622032170578, + "float64": 1.732492597787057e-08, + "float32_max_layer": 0.0006493605324067175, + "float32_min_layer": 2.604838300612755e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "USPTO_Backgrounds|s2": { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "theta_32": { + "float32": 0.0003091964860616897, + "float64": 1.416744789544734e-08, + "float32_max_layer": 0.0005510359769687057, + "float32_min_layer": 4.5705448428634554e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Pile-CC|s0": { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "theta_32": { + "float32": 0.00037359047564677894, + "float64": 1.385618307047693e-08, + "float32_max_layer": 0.0006052301614545286, + "float32_min_layer": 0.00020395449246279895 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Pile-CC|s1": { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "theta_32": { + "float32": 0.00030033288315350475, + "float64": 1.6398275248699866e-08, + "float32_max_layer": 0.0005769651615992188, + "float32_min_layer": 8.701262413524091e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Pile-CC|s2": { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "theta_32": { + "float32": 0.0003979983750748663, + "float64": 1.6460400014061635e-08, + "float32_max_layer": 0.0006565966177731752, + "float32_min_layer": 0.00017312722047790885 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "StackExchange|s0": { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "theta_32": { + "float32": 0.000401071236970333, + "float64": 1.1612080217829672e-08, + "float32_max_layer": 0.0006889902288094163, + "float32_min_layer": 0.00017312722047790885 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "StackExchange|s1": { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "theta_32": { + "float32": 0.0003338489399399035, + "float64": 1.504088066496445e-08, + "float32_max_layer": 0.0006363721331581473, + "float32_min_layer": 0.00016609243175480515 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "StackExchange|s2": { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "theta_32": { + "float32": 0.0003309558759121081, + "float64": 1.5863950452687596e-08, + "float32_max_layer": 0.0006797167006880045, + "float32_min_layer": 0.00011051393084926531 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Github|s0": { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "theta_32": { + "float32": 0.0004431792622199282, + "float64": 1.4711223208867249e-08, + "float32_max_layer": 0.0006653249729424715, + "float32_min_layer": 0.00023528341262135655 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Github|s1": { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "theta_32": { + "float32": 0.00032374119977108564, + "float64": 1.3284293976749074e-08, + "float32_max_layer": 0.0005158634157851338, + "float32_min_layer": 0.00016327228513546288 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "Github|s2": { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "theta_32": { + "float32": 0.0003119326085232807, + "float64": 1.639768125186644e-08, + "float32_max_layer": 0.0005656667053699493, + "float32_min_layer": 5.590671571553685e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_News_2024|s0": { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "theta_32": { + "float32": 0.0004083297390025109, + "float64": 1.1936339481362e-08, + "float32_max_layer": 0.0006693445611745119, + "float32_min_layer": 0.00026044953847303987 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_News_2024|s1": { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "theta_32": { + "float32": 0.0003676565829664469, + "float64": 1.5108339483668996e-08, + "float32_max_layer": 0.000646946718916297, + "float32_min_layer": 0.0001577975053805858 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_News_2024|s2": { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "theta_32": { + "float32": 0.0003824657553475565, + "float64": 1.4810452471587455e-08, + "float32_max_layer": 0.0009769743774086237, + "float32_min_layer": 3.2368778192903847e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_arXiv_2023|s0": { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "theta_32": { + "float32": 0.0003802260919026314, + "float64": 9.925541985693294e-09, + "float32_max_layer": 0.0006047913921065629, + "float32_min_layer": 0.00010064321395475417 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_arXiv_2023|s1": { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "theta_32": { + "float32": 0.0003397518499136473, + "float64": 1.236161577491288e-08, + "float32_max_layer": 0.000870111514814198, + "float32_min_layer": 0.0001126454517361708 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_arXiv_2023|s2": { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "theta_32": { + "float32": 0.0003069003466337633, + "float64": 1.3518316187654634e-08, + "float32_max_layer": 0.0006977830780670047, + "float32_min_layer": 9.148182289209217e-05 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_CommonPile|s0": { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "theta_32": { + "float32": 0.0003957398641012752, + "float64": 1.5405293302507714e-08, + "float32_max_layer": 0.0006753915804438293, + "float32_min_layer": 0.00017878838116303086 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_CommonPile|s1": { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "theta_32": { + "float32": 0.0004019357908803683, + "float64": 1.173588897947812e-08, + "float32_max_layer": 0.0007019342156127095, + "float32_min_layer": 0.00018514538533054292 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + }, + "OOD_CommonPile|s2": { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "theta_32": { + "float32": 0.00033585767064673395, + "float64": 1.3107076237866253e-08, + "float32_max_layer": 0.0004825556534342468, + "float32_min_layer": 0.0001355224521830678 + }, + "d_rel": { + "float32": 0.0, + "float64": 0.0, + "float32_max_layer": 0.0 + } + } + }, + "controls": { + "K1_random_reproduces_floor": { + "required": "RAND_theta within 10x of FLOOR_theta \u2014 the floor is arithmetic, not these operators", + "RAND_theta": 0.00035187862410496635, + "FLOOR_theta": 0.0004431792622199282, + "ratio": 0.7939871156027742, + "fires": true, + "os1b_reference": { + "theta_self": 0.0003735905, + "random": 0.0002859 + } + }, + "K2_float64_collapses_it": { + "required": "F64_theta <= FLOOR_theta / 100 \u2014 the floor tracks precision, not structure", + "F64_theta": 1.732492597787057e-08, + "FLOOR_theta_over_100": 4.431792622199282e-06, + "ratio_f32_over_f64": 25580.44188968015, + "fires": true + }, + "K3_metric_can_still_fail": { + "required": "min over distinct operator pairs of theta_32(A,B) >= 100 * FLOOR_theta. IF THIS FAILS THE METRIC IS BLIND AND NO VERDICT ISSUES.", + "n_pairs": 276, + "min_distinct_pair_theta": 0.07482731714844704, + "min_pair": "OOD_arXiv_2023|s0 vs OOD_arXiv_2023|s1", + "threshold_100x_floor": 0.04431792622199282, + "ratio_min_pair_over_floor": 168.84209963622777, + "fires": true + }, + "K4_scope_is_os1s": { + "required": "the 24 operator paths are OS1's panel, asserted before measuring", + "n_operators": 24, + "os1_scope_field_is_prose_not_a_path_list": true, + "os1_provenance_inputs_recorded": 0, + "panel_source": "experiments/os1_operator_space.py:52-55 (INSTREAM + OOD, SEEDS)", + "cross_check": { + "os1_n_BETWEEN_corpus": 84, + "expected_from_panel": 84, + "os1_n_WITHIN_seed_block": 24, + "expected_within": 24 + }, + "missing_paths": [], + "fires": true + }, + "K5_band_rule": { + "required": "band == floor(0.38L)..floor(0.92L) intersected with layers strictly below the penultimate target layer, at L=24. AMENDED: the registration wrote this rule without its second clause; see the 2026-08-30 amendment.", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "rule_band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "literal_registered_arithmetic": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "literal_reading_is_unrealisable": "layer 22 is absent from every stored operator; os1_operator_space.py:63 asserts source_layers == [9..21]", + "target_layer": 22, + "fires": true + } + }, + "controls_fired": { + "K1": true, + "K2": true, + "K3": true, + "K4": true, + "K5": true + }, + "OUTCOME": "UNCLEAR", + "VERDICT": "UNCLEAR \u2014 d_rel(A,A) is exactly 0.0 on all 24 operators in float32, so C1's d_rel half WAS fireable and did fire; only its theta half could not (FLOOR_theta = 4.431793e-04 against a predicted floor of 4.882812e-04). The registered rule requires the retirement conditions to hold for BOTH metrics, and a metric that returns an exact zero has no floor for a random matrix to reproduce: the RAND-within-10x condition is degenerate (0 against 0), not satisfied. DECLARED BIAS (3) of the registration states this outcome in advance. Report and stop. Do not re-cut. OS1's UNCLEAR stands, and C1 is NOT retired.", + "scope": "Adjudicates OS1's C1 only. Says nothing about whether the corpus displacement OS1 measured is real or matters for any readout, and changes no OS1 number.", + "provenance": { + "schema_version": 1, + "experiment": "OS1c", + "utc": "2026-08-31T06:00:37+00:00", + "script": { + "path": "experiments/os1c_selfdistance_adjudication.py", + "sha256": "c1eee034653aede741b9f44172f848c94b2b1750b5c0761437812243e6e3de87", + "n_lines": 386 + }, + "git": { + "commit": "9770d208af381e9579ae259d49d95c0c0a5b9a33", + "commit_short": "9770d208af38", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "research-paper-writing/" + ], + "dirty_diff_sha256": null, + "describe": "9770d20" + }, + "argv": [ + "experiments/os1c_selfdistance_adjudication.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s0.pt", + "sha256": "50168cf38a9aab76190187b3669a4f26b0cf0ae69968e2decfee4e7ce0a26f8a", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s1.pt", + "sha256": "a6614fc65dd71659e2d1cb5799c37bffb291be1310fb06b69fb23921fcf4c9cc", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Wikipedia_en_410m_n200_s2.pt", + "sha256": "c0702e058e3c41fb1e8b10020fa27e6ac96d34862f0648f9e55f13bd33776ed5", + "bytes": 27268444, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s0.pt", + "sha256": "75162b82ab15f9144b3776aabf62d0a1c732a474650e48b04c65bc77614aa942", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s1.pt", + "sha256": "860d4c8da99be8789db610f1046fa97d8e5f6a0b7c0b2559a64825451dc33c5f", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_USPTO_Backgrounds_410m_n200_s2.pt", + "sha256": "1a727ecb9d2a8dfcf76db138ef90c52cd2fe98bcef0be07ae759969b3d9c67e1", + "bytes": 27268539, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s0.pt", + "sha256": "4f00cc3d7450c3646b78f002e794572ca27d6ab175c70626d2d5fb7fccf28e20", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s1.pt", + "sha256": "efc6a88531fdcd4423514fd0e4f65a02b55638653c351ad0af35cd236aad2fc2", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Pile-CC_410m_n200_s2.pt", + "sha256": "058810065ae8bc499931d6aa3783dc1ead43d08e322395f7f142b90dfefae028", + "bytes": 27268349, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s0.pt", + "sha256": "6ee98aa7e9242d0c4d6ff26b1ed918180f5b37822e30181c6d0fc9fe826c0b79", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s1.pt", + "sha256": "9182d6655dc5808e47b92fe4611fc522226db741c4e6b900f8ba0e1eba7ca7b3", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_StackExchange_410m_n200_s2.pt", + "sha256": "7e21bde15c9634ffff62ec1983ad595f0604915dc8a8f9169a165e33edaee8c0", + "bytes": 27268463, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s0.pt", + "sha256": "b4acd10402749791b713e16738ba1d165c04d7134df139418618fd05a4edcbdb", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s1.pt", + "sha256": "0ec0403597cbdc971dc0348fae888a10aaa0602b8c9f9268ba008af83e07694b", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_INSTREAM_Github_410m_n200_s2.pt", + "sha256": "f5057978c3f1e94930b3c3892ef560c79cc2c68f90062d6b07aeefdec7828847", + "bytes": 27268330, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s0.pt", + "sha256": "03b0011216c985bd1fe5cb7e94a52deacce0bacf887fff43663c16190541f189", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s1.pt", + "sha256": "76e560c6f4e868d1f86c6e5306b014385493d98756ed3d085f01fe9f05d42acd", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_News_2024_410m_n200_s2.pt", + "sha256": "7909266ad37727f1c1493b42c7f10a9a975d120c4e74b20bbfd01807f9d124ed", + "bytes": 27268292, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s0.pt", + "sha256": "2054bbae85401e42f912b5782fbcd362f34e0f2f5834645024b804c6594acd60", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s1.pt", + "sha256": "1452adb3b19db0a93b70d2467a77db926f76a8e478c7fe59695592369e602818", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_arXiv_2023_410m_n200_s2.pt", + "sha256": "3df1bf622c53345cf46d2dd2255728bc238ce13b26427ad0d4125a81eafc1a1c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s0.pt", + "sha256": "305c1c5444bdca99bc291b541ba8b72d933f0463673cfd843dd0c67c5afac91c", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s1.pt", + "sha256": "75aef57fc6e12521f6cd37b9bbdb0ad1886d782515a8161b0526daa1ed17d9c8", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/e48/lens_OOD_CommonPile_410m_n200_s2.pt", + "sha256": "fd71e7b7690a7377d7693472c7e12c5d4362d75e0417bc337222c3e183d92153", + "bytes": 27268311, + "exists": true + }, + { + "path": "results/os1_operator_space_410m.json", + "sha256": "66ace1a2b552945149366de4d90b51970963058957eb19a218b7c99c4ce81f84", + "bytes": 8689, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "5338e4e5d66b016de93d978b0cd515582627848d338643a0b1b77ecc048e1f88" + } +} \ No newline at end of file diff --git a/results/paper_clustered_derangement_t.json b/results/paper_clustered_derangement_t.json new file mode 100644 index 0000000000000000000000000000000000000000..4819ed5d1b65995f1d390262a8463f861b822837 --- /dev/null +++ b/results/paper_clustered_derangement_t.json @@ -0,0 +1,228 @@ +{ + "experiment": "PAPER \u2014 corpus-clustered paired t on the derangement gap", + "status": "RECOMPUTATION from stored per-corpus gaps; measures nothing new", + "why": "the draw count (84 of 120) is descriptive because five derangements of one operator share an operator and a cache; the corpus is the replication unit", + "statistic": "one-sample paired t over the 8 per-corpus values of (jp_mean - shuf_mean), df = 7, two-sided p from Student's t", + "source_key": "C2_derangement..per_corpus..gap", + "by_readout": { + "corrected": { + "file": "results/e54_aggregation_audit_rstrip_v2.json", + "min": { + "n_corpora": 8, + "per_corpus_gap": { + "Github": 0.008090302099784202, + "OOD_CommonPile": -0.007885783364375398, + "OOD_News_2024": -0.007111757447322231, + "OOD_arXiv_2023": -0.0010599334662159121, + "Pile-CC": -0.012404273003339739, + "StackExchange": -0.0006210960696140799, + "USPTO_Backgrounds": -0.00401597244044144, + "Wikipedia_en": -0.01677957579493522 + }, + "mean_gap": -0.005223511185807477, + "sd_gap": 0.007681076142808633, + "t": -1.923470155700173, + "df": 7, + "p_two_sided": 0.0958390634744094, + "clears_05": false, + "n_draws_shuf_beats_jp": 84, + "n_draws": 120, + "n_corpora_shuf_beats_on_mean": 7 + }, + "persist": { + "n_corpora": 8, + "per_corpus_gap": { + "Github": 0.0190823487068216, + "OOD_CommonPile": 0.027265302973488975, + "OOD_News_2024": 0.026893035499379034, + "OOD_arXiv_2023": 0.02673099617784222, + "Pile-CC": 0.02854199816628049, + "StackExchange": 0.03240535648384442, + "USPTO_Backgrounds": 0.033592421001133826, + "Wikipedia_en": 0.025456481482833623 + }, + "mean_gap": 0.027495992561453023, + "sd_gap": 0.004445116490390382, + "t": 17.49569698584057, + "df": 7, + "p_two_sided": 4.905678869008232e-07, + "clears_05": true, + "n_draws_shuf_beats_jp": 0, + "n_draws": 120, + "n_corpora_shuf_beats_on_mean": 0 + } + }, + "legacy": { + "file": "results/e54_aggregation_audit.json", + "min": { + "n_corpora": 8, + "per_corpus_gap": { + "Github": -0.006719535738229754, + "OOD_CommonPile": -0.00794652586181957, + "OOD_News_2024": -0.03211234564582505, + "OOD_arXiv_2023": -0.008748255235453473, + "Pile-CC": -0.019164425681034714, + "StackExchange": -0.0063389639804760645, + "USPTO_Backgrounds": 0.0011352714647849538, + "Wikipedia_en": -0.027178369810183844 + }, + "mean_gap": -0.013384143811029689, + "sd_gap": 0.011528610403094932, + "t": -3.283663344756407, + "df": 7, + "p_two_sided": 0.013419144852786318, + "clears_05": true, + "n_draws_shuf_beats_jp": 104, + "n_draws": 120, + "n_corpora_shuf_beats_on_mean": 7 + }, + "persist": { + "n_corpora": 8, + "per_corpus_gap": { + "Github": 0.010767463998248177, + "OOD_CommonPile": 0.014681099609782297, + "OOD_News_2024": 0.015046576190118989, + "OOD_arXiv_2023": 0.01635738908623656, + "Pile-CC": 0.01854549055142949, + "StackExchange": 0.01933951349152873, + "USPTO_Backgrounds": 0.021970295277424154, + "Wikipedia_en": 0.018278182124098142 + }, + "mean_gap": 0.016873251291108317, + "sd_gap": 0.0034334810454757416, + "t": 13.89981799879624, + "df": 7, + "p_two_sided": 2.3583430174303775e-06, + "clears_05": true, + "n_draws_shuf_beats_jp": 0, + "n_draws": 120, + "n_corpora_shuf_beats_on_mean": 0 + } + } + }, + "VERDICT": "At the CORRECTED readout the corpus-clustered paired t on `min` is t(7) = -1.92, p = 0.096, which does NOT clear 5% \u2014 the descriptive counts are 84 of 120 draws and 7 of 8 corpora. At the LEGACY readout it is t(7) = -3.28, p = 0.013. The readout correction moved this from significant to not, so the -3.3 carried in CONTEXT.md section 5 and RESULTS_TAXONOMY.md section 1.2 is a LEGACY-readout number. `persist` is unambiguous at both readouts: t(7) = 17.50, p = 4.91e-07 corrected.", + "scope": "A derangement is a null for layer-to-derivative CORRESPONDENCE, not for the claim that the Jacobians carry layer-specific information. A t that does not clear 5% means the evidence that `min` actively PREFERS the derangement is weaker than the draw count suggests; it does not rehabilitate `min`, because R9 shows separately that no operator out of eight beats all fifteen derangements of itself under `min`.", + "provenance": { + "schema_version": 1, + "experiment": "PAPER_CLUSTERED_T", + "utc": "2026-08-31T23:20:36+00:00", + "script": { + "path": "tools/paper_clustered_derangement_t.py", + "sha256": "e1affc82d9d7fa707f245ac75d8db4f14af51f2ac09270b2caacd81205d3ffb3", + "n_lines": 200 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "tools/paper_clustered_derangement_t.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e54_aggregation_audit_rstrip_v2.json", + "sha256": "e2f0b3fa2184ebcc3792a393ea823f883deaabd16a5c18e7c224b093823ca09d", + "bytes": 74486, + "exists": true + }, + { + "path": "results/e54_aggregation_audit.json", + "sha256": "1a1d45be09127b311a5d7ab534a0a1a52f4ab2f78d73b198f12090b27b2bb445", + "bytes": 69086, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "1a9351e1a7df21ed48549aecef17fc95748f424607739296b8e73233e4a7c481" + } +} \ No newline at end of file diff --git a/results/postreview_completion.json b/results/postreview_completion.json new file mode 100644 index 0000000000000000000000000000000000000000..3906980a0e51de9023ec38048bbb4310b1c25a40 --- /dev/null +++ b/results/postreview_completion.json @@ -0,0 +1,295 @@ +{ + "experiment": "POSTREVIEW completion ledger", + "slate": "specs/POSTREVIEW_EXPERIMENTS.md", + "slate_sha256": "8e89d9085020a236e83fdae404c555f443e8a7d1b3883d0d800938d15f9890dc", + "item_headings_parsed_from_the_slate": [ + "P0", + "R0", + "R1", + "R2", + "R3", + "R4", + "R4b", + "R5", + "R6", + "R7" + ], + "n_items_tracked": 12, + "n_with_prereg_and_results": 11, + "items_incomplete": [], + "operator_only_outstanding": [ + "R0 \u2014 retraction of the corpus-axis crossover; the SLATE itself marks it 'OPERATOR ONLY. DO NOT EXECUTE.' Its MEASUREMENT is confirmed twice (R1, R2) but the canonical documents were deliberately not edited." + ], + "tier_C_not_started_by_design": [ + "move the task battery (three-factor P x Q x T)", + "canonical N=1000 protocol arm", + "external validation of persist against causal interventions", + "a second model family / instruction-tuned models", + "full corpus-resampling hierarchical design (R6 is the two-source version)", + "E66's fitter discrepancy resolved by refit" + ], + "items": [ + { + "item": "R0", + "what": "retract the corpus-axis crossover \u2014 OPERATOR ONLY, not an agent task", + "prereg": null, + "prereg_exists": false, + "prereg_sha256": null, + "results_files_expected": [], + "results_files_present": [], + "verdict": "OPERATOR-ONLY \u2014 not an agent task" + }, + { + "item": "P0", + "what": "port a process pool into t52_factorial.py", + "prereg": "specs/experiments/P0_t52_parallel_port.md", + "prereg_exists": true, + "prereg_sha256": "d2cd8cb4b3296249ace1ae995b238b2035b21d7884858596777202971869565e", + "results_files_expected": [ + "results/e52_factorial_410m_pooled.json" + ], + "results_files_present": [ + "results/e52_factorial_410m_pooled.json" + ], + "verdict": "**ACCEPT on the decisive clause, with C2's full-grid arms outstanding.**" + }, + { + "item": "R1", + "what": "re-score the 8x8 grid at the correct readout token", + "prereg": "specs/experiments/R1_grid_rstrip.md", + "prereg_exists": true, + "prereg_sha256": "3b13a90cd3730455c6d1328a84d50653f3f146e39ed115f08dc2f77e872fcd07", + "results_files_expected": [ + "results/e52_factorial_410m_rstrip.json", + "results/e57_grid_variance_ci_rstrip.json" + ], + "results_files_present": [ + "results/e52_factorial_410m_rstrip.json", + "results/e57_grid_variance_ci_rstrip.json" + ], + "verdict": "**QUALIFIED \u2014 the ordering survives, the magnitude does not.**" + }, + { + "item": "R2", + "what": "the declared-but-never-run --rstrip sensitivity arm on E48", + "prereg": "specs/experiments/R2_e48_rstrip_arm.md", + "prereg_exists": true, + "prereg_sha256": "362df3c95ccc553b565ea9c73bcbdfbcdc3d3030d4cf9cb354eb2a5e17be4637", + "results_files_expected": [ + "results/e48_crossover_410m_rstrip.json" + ], + "results_files_present": [ + "results/e48_crossover_410m_rstrip.json" + ], + "verdict": "**DISCLOSED AND STORED \u2014 no branch, as registered.**" + }, + { + "item": "R3", + "what": "close D2 \u2014 re-score ladder410 on CPU and re-derive", + "prereg": "specs/experiments/R3_ladder410_cpu.md", + "prereg_exists": true, + "prereg_sha256": "5e7c31cad6092e0986b6b8a2c2f69f84c20035911746bed9f00fcaf11bbbf809", + "results_files_expected": [ + "results/r3_close_d2.json" + ], + "results_files_present": [ + "results/r3_close_d2.json" + ], + "verdict": "**CLEARED.**" + }, + { + "item": "R4", + "what": "R4a/b-table/c/e/f/g/h/i/j/k/l \u2014 corrections that are pure recomputation", + "prereg": "specs/experiments/R4_corrections.md", + "prereg_exists": true, + "prereg_sha256": "64c4c9b5ec1d03a60251b8aa1e67ea7bdfd3c91dec63e8eadec51aa935488783", + "results_files_expected": [ + "results/r4_corrections.json", + "results/r4e_control_power.json", + "results/r4g_e28_provenance.json" + ], + "results_files_present": [ + "results/r4_corrections.json", + "results/r4e_control_power.json", + "results/r4g_e28_provenance.json" + ], + "verdict": "**COMPLETE \u2014 11 items landed, 9 of 10 checkable values reproduce the slate exactly, 1 disagrees.**" + }, + { + "item": "R4b", + "what": "re-score E36 \u2014 S3's READ-axis rejection is a logit-lens comparison", + "prereg": "specs/experiments/R4b_e36_rstrip.md", + "prereg_exists": true, + "prereg_sha256": "bdf53d16f6ef3c9c3360d9b0ac2ed5aaf93da9e5e44946f2a05f6372007fb721", + "results_files_expected": [ + "results/r4b_e36_flatness.json" + ], + "results_files_present": [ + "results/r4b_e36_flatness.json" + ], + "verdict": "**REJECT OVERTURNED \u2014. Flagged for adjudication.**" + }, + { + "item": "R5", + "what": "corpus-axis uncertainty, stated as such", + "prereg": "specs/experiments/R5_corpus_axis_uncertainty.md", + "prereg_exists": true, + "prereg_sha256": "7575ee49b8c025d8843f3cc0acb8e8858af824fe5b1b3bd7b9ad1e83874f53f4", + "results_files_expected": [ + "results/r5_corpus_axis_uncertainty.json", + "results/r5_corpus_axis_uncertainty_rstrip.json" + ], + "results_files_present": [ + "results/r5_corpus_axis_uncertainty.json", + "results/r5_corpus_axis_uncertainty_rstrip.json" + ], + "verdict": "**DELIVERED \u2014 descriptive, as registered; no branch.**" + }, + { + "item": "R6", + "what": "within-source resampling \u2014 is corpus a factor or a bundle?", + "prereg": "specs/experiments/R6_within_source_resampling.md", + "prereg_exists": true, + "prereg_sha256": "7c8d4c6438dcc18f4c136805ab97c68ffc997ba07acbf40ab6986ed2c99afbea", + "results_files_expected": [ + "results/r6_within_source_410m.json", + "results/r6_within_source_410m_cpu.json" + ], + "results_files_present": [ + "results/r6_within_source_410m.json", + "results/r6_within_source_410m_cpu.json" + ], + "verdict": "**ACCEPT \u2014 corpus identity is a real factor, not a bundle artifact of sampling.**" + }, + { + "item": "R7", + "what": "length- and format-matched fitting pools", + "prereg": "specs/experiments/R7_length_matched_pools.md", + "prereg_exists": true, + "prereg_sha256": "579f9f4ff68b4bbc53c95ea2cf6a112327f1726666fee77e7f7963be9b8c6256", + "results_files_expected": [ + "results/r7_matched_pools_410m.json" + ], + "results_files_present": [ + "results/r7_matched_pools_410m.json" + ], + "verdict": "**DIRECTIONAL RESULT (REQUIRES C1) \u2014 ACCEPT on the primary rule, ungraded pending a certified match.**" + }, + { + "item": "R8", + "what": "slate \u00a75 item S-1, self-promoted to Tier A by R1's QUALIFIED verdict \u2014 closes S2", + "prereg": "specs/experiments/R8_ladder_rstrip_S2.md", + "prereg_exists": true, + "prereg_sha256": "4194225815ef6a817213308cc4d94368f4fa586d1bea60b27dfb4d1fea755a47", + "results_files_expected": [ + "results/r8_ladder_flatness.json" + ], + "results_files_present": [ + "results/r8_ladder_flatness.json" + ], + "verdict": "**UNCLEAR \u2014 report and stop. The rule is not re-cut.**" + }, + { + "item": "R9", + "what": "permutation-calibrated min \u2014 dissolves the metric circularity", + "prereg": "specs/experiments/R9_permutation_calibrated_min.md", + "prereg_exists": true, + "prereg_sha256": "20fb005dfac0042c31842640d5b644558cad4515409f18155f831b0992f09fa4", + "results_files_expected": [ + "results/r9_permutation_calibrated_min.json" + ], + "results_files_present": [ + "results/r9_permutation_calibrated_min.json" + ], + "verdict": "CALIBRATED. Under the SOURCE'S OWN statistic (`min`), the real operator exceeds every one of its 15 own layer-derangement draws on 0 of 8 corpora (median z = -0.71). Under `persist` it does so on 8 of 8 (median z = +12.14). The published statistic, read against its own null rather than as a raw scor" + } + ], + "provenance": { + "schema_version": 1, + "experiment": "POSTREVIEW", + "utc": "2026-08-20T18:29:44+00:00", + "script": { + "path": "tools/postreview_completion.py", + "sha256": "0b1e5c38d856e111dcb6b3c2276dae145abafa4ca64c3f11d5c282abe293be8c", + "n_lines": 128 + }, + "git": { + "commit": "86eacfe5c7ff916f710aa5c5c2df1d3975485320", + "commit_short": "86eacfe5c7ff", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "experiments/d2_device_crossvalidation.py", + "figures/fB_band_width.png", + "figures/make_figures.py", + "igures/f0_grid_single.png", + "paper/.gitignore", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/audit/", + "results/d2_device_crossvalidation.json", + "results/postreview_completion.json", + "results/r7_matched_pools_410m.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/RESULTS_TAXONOMY.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "specs/experiments/R2_e48_rstrip_arm.md", + "specs/experiments/R4_corrections.md", + "specs/experiments/R5_corpus_axis_uncertainty.md", + "specs/experiments/R6_within_source_resampling.md", + "specs/experiments/R7_length_matched_pools.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/postreview_completion.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "7609f84fff2b5213adeb551ee79226a7b3a7d987713b98d5a27c09d7daa5d6fd", + "describe": "86eacfe-dirty" + }, + "argv": [ + "tools/postreview_completion.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "specs/POSTREVIEW_EXPERIMENTS.md", + "sha256": "8e89d9085020a236e83fdae404c555f443e8a7d1b3883d0d800938d15f9890dc", + "bytes": 61392, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "572f3d596fa58396840a5c8a4c7dfd63236919e0966dc55f440090148d502154", + "payload_sha256_before_release_redaction": "fde5900f67638e3d16fc1cb3ebd9cf74d5cf2914a7262f329c1d18df35a7e18e", + "release_redaction": "internal operator-protocol wording removed for the anonymous release; no number, threshold or verdict changed" + } +} \ No newline at end of file diff --git a/results/r10_t59_readout_reconciliation.json b/results/r10_t59_readout_reconciliation.json new file mode 100644 index 0000000000000000000000000000000000000000..1abe40124d65995ab3f73be062bfd98e0b6373b8 --- /dev/null +++ b/results/r10_t59_readout_reconciliation.json @@ -0,0 +1,312 @@ +{ + "experiment": "R10 \u2014 E59 readout-convention reconciliation", + "status": "RECOMPUTATION \u2014 no model, no GPU, no scoring, no fitting", + "recomputes_not_remeasures": true, + "question": "is the stored E59 dose ladder scored under the LEGACY readout while the headline factorial it is cited to bound is scored under the CORRECTED one?", + "inputs_used": [ + "results/e59_read_dose_410m.json", + "results/e52_factorial_410m.json", + "results/e52_factorial_410m_rstrip.json", + "results/e57_grid_variance_ci_rstrip.json", + "experiments/t59_read_dose.py" + ], + "corpora": [ + "Wikipedia_en", + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Github", + "OOD_News_2024", + "OOD_arXiv_2023" + ], + "ALLEGATION_UPHELD": true, + "evidence": { + "1_argv": { + "what": "the command line that produced the stored result", + "argv": [ + "experiments/t59_read_dose.py", + "--device", + "cpu" + ], + "carries_rstrip_flag": false, + "verdict": "no --rstrip in argv" + }, + "2_code_path": { + "what": "did the script that ran contain a .rstrip() on the prompt?", + "sha256_recorded_in_result": "76473cf24c5b8b440ada67cc3229206f077355a4c024e4bda3201ed0089d2129", + "sha256_of_prefix_script_literal": "76473cf24c5b8b440ada67cc3229206f077355a4c024e4bda3201ed0089d2129", + "result_matches_prefix_script": true, + "sha256_on_disk_now": "33facd62d2f61f30c4145f2603a6767c6168c28b378c8d6865fe2f2c24093669", + "on_disk_is_still_the_prefix_script": false, + "note": "The stored result names the pre-fix script. Once --rstrip is added the on-disk hash moves, which is expected and is why the pre-fix hash is pinned as a literal above. If BOTH are true the fix has not been applied yet; if only the first is true the fix is in and the evidence stands." + }, + "3_result_metadata": { + "what": "corrected artifacts in this repository declare the convention in-payload", + "e59_has_rstrip_key": false, + "e48_rstrip_declares": true, + "verdict": "the stored E59 payload declares no readout convention at all" + } + }, + "by_aggregation": { + "min": { + "E59_at_dose128_STORED_legacy_scorer": { + "fit_pct": 77.88156213345998, + "read_pct": 19.309979397701433, + "residual_pct": 2.8084584688385887, + "row_means": { + "Wikipedia_en": 0.13037704450507012, + "USPTO_Backgrounds": 0.15757397184118865, + "Pile-CC": 0.1444637641547218, + "StackExchange": 0.14656658471221962, + "Github": 0.11268684133177712, + "OOD_News_2024": 0.13503231443760413, + "OOD_arXiv_2023": 0.1415638420316908 + }, + "row_mean_over_corpora": 0.13832348043061032 + }, + "E52_LEGACY_restricted_to_E59_corpora": { + "fit_pct": 77.20909139763098, + "read_pct": 20.313039456100714, + "residual_pct": 2.4778691462682962, + "row_mean_over_corpora": 0.13860723340626205 + }, + "E52_CORRECTED_restricted_to_E59_corpora": { + "fit_pct": 57.55685418677557, + "read_pct": 40.24133072996441, + "residual_pct": 2.2018150832600214, + "row_mean_over_corpora": 0.248280280304739, + "IS_THE_CORRECTED_DOSE128_ANCHOR": true, + "why": "E52 --rstrip is scored at prefix_tokens=128, the same read dose as E59's first rung, under the corrected readout. Restricting its 8x8 matrix to E59's seven corpora is a recomputation of stored values." + }, + "evidence_4_which_arm_does_E59_match": { + "rms_row_mean_distance_to_E52_legacy": 0.00045233575340530163, + "rms_row_mean_distance_to_E52_corrected": 0.1101164291015297, + "ratio_corrected_over_legacy": 243.43958723701252, + "matches": "legacy", + "falsified_if": "E59 sat closer to the corrected arm, or the two arms were not separated by much more than the pool effect" + }, + "pool_effect_at_the_legacy_readout": { + "what": "E59 draws prefixes only from documents of >= 768 tokens; E52 draws from the unrestricted pool. At the legacy readout both exist, so their difference isolates the pool, holding the readout fixed.", + "fit_pct_delta_E59_minus_E52": 0.6724707358289947, + "read_pct_delta_E59_minus_E52": -1.0030600583992815, + "row_mean_delta": -0.0002837529756517243 + }, + "corrected_E59_dose128_ESTIMATE": { + "status": "ESTIMATE, NOT A MEASUREMENT \u2014 reported so the size of the gap is visible, never to be cited as a result", + "method": "corrected E52 anchor + the legacy-measured pool effect", + "fit_pct": 58.22932492260456, + "read_pct": 39.23827067156513, + "assumes": "the pool effect is additive across readout conventions, which is untested" + }, + "corrected_doses_384_and_768": { + "status": "NOT RUN", + "blocker": "needs the read-context prefix pools, built from corpora/*.jsonl, which this project deliberately does not redistribute", + "to_run": "bash repro/06_data.sh --build # then: python experiments/t59_read_dose.py --rstrip --device cpu" + } + }, + "persist": { + "E59_at_dose128_STORED_legacy_scorer": { + "fit_pct": 88.68478030369347, + "read_pct": 9.45683761637228, + "residual_pct": 1.8583820799342452, + "row_means": { + "Wikipedia_en": 0.038766393582854004, + "USPTO_Backgrounds": 0.047598725785723044, + "Pile-CC": 0.04712269108128985, + "StackExchange": 0.04512989729576345, + "Github": 0.02580937210078691, + "OOD_News_2024": 0.03705063803742329, + "OOD_arXiv_2023": 0.039499257347132596 + }, + "row_mean_over_corpora": 0.04013956789013902 + }, + "E52_LEGACY_restricted_to_E59_corpora": { + "fit_pct": 91.10865204439243, + "read_pct": 7.5957558760329, + "residual_pct": 1.2955920795746683, + "row_mean_over_corpora": 0.04005158821272911 + }, + "E52_CORRECTED_restricted_to_E59_corpora": { + "fit_pct": 50.85297785477784, + "read_pct": 48.11818900866957, + "residual_pct": 1.0288331365525942, + "row_mean_over_corpora": 0.10186067436728838, + "IS_THE_CORRECTED_DOSE128_ANCHOR": true, + "why": "E52 --rstrip is scored at prefix_tokens=128, the same read dose as E59's first rung, under the corrected readout. Restricting its 8x8 matrix to E59's seven corpora is a recomputation of stored values." + }, + "evidence_4_which_arm_does_E59_match": { + "rms_row_mean_distance_to_E52_legacy": 0.0002731634085451984, + "rms_row_mean_distance_to_E52_corrected": 0.06177973256184672, + "ratio_corrected_over_legacy": 226.16401256255543, + "matches": "legacy", + "falsified_if": "E59 sat closer to the corrected arm, or the two arms were not separated by much more than the pool effect" + }, + "pool_effect_at_the_legacy_readout": { + "what": "E59 draws prefixes only from documents of >= 768 tokens; E52 draws from the unrestricted pool. At the legacy readout both exist, so their difference isolates the pool, holding the readout fixed.", + "fit_pct_delta_E59_minus_E52": -2.4238717406989565, + "read_pct_delta_E59_minus_E52": 1.8610817403393796, + "row_mean_delta": 8.797967740990786e-05 + }, + "corrected_E59_dose128_ESTIMATE": { + "status": "ESTIMATE, NOT A MEASUREMENT \u2014 reported so the size of the gap is visible, never to be cited as a result", + "method": "corrected E52 anchor + the legacy-measured pool effect", + "fit_pct": 48.429106114078884, + "read_pct": 49.97927074900895, + "assumes": "the pool effect is additive across readout conventions, which is untested" + }, + "corrected_doses_384_and_768": { + "status": "NOT RUN", + "blocker": "needs the read-context prefix pools, built from corpora/*.jsonl, which this project deliberately does not redistribute", + "to_run": "bash repro/06_data.sh --build # then: python experiments/t59_read_dose.py --rstrip --device cpu" + } + } + }, + "controls": { + "C1_shares_reproduces_the_stored_decomposition": { + "requirement": "shares() on the full 8x8 corrected matrix reproduces E57's stored fit/read point estimates to < 0.01 percentage points", + "fails_if": "any difference >= 0.01 pp, which would mean the SS decomposition here is not the one the paper reports", + "per_agg": { + "min": { + "recomputed_fit_pct": 53.35344735234028, + "recomputed_read_pct": 44.60936175509925, + "e57_stored_fit_pct": 53.35344735234028, + "e57_stored_read_pct": 44.60936175509925, + "abs_diff_fit": 0.0, + "abs_diff_read": 0.0 + }, + "persist": { + "recomputed_fit_pct": 50.697184677169346, + "recomputed_read_pct": 48.13211438274321, + "e57_stored_fit_pct": 50.697184677169346, + "e57_stored_read_pct": 48.13211438274321, + "abs_diff_fit": 0.0, + "abs_diff_read": 0.0 + } + }, + "max_abs_diff_pp": 0.0, + "fires": true + }, + "C2_e59_selfconsistent": { + "requirement": "shares() on E59's stored dose-128 matrix returns E59's stored fit_pct/read_pct to < 1e-9", + "fails_if": "any difference, which would mean this tool is misreading E59", + "per_agg": { + "min": { + "abs_diff_fit": 0.0, + "abs_diff_read": 0.0 + }, + "persist": { + "abs_diff_fit": 0.0, + "abs_diff_read": 0.0 + } + }, + "max_abs_diff_pp": 0.0, + "fires": true + }, + "C3_the_two_arms_are_separable": { + "requirement": "the legacy and corrected e52 arms must be separated by much more than E59's distance to the one it matches, or evidence 4 discriminates nothing", + "fails_if": "separation_over_E59_legacy_distance is of order 1 on either aggregation", + "per_agg": { + "min": { + "legacy_row_mean": 0.13860723340626205, + "corrected_row_mean": 0.248280280304739, + "ratio_corrected_over_legacy": 1.791250530028414, + "abs_separation": 0.10967304689847696, + "E59_distance_to_legacy": 0.00045233575340530163, + "separation_over_E59_legacy_distance": 242.4593812733785 + }, + "persist": { + "legacy_row_mean": 0.04005158821272911, + "corrected_row_mean": 0.10186067436728838, + "ratio_corrected_over_legacy": 2.543236833113031, + "abs_separation": 0.06180908615455927, + "E59_distance_to_legacy": 0.0002731634085451984, + "separation_over_E59_legacy_distance": 226.27147055946975 + } + }, + "fires": true + } + }, + "VERDICT": "UPHELD. The stored E59 dose ladder is legacy-scored on all four independent lines of evidence. Its dose-128 rung is therefore not an anchor for the corrected factorial, and Appendix D's dose numbers cannot bound a corrected headline. The corrected dose-128 decomposition IS available and is reported here from the corrected 8x8 factorial restricted to E59's seven corpora. The corrected 384 and 768 rungs are NOT RUN: they need corpus plaintext this project does not redistribute. The corrected DOSE TREND is therefore unresolved, and E59 cannot stand as a robustness claim for the corrected factorial until it is re-run.", + "provenance": { + "schema_version": 1, + "experiment": "R10", + "utc": "2026-09-02T01:02:21+00:00", + "script": { + "path": "tools/r10_t59_readout_reconciliation.py", + "sha256": "0294ce9dbdd32e747bdaf391b6839984aabc5c1b9ecc71f9dc9d36de7157ddf6", + "n_lines": 321 + }, + "git": { + "commit": "271454ebce2fcff5c6e8efc26ed26536367d5936", + "commit_short": "271454ebce2f", + "git_repository_present": true, + "branch": "main", + "dirty": true, + "dirty_files": [ + "docs/experiments/DA1_OUTCOME.md", + "docs/experiments/preregs/DA1_derangement_adjudication.md", + "experiments/da1_derangement_adjudication.py", + "experiments/t59_read_dose.py", + "ocs/reproducibility/SCRIPT_PROVENANCE.md", + "repro/exp/da1_derangement.sh", + "repro/exp/r10_t59_readout.sh", + "repro/scripts_at_run/76473cf24c5b__t59_read_dose.py", + "tools/da1b_clustered_t.py", + "tools/emit_exp_modules.py", + "tools/r10_t59_readout_reconciliation.py" + ], + "dirty_diff_sha256": "4b9a825f5c9d7c0df2ccb7fe9a4474b14aabbc47236f6ed60d8740097d85eb3b", + "describe": "271454e-dirty" + }, + "argv": [ + "tools/r10_t59_readout_reconciliation.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e59_read_dose_410m.json", + "sha256": "9f07e0a22319064ed20c1f7306394d4d82502d54c88b1a3366daa3eb3a3ce170", + "bytes": 43949, + "exists": true + }, + { + "path": "results/e52_factorial_410m.json", + "sha256": "81c0a06357e4ca91d2fa9dd9a748ff4696d02e9f78ad2e7edd65a60334c100fb", + "bytes": 41197, + "exists": true + }, + { + "path": "results/e52_factorial_410m_rstrip.json", + "sha256": "e3e42850ce7f436eafb50fa2f6ba7f94559b321540075023badd6e1cd6ef7e19", + "bytes": 45765, + "exists": true + }, + { + "path": "results/e57_grid_variance_ci_rstrip.json", + "sha256": "91d1a2853b98f843e09417f9a7f24bad5e0b8d9c48a817e57a2e375eed5a1bbe", + "bytes": 6982, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "05f00ba9128e3270da2f095b449513e377339f52c5dbd13284b1c7371275681e" + } +} \ No newline at end of file diff --git a/results/r3_close_d2.json b/results/r3_close_d2.json new file mode 100644 index 0000000000000000000000000000000000000000..3ca715f3c8b1c301898891f88f6bf3221bdd5f29 --- /dev/null +++ b/results/r3_close_d2.json @@ -0,0 +1,684 @@ +{ + "experiment": "R3 \u2014 closing D2 by re-scoring the 410M ladder on CPU", + "re_derived_from_the_cpu_cells": { + "E51_from_cpu_cells": { + "cuda": { + "frac_set_main": 91.27829399393488, + "frac_corpus_main": 1.9064501148312791, + "frac_interaction": 6.780626428099203 + }, + "cpu": { + "frac_set_main": 91.27840707082612, + "frac_corpus_main": 1.9063561844296986, + "frac_interaction": 6.780610811206689 + }, + "max_abs_move_pp": 0.00011307689123940534 + }, + "E53_from_cpu_cells": { + "C1_vs_e54_as_reported": { + "n_quantities_checked": 24, + "max_abs_diff": 0.14555705543928354, + "tolerance": 1e-12, + "fires": false, + "reference": "results/e54_aggregation_audit.json" + }, + "WHY_C1_DOES_NOT_FIRE_AND_WHY_IT_IS_NOT_A_DEFECT": "E53's C1 compares E53 against E54 at a 1e-12 tolerance, and E54 is derived from the CUDA ladder. On the CPU arm the two sides are therefore scored on different devices, which is the very thing R3 exists to measure, so an equality check at 1e-12 MUST fail. The whole 1.456e-01 sits in ONE quantity, 410m|min|spread_over_seed_sd (55.523 CPU vs 55.378 CUDA, 0.26% relative): it is a RATIO whose denominator is a pooled seed SD of ~3.7e-4, so a 1.6e-05 change in the asymptotes is amplified onto a scale of ~55. The paper-cited asymptotes themselves move by at most 1.565e-05. The 1B rows are identical (0.0 and 7.1e-15) because the 1B ladder was not re-scored, which is the internal control that the CPU/CUDA difference is the only thing moving. To make E53's C1 fire on a CPU arm, E54 must also be re-derived from the CPU ladder.", + "per_aggregation": { + "persist": { + "spread_over_seed_sd_cpu": 58.32948750281687, + "spread_over_seed_sd_cuda_via_e54": 58.381738635199795, + "max_asymptote_abs_move": 1.7265808217384326e-06 + }, + "min": { + "spread_over_seed_sd_cpu": 55.52328994550481, + "spread_over_seed_sd_cuda_via_e54": 55.37773289006552, + "max_asymptote_abs_move": 1.5645383527052026e-05 + } + } + } + }, + "prereg": "specs/experiments/R3_ladder410_cpu.md", + "prereg_sha256": "e58e065502a47a0810dabd9435ddc07d35907187b880950c5077e6466d27f303", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "CLEARED if every paper-cited ladder number moves by less than one pooled seed SD (3.5e-3) and no reported ordering changes. Otherwise STOP and alert: the CUDA ladder is not citable and the affected paper claims come out.", + "declared_bias": "re-scoring tests the READS, not the FITS. The fits stay TRACED-NOT-RERUN and a refit needs a GPU. Separately, this arm is scored at the UNSTRIPPED readout on purpose: its question is a device comparison, and changing two things at once would confound them.", + "n_cells_cpu": 15, + "n_cells_compared": 15, + "worst_over_all_aggregations": { + "cell": "Github|s2", + "N": 100, + "set": "typo", + "aggregation": "min", + "cpu": 0.27529761833803995, + "cuda": 0.2738095223903656, + "abs_diff": 0.0014880959476743505 + }, + "by_aggregation": { + "persist": { + "n_values": 1025, + "worst": { + "cell": "Pile-CC|s1", + "N": 50, + "set": "typo", + "aggregation": "persist", + "cpu": 0.16964286006987095, + "cuda": 0.16815476465438092, + "abs_diff": 0.001488095415490026 + }, + "worst_at_N200": { + "abs_diff": 0.0 + } + }, + "min": { + "n_values": 1025, + "worst": { + "cell": "Github|s2", + "N": 100, + "set": "typo", + "aggregation": "min", + "cpu": 0.27529761833803995, + "cuda": 0.2738095223903656, + "abs_diff": 0.0014880959476743505 + }, + "worst_at_N200": { + "cell": "USPTO_Backgrounds|s2", + "N": 200, + "set": "multihop", + "aggregation": "min", + "cpu": 0.12205270544758864, + "cuda": 0.12343966774642467, + "abs_diff": 0.0013869622988360308 + } + }, + "mean": { + "n_values": 1025, + "worst": { + "cell": "Github|s1", + "N": 25, + "set": "typo", + "aggregation": "mean", + "cpu": 0.10267856956592628, + "cuda": 0.10290750887777124, + "abs_diff": 0.00022893931184496397 + }, + "worst_at_N200": { + "cell": "Pile-CC|s2", + "N": 200, + "set": "typo", + "aggregation": "mean", + "cpu": 0.15785256108003004, + "cuda": 0.15796703020376818, + "abs_diff": 0.00011446912373813678 + } + }, + "best1L": { + "n_values": 1025, + "worst": { + "cell": "Github|s2", + "N": 100, + "set": "typo", + "aggregation": "best1L", + "cpu": 0.16071428890739167, + "cuda": 0.159226194024086, + "abs_diff": 0.001488094883305674 + }, + "worst_at_N200": { + "cell": "USPTO_Backgrounds|s2", + "N": 200, + "set": "multihop", + "aggregation": "best1L", + "cpu": 0.08183079212903976, + "cuda": 0.08321775469396796, + "abs_diff": 0.001386962564928193 + } + } + }, + "paper_cited_numbers": { + "persist": { + "asymptotes_cpu": { + "Pile-CC": { + "mean": 0.04571574828415005, + "seed_sd": 0.00014986174968815449, + "per_seed": [ + 0.04550649982004885, + 0.04579123274556228, + 0.045849512286839034 + ] + }, + "StackExchange": { + "mean": 0.04567387651994216, + "seed_sd": 0.0002446595627485649, + "per_seed": [ + 0.046019043400883676, + 0.04552208540238896, + 0.04548050075655385 + ] + }, + "Wikipedia_en": { + "mean": 0.03860496947484109, + "seed_sd": 0.0004062172167933992, + "per_seed": [ + 0.03872150797967096, + 0.03903386828736687, + 0.03805953215748545 + ] + }, + "Github": { + "mean": 0.028084053793529264, + "seed_sd": 0.0003041462666830284, + "per_seed": [ + 0.028301037585704913, + 0.0282971920464568, + 0.02765393174842608 + ] + }, + "USPTO_Backgrounds": { + "mean": 0.049735264846729854, + "seed_sd": 0.0004104842356619336, + "per_seed": [ + 0.05013739873492529, + 0.049171620719726125, + 0.04989677508553815 + ] + } + }, + "asymptotes_cuda": { + "Pile-CC": { + "mean": 0.04571574828415005, + "seed_sd": 0.00014986174968815449, + "per_seed": [ + 0.04550649982004885, + 0.04579123274556228, + 0.045849512286839034 + ] + }, + "StackExchange": { + "mean": 0.0456756031007639, + "seed_sd": 0.00024330332105846066, + "per_seed": [ + 0.046019043400883676, + 0.04552208540238896, + 0.04548568049901906 + ] + }, + "Wikipedia_en": { + "mean": 0.03860496947484109, + "seed_sd": 0.0004062172167933992, + "per_seed": [ + 0.03872150797967096, + 0.03903386828736687, + 0.03805953215748545 + ] + }, + "Github": { + "mean": 0.028084053793529264, + "seed_sd": 0.0003041462666830284, + "per_seed": [ + 0.028301037585704913, + 0.0282971920464568, + 0.02765393174842608 + ] + }, + "USPTO_Backgrounds": { + "mean": 0.049735264846729854, + "seed_sd": 0.0004104842356619336, + "per_seed": [ + 0.05013739873492529, + 0.049171620719726125, + 0.04989677508553815 + ] + } + }, + "per_corpus_abs_move": { + "Pile-CC": 0.0, + "StackExchange": 1.7265808217384326e-06, + "Wikipedia_en": 0.0, + "Github": 0.0, + "USPTO_Backgrounds": 0.0 + }, + "max_abs_move": 1.7265808217384326e-06, + "moves_less_than_one_pooled_seed_sd": true, + "ordering_cpu": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "ordering_cuda": [ + "USPTO_Backgrounds", + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github" + ], + "ordering_unchanged": true, + "spread_cpu": 0.02165121105320059, + "pooled_seed_sd_cpu": 0.0003030738063150161, + "spread_over_seed_sd_cpu": 71.43874066997475 + }, + "min": { + "asymptotes_cpu": { + "Pile-CC": { + "mean": 0.1449443925970367, + "seed_sd": 0.0003498542782962104, + "per_seed": [ + 0.14479848204683216, + 0.1454267738209594, + 0.14460792192331856 + ] + }, + "StackExchange": { + "mean": 0.14538949898881268, + "seed_sd": 0.0010453292155005974, + "per_seed": [ + 0.1467122168660316, + 0.14415640338343017, + 0.14529987671697625 + ] + }, + "Wikipedia_en": { + "mean": 0.13042415689524947, + "seed_sd": 9.156387332279143e-05, + "per_seed": [ + 0.13053648250595648, + 0.13042378953981157, + 0.1303121986399804 + ] + }, + "Github": { + "mean": 0.11288843376397276, + "seed_sd": 0.00028568293320947513, + "per_seed": [ + 0.11265755031181841, + 0.11329100217138018, + 0.11271674880871967 + ] + }, + "USPTO_Backgrounds": { + "mean": 0.15532431357377577, + "seed_sd": 0.0013477694219290323, + "per_seed": [ + 0.1562625119492175, + 0.15341835618906077, + 0.15629207258304906 + ] + } + }, + "asymptotes_cuda": { + "Pile-CC": { + "mean": 0.14493902100437375, + "seed_sd": 0.00034728237752097024, + "per_seed": [ + 0.14479042468149037, + 0.14541871640831233, + 0.14460792192331856 + ] + }, + "StackExchange": { + "mean": 0.14538949898881268, + "seed_sd": 0.0010453292155005974, + "per_seed": [ + 0.1467122168660316, + 0.14415640338343017, + 0.14529987671697625 + ] + }, + "Wikipedia_en": { + "mean": 0.13042243031696196, + "seed_sd": 8.945405033802688e-05, + "per_seed": [ + 0.1305313027710939, + 0.13042378953981157, + 0.1303121986399804 + ] + }, + "Github": { + "mean": 0.11288307417108089, + "seed_sd": 0.00028898531403988997, + "per_seed": [ + 0.11265755031181841, + 0.11329100217138018, + 0.11270067003004405 + ] + }, + "USPTO_Backgrounds": { + "mean": 0.15533995895730282, + "seed_sd": 0.0013588985237042717, + "per_seed": [ + 0.15627862671538004, + 0.15341835618906077, + 0.15632289396746765 + ] + } + }, + "per_corpus_abs_move": { + "Pile-CC": 5.371592662950508e-06, + "StackExchange": 0.0, + "Wikipedia_en": 1.7265782875086622e-06, + "Github": 5.3595928918709346e-06, + "USPTO_Backgrounds": 1.5645383527052026e-05 + }, + "max_abs_move": 1.5645383527052026e-05, + "moves_less_than_one_pooled_seed_sd": true, + "ordering_cpu": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "ordering_cuda": [ + "USPTO_Backgrounds", + "StackExchange", + "Pile-CC", + "Wikipedia_en", + "Github" + ], + "ordering_unchanged": true, + "spread_cpu": 0.04243587980980301, + "pooled_seed_sd_cpu": 0.0006240399444516214, + "spread_over_seed_sd_cpu": 68.00186460354519 + } + }, + "controls": { + "C1_partial_rescore_bit_identical": { + "required": "max_abs_diff = 0.0", + "checked": true, + "cells": [ + "Github|s0" + ], + "max_abs_diff": 0.0, + "worst_at": null, + "fires": true + }, + "C2_persist_agrees_with_cuda_at_N200": { + "required": "persist at N=200 must agree with CUDA to better than 3.4e-09 (D2's own measured persist agreement, which was taken at rung 200)", + "observed_max_abs_diff_at_N200": 0.0, + "worst": { + "abs_diff": 0.0 + }, + "fires": true, + "SCOPE_NOTE": "D2 compared ONLY the N=200 rung, so its 3.4e-9 persist figure is a statement about that rung. The whole-ladder persist worst is reported separately below and is NOT held to this threshold -- extending a rung-200 number to every rung would be reinterpreting it.", + "whole_ladder_persist_worst": { + "cell": "Pile-CC|s1", + "N": 50, + "set": "typo", + "aggregation": "persist", + "cpu": 0.16964286006987095, + "cuda": 0.16815476465438092, + "abs_diff": 0.001488095415490026 + } + } + }, + "VERDICT": "CLEARED \u2014 every paper-cited ladder number moves by less than one pooled seed SD (3.5e-03) and no reported ordering changes. Worst move 1.565e-05. The CUDA ladder's numbers are citable; the CPU numbers are the ones to print.", + "provenance": { + "schema_version": 1, + "experiment": "R3", + "utc": "2026-08-20T09:02:40+00:00", + "script": { + "path": "tools/r3_close_d2.py", + "sha256": "4bc0e3d5180fd47f71990c8d647185da81cc8bc3d15a088e6f9a3b90fc5f9d36", + "n_lines": 248 + }, + "git": { + "commit": "b197ac445bf89d5b778d5e16ce9d1c325c09473e", + "commit_short": "b197ac445bf8", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/ladder410_cpu/", + "results/r3_close_d2.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/r3_close_d2.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "acffd7dd0f0f8f5c8a110813cfaa8fbb3c567f8166d5d188791d872dc6241c3c", + "describe": "b197ac4-dirty" + }, + "argv": [ + "tools/r3_close_d2.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410_cpu/e51_interaction_cpu.json", + "sha256": "67b280e71e18f33f90a9d8f09deb25d926f9233711886bc241815e8baea294d2", + "bytes": 29031, + "exists": true + }, + { + "path": "results/ladder410_cpu/e53_ladder_summary_cpu.json", + "sha256": "3136274d819c32d7646fa82e38332df07f91e85ef387d3978ad8f88c136de44d", + "bytes": 34803, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Github_s0.json", + "sha256": "4a6e725faf9298317522b45e07e59cccf642fce50a8b5e00c8f42ac8babf782c", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Github_s1.json", + "sha256": "800a48a767c095a5643413d8693e0a94e0b871915df8445b75f2df552f7f648d", + "bytes": 15765, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Github_s2.json", + "sha256": "2efceaa5838a92c5707600a2d680d0411d485064a816c61f4536f0364a4cb1c4", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Pile-CC_s0.json", + "sha256": "4486882a4bf78c6f75e66c8c7e17b56ba23b746da708a09474fe18e9faf4d82d", + "bytes": 10906, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Pile-CC_s1.json", + "sha256": "f288e7b11b2ab2837d713239c3644568e4071ea1ebd326baf276ab9acf2c89b9", + "bytes": 10918, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Pile-CC_s2.json", + "sha256": "766eda99f7f2ea462a73ad816ad44029508d8ea7b6279561bdd033638fc70984", + "bytes": 10909, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_StackExchange_s0.json", + "sha256": "3922aed1bba3b34bc29cbd8e12866108980417e97c02a16ba9d70d63bb1946c5", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_StackExchange_s1.json", + "sha256": "09a2c3a17459e3abc354f0774267ea3747782657616c8f9f5dec1de889f9012c", + "bytes": 15791, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_StackExchange_s2.json", + "sha256": "69c734ca706ba24daabe14ada761252363afbc676b1948cd03d09ad6d2509002", + "bytes": 15781, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_USPTO_Backgrounds_s0.json", + "sha256": "dca4188fad0ee8ad338f3b0b21e493c2bb4de41f0257eb3c65c3fbddd31a4bae", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_USPTO_Backgrounds_s1.json", + "sha256": "2dd8d1c35a0f4f8ce4425f28997662fdf166344de413b51c51da02786b3a2f89", + "bytes": 10900, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_USPTO_Backgrounds_s2.json", + "sha256": "4c27aca6932a25a0ec89ce00fca4709fa00ada0f4360fdde2f19cc0039e1430b", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Wikipedia_en_s0.json", + "sha256": "df4f2d634f7a662598cd49999b5a355203684b961c36ea78ff679ca20891a7de", + "bytes": 15637, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Wikipedia_en_s1.json", + "sha256": "d8b27dbb61408fa4b77a5dfc9f4303bd951b958797566eec40e0f66fdf06537f", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410_cpu/ladder_Wikipedia_en_s2.json", + "sha256": "97fde88911ffbc01634df8d0df6e59357efa5e09a4ff2f9ffac3e7d01d3ddd20", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "54c1ab82af1c20c24d967f8b190aab5ff392daf4ed7d503dc4d4e307bd7c878e" + } +} \ No newline at end of file diff --git a/results/r4_corrections.json b/results/r4_corrections.json new file mode 100644 index 0000000000000000000000000000000000000000..6f44a9c4f23e85c91ef01f6d36911f552015a119 --- /dev/null +++ b/results/r4_corrections.json @@ -0,0 +1,1818 @@ +{ + "experiment": "R4 \u2014 the post-review corrections that are pure recomputation", + "prereg": "specs/experiments/R4_corrections.md", + "prereg_sha256": "78dd3d8e4a0ebdd3a4d2b77fac02abf8fdac02f9b645e1f5708cddc6a48c88c0", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "Grouped because each is minutes of work, each closes a named finding, and none has a losing branch. Each still gets a results file and a line in RESULTS_TAXONOMY.md; none may be reported from memory (\u00a76.2).", + "standing_rule": "a recompute that does not reproduce the slate's stated value is a disagreement to REPORT, not a number to adopt", + "recomputes_not_remeasures": true, + "agreement_with_the_slate": { + "R4a_containment_by_coverage": true, + "R4b_table_four_arm_intervals": true, + "R4j_leave_two_out_reversal": true, + "R4k_item_counts": true, + "R4l_registration_evidence": false, + "R4h.M-7_effective_rank_minimum": true, + "R4h.M-20_random_beats_top": true, + "R4h.M-21_max_abs_rho": true, + "R4h.M-22_two_sets_exactly_zero": true, + "R4h.M-22b_fit_prefix_overlap": true + }, + "n_items_agreeing": 9, + "n_items_disagreeing": 1, + "DISAGREEMENTS": [ + "R4l \u2014 five candidates rest on in-file docstrings, not three, and E47 contributes no predictor row at all (see R4l.DISAGREEMENT_WITH_THE_SLATE)" + ], + "R4a_containment_by_coverage": { + "what": "the k=32 containment row at FULL 20/20 shard coverage for every corpus, plus the whole coverage curve, in one table keyed by shard count", + "finding": "F-8 \u2014 the text quoted a k=32 row computed at partial coverage next to one at full", + "k_values": [ + 8, + 13, + 20, + 32 + ], + "n_shards_total_in_stream": 20, + "primary_k": 32, + "k32_at_full_coverage": { + "CONTROL_PubMed_2023": 0.7927273508312213, + "Github": 0.9340004374453194, + "OOD_CommonPile": 0.00125, + "OOD_News_2024": 0.0007487928942136383, + "OOD_Wikipedia_2023": 0.26957534299550123, + "OOD_arXiv_2023": 8.598932872430531e-05, + "Pile-CC": 0.9276060773863086, + "StackExchange": 0.9179914195561899, + "TRAP_FineWeb": 0.06309446548070848, + "USPTO_Backgrounds": 0.9267616923398619, + "Wikipedia_en": 0.9215951727072293, + "_CONTROL_random": 0.0 + }, + "in_stream_range_at_k32_full": [ + 0.9179914195561899, + 0.9340004374453194 + ], + "ood_range_at_k32_full": [ + 8.598932872430531e-05, + 0.00125 + ], + "slate_required": { + "OOD_Wikipedia_2023": 0.26958, + "CONTROL_PubMed_2023": 0.79273 + }, + "observed": { + "OOD_Wikipedia_2023": 0.26958, + "CONTROL_PubMed_2023": 0.79273 + }, + "agrees_with_slate": true, + "full_table_by_k_and_shard_count": { + "8": { + "CONTROL_PubMed_2023": { + "n_shards_1": 0.10180322858492401, + "n_shards_2": 0.17104623378066738, + "n_shards_3": 0.2401190575421633, + "n_shards_4": 0.29884901781000806, + "n_shards_5": 0.34787169014475844, + "n_shards_6": 0.3962275291044983, + "n_shards_7": 0.4450834930954961, + "n_shards_8": 0.49331430079742156, + "n_shards_9": 0.5308723014086855, + "n_shards_10": 0.5657178183434748, + "n_shards_11": 0.6022408379872746, + "n_shards_12": 0.6359124503348059, + "n_shards_13": 0.6579769942485622, + "n_shards_14": 0.6892209163401961, + "n_shards_15": 0.7159532938790253, + "n_shards_16": 0.741612486454947, + "n_shards_17": 0.7640278125086827, + "n_shards_18": 0.7836820316190158, + "n_shards_19": 0.8058021449806896, + "n_shards_20": 0.8239386235447751 + }, + "Github": { + "n_shards_1": 0.5724903725754247, + "n_shards_2": 0.6574337893070616, + "n_shards_3": 0.7083374678101448, + "n_shards_4": 0.7499468424409951, + "n_shards_5": 0.7798426536253455, + "n_shards_6": 0.8057504666052402, + "n_shards_7": 0.8265031776407494, + "n_shards_8": 0.8478370779880454, + "n_shards_9": 0.8659200982824202, + "n_shards_10": 0.8820091194745671, + "n_shards_11": 0.8951685685259999, + "n_shards_12": 0.906338743591561, + "n_shards_13": 0.9166111465494838, + "n_shards_14": 0.9256172183240012, + "n_shards_15": 0.9340232003213079, + "n_shards_16": 0.9435679353604083, + "n_shards_17": 0.9540246178562147, + "n_shards_18": 0.9604129751695135, + "n_shards_19": 0.9665650766650129, + "n_shards_20": 0.9736669265480662 + }, + "OOD_CommonPile": { + "n_shards_1": 0.0357996791277796, + "n_shards_2": 0.052184863801818276, + "n_shards_3": 0.06627563958804147, + "n_shards_4": 0.08005934410957959, + "n_shards_5": 0.09085169147963497, + "n_shards_6": 0.10326910138527093, + "n_shards_7": 0.1142719133299981, + "n_shards_8": 0.12280435420152845, + "n_shards_9": 0.1310883778701675, + "n_shards_10": 0.1386685527972812, + "n_shards_11": 0.14462366518881434, + "n_shards_12": 0.15104455983576862, + "n_shards_13": 0.15655114116652577, + "n_shards_14": 0.16218193109872858, + "n_shards_15": 0.1679196784377318, + "n_shards_16": 0.17307088515879723, + "n_shards_17": 0.17806683112805563, + "n_shards_18": 0.18184829299428987, + "n_shards_19": 0.18519157451653526, + "n_shards_20": 0.18745838149291838 + }, + "OOD_News_2024": { + "n_shards_1": 0.029309548949175205, + "n_shards_2": 0.03802707047768622, + "n_shards_3": 0.04525364644958641, + "n_shards_4": 0.050097893044553275, + "n_shards_5": 0.053965862122024075, + "n_shards_6": 0.05695836494739244, + "n_shards_7": 0.05950517586259955, + "n_shards_8": 0.06192995208811966, + "n_shards_9": 0.06405760037353227, + "n_shards_10": 0.06572363918056359, + "n_shards_11": 0.06733131357078807, + "n_shards_12": 0.06880634155917886, + "n_shards_13": 0.07010097044107581, + "n_shards_14": 0.07163436284627343, + "n_shards_15": 0.07265839306842962, + "n_shards_16": 0.07348080075979858, + "n_shards_17": 0.07420239718577394, + "n_shards_18": 0.07500358145284951, + "n_shards_19": 0.07556600219662442, + "n_shards_20": 0.07613372879647266 + }, + "OOD_Wikipedia_2023": { + "n_shards_1": 0.11954121794227443, + "n_shards_2": 0.18290970977771462, + "n_shards_3": 0.2335608295234365, + "n_shards_4": 0.2792137343459167, + "n_shards_5": 0.32380599161880463, + "n_shards_6": 0.3554555758518976, + "n_shards_7": 0.3813582527738013, + "n_shards_8": 0.40651223730775765, + "n_shards_9": 0.4282728429167114, + "n_shards_10": 0.4489381392914458, + "n_shards_11": 0.465811447367965, + "n_shards_12": 0.483551297560164, + "n_shards_13": 0.4999774699049923, + "n_shards_14": 0.5158663861380991, + "n_shards_15": 0.5293497814580784, + "n_shards_16": 0.5400498435024939, + "n_shards_17": 0.5506355219876397, + "n_shards_18": 0.5611726741143073, + "n_shards_19": 0.5679213041112224, + "n_shards_20": 0.5736300835693217 + }, + "OOD_arXiv_2023": { + "n_shards_1": 0.02355505480735819, + "n_shards_2": 0.030044891841899287, + "n_shards_3": 0.034386659506435435, + "n_shards_4": 0.03776319992137785, + "n_shards_5": 0.0401850428385403, + "n_shards_6": 0.0423506617949015, + "n_shards_7": 0.044116150182340205, + "n_shards_8": 0.045692103037131414, + "n_shards_9": 0.047075010442003885, + "n_shards_10": 0.04821924347243135, + "n_shards_11": 0.04925817898472133, + "n_shards_12": 0.05028307482792631, + "n_shards_13": 0.05119916323572253, + "n_shards_14": 0.051950285531770016, + "n_shards_15": 0.0527014078278175, + "n_shards_16": 0.05336127227481248, + "n_shards_17": 0.05388074003095747, + "n_shards_18": 0.05438265820074621, + "n_shards_19": 0.054845967280551196, + "n_shards_20": 0.05521801851130369 + }, + "Pile-CC": { + "n_shards_1": 0.15484516883353286, + "n_shards_2": 0.23412656738988888, + "n_shards_3": 0.30552813425468905, + "n_shards_4": 0.37363214383230764, + "n_shards_5": 0.433747803378771, + "n_shards_6": 0.4894457163260591, + "n_shards_7": 0.5307176912898279, + "n_shards_8": 0.572861313561992, + "n_shards_9": 0.613566192686564, + "n_shards_10": 0.6568755119615215, + "n_shards_11": 0.6940833000777131, + "n_shards_12": 0.7317636680599021, + "n_shards_13": 0.7671651509105039, + "n_shards_14": 0.8026226432267053, + "n_shards_15": 0.8249249123101804, + "n_shards_16": 0.8552190320164108, + "n_shards_17": 0.8781549081794824, + "n_shards_18": 0.8997290542101615, + "n_shards_19": 0.9232460285788299, + "n_shards_20": 0.9434829486183165 + }, + "StackExchange": { + "n_shards_1": 0.17365869379405047, + "n_shards_2": 0.25994857567222945, + "n_shards_3": 0.3301490885142993, + "n_shards_4": 0.3869225272184623, + "n_shards_5": 0.44562262670421904, + "n_shards_6": 0.497705522160102, + "n_shards_7": 0.546635699973377, + "n_shards_8": 0.5924692084576906, + "n_shards_9": 0.6379489119621113, + "n_shards_10": 0.6779078565723654, + "n_shards_11": 0.7121954124455281, + "n_shards_12": 0.7454320624378215, + "n_shards_13": 0.7758733027870024, + "n_shards_14": 0.8026048453767147, + "n_shards_15": 0.8255496237756946, + "n_shards_16": 0.8579665671809099, + "n_shards_17": 0.8842742443986716, + "n_shards_18": 0.9041608866843219, + "n_shards_19": 0.924152619558059, + "n_shards_20": 0.9371418162456037 + }, + "TRAP_FineWeb": { + "n_shards_1": 0.04889778589110622, + "n_shards_2": 0.06828333955662504, + "n_shards_3": 0.08378625067418993, + "n_shards_4": 0.094880996833036, + "n_shards_5": 0.10673636753377864, + "n_shards_6": 0.11854333485458242, + "n_shards_7": 0.12580384184541343, + "n_shards_8": 0.13302977499343097, + "n_shards_9": 0.14049426765686152, + "n_shards_10": 0.1474989282108728, + "n_shards_11": 0.15262968648439337, + "n_shards_12": 0.1580646945746726, + "n_shards_13": 0.16252817768189298, + "n_shards_14": 0.16650071222116195, + "n_shards_15": 0.17046287460758688, + "n_shards_16": 0.17528592568006748, + "n_shards_17": 0.1786499605858192, + "n_shards_18": 0.1824876571381156, + "n_shards_19": 0.18435118726576222, + "n_shards_20": 0.1872242736035625 + }, + "USPTO_Backgrounds": { + "n_shards_1": 0.20597797096434697, + "n_shards_2": 0.3047734382884247, + "n_shards_3": 0.38791214336173463, + "n_shards_4": 0.4523304932024278, + "n_shards_5": 0.5111694995891182, + "n_shards_6": 0.5624612545593473, + "n_shards_7": 0.6120121678704785, + "n_shards_8": 0.6505773971714026, + "n_shards_9": 0.6942245865951588, + "n_shards_10": 0.7280034023903234, + "n_shards_11": 0.7587222294306762, + "n_shards_12": 0.7949086688868705, + "n_shards_13": 0.8208627077836887, + "n_shards_14": 0.8461751942678373, + "n_shards_15": 0.8691989965831928, + "n_shards_16": 0.8879626025402593, + "n_shards_17": 0.9108746738174531, + "n_shards_18": 0.9253240200106685, + "n_shards_19": 0.9389660193474907, + "n_shards_20": 0.9508455516629903 + }, + "Wikipedia_en": { + "n_shards_1": 0.16650006594611855, + "n_shards_2": 0.25691219447857444, + "n_shards_3": 0.3255759872828117, + "n_shards_4": 0.39258765627494674, + "n_shards_5": 0.44922148871627204, + "n_shards_6": 0.503252184899033, + "n_shards_7": 0.5536315486231145, + "n_shards_8": 0.6009045030786425, + "n_shards_9": 0.6396773499378718, + "n_shards_10": 0.6797482940780386, + "n_shards_11": 0.719010530553878, + "n_shards_12": 0.7561416661460394, + "n_shards_13": 0.7907078448114289, + "n_shards_14": 0.82202530942613, + "n_shards_15": 0.8490215678516143, + "n_shards_16": 0.8685520314875362, + "n_shards_17": 0.8842194409157487, + "n_shards_18": 0.9079149225653734, + "n_shards_19": 0.9278653588510104, + "n_shards_20": 0.9438000236017687 + }, + "_CONTROL_random": { + "n_shards_1": 0.0, + "n_shards_2": 0.0, + "n_shards_3": 0.0, + "n_shards_4": 0.0, + "n_shards_5": 0.0, + "n_shards_6": 0.0, + "n_shards_7": 0.0, + "n_shards_8": 0.0, + "n_shards_9": 0.0, + "n_shards_10": 0.0, + "n_shards_11": 0.0, + "n_shards_12": 0.0, + "n_shards_13": 0.0, + "n_shards_14": 0.0, + "n_shards_15": 0.0, + "n_shards_16": 0.0, + "n_shards_17": 0.0, + "n_shards_18": 0.0, + "n_shards_19": 0.0, + "n_shards_20": 0.0 + } + }, + "13": { + "CONTROL_PubMed_2023": { + "n_shards_1": 0.06655420230635867, + "n_shards_2": 0.1278280949880222, + "n_shards_3": 0.19475637917514946, + "n_shards_4": 0.2521887386966124, + "n_shards_5": 0.3011107354305898, + "n_shards_6": 0.35009388016430826, + "n_shards_7": 0.3997280712482105, + "n_shards_8": 0.44926154797959816, + "n_shards_9": 0.4881624666383707, + "n_shards_10": 0.5246534347191149, + "n_shards_11": 0.5637593790240779, + "n_shards_12": 0.5998151172243125, + "n_shards_13": 0.6235729135942795, + "n_shards_14": 0.6567906652182983, + "n_shards_15": 0.6855158373319329, + "n_shards_16": 0.7130288402742307, + "n_shards_17": 0.7371175552310315, + "n_shards_18": 0.7585193551403886, + "n_shards_19": 0.7834749329170474, + "n_shards_20": 0.8032293337745581 + }, + "Github": { + "n_shards_1": 0.48856291521883943, + "n_shards_2": 0.5773274760095781, + "n_shards_3": 0.6349855067245197, + "n_shards_4": 0.681494517761014, + "n_shards_5": 0.7156167293808401, + "n_shards_6": 0.7456385143042328, + "n_shards_7": 0.7714293430315251, + "n_shards_8": 0.7976027582233585, + "n_shards_9": 0.8204274165961507, + "n_shards_10": 0.8407540104063518, + "n_shards_11": 0.8576103199323047, + "n_shards_12": 0.8716669967412636, + "n_shards_13": 0.88539960030967, + "n_shards_14": 0.8968456871252903, + "n_shards_15": 0.9080352159588067, + "n_shards_16": 0.9212727076319248, + "n_shards_17": 0.9346857389770088, + "n_shards_18": 0.9427965720252777, + "n_shards_19": 0.9515240444340421, + "n_shards_20": 0.9606656104279567 + }, + "OOD_CommonPile": { + "n_shards_1": 0.00485002640569932, + "n_shards_2": 0.008216303992469885, + "n_shards_3": 0.011891546224344258, + "n_shards_4": 0.015333268666314591, + "n_shards_5": 0.0183007663041721, + "n_shards_6": 0.02209097212492231, + "n_shards_7": 0.025349471347121778, + "n_shards_8": 0.027637965288181385, + "n_shards_9": 0.030073756327488155, + "n_shards_10": 0.032470028633118854, + "n_shards_11": 0.03418729724195165, + "n_shards_12": 0.03639316110350675, + "n_shards_13": 0.03782661335230233, + "n_shards_14": 0.03952591890037327, + "n_shards_15": 0.04170663447686178, + "n_shards_16": 0.04352449622596093, + "n_shards_17": 0.045371098872279046, + "n_shards_18": 0.04649199386381844, + "n_shards_19": 0.04765600020118628, + "n_shards_20": 0.048381707855964996 + }, + "OOD_News_2024": { + "n_shards_1": 0.0008914215713082569, + "n_shards_2": 0.001175801459087578, + "n_shards_3": 0.002220350662277008, + "n_shards_4": 0.0026359828059544775, + "n_shards_5": 0.002778172749844138, + "n_shards_6": 0.00290395616174653, + "n_shards_7": 0.0029422380697168233, + "n_shards_8": 0.0030406772616404345, + "n_shards_9": 0.003215680269504632, + "n_shards_10": 0.0033086506174324873, + "n_shards_11": 0.0033469325254027803, + "n_shards_12": 0.0034399028733306356, + "n_shards_13": 0.0036969385411311758, + "n_shards_14": 0.0040578822448510835, + "n_shards_15": 0.004068819932842596, + "n_shards_16": 0.004123508372800158, + "n_shards_17": 0.004150852592778939, + "n_shards_18": 0.004221947564723769, + "n_shards_19": 0.00427663600468133, + "n_shards_20": 0.004347730976626161 + }, + "OOD_Wikipedia_2023": { + "n_shards_1": 0.055348043872527855, + "n_shards_2": 0.09854046117972191, + "n_shards_3": 0.13488139447850994, + "n_shards_4": 0.17124391858824883, + "n_shards_5": 0.20917897342890865, + "n_shards_6": 0.2360631315312203, + "n_shards_7": 0.25875507384057345, + "n_shards_8": 0.280795693352909, + "n_shards_9": 0.3000942798744854, + "n_shards_10": 0.3195655928836687, + "n_shards_11": 0.33522612776002536, + "n_shards_12": 0.3522144975098598, + "n_shards_13": 0.3681845006765121, + "n_shards_14": 0.3839278003281803, + "n_shards_15": 0.3978574718599764, + "n_shards_16": 0.4083865906670121, + "n_shards_17": 0.41947347209027835, + "n_shards_18": 0.4297435011659038, + "n_shards_19": 0.4364438494976538, + "n_shards_20": 0.44222298989550046 + }, + "OOD_arXiv_2023": { + "n_shards_1": 0.0010665744223922976, + "n_shards_2": 0.001531399086205157, + "n_shards_3": 0.0017656130640953576, + "n_shards_4": 0.0019818105821478504, + "n_shards_5": 0.002104322509044263, + "n_shards_6": 0.0022412476038108416, + "n_shards_7": 0.0024250154941554605, + "n_shards_8": 0.002536717545149248, + "n_shards_9": 0.0026231965523702454, + "n_shards_10": 0.002731295311396492, + "n_shards_11": 0.002785344690909615, + "n_shards_12": 0.0028429973623902796, + "n_shards_13": 0.0029799224571568587, + "n_shards_14": 0.0029871290410919417, + "n_shards_15": 0.003070004756345397, + "n_shards_16": 0.003174500223404102, + "n_shards_17": 0.003221343018982142, + "n_shards_18": 0.003232152894884767, + "n_shards_19": 0.0032898055663654314, + "n_shards_20": 0.0033186319021057636 + }, + "Pile-CC": { + "n_shards_1": 0.11586139686764775, + "n_shards_2": 0.18759931782066247, + "n_shards_3": 0.25664997581826976, + "n_shards_4": 0.32429336620594107, + "n_shards_5": 0.3854749619092294, + "n_shards_6": 0.4433038425314817, + "n_shards_7": 0.4859582328791532, + "n_shards_8": 0.530118072297919, + "n_shards_9": 0.5741288213497405, + "n_shards_10": 0.6206886520412653, + "n_shards_11": 0.660972141919484, + "n_shards_12": 0.7019065385216781, + "n_shards_13": 0.7402445809287966, + "n_shards_14": 0.7790008036334414, + "n_shards_15": 0.803142534027149, + "n_shards_16": 0.8367496845466017, + "n_shards_17": 0.8622368645932197, + "n_shards_18": 0.8867422300282545, + "n_shards_19": 0.9131530430798434, + "n_shards_20": 0.9354002349082366 + }, + "StackExchange": { + "n_shards_1": 0.1010960349112768, + "n_shards_2": 0.1818392000462401, + "n_shards_3": 0.25172316629096586, + "n_shards_4": 0.31005071961158315, + "n_shards_5": 0.3721316686896711, + "n_shards_6": 0.427796803652968, + "n_shards_7": 0.48174599734119417, + "n_shards_8": 0.5322127333680134, + "n_shards_9": 0.5831960580313277, + "n_shards_10": 0.6275793306745275, + "n_shards_11": 0.6669737298422056, + "n_shards_12": 0.7045221374487024, + "n_shards_13": 0.7400041905092192, + "n_shards_14": 0.7708766834287035, + "n_shards_15": 0.7970782035720478, + "n_shards_16": 0.8345399109878042, + "n_shards_17": 0.864263626379978, + "n_shards_18": 0.887914715912375, + "n_shards_19": 0.9111756545864401, + "n_shards_20": 0.9260953124096873 + }, + "TRAP_FineWeb": { + "n_shards_1": 0.018117363372982313, + "n_shards_2": 0.028842381817854635, + "n_shards_3": 0.03806301848086232, + "n_shards_4": 0.044534019542567166, + "n_shards_5": 0.052700149358478346, + "n_shards_6": 0.06161847006532184, + "n_shards_7": 0.06641953536916737, + "n_shards_8": 0.07176045059473467, + "n_shards_9": 0.0773604937827284, + "n_shards_10": 0.08284176998794335, + "n_shards_11": 0.08668190243112414, + "n_shards_12": 0.09097550880855122, + "n_shards_13": 0.09444854330496122, + "n_shards_14": 0.0974573068686905, + "n_shards_15": 0.10073239639380252, + "n_shards_16": 0.10492523078584154, + "n_shards_17": 0.10761728239549405, + "n_shards_18": 0.11101473790286301, + "n_shards_19": 0.1123319717118641, + "n_shards_20": 0.11472170736535243 + }, + "USPTO_Backgrounds": { + "n_shards_1": 0.15446205415137354, + "n_shards_2": 0.24780163281138504, + "n_shards_3": 0.3295958956493737, + "n_shards_4": 0.393949595822462, + "n_shards_5": 0.4553867772284655, + "n_shards_6": 0.5097138005862313, + "n_shards_7": 0.5630381562624648, + "n_shards_8": 0.6053295080707429, + "n_shards_9": 0.6530184322522331, + "n_shards_10": 0.6904318423835679, + "n_shards_11": 0.7246798965129853, + "n_shards_12": 0.7651049690600173, + "n_shards_13": 0.7937505260345957, + "n_shards_14": 0.8213751184721394, + "n_shards_15": 0.8477957778176243, + "n_shards_16": 0.8697263888228169, + "n_shards_17": 0.8953163709284923, + "n_shards_18": 0.9116920566031522, + "n_shards_19": 0.9278372018984817, + "n_shards_20": 0.9416000966073971 + }, + "Wikipedia_en": { + "n_shards_1": 0.099935954635733, + "n_shards_2": 0.18004663078207306, + "n_shards_3": 0.2451282346507009, + "n_shards_4": 0.31316024294061773, + "n_shards_5": 0.37257131343369504, + "n_shards_6": 0.42934860827264615, + "n_shards_7": 0.48454995538412826, + "n_shards_8": 0.5363763276819896, + "n_shards_9": 0.5800639014421001, + "n_shards_10": 0.6254965314757779, + "n_shards_11": 0.6700044615871736, + "n_shards_12": 0.7122887942200857, + "n_shards_13": 0.7522343916410006, + "n_shards_14": 0.7886538959730578, + "n_shards_15": 0.8198993984053424, + "n_shards_16": 0.8421965401111079, + "n_shards_17": 0.8606941366109209, + "n_shards_18": 0.8883775654126248, + "n_shards_19": 0.911973604674592, + "n_shards_20": 0.9313131458507239 + }, + "_CONTROL_random": { + "n_shards_1": 0.0, + "n_shards_2": 0.0, + "n_shards_3": 0.0, + "n_shards_4": 0.0, + "n_shards_5": 0.0, + "n_shards_6": 0.0, + "n_shards_7": 0.0, + "n_shards_8": 0.0, + "n_shards_9": 0.0, + "n_shards_10": 0.0, + "n_shards_11": 0.0, + "n_shards_12": 0.0, + "n_shards_13": 0.0, + "n_shards_14": 0.0, + "n_shards_15": 0.0, + "n_shards_16": 0.0, + "n_shards_17": 0.0, + "n_shards_18": 0.0, + "n_shards_19": 0.0, + "n_shards_20": 0.0 + } + }, + "20": { + "CONTROL_PubMed_2023": { + "n_shards_1": 0.062416592420378024, + "n_shards_2": 0.12189647326942417, + "n_shards_3": 0.18819044268632631, + "n_shards_4": 0.24494774718278353, + "n_shards_5": 0.29342643116891065, + "n_shards_6": 0.3421804318648501, + "n_shards_7": 0.39172979194469193, + "n_shards_8": 0.4411300221400521, + "n_shards_9": 0.4799993881850893, + "n_shards_10": 0.5167388735722726, + "n_shards_11": 0.556028862368412, + "n_shards_12": 0.592206242806395, + "n_shards_13": 0.6163691079356218, + "n_shards_14": 0.6498468550801668, + "n_shards_15": 0.6790992554977305, + "n_shards_16": 0.7065353303991709, + "n_shards_17": 0.7307593770194671, + "n_shards_18": 0.7523258526214357, + "n_shards_19": 0.7777467621607773, + "n_shards_20": 0.7977225189949411 + }, + "Github": { + "n_shards_1": 0.4280769845437319, + "n_shards_2": 0.516079541562978, + "n_shards_3": 0.5773521152485103, + "n_shards_4": 0.6259446131366864, + "n_shards_5": 0.6629392023013173, + "n_shards_6": 0.696367662838748, + "n_shards_7": 0.7261888997968083, + "n_shards_8": 0.7555169973288282, + "n_shards_9": 0.7820278075843017, + "n_shards_10": 0.8059450697472661, + "n_shards_11": 0.8259126503961097, + "n_shards_12": 0.8425378416017899, + "n_shards_13": 0.8589118969886532, + "n_shards_14": 0.8717061254309262, + "n_shards_15": 0.8852172324832767, + "n_shards_16": 0.901340152051323, + "n_shards_17": 0.9176776785918129, + "n_shards_18": 0.9270975548503458, + "n_shards_19": 0.9376726558754367, + "n_shards_20": 0.9485445537773111 + }, + "OOD_CommonPile": { + "n_shards_1": 0.00040138535287506595, + "n_shards_2": 0.0007339617881144063, + "n_shards_3": 0.0012882558468466402, + "n_shards_4": 0.0016361231526716973, + "n_shards_5": 0.0018845997996895954, + "n_shards_6": 0.002553575387814705, + "n_shards_7": 0.003027592375971926, + "n_shards_8": 0.0032875371759291117, + "n_shards_9": 0.003623936328814881, + "n_shards_10": 0.003922108305236358, + "n_shards_11": 0.004052080705214951, + "n_shards_12": 0.004373188987515004, + "n_shards_13": 0.004449643340443588, + "n_shards_14": 0.004606374763947185, + "n_shards_15": 0.0052868185050115825, + "n_shards_16": 0.005466486234393755, + "n_shards_17": 0.005623217657897353, + "n_shards_18": 0.005695849293179507, + "n_shards_19": 0.005757012775522374, + "n_shards_20": 0.005806708104925954 + }, + "OOD_News_2024": { + "n_shards_1": 0.00012684794391014552, + "n_shards_2": 0.00012684794391014552, + "n_shards_3": 0.0007553218478285939, + "n_shards_4": 0.0009455937636938122, + "n_shards_5": 0.0009513595793260915, + "n_shards_6": 0.0009513595793260915, + "n_shards_7": 0.0009513595793260915, + "n_shards_8": 0.0009513595793260915, + "n_shards_9": 0.0009686570262229295, + "n_shards_10": 0.0009801886574874882, + "n_shards_11": 0.0009801886574874882, + "n_shards_12": 0.0010205493669134436, + "n_shards_13": 0.0012857768859982934, + "n_shards_14": 0.0014933462487603496, + "n_shards_15": 0.0014933462487603496, + "n_shards_16": 0.0014933462487603496, + "n_shards_17": 0.0015048778800249084, + "n_shards_18": 0.0015048778800249084, + "n_shards_19": 0.0015106436956571878, + "n_shards_20": 0.0015279411425540258 + }, + "OOD_Wikipedia_2023": { + "n_shards_1": 0.03568873637619119, + "n_shards_2": 0.06802244844088921, + "n_shards_3": 0.09550151301267411, + "n_shards_4": 0.12488953668529719, + "n_shards_5": 0.15561650962704524, + "n_shards_6": 0.1778507186332006, + "n_shards_7": 0.19670312434248027, + "n_shards_8": 0.21456853316194782, + "n_shards_9": 0.230329878844219, + "n_shards_10": 0.24726949988332014, + "n_shards_11": 0.2608043642095035, + "n_shards_12": 0.27564757325008893, + "n_shards_13": 0.28978305196270837, + "n_shards_14": 0.30401799547817704, + "n_shards_15": 0.3171894307170265, + "n_shards_16": 0.32649321535277487, + "n_shards_17": 0.33700205433073577, + "n_shards_18": 0.34561341091587955, + "n_shards_19": 0.3514053228971802, + "n_shards_20": 0.35669225972555363 + }, + "OOD_arXiv_2023": { + "n_shards_1": 6.891666826195992e-05, + "n_shards_2": 0.00015314815169324426, + "n_shards_3": 0.00016080555927790646, + "n_shards_4": 0.00016463426307023756, + "n_shards_5": 0.00016463426307023756, + "n_shards_6": 0.00016463426307023756, + "n_shards_7": 0.0001952638934088864, + "n_shards_8": 0.00020675000478587974, + "n_shards_9": 0.00020675000478587974, + "n_shards_10": 0.00021823611616287306, + "n_shards_11": 0.00021823611616287306, + "n_shards_12": 0.00021823611616287306, + "n_shards_13": 0.0003062963033864885, + "n_shards_14": 0.0003062963033864885, + "n_shards_15": 0.0003139537109711507, + "n_shards_16": 0.0003445833413097996, + "n_shards_17": 0.00035224074889446177, + "n_shards_18": 0.00035224074889446177, + "n_shards_19": 0.00035224074889446177, + "n_shards_20": 0.00035224074889446177 + }, + "Pile-CC": { + "n_shards_1": 0.10712866013916923, + "n_shards_2": 0.17629260272703765, + "n_shards_3": 0.24451099730229364, + "n_shards_4": 0.3113477339652424, + "n_shards_5": 0.37259608890466944, + "n_shards_6": 0.4307144869533671, + "n_shards_7": 0.47357289789239276, + "n_shards_8": 0.5177627948083625, + "n_shards_9": 0.5628210521440784, + "n_shards_10": 0.6100637183579112, + "n_shards_11": 0.6511236583831514, + "n_shards_12": 0.6928165366698957, + "n_shards_13": 0.7315415520109297, + "n_shards_14": 0.7709496775488308, + "n_shards_15": 0.7955223477378246, + "n_shards_16": 0.8299480141562046, + "n_shards_17": 0.8564773898243596, + "n_shards_18": 0.8820959518966921, + "n_shards_19": 0.9090537187052553, + "n_shards_20": 0.9320556213794194 + }, + "StackExchange": { + "n_shards_1": 0.0840036465036465, + "n_shards_2": 0.1630125022982166, + "n_shards_3": 0.23242630385487528, + "n_shards_4": 0.29077036219893365, + "n_shards_5": 0.3528183796040939, + "n_shards_6": 0.4093085432371147, + "n_shards_7": 0.4645078752221609, + "n_shards_8": 0.5159151498437212, + "n_shards_9": 0.5680693448550591, + "n_shards_10": 0.6135357296071582, + "n_shards_11": 0.6540494576208862, + "n_shards_12": 0.6925292639578354, + "n_shards_13": 0.7292356131641846, + "n_shards_14": 0.7613914935343506, + "n_shards_15": 0.7883304222589936, + "n_shards_16": 0.827005577005577, + "n_shards_17": 0.857334375191518, + "n_shards_18": 0.8820670159955875, + "n_shards_19": 0.9063629956487099, + "n_shards_20": 0.9220444934730649 + }, + "TRAP_FineWeb": { + "n_shards_1": 0.0120624399652516, + "n_shards_2": 0.020232906121917898, + "n_shards_3": 0.027106047675722427, + "n_shards_4": 0.03218436551642321, + "n_shards_5": 0.03867481544696468, + "n_shards_6": 0.04598039853505646, + "n_shards_7": 0.04941696931195873, + "n_shards_8": 0.05336634686403349, + "n_shards_9": 0.058016050086679656, + "n_shards_10": 0.06232133084838906, + "n_shards_11": 0.06545174832668088, + "n_shards_12": 0.06925570306191568, + "n_shards_13": 0.0721488517337844, + "n_shards_14": 0.07475880860443845, + "n_shards_15": 0.07755628437049142, + "n_shards_16": 0.08126839311614308, + "n_shards_17": 0.08341146620641621, + "n_shards_18": 0.08632757637568071, + "n_shards_19": 0.08733788226109519, + "n_shards_20": 0.08935849403192414 + }, + "USPTO_Backgrounds": { + "n_shards_1": 0.13277103572426208, + "n_shards_2": 0.22183726993388994, + "n_shards_3": 0.3017706943108104, + "n_shards_4": 0.36564604736335704, + "n_shards_5": 0.4278531301406003, + "n_shards_6": 0.4830767559514572, + "n_shards_7": 0.5374856451162358, + "n_shards_8": 0.5810701759831155, + "n_shards_9": 0.6305246904000745, + "n_shards_10": 0.669674726093299, + "n_shards_11": 0.7055347776156926, + "n_shards_12": 0.7477264967876098, + "n_shards_13": 0.7775846550172258, + "n_shards_14": 0.806542723237841, + "n_shards_15": 0.8345502653713647, + "n_shards_16": 0.8580845153480865, + "n_shards_17": 0.885036779540023, + "n_shards_18": 0.9024527452745275, + "n_shards_19": 0.9200238989416183, + "n_shards_20": 0.9348792637884478 + }, + "Wikipedia_en": { + "n_shards_1": 0.08057673400840848, + "n_shards_2": 0.1566355397604465, + "n_shards_3": 0.21928210464302245, + "n_shards_4": 0.28759425102236774, + "n_shards_5": 0.34699295723461476, + "n_shards_6": 0.4041078334984679, + "n_shards_7": 0.4606833127393335, + "n_shards_8": 0.5141333496555127, + "n_shards_9": 0.5596186730833235, + "n_shards_10": 0.6067068855368913, + "n_shards_11": 0.652261068159126, + "n_shards_12": 0.6961473280719808, + "n_shards_13": 0.7377076774176272, + "n_shards_14": 0.7758480201068848, + "n_shards_15": 0.8084911037746315, + "n_shards_16": 0.8316622226982858, + "n_shards_17": 0.851386556389942, + "n_shards_18": 0.8804451364368427, + "n_shards_19": 0.9050661239542009, + "n_shards_20": 0.9261944201099451 + }, + "_CONTROL_random": { + "n_shards_1": 0.0, + "n_shards_2": 0.0, + "n_shards_3": 0.0, + "n_shards_4": 0.0, + "n_shards_5": 0.0, + "n_shards_6": 0.0, + "n_shards_7": 0.0, + "n_shards_8": 0.0, + "n_shards_9": 0.0, + "n_shards_10": 0.0, + "n_shards_11": 0.0, + "n_shards_12": 0.0, + "n_shards_13": 0.0, + "n_shards_14": 0.0, + "n_shards_15": 0.0, + "n_shards_16": 0.0, + "n_shards_17": 0.0, + "n_shards_18": 0.0, + "n_shards_19": 0.0, + "n_shards_20": 0.0 + } + }, + "32": { + "CONTROL_PubMed_2023": { + "n_shards_1": 0.061166716783366985, + "n_shards_2": 0.11960565316379569, + "n_shards_3": 0.1853258301473431, + "n_shards_4": 0.24163838652863096, + "n_shards_5": 0.2897504188324241, + "n_shards_6": 0.338141672752266, + "n_shards_7": 0.38725031143949484, + "n_shards_8": 0.43627303578332405, + "n_shards_9": 0.4747282958889987, + "n_shards_10": 0.5114910434297006, + "n_shards_11": 0.5506550968684222, + "n_shards_12": 0.5867906697023068, + "n_shards_13": 0.6109755573693028, + "n_shards_14": 0.644417715537609, + "n_shards_15": 0.673950771081232, + "n_shards_16": 0.7012242793934448, + "n_shards_17": 0.7253146612827012, + "n_shards_18": 0.7469478929507282, + "n_shards_19": 0.7725804373040079, + "n_shards_20": 0.7927273508312213 + }, + "Github": { + "n_shards_1": 0.3742096158434741, + "n_shards_2": 0.4577368169887855, + "n_shards_3": 0.5195011134971765, + "n_shards_4": 0.5686640022269943, + "n_shards_5": 0.6077905034597948, + "n_shards_6": 0.6443569553805775, + "n_shards_7": 0.6786467032530025, + "n_shards_8": 0.7105752405949256, + "n_shards_9": 0.7410721387099339, + "n_shards_10": 0.7680744452397996, + "n_shards_11": 0.7919400699912511, + "n_shards_12": 0.8114014157321244, + "n_shards_13": 0.8298983933826454, + "n_shards_14": 0.844239640499483, + "n_shards_15": 0.8595303427980593, + "n_shards_16": 0.8783255786208543, + "n_shards_17": 0.8980553567167741, + "n_shards_18": 0.9088274477054005, + "n_shards_19": 0.9214288554839736, + "n_shards_20": 0.9340004374453194 + }, + "OOD_CommonPile": { + "n_shards_1": 0.0, + "n_shards_2": 2.577319587628866e-05, + "n_shards_3": 0.00012457044673539519, + "n_shards_4": 0.00018900343642611685, + "n_shards_5": 0.00019329896907216494, + "n_shards_6": 0.0006443298969072165, + "n_shards_7": 0.0007259450171821306, + "n_shards_8": 0.0007259450171821306, + "n_shards_9": 0.000743127147766323, + "n_shards_10": 0.0007860824742268041, + "n_shards_11": 0.0007860824742268041, + "n_shards_12": 0.0007989690721649485, + "n_shards_13": 0.0007989690721649485, + "n_shards_14": 0.0007989690721649485, + "n_shards_15": 0.00125, + "n_shards_16": 0.00125, + "n_shards_17": 0.00125, + "n_shards_18": 0.00125, + "n_shards_19": 0.00125, + "n_shards_20": 0.00125 + }, + "OOD_News_2024": { + "n_shards_1": 1.936533347104237e-05, + "n_shards_2": 1.936533347104237e-05, + "n_shards_3": 0.0004324924475199463, + "n_shards_4": 0.0005486844483462006, + "n_shards_5": 0.0005486844483462006, + "n_shards_6": 0.0005486844483462006, + "n_shards_7": 0.0005486844483462006, + "n_shards_8": 0.0005486844483462006, + "n_shards_9": 0.0005486844483462006, + "n_shards_10": 0.0005486844483462006, + "n_shards_11": 0.0005486844483462006, + "n_shards_12": 0.0005486844483462006, + "n_shards_13": 0.0007423377830566242, + "n_shards_14": 0.0007487928942136383, + "n_shards_15": 0.0007487928942136383, + "n_shards_16": 0.0007487928942136383, + "n_shards_17": 0.0007487928942136383, + "n_shards_18": 0.0007487928942136383, + "n_shards_19": 0.0007487928942136383, + "n_shards_20": 0.0007487928942136383 + }, + "OOD_Wikipedia_2023": { + "n_shards_1": 0.022133038856352236, + "n_shards_2": 0.04430904614379815, + "n_shards_3": 0.06429366344546661, + "n_shards_4": 0.08671029394703711, + "n_shards_5": 0.11009801099132467, + "n_shards_6": 0.12712640023374827, + "n_shards_7": 0.14158527729677006, + "n_shards_8": 0.15459182138882563, + "n_shards_9": 0.1662061883134461, + "n_shards_10": 0.17953069879559488, + "n_shards_11": 0.1900794486290922, + "n_shards_12": 0.2019688135127122, + "n_shards_13": 0.21365622677019194, + "n_shards_14": 0.226095587571811, + "n_shards_15": 0.23774432924130642, + "n_shards_16": 0.24528528889824647, + "n_shards_17": 0.25429147205548086, + "n_shards_18": 0.2606894714453291, + "n_shards_19": 0.2651152198479777, + "n_shards_20": 0.26957534299550123 + }, + "OOD_arXiv_2023": { + "n_shards_1": 0.0, + "n_shards_2": 0.0, + "n_shards_3": 0.0, + "n_shards_4": 0.0, + "n_shards_5": 0.0, + "n_shards_6": 0.0, + "n_shards_7": 0.0, + "n_shards_8": 0.0, + "n_shards_9": 0.0, + "n_shards_10": 0.0, + "n_shards_11": 0.0, + "n_shards_12": 0.0, + "n_shards_13": 8.598932872430531e-05, + "n_shards_14": 8.598932872430531e-05, + "n_shards_15": 8.598932872430531e-05, + "n_shards_16": 8.598932872430531e-05, + "n_shards_17": 8.598932872430531e-05, + "n_shards_18": 8.598932872430531e-05, + "n_shards_19": 8.598932872430531e-05, + "n_shards_20": 8.598932872430531e-05 + }, + "Pile-CC": { + "n_shards_1": 0.09955354052744514, + "n_shards_2": 0.16657437530283103, + "n_shards_3": 0.23348705613622206, + "n_shards_4": 0.2993787637571814, + "n_shards_5": 0.36025645462725825, + "n_shards_6": 0.41824859832491174, + "n_shards_7": 0.4614063473385478, + "n_shards_8": 0.5054509586765419, + "n_shards_9": 0.5512173807710944, + "n_shards_10": 0.5990603585519485, + "n_shards_11": 0.6408813248425279, + "n_shards_12": 0.6832819962621998, + "n_shards_13": 0.7223428739530698, + "n_shards_14": 0.7621478507648647, + "n_shards_15": 0.7870665190004845, + "n_shards_16": 0.8223117256177753, + "n_shards_17": 0.8498606977227106, + "n_shards_18": 0.876626635287603, + "n_shards_19": 0.9038122101474354, + "n_shards_20": 0.9276060773863086 + }, + "StackExchange": { + "n_shards_1": 0.07686719226900293, + "n_shards_2": 0.15416426649242118, + "n_shards_3": 0.2230184591311226, + "n_shards_4": 0.2813061533302955, + "n_shards_5": 0.34294262795436375, + "n_shards_6": 0.3998418034407752, + "n_shards_7": 0.4551891050717473, + "n_shards_8": 0.5066674691129815, + "n_shards_9": 0.559272123874784, + "n_shards_10": 0.6051534248695308, + "n_shards_11": 0.6458245565767641, + "n_shards_12": 0.6847718616467918, + "n_shards_13": 0.7220297306359673, + "n_shards_14": 0.7550102741787105, + "n_shards_15": 0.7822218018932001, + "n_shards_16": 0.8213711514818031, + "n_shards_17": 0.8520174360120711, + "n_shards_18": 0.8771655303453672, + "n_shards_19": 0.9017547781379234, + "n_shards_20": 0.9179914195561899 + }, + "TRAP_FineWeb": { + "n_shards_1": 0.007237811980212063, + "n_shards_2": 0.013104565752771127, + "n_shards_3": 0.01772490297291838, + "n_shards_4": 0.02130942505813029, + "n_shards_5": 0.02606299990974225, + "n_shards_6": 0.0317105562886013, + "n_shards_7": 0.033653246915119034, + "n_shards_8": 0.036180463924836787, + "n_shards_9": 0.039562980568795744, + "n_shards_10": 0.04249850644913116, + "n_shards_11": 0.04496125363717244, + "n_shards_12": 0.04796554732729609, + "n_shards_13": 0.04993402588248441, + "n_shards_14": 0.05192399437823155, + "n_shards_15": 0.05381081115929633, + "n_shards_16": 0.056815104849419984, + "n_shards_17": 0.05843114837944358, + "n_shards_18": 0.06064891024511426, + "n_shards_19": 0.061435442069567234, + "n_shards_20": 0.06309446548070848 + }, + "USPTO_Backgrounds": { + "n_shards_1": 0.12033439046490556, + "n_shards_2": 0.2053173137241235, + "n_shards_3": 0.2827638182198369, + "n_shards_4": 0.3454772774796125, + "n_shards_5": 0.4081210357566042, + "n_shards_6": 0.4635638112497386, + "n_shards_7": 0.5176909806928278, + "n_shards_8": 0.5611234055900188, + "n_shards_9": 0.6115782045026835, + "n_shards_10": 0.651882797797449, + "n_shards_11": 0.6885629399874538, + "n_shards_12": 0.731903882344741, + "n_shards_13": 0.762703004112358, + "n_shards_14": 0.7927179898236565, + "n_shards_15": 0.8213345995678539, + "n_shards_16": 0.8459216212448596, + "n_shards_17": 0.8741113124695058, + "n_shards_18": 0.8923729699588764, + "n_shards_19": 0.9112096605562139, + "n_shards_20": 0.9267616923398619 + }, + "Wikipedia_en": { + "n_shards_1": 0.07066448338254318, + "n_shards_2": 0.14398071145837363, + "n_shards_3": 0.2045797392952462, + "n_shards_4": 0.2726569450353924, + "n_shards_5": 0.3318978670001762, + "n_shards_6": 0.3885042354852435, + "n_shards_7": 0.4455060018824379, + "n_shards_8": 0.49941334983689833, + "n_shards_9": 0.5458253286745145, + "n_shards_10": 0.593750134306356, + "n_shards_11": 0.6392423832179374, + "n_shards_12": 0.6840942594240084, + "n_shards_13": 0.7268702965914121, + "n_shards_14": 0.7660533701225304, + "n_shards_15": 0.799593427799052, + "n_shards_16": 0.8232872179029298, + "n_shards_17": 0.8439811412387129, + "n_shards_18": 0.8737821099635976, + "n_shards_19": 0.8991993192279426, + "n_shards_20": 0.9215951727072293 + }, + "_CONTROL_random": { + "n_shards_1": 0.0, + "n_shards_2": 0.0, + "n_shards_3": 0.0, + "n_shards_4": 0.0, + "n_shards_5": 0.0, + "n_shards_6": 0.0, + "n_shards_7": 0.0, + "n_shards_8": 0.0, + "n_shards_9": 0.0, + "n_shards_10": 0.0, + "n_shards_11": 0.0, + "n_shards_12": 0.0, + "n_shards_13": 0.0, + "n_shards_14": 0.0, + "n_shards_15": 0.0, + "n_shards_16": 0.0, + "n_shards_17": 0.0, + "n_shards_18": 0.0, + "n_shards_19": 0.0, + "n_shards_20": 0.0 + } + } + }, + "why_keyed": "stored keyed by BOTH k and shard count so a value can never again be quoted at one coverage beside a value at another", + "formatting_floor": { + "note": "fraction of OOD_CommonPile's own k=32 k-grams that survive re-wrapping the IDENTICAL text \u2014 the band in which a containment value cannot be read as absence", + "80col_its_own_wrap": 0.2877, + "72col": 0.0243, + "100col": 0.0132, + "single_line_reflow": 0.0014 + }, + "reading": "A corpus is 'not found' only if its containment is BELOW the formatting floor AND flat in coverage. Containment inside 0.001-0.03 at k=32 is consistent with present-but-reformatted and must not be called absence." + }, + "R4b_table_four_arm_intervals": { + "what": "all four hierarchical intervals, 410M/160M x min/persist, J-minus-logit and J-minus-shuffled", + "finding": "F-5 \u2014 each file reported only itself, so the pattern across them was invisible", + "arms": { + "410m|min": { + "file": "results/t22_bootstrap_ci_410m.json", + "J_minus_logit": { + "point": 0.05566275119781494, + "ci_lo": 0.006518487352877855, + "ci_hi": 0.11540429294109344, + "excludes_zero": true + }, + "J_minus_shuffled": { + "point": 0.018828706815838814, + "ci_lo": -0.023924075067043304, + "ci_hi": 0.05630238726735115, + "excludes_zero": false + }, + "beats_free_baseline": true, + "clears_derangement_control": false, + "hurts_vs_free_baseline": false + }, + "410m|persist": { + "file": "results/t22_bootstrap_ci_persist_410m.json", + "J_minus_logit": { + "point": 0.021596459671854973, + "ci_lo": -0.002151099033653736, + "ci_hi": 0.05984563007950783, + "excludes_zero": false + }, + "J_minus_shuffled": { + "point": 0.041693612933158875, + "ci_lo": 0.013362868689000607, + "ci_hi": 0.07319377362728119, + "excludes_zero": true + }, + "beats_free_baseline": false, + "clears_derangement_control": true, + "hurts_vs_free_baseline": false + }, + "160m|min": { + "file": "results/t22_bootstrap_ci_160m.json", + "J_minus_logit": { + "point": -0.0207336638122797, + "ci_lo": -0.04180724173784256, + "ci_hi": -0.0023532172199338675, + "excludes_zero": true + }, + "J_minus_shuffled": { + "point": 0.018255183473229408, + "ci_lo": -0.013131593354046345, + "ci_hi": 0.05649133026599884, + "excludes_zero": false + }, + "beats_free_baseline": false, + "clears_derangement_control": false, + "hurts_vs_free_baseline": true + }, + "160m|persist": { + "file": "results/t22_bootstrap_ci_persist_160m.json", + "J_minus_logit": { + "point": -0.01768648810684681, + "ci_lo": -0.06020132079720497, + "ci_hi": 0.012619917280972004, + "excludes_zero": false + }, + "J_minus_shuffled": { + "point": 0.021994220092892647, + "ci_lo": 0.0002949030895251781, + "ci_hi": 0.05621275305747986, + "excludes_zero": true + }, + "beats_free_baseline": false, + "clears_derangement_control": true, + "hurts_vs_free_baseline": false + } + }, + "THE_PATTERN": "no arm both beats the free baseline AND clears the derangement control: at 410M `min` beats the logit lens (+0.0557 [+0.0065,+0.1154]) and FAILS the derangement control (+0.0188 [-0.0239,+0.0563]), while `persist` clears the derangement control (+0.0417 [+0.0134,+0.0732]) and does NOT beat the logit lens (+0.0216 [-0.0022,+0.0598]). The aggregation that wins the comparison the paper cares about is the one that fails the control the paper relies on.", + "no_arm_does_both": true, + "agrees_with_slate": true, + "SLATE_CORRECTION_CONFIRMED": "the slate's own \u00a72 note is right and the original F-5 framing is wrong at 160M: NEITHER aggregation beats the logit lens there. Under `min` the interval excludes zero but is NEGATIVE (-0.0207 [-0.0418,-0.0024]) -- the lens measurably HURTS -- and under `persist` it includes zero.", + "160m_min_is_negative": true + }, + "R4c_all_four_aggregations": { + "what": "the corpus x eval-set variance decomposition under all four aggregations", + "finding": "F-5 second half \u2014 best1L and mean exist in every ladder cell and are never reported", + "by_scale": { + "410m": { + "persist": { + "frac_set_main_pct": 91.27829399393488, + "frac_corpus_main_pct": 1.9064501148312791, + "frac_interaction_pct": 6.780626428099203, + "frac_residual_seed_pct": 0.034629463134648465, + "interaction_over_corpus_main": 3.556676555735259 + }, + "min": { + "frac_set_main_pct": 96.06544098391524, + "frac_corpus_main_pct": 1.911884671540871, + "frac_interaction_pct": 1.9989092146925354, + "frac_residual_seed_pct": 0.023765129851367858, + "interaction_over_corpus_main": 1.0455176739722107 + }, + "mean": { + "frac_set_main_pct": 94.18015768908394, + "frac_corpus_main_pct": 1.6504107475753969, + "frac_interaction_pct": 4.145089404552656, + "frac_residual_seed_pct": 0.024342158787985454, + "interaction_over_corpus_main": 2.511550176610379 + }, + "best1L": { + "frac_set_main_pct": 92.93385922614715, + "frac_corpus_main_pct": 2.553416835551286, + "frac_interaction_pct": 4.4645987524086, + "frac_residual_seed_pct": 0.04812518589297392, + "interaction_over_corpus_main": 1.7484801894653001 + } + }, + "1b": { + "persist": { + "frac_set_main_pct": 88.57950497345804, + "frac_corpus_main_pct": 2.2117202251311614, + "frac_interaction_pct": 9.158325214755473, + "frac_residual_seed_pct": 0.05044958665530746, + "interaction_over_corpus_main": 4.1408154208167804 + }, + "min": { + "frac_set_main_pct": 94.14618752120758, + "frac_corpus_main_pct": 1.9252567551293127, + "frac_interaction_pct": 3.896419437299789, + "frac_residual_seed_pct": 0.03213628636333453, + "interaction_over_corpus_main": 2.0238440545235643 + }, + "mean": { + "frac_set_main_pct": 89.08620460630291, + "frac_corpus_main_pct": 2.634040417272565, + "frac_interaction_pct": 8.227329209779771, + "frac_residual_seed_pct": 0.05242576664476654, + "interaction_over_corpus_main": 3.123463541344903 + }, + "best1L": { + "frac_set_main_pct": 95.00877329965921, + "frac_corpus_main_pct": 1.7035403113063254, + "frac_interaction_pct": 3.229682343981307, + "frac_residual_seed_pct": 0.05800404505314376, + "interaction_over_corpus_main": 1.8958649364186106 + } + } + }, + "missing_inputs": [], + "THE_REPORTED_SHARE_IS_THE_ARGMAX": { + "reported_aggregation": "persist", + "argmax_aggregation": "persist", + "reported_is_argmax": true, + "interaction_pct_by_agg": { + "persist": 6.781, + "min": 1.999, + "mean": 4.145, + "best1L": 4.465 + }, + "range_pp": 4.782, + "what_survives": "the ORDERING survives all four -- the interaction exceeds the corpus main effect under every aggregation (ratio 1.05-3.56) -- but the MAGNITUDE does not: 6.78% under `persist` against 2.00% under `min`, a 3.4x range, and the reported value is the largest of the four.", + "ordering_holds_under_all_four": true + } + }, + "R4h_numeric_corrections": { + "M-7_effective_rank_minimum": { + "claim_as_written": "effective rank never falls below 726", + "observed_minimum": 725.887939453125, + "at_checkpoint": "step143000", + "n_checkpoints": 19, + "n_below_726": 1, + "slate_required": 725.888, + "agrees_with_slate": true, + "verdict": "the claim is FALSE; one checkpoint of 19 falls below, at 725.888" + }, + "M-20_random_beats_top": { + "claim_as_written": "random better on one pair", + "observed_n_pairs": 2, + "of": 5, + "which_pairs": [ + "Github->USPTO_Backgrounds", + "Github->StackExchange" + ], + "all_are_github_pairs": true, + "selectivity_top": { + "Github->USPTO_Backgrounds": 0.6156383234735877, + "Github->StackExchange": 0.037886105164915744, + "Wikipedia_en->USPTO_Backgrounds": 0.9498469042702207, + "Wikipedia_en->StackExchange": 0.8368777257857798, + "Pile-CC->StackExchange": 0.7558589460641069 + }, + "selectivity_random": { + "Github->USPTO_Backgrounds": 0.8683138881706304, + "Github->StackExchange": 0.35789987398865764, + "Wikipedia_en->USPTO_Backgrounds": 0.9271887199762486, + "Wikipedia_en->StackExchange": 0.5328194939641212, + "Pile-CC->StackExchange": 0.6701019020353106 + }, + "slate_required": "2 of 5, both Github pairs", + "agrees_with_slate": true, + "verdict": "the claim UNDERSTATES: random beats top-k on 2 of 5 pairs, and both are Github" + }, + "M-21_max_abs_rho": { + "claim_as_written": "|rho| <= 0.28", + "observed_max_abs_spearman": 0.2818181818181818, + "slate_required": 0.2818, + "agrees_with_slate": true, + "verdict": "the claim is FALSE as written by 0.0018: the maximum is 0.28182 (StackExchange), which exceeds 0.28. Write <= 0.29, or print the value" + }, + "M-22_two_sets_exactly_zero": { + "claim_as_written": "two eval sets give exactly zero", + "observed": { + "order-ops": -1.2883919615498084e-05, + "poetry": -2.892319614710725e-06 + }, + "slate_required": { + "order-ops": -1.29e-05, + "poetry": -2.89e-06 + }, + "agrees_with_slate": true, + "verdict": "neither is exactly zero; both are small and NEGATIVE. 'Exactly zero' asserts a structural fact the numbers do not carry" + }, + "M-22b_fit_prefix_overlap": { + "claim_as_written": "the repo previously asserted ~25%", + "observed_mean_over_corpora": 0.17042513863216266, + "per_corpus": { + "Github": 0.1700554528650647, + "Wikipedia_en": 0.1700554528650647, + "StackExchange": 0.1700554528650647, + "Pile-CC": 0.17067159581022798, + "USPTO_Backgrounds": 0.17128773875539124 + }, + "slate_required": 0.1704, + "agrees_with_slate": true, + "verdict": "the measurement is 0.17043, so a ~17% guess was VINDICATED to 0.04 pp rather than corrected. The superseded ~25% appears in two live places, not one: paper/CONTEXT.md and specs/RESULTS_TAXONOMY.md -- and RESULTS_TAXONOMY.md:66 already states the correction in place, which the reviewer's note missed", + "note_on_the_reviewers_claim": "checked directly: specs/RESULTS_TAXONOMY.md:66 reads 'measured at 0.1704, not the ~25% previously asserted', i.e. it is a corrected restatement, not an uncorrected one" + }, + "M-16_E52_diagonal_excess_D": { + "claim_as_written": "D is mislabelled", + "verdict": "PARTLY REFUTED, as the slate says. specs/archive/prereg/PREREG_E52_FACTORIAL.md:69 defines D verbatim as implemented, so it is not an implementation-vs-prereg break. It is a redundant estimand -- D = 8/7 x the diagonal-residual mean identically -- plus a misleading English gloss, and it inflates the reported figure by 14.3%. The C4 permutation p is computed on D itself, so the 8/7 factor cancels and the INFERENCE is untouched. Fix the gloss, not the number", + "action": "gloss only; the stored D and its p-value stand" + } + }, + "R4i_1b_corrected_band": { + "what": "the 1B replication, re-derived on the declared band [6,13] instead of the used [5,13]", + "finding": "R4i, new 2026-08-19 \u2014 the 1B arm ran one layer below the paper's own band rule", + "bands_declared_vs_used": { + "410m": { + "n_layers": 24, + "L38_L92": [ + 9, + 22 + ], + "target_layer_index": 22, + "declared_by_the_papers_own_rule": [ + 9, + 21 + ], + "used_in_the_ladder": [ + 9, + 21 + ], + "matches": true + }, + "1b": { + "n_layers": 16, + "L38_L92": [ + 6, + 15 + ], + "target_layer_index": 14, + "declared_by_the_papers_own_rule": [ + 6, + 13 + ], + "used_in_the_ladder": [ + 5, + 13 + ], + "matches": false + } + }, + "C1_reproduces_e53_on_the_stored_band": { + "required": "the stored-band re-derivation must reproduce e53's 1B spread_over_seed_sd exactly", + "e53_stored": 35.51615385197748, + "recomputed": 35.51615385197748, + "fires": true + }, + "on_the_used_band_5_13": { + "spread": 0.03149575506159592, + "pooled_seed_sd": 0.0008868008397773706, + "spread_over_seed_sd": 35.51615385197748, + "n_pairs_separating": 9, + "n_pairs": 10 + }, + "on_the_declared_band_6_13": { + "spread": 0.028745208793718897, + "pooled_seed_sd": 0.0009264209495856735, + "spread_over_seed_sd": 31.028237008861595, + "n_pairs_separating": 9, + "n_pairs": 10 + }, + "interaction_share": { + "band_6_13": { + "frac_interaction": 9.191520642250026, + "frac_corpus_main": 2.2424534939728105, + "frac_set_main": 88.50952415142129, + "interaction_over_corpus_main": 4.098867899358752, + "units": "percent, converted from t51's fractions" + }, + "band_5_13_stored": { + "frac_interaction": 9.158325214755475, + "frac_corpus_main": 2.2117202251311614 + }, + "delta_pp": 0.033195427494550955 + }, + "VERDICT": "the 1B claim SURVIVES the band correction. Pair separation is 9 of 10 on the declared band and 9 of 10 on the used band -- unchanged, and the same pair (Github vs Wikipedia_en) is the one that does not separate in both. The multiplier moves 35.52x -> 31.03x and the interaction share moves 9.158% -> 9.192%. Report the corrected band's numbers; the deviation is immaterial but must not stay undisclosed", + "convention_note": "pooled_seed_sd is the ARITHMETIC MEAN of per-corpus seed SDs, which is e53's convention, verified by reproducing its stored 35.52x exactly. An RMS pooling gives 29.7x / 27.4x -- the multiplier is convention-dependent and the convention must travel with it" + }, + "R4j_leave_two_out_reversal": { + "what": "the drop-both-extremes leave-two-out split, both aggregations, with magnitudes", + "finding": "R4j, new \u2014 the paper says '27/28, the exception being...' without saying by how much", + "drop_both_extremes_github_and_uspto": { + "persist": { + "fit_pct": 78.67789669144574, + "read_pct": 16.686938451159083, + "interaction_pct": 4.63516485739523, + "fit_span": 0.010451911164758111, + "read_span": 0.0049434692887761825, + "span_ratio": 2.114286658660645, + "fit_dominates_read": true + }, + "min": { + "fit_pct": 42.0114078749595, + "read_pct": 47.61343085472334, + "interaction_pct": 10.375161270317184, + "fit_span": 0.01565969237298878, + "read_span": 0.014799208839044536, + "span_ratio": 1.0581438875079623, + "fit_dominates_read": false + } + }, + "under_min_the_read_axis_WINS_by_pp": 5.602022979763838, + "slate_required": { + "fit_pct": 42.011, + "read_pct": 47.613, + "residual": 10.375, + "fit_dominates_read": false + }, + "agrees_with_slate": true, + "VERDICT": "under `min`, dropping both extremes REVERSES the headline: read 47.613% beats fit 42.011% by 5.602 percentage points. Under `persist` the ordering holds (78.678% vs 16.687%). The abstract's '28/28 leave-two-out splits' carries no aggregation qualifier and is `persist`-only" + }, + "R4k_item_counts": { + "what": "the item and pair counts, per set, released vs scorable vs admitted", + "finding": "R4k, new \u2014 '541 items' is the SIX-set scorable count; only five sets are scored", + "per_set": { + "multihop": { + "released": 93, + "scorable": 93, + "pairs": 103, + "admitted": true + }, + "multilingual": { + "released": 107, + "scorable": 107, + "pairs": 394, + "admitted": true + }, + "order-ops": { + "released": 55, + "scorable": 55, + "pairs": 110, + "admitted": true + }, + "poetry": { + "released": 98, + "scorable": 98, + "pairs": 98, + "admitted": true + }, + "typo": { + "released": 96, + "scorable": 96, + "pairs": 96, + "admitted": true + }, + "association": { + "released": 102, + "scorable": 92, + "pairs": 92, + "admitted": false + } + }, + "all_six": { + "released": 551, + "scorable": 541, + "pairs": 893 + }, + "admitted_five": { + "released": 449, + "scorable": 449, + "pairs": 801 + }, + "slate_required": { + "admitted_released": 449, + "admitted_pairs": 801, + "all_pairs": 893 + }, + "agrees_with_slate": true, + "VERDICT": "t52_factorial.py builds `items` by looping EVAL_SETS, not ADMITTED, so n_items = 541 is 551 released minus the 10 association items with no scorable intermediate. The scored quantity is the admitted five, which hold 449 released items and 801 of the 893 pairs. Calling 541 'the readout task' overstates coverage by one whole eval set" + }, + "R4l_registration_evidence": { + "what": "the registration evidence behind each of the twenty candidate predictors", + "finding": "C-1 \u2014 6 of 45 registration claims are UNVERIFIABLE against git", + "n_candidates": 20, + "evidence_by_source_file": { + "e31_local_bakeoff_410m.json": { + "class": "criterion-only", + "evidence": "the file's `prereg` field is null; specs/experiments/E31_predictor_bakeoff.md records a pre-registered CRITERION (|r|>=0.8 leave-one-corpus-out) applied uniformly, but no registration of the individual candidates" + }, + "e38_jgeometry.json": { + "class": "independent-file", + "evidence": "specs/archive/prereg/PREREG_E38_JGEOMETRY.md \u2014 an independent timestamped document" + }, + "e34a_dact_vs_floor_410m.json": { + "class": "inherited-rule", + "evidence": "PLAN.tex section 6 (E34), quoted verbatim" + }, + "e34b_jeval_dact_410m.json": { + "class": "inherited-rule", + "evidence": "PLAN.tex section 6 (E34), rule unchanged" + }, + "e44_dread_410m.json": { + "class": "in-file-docstring", + "evidence": "`prereg`: 'in-file docstring, fixed before running'; no provenance block" + }, + "e45_disagreement_geometry.json": { + "class": "in-file-docstring", + "evidence": "same; no provenance block" + }, + "e50_concept_energy.json": { + "class": "in-file-docstring", + "evidence": "same; no provenance block" + } + }, + "candidates_by_evidence_class": { + "criterion-only": [ + "condition_number", + "dispersion_L0", + "dispersion_band_mean", + "dispersion_slope", + "eval_vocab_overlap", + "mean_prompt_tokens", + "model_cross_entropy", + "stable_rank", + "top8_spectral_share", + "type_token_ratio", + "unembed_alignment" + ], + "independent-file": [ + "layer_cos", + "rho_I" + ], + "inherited-rule": [ + "D_act(J^P, I)", + "D_act(J^P, J^eval)" + ], + "in-file-docstring": [ + "concept-conditional activation energy", + "disagreement geometry s_c (post_norm)", + "disagreement geometry s_c (pre_norm)", + "dread_vs_I", + "dread_vs_Jeval" + ] + }, + "counts_by_evidence_class": { + "criterion-only": 11, + "independent-file": 2, + "inherited-rule": 2, + "in-file-docstring": 5 + }, + "slate_required": "three rows of tab:predictors have no locatable registering text (E45, E47, E50 family)", + "agrees_with_slate": false, + "DISAGREEMENT_WITH_THE_SLATE": { + "slate_says": 3, + "recomputed": 5, + "why": "counting only rows whose registration is an in-file docstring with no independent timestamp and no provenance block gives FIVE candidates, not three: 2 from E44 (dread_vs_I, dread_vs_Jeval), 2 from E45, 1 from E50. E47 contributes no candidate row to the registry at all -- its rows are the ablation pairs, not predictors -- so the slate's 'E45, E47, E50 family' mislabels which files are implicated. Reported, not adopted", + "and_it_is_larger_still": "11 further candidates come from E31, whose results file carries `prereg: null`. Their CRITERION is pre-registered and uniformly applied, which is the defensible claim; the individual candidates are not individually registered" + }, + "VERDICT": "'Twenty pre-registered candidate predictors' is not supportable as written. What IS supportable, and is the stronger sentence: twenty candidates were tested against a criterion fixed in advance and applied uniformly -- the |r| >= 0.8 leave-one-corpus-out rule that killed the programme's own former headline F19 -- and none passed. The conclusion is unchanged either way" + }, + "provenance": { + "schema_version": 1, + "experiment": "R4", + "utc": "2026-08-20T05:36:53+00:00", + "script": { + "path": "tools/r4_recomputes.py", + "sha256": "13686b9447d8e29444316102a38b7087a899d98f1e2eb95744b924d02d1d2d91", + "n_lines": 421 + }, + "git": { + "commit": "a4fc687ed7027ca729606455391b9463310911e8", + "commit_short": "a4fc687ed702", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t52_factorial.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/30_repo_health.sh", + "results/d2_device_crossvalidation.json", + "results/r4_corrections.json", + "results/r4c/", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "src/provenance.py", + "tests/test_provenance_hash.py", + "tmp/", + "tools/make_paper_tables.py", + "tools/r4_recomputes.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "d29f669891e169cd963ff594a3800218616b3c18e199537f16ffeba337c81d24", + "describe": "a4fc687-dirty" + }, + "argv": [ + "tools/r4_recomputes.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48b_exposure_growth.json", + "sha256": "30ba5ecd395f1cdff653201c634a135a6d0d41e4b12c359a60c307cd85ad7d95", + "bytes": 32715, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_410m.json", + "sha256": "5439e0b15e315208d59f52f6dc990d36f1670ea4a1d101ebdc3b346a5930633a", + "bytes": 9262, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_persist_410m.json", + "sha256": "27d2b7110d23db75dc1d61fe9d3987119eab68a5d18add13b73dbf2c57e6de79", + "bytes": 9415, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_160m.json", + "sha256": "bee341836fa3252d2c7b3bbb955134363025531c90c658ae7d1fffb8152fe668", + "bytes": 8994, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_persist_160m.json", + "sha256": "bd3d3c61ed3b4ebe87796c06a9bb31345cff80c5573e9c8b4bfb683578daca5e", + "bytes": 9212, + "exists": true + }, + { + "path": "results/r4c/e51_interaction_persist.json", + "sha256": "26798b0e25bbf84350a2e3f70dfe5646360362da934def6a14d5ea6105735f82", + "bytes": 28946, + "exists": true + }, + { + "path": "results/r4c/e51_interaction_min.json", + "sha256": "44c9c41d36a5e495e024e49e758b77089606da680fee9b80c1f8bbfdf8baa0e1", + "bytes": 28935, + "exists": true + }, + { + "path": "results/r4c/e51_interaction_mean.json", + "sha256": "ab5541171c474655fe868136095fce1d7dadd7975ff5cc291293f3463e8b37f5", + "bytes": 28962, + "exists": true + }, + { + "path": "results/r4c/e51_interaction_best1L.json", + "sha256": "a85ac3280008dab42254a5da5b8c3c043f5ff441f48c1592f91ed2c60e752e23", + "bytes": 28914, + "exists": true + }, + { + "path": "results/e65_ckpt_geometry_410m.json", + "sha256": "209935237e5fc4aecdb511820f1bbbcc69cbafcfc76a178ecd1c6c4410f2a1c2", + "bytes": 7520, + "exists": true + }, + { + "path": "results/e47_ablation_pairs.json", + "sha256": "16aada1aaac0d41f41771462256600ae57ed0f05b1492d7214472671c7abe34c", + "bytes": 11628, + "exists": true + }, + { + "path": "results/e53_ladder_summary.json", + "sha256": "511c9d9b65199907f1785c2a3f5a147bce9c73352f990bd918a2201ce9cc9bf7", + "bytes": 33794, + "exists": true + }, + { + "path": "results/e52_factorial_410m.json", + "sha256": "6e81f3cae37d0c1906f8912af4bf8e75db0a0f259b2869f209701b74b6682590", + "bytes": 41190, + "exists": true + }, + { + "path": "results/e58_algebra_audit.json", + "sha256": "56931b57eaf55dec1f85f7c8bca9bc69c10c9a3b22d541e9f5ff0a2b62c16fef", + "bytes": 9382, + "exists": true + }, + { + "path": "results/e62_band_adjudication.json", + "sha256": "39820894aca707c687b10995830e39a7df50bfd0e11bf830596c0591fa9072e3", + "bytes": 8434, + "exists": true + }, + { + "path": "results/e54_aggregation_audit.json", + "sha256": "886885b41660dd01808aa626d5f87dddf6d5402f2c9b2f8241ac4470df82276b", + "bytes": 69079, + "exists": true + }, + { + "path": "results/e56_predictor_registry.json", + "sha256": "a28bdfac4ef4fe3fc83f5e28e04b6cfe400f4a06b5bc05a4f49d9dfc29d1ee18", + "bytes": 18048, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "9250ac0181df4d119a80dee7cf7fd5825357e061a28ef77e3111f03451520fe0" + } +} \ No newline at end of file diff --git a/results/r4b_e36_flatness.json b/results/r4b_e36_flatness.json new file mode 100644 index 0000000000000000000000000000000000000000..2eb0ad240d7071b70dc9b0ff6f40e70b741c6ab4 --- /dev/null +++ b/results/r4b_e36_flatness.json @@ -0,0 +1,588 @@ +{ + "experiment": "R4b \u2014 E36's shift-axis slopes at the corrected readout", + "prereg": "specs/experiments/R4b_e36_rstrip.md", + "prereg_sha256": "9d6b5847fa32234d38b8486761dce0552ae4749eb75ec2f2ddaa58abfdbdfe31", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "REJECT STANDS: the fitted operator is flatter on >= 4 of 5 rungs. Section 3.1 is unchanged. | REJECT OVERTURNED: flatter on <= 1 of 5. STOP and alert. | UNCLEAR: 2 or 3 of 5. Report and stop. Do not re-cut.", + "WORDING_AMBIGUITY_IN_THE_RULE": "the rule says '5 rungs'; E36 has eleven-plus rungs and five FIT CORPORA, and the published 5-of-5 is over corpora (CONTEXT.md \u00a73.1 lists five slopes, one per fitting corpus). Adjudicated over the five fit corpora, which is the only reading in which 5 is the denominator. Flagged, not resolved.", + "PRIMARY_slope_secondary": { + "persist": { + "n_flatter_stripped": 1, + "n_flatter_unstripped": 5 + }, + "min": { + "n_flatter_stripped": 3, + "n_flatter_unstripped": 1 + } + }, + "E36_PRIMARY_reject_s3_crossings": { + "crosses_per_corpus": { + "unstripped": { + "Pile-CC": false, + "StackExchange": false, + "Wikipedia_en": false, + "Github": false, + "USPTO_Backgrounds": false + }, + "stripped": { + "Pile-CC": false, + "StackExchange": false, + "Wikipedia_en": false, + "Github": false, + "USPTO_Backgrounds": false + } + }, + "n_crossing": { + "unstripped": 0, + "stripped": 0 + }, + "published": "0 of 5 -> REJECT S3" + }, + "by_aggregation": { + "persist": { + "unstripped": { + "slopes": { + "logit": { + "slope_linear": -0.003720899629554985, + "slope_log10": -0.0005296062512720639, + "level_mean": 0.031253348919562995 + }, + "J|Pile-CC": { + "slope_linear": -0.0011535435993366304, + "slope_log10": 0.00013903741903277346, + "level_mean": 0.048237061704452515 + }, + "J|StackExchange": { + "slope_linear": 0.0002240976423130857, + "slope_log10": 0.00043591080099479097, + "level_mean": 0.04689619833829276 + }, + "J|Wikipedia_en": { + "slope_linear": -0.0009354827291769033, + "slope_log10": 0.0002869798762885821, + "level_mean": 0.03964683171079466 + }, + "J|Github": { + "slope_linear": -0.0014550596366756292, + "slope_log10": -1.0939804428585397e-06, + "level_mean": 0.026799873580845693 + }, + "J|USPTO_Backgrounds": { + "slope_linear": -0.000885040945743085, + "slope_log10": 0.0002350879878021014, + "level_mean": 0.04893984228838235 + } + }, + "flatness": { + "per_corpus": { + "Pile-CC": { + "slope_linear": -0.0011535435993366304, + "logit_slope_linear": -0.003720899629554985, + "abs_slope": 0.0011535435993366304, + "abs_logit_slope": 0.003720899629554985, + "flatter_than_logit": true + }, + "StackExchange": { + "slope_linear": 0.0002240976423130857, + "logit_slope_linear": -0.003720899629554985, + "abs_slope": 0.0002240976423130857, + "abs_logit_slope": 0.003720899629554985, + "flatter_than_logit": true + }, + "Wikipedia_en": { + "slope_linear": -0.0009354827291769033, + "logit_slope_linear": -0.003720899629554985, + "abs_slope": 0.0009354827291769033, + "abs_logit_slope": 0.003720899629554985, + "flatter_than_logit": true + }, + "Github": { + "slope_linear": -0.0014550596366756292, + "logit_slope_linear": -0.003720899629554985, + "abs_slope": 0.0014550596366756292, + "abs_logit_slope": 0.003720899629554985, + "flatter_than_logit": true + }, + "USPTO_Backgrounds": { + "slope_linear": -0.000885040945743085, + "logit_slope_linear": -0.003720899629554985, + "abs_slope": 0.000885040945743085, + "abs_logit_slope": 0.003720899629554985, + "flatter_than_logit": true + } + }, + "n_flatter": 5, + "n_corpora": 5 + } + }, + "stripped": { + "slopes": { + "logit": { + "slope_linear": -0.0037284686515085515, + "slope_log10": -5.887753023934386e-05, + "level_mean": 0.07387016320417664 + }, + "J|Pile-CC": { + "slope_linear": -0.0048877667147762035, + "slope_log10": -0.0006226113130866229, + "level_mean": 0.10972794660465848 + }, + "J|StackExchange": { + "slope_linear": -0.0036664399547795337, + "slope_log10": -3.4500043359785385e-05, + "level_mean": 0.11132412637922574 + }, + "J|Wikipedia_en": { + "slope_linear": -0.003751919371114022, + "slope_log10": -0.00021278116380330815, + "level_mean": 0.1012678702841654 + }, + "J|Github": { + "slope_linear": -0.006302421723152167, + "slope_log10": -0.000664630277617062, + "level_mean": 0.09380335117845486 + }, + "J|USPTO_Backgrounds": { + "slope_linear": -0.004609631566017049, + "slope_log10": -0.00033176743326874674, + "level_mean": 0.11131394055714323 + } + }, + "flatness": { + "per_corpus": { + "Pile-CC": { + "slope_linear": -0.0048877667147762035, + "logit_slope_linear": -0.0037284686515085515, + "abs_slope": 0.0048877667147762035, + "abs_logit_slope": 0.0037284686515085515, + "flatter_than_logit": false + }, + "StackExchange": { + "slope_linear": -0.0036664399547795337, + "logit_slope_linear": -0.0037284686515085515, + "abs_slope": 0.0036664399547795337, + "abs_logit_slope": 0.0037284686515085515, + "flatter_than_logit": true + }, + "Wikipedia_en": { + "slope_linear": -0.003751919371114022, + "logit_slope_linear": -0.0037284686515085515, + "abs_slope": 0.003751919371114022, + "abs_logit_slope": 0.0037284686515085515, + "flatter_than_logit": false + }, + "Github": { + "slope_linear": -0.006302421723152167, + "logit_slope_linear": -0.0037284686515085515, + "abs_slope": 0.006302421723152167, + "abs_logit_slope": 0.0037284686515085515, + "flatter_than_logit": false + }, + "USPTO_Backgrounds": { + "slope_linear": -0.004609631566017049, + "logit_slope_linear": -0.0037284686515085515, + "abs_slope": 0.004609631566017049, + "abs_logit_slope": 0.0037284686515085515, + "flatter_than_logit": false + } + }, + "n_flatter": 1, + "n_corpora": 5 + } + }, + "rungs": [ + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "USPTO_Backgrounds", + "Github", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ] + }, + "min": { + "unstripped": { + "slopes": { + "logit": { + "slope_linear": -0.0010699323813758122, + "slope_log10": 0.0009282505416221876, + "level_mean": 0.12512224580753933 + }, + "J|Pile-CC": { + "slope_linear": -0.004014659692830054, + "slope_log10": -0.00033377197965353794, + "level_mean": 0.14627732089861775 + }, + "J|StackExchange": { + "slope_linear": -0.0035589979944138225, + "slope_log10": -0.00028825210444100556, + "level_mean": 0.148409635133364 + }, + "J|Wikipedia_en": { + "slope_linear": -0.004603344091202535, + "slope_log10": -0.000310141050106287, + "level_mean": 0.13278038299670725 + }, + "J|Github": { + "slope_linear": -0.0035714446311215964, + "slope_log10": -0.00029611400443362804, + "level_mean": 0.11407741325145418 + }, + "J|USPTO_Backgrounds": { + "slope_linear": -4.247891056195874e-05, + "slope_log10": 0.0010588073618437773, + "level_mean": 0.160057055566347 + } + }, + "flatness": { + "per_corpus": { + "Pile-CC": { + "slope_linear": -0.004014659692830054, + "logit_slope_linear": -0.0010699323813758122, + "abs_slope": 0.004014659692830054, + "abs_logit_slope": 0.0010699323813758122, + "flatter_than_logit": false + }, + "StackExchange": { + "slope_linear": -0.0035589979944138225, + "logit_slope_linear": -0.0010699323813758122, + "abs_slope": 0.0035589979944138225, + "abs_logit_slope": 0.0010699323813758122, + "flatter_than_logit": false + }, + "Wikipedia_en": { + "slope_linear": -0.004603344091202535, + "logit_slope_linear": -0.0010699323813758122, + "abs_slope": 0.004603344091202535, + "abs_logit_slope": 0.0010699323813758122, + "flatter_than_logit": false + }, + "Github": { + "slope_linear": -0.0035714446311215964, + "logit_slope_linear": -0.0010699323813758122, + "abs_slope": 0.0035714446311215964, + "abs_logit_slope": 0.0010699323813758122, + "flatter_than_logit": false + }, + "USPTO_Backgrounds": { + "slope_linear": -4.247891056195874e-05, + "logit_slope_linear": -0.0010699323813758122, + "abs_slope": 4.247891056195874e-05, + "abs_logit_slope": 0.0010699323813758122, + "flatter_than_logit": true + } + }, + "n_flatter": 1, + "n_corpora": 5 + } + }, + "stripped": { + "slopes": { + "logit": { + "slope_linear": 0.0021193657987416048, + "slope_log10": 0.00200160581744663, + "level_mean": 0.1990344447394212 + }, + "J|Pile-CC": { + "slope_linear": -0.001888259914374931, + "slope_log10": 0.0010884349751549287, + "level_mean": 0.2590676621449265 + }, + "J|StackExchange": { + "slope_linear": 0.00014062084151355905, + "slope_log10": 0.0018836771421971345, + "level_mean": 0.2655780210075053 + }, + "J|Wikipedia_en": { + "slope_linear": -0.0017812876146258553, + "slope_log10": 0.0009754492560197017, + "level_mean": 0.23547124836706754 + }, + "J|Github": { + "slope_linear": 0.00374649675334448, + "slope_log10": 0.0028687089581596976, + "level_mean": 0.2293679065437931 + }, + "J|USPTO_Backgrounds": { + "slope_linear": 0.0028406448262535064, + "slope_log10": 0.002393799693174576, + "level_mean": 0.2592142216635473 + } + }, + "flatness": { + "per_corpus": { + "Pile-CC": { + "slope_linear": -0.001888259914374931, + "logit_slope_linear": 0.0021193657987416048, + "abs_slope": 0.001888259914374931, + "abs_logit_slope": 0.0021193657987416048, + "flatter_than_logit": true + }, + "StackExchange": { + "slope_linear": 0.00014062084151355905, + "logit_slope_linear": 0.0021193657987416048, + "abs_slope": 0.00014062084151355905, + "abs_logit_slope": 0.0021193657987416048, + "flatter_than_logit": true + }, + "Wikipedia_en": { + "slope_linear": -0.0017812876146258553, + "logit_slope_linear": 0.0021193657987416048, + "abs_slope": 0.0017812876146258553, + "abs_logit_slope": 0.0021193657987416048, + "flatter_than_logit": true + }, + "Github": { + "slope_linear": 0.00374649675334448, + "logit_slope_linear": 0.0021193657987416048, + "abs_slope": 0.00374649675334448, + "abs_logit_slope": 0.0021193657987416048, + "flatter_than_logit": false + }, + "USPTO_Backgrounds": { + "slope_linear": 0.0028406448262535064, + "logit_slope_linear": 0.0021193657987416048, + "abs_slope": 0.0028406448262535064, + "abs_logit_slope": 0.0021193657987416048, + "flatter_than_logit": false + } + }, + "n_flatter": 3, + "n_corpora": 5 + } + }, + "rungs": [ + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "USPTO_Backgrounds", + "Github", + "CONTROL_PubMed_2023", + "OOD_Wikipedia_2023", + "TRAP_FineWeb", + "OOD_News_2024", + "OOD_arXiv_2023", + "OOD_CommonPile" + ] + } + }, + "controls": { + "C1_unstripped_is_the_stored_run": { + "required": "the unstripped arm read here is results/e36_qladder_410m.json as stored", + "file": "results/e36_qladder_410m.json", + "sha256": "d0cc7b6381ac515f6d53401705e246d8c20c67884cda25e4b1d35d5312f45153", + "rstrip_flag_in_file": "absent (pre-R4b run)", + "fires": true + }, + "C1b_unstripped_slopes_reproduce_e53": { + "required": "max_abs_diff = 0.0", + "checked": true, + "max_abs_diff": 0.0, + "worst_at": null, + "fires": true + }, + "C2_own_derangement_loses": { + "per_rung_per_corpus": { + "Q0": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Pile-CC": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "StackExchange": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Wikipedia_en": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "USPTO_Backgrounds": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "Github": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "CONTROL_PubMed_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_Wikipedia_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "TRAP_FineWeb": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_News_2024": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_arXiv_2023": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "OOD_CommonPile": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + }, + "SHUFFLED_Pile-CC": { + "Github": true, + "Wikipedia_en": true, + "StackExchange": true, + "Pile-CC": true, + "USPTO_Backgrounds": true + } + }, + "n_cells": 65, + "n_cells_where_derangement_wins": 0, + "required": "0 cells where the derangement is at or above its own operator", + "fires": true, + "like_for_like_evidence_already_on_disk": "e33_logit_baseline_410m_v2: Github 0.027926 vs its own jshuf 0.018142; e54 C2: 0 of 15 and 0 of 120" + }, + "C3_report_C2_shuf_below_everything": { + "unstripped": false, + "stripped": false, + "note": "a RECORDER, not a gate -- its power is nil because it compares Pile-CC's derangement to every corpus's operator (R4d/F-7). Reported whichever way it fires, as the rule requires." + } + }, + "stripped_verdict_string_from_the_run": "REJECT S3 (the publishable null) \u2014 no fitting corpus produces a J-lens curve that goes strictly below the logit lens at any measured rung. Shift in the READ distribution degrades both lenses together; the averaged derivative is no more shift-fragile than not fitting at all.", + "unstripped_verdict_string_from_the_run": "REJECT S3 (the publishable null) \u2014 no fitting corpus produces a J-lens curve that goes strictly below the logit lens at any measured rung. Shift in the READ distribution degrades both lenses together; the averaged derivative is no more shift-fragile than not fitting at all.", + "VERDICT": "REJECT OVERTURNED \u2014 flatter on only 1 of 5. Flagged for adjudication. S3's first half is no longer rejected on the read axis, which reopens the subthesis and changes the paper's framing.", + "provenance": { + "schema_version": 1, + "experiment": "R4b", + "utc": "2026-08-20T13:19:10+00:00", + "script": { + "path": "tools/r4b_e36_flatness.py", + "sha256": "5d696c6e0f814d67bed3a8170b554794cd901d09d2952ba2c48f1ae513f09191", + "n_lines": 194 + }, + "git": { + "commit": "0085188b142f807a95aed2394063dae7009cf1b7", + "commit_short": "0085188b142f", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/e36_qladder_410m_c1arm.json", + "results/e36_qladder_410m_rstrip.json", + "results/e52_factorial_410m_rstrip_w1.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "aee92baa8be53162e66b7c00eec04f9286b40c2afaf4dbb2dbe8c1782597cc28", + "describe": "0085188-dirty" + }, + "argv": [ + "tools/r4b_e36_flatness.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e36_qladder_410m.json", + "sha256": "d0cc7b6381ac515f6d53401705e246d8c20c67884cda25e4b1d35d5312f45153", + "bytes": 80656, + "exists": true + }, + { + "path": "results/e36_qladder_410m_rstrip.json", + "sha256": "6a40a858516cc56473bc4b9145a7d232a5a64bee8cfb9e2a405e78b4bd8362d2", + "bytes": 111957, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "40491de3b427b0d91698599c34b6a0fec4332e7cbfbcc63a3987aadabeee1b28", + "payload_sha256_before_release_redaction": "ad6aee81ac7bc8ab098176836f2ce4c399ffcb97fee360b9567760264d57dad8", + "release_redaction": "internal operator-protocol wording removed for the anonymous release; no number, threshold or verdict changed" + } +} \ No newline at end of file diff --git a/results/r4c/e51_interaction_best1L.json b/results/r4c/e51_interaction_best1L.json new file mode 100644 index 0000000000000000000000000000000000000000..71d37f3d6d280967f1c3da6ea4bc1b755f89e8d2 --- /dev/null +++ b/results/r4c/e51_interaction_best1L.json @@ -0,0 +1,755 @@ +{ + "experiment": "E51 \u2014 the corpus x concept-set interaction decomposition, STORED", + "why": "the programme's headline interaction numbers (6.8% / 1.9% / 4614x at 410M, 9.2% / 2.2% at 1B) appear in six documents including the paper abstract and in NO results JSON; no committed script computed them. This stores the decomposition and reports whether it reproduces them.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json" + ], + "aggregation": "best1L", + "n_min": 75, + "decomposition": "balanced two-factor (eval set, corpus) with seed as the replication unit, after averaging over the NESTED N axis (SPINE A1.1)", + "scales": { + "410m": { + "admitted5": { + "grand_mean": 0.0952893483853045, + "SS_total": 0.2951703263999987, + "frac_set_main": 0.9293385922614714, + "frac_corpus_main": 0.025534168355512858, + "frac_interaction": 0.044645987524085995, + "frac_residual_seed": 0.00048125185892973916, + "SS_set": 0.2743131756139338, + "SS_corpus": 0.007536928807849248, + "SS_interaction": 0.013178170709934732, + "SS_residual_seed": 0.00014205126828089723, + "interaction_over_noise_floor": 92.77052482119166, + "candidate_noise_floors": { + "seed_x_set": 395.530142033832, + "seed_x_corpus": 605.3364578998096, + "seed_main": 7192.797219195682, + "three_way_residual": 154.7980032825852, + "full_two_factor_residual": 92.77052482119166, + "mean_squares_F": 289.90789006622396 + }, + "SS_seed_main": 1.8321343294324583e-06, + "SS_seed_x_set": 3.33177406965043e-05, + "SS_seed_x_corpus": 2.176999342754914e-05, + "SS_three_way_residual": 8.513139982741158e-05, + "interaction_over_corpus_main": 1.7484801894653001, + "per_corpus_main": { + "Github": -0.016120271068080436, + "Pile-CC": 0.008444525683699952, + "StackExchange": 0.00935537426268579, + "USPTO_Backgrounds": 0.005577161601510172, + "Wikipedia_en": -0.007256790479815492 + }, + "per_set_main": { + "multihop": -0.013032194670607941, + "multilingual": 0.02185845809945737, + "order-ops": -0.021994494814960713, + "poetry": -0.08579947535921444, + "typo": 0.0989677067453257 + }, + "interaction_cells": { + "Github|multihop": -0.000619413115602277, + "Github|multilingual": -0.021958852379539887, + "Github|order-ops": 0.020469202489880045, + "Github|poetry": 0.018326913159008362, + "Github|typo": -0.016217850153746258, + "Pile-CC|multihop": -8.679154714059945e-05, + "Pile-CC|multilingual": 0.020125176429675345, + "Pile-CC|order-ops": -0.01160950948218853, + "Pile-CC|poetry": -0.010753748666365553, + "Pile-CC|typo": 0.002324873266019392, + "StackExchange|multihop": -0.0009315942108591713, + "StackExchange|multilingual": 0.015248950219059185, + "StackExchange|order-ops": -0.028012008795575236, + "StackExchange|poetry": -0.007218147178468429, + "StackExchange|typo": 0.020912799965843665, + "USPTO_Backgrounds|multihop": -0.0049247757771748585, + "USPTO_Backgrounds|multilingual": -0.010499252542256407, + "USPTO_Backgrounds|order-ops": 0.018867283532901402, + "USPTO_Backgrounds|poetry": -0.006320678988627898, + "USPTO_Backgrounds|typo": 0.002877423775157789, + "Wikipedia_en|multihop": 0.00656257465077692, + "Wikipedia_en|multilingual": -0.0029160217269382227, + "Wikipedia_en|order-ops": 0.0002850322549823453, + "Wikipedia_en|poetry": 0.005965661674453546, + "Wikipedia_en|typo": -0.009897246853274574 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.044645987524085995, + "null2_within_cell_mean": 0.019126723757405643, + "null2_max": 0.04105372702395432, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.044645987524085995, + "fires": null + }, + "C4_F_test": { + "F": 289.90789006622396, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.08023361985688537, + "SS_total": 0.39722306543955493, + "frac_set_main": 0.947369226613684, + "frac_corpus_main": 0.016717690866194745, + "frac_interaction": 0.03554681239956558, + "frac_residual_seed": 0.0003662701205557535, + "SS_set": 0.37631690829858794, + "SS_corpus": 0.006640652412940725, + "SS_interaction": 0.014120013787960222, + "SS_residual_seed": 0.00014549094006607174, + "interaction_over_noise_floor": 97.05081142198893, + "candidate_noise_floors": { + "seed_x_set": 422.8959989977643, + "seed_x_corpus": 689.9431251190646, + "seed_main": 5645.117290660995, + "three_way_residual": 158.41096743350974, + "full_two_factor_residual": 97.05081142198893, + "mean_squares_F": 291.1524342659668 + }, + "SS_seed_main": 2.5012790808296717e-06, + "SS_seed_x_set": 3.338885641250739e-05, + "SS_seed_x_corpus": 2.0465475013644797e-05, + "SS_three_way_residual": 8.913532955909036e-05, + "interaction_over_corpus_main": 2.126299181153401, + "per_corpus_main": { + "Github": -0.013797247372300148, + "Pile-CC": 0.007006843039990288, + "StackExchange": 0.00800551236758483, + "USPTO_Backgrounds": 0.005077460387643756, + "Wikipedia_en": -0.006292568422918712 + }, + "per_set_main": { + "multihop": 0.002023533857811191, + "multilingual": 0.0369141866278765, + "order-ops": -0.006938766286541581, + "poetry": -0.0707437468307953, + "typo": 0.11402343527374484, + "association": -0.07527864264209563 + }, + "interaction_cells": { + "Github|multihop": -0.002942436811382565, + "Github|multilingual": -0.024281876075320175, + "Github|order-ops": 0.018146178794099757, + "Github|poetry": 0.016003889463228074, + "Github|typo": -0.018540873849526546, + "Github|association": 0.011615118478901412, + "Pile-CC|multihop": 0.0013508910965690651, + "Pile-CC|multilingual": 0.02156285907338501, + "Pile-CC|order-ops": -0.010171826838478865, + "Pile-CC|poetry": -0.009316066022655889, + "Pile-CC|typo": 0.0037625559097290567, + "Pile-CC|association": -0.007188413218548406, + "StackExchange|multihop": 0.0004182676842417882, + "StackExchange|multilingual": 0.016598812114160144, + "StackExchange|order-ops": -0.026662146900474276, + "StackExchange|poetry": -0.005868285283367469, + "StackExchange|typo": 0.022262661860944624, + "StackExchange|association": -0.006749309475504811, + "USPTO_Backgrounds|multihop": -0.004425074563308443, + "USPTO_Backgrounds|multilingual": -0.009999551328389991, + "USPTO_Backgrounds|order-ops": 0.019366984746767818, + "USPTO_Backgrounds|poetry": -0.005820977774761482, + "USPTO_Backgrounds|typo": 0.003377124989024205, + "USPTO_Backgrounds|association": -0.002498506069332121, + "Wikipedia_en|multihop": 0.00559835259388014, + "Wikipedia_en|multilingual": -0.0038802437838350023, + "Wikipedia_en|order-ops": -0.0006791898019144343, + "Wikipedia_en|poetry": 0.005001439617556766, + "Wikipedia_en|typo": -0.010861468910171354, + "Wikipedia_en|association": 0.004821110284483884 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 1.0, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.03554681239956558, + "null2_within_cell_mean": 0.014564699868540195, + "null2_max": 0.02790431544972496, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.03554681239956558, + "fires": null + }, + "C4_F_test": { + "F": 291.1524342659668, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + }, + "1b": { + "admitted5": { + "grand_mean": 0.10966614778419691, + "SS_total": 0.7605917598758213, + "frac_set_main": 0.9500877329965921, + "frac_corpus_main": 0.017035403113063254, + "frac_interaction": 0.03229682343981307, + "frac_residual_seed": 0.0005800404505314376, + "SS_set": 0.7226289008763074, + "SS_corpus": 0.012956987233958824, + "SS_interaction": 0.0245646977784861, + "SS_residual_seed": 0.0004411739870688703, + "interaction_over_noise_floor": 55.68029507290823, + "candidate_noise_floors": { + "seed_x_set": 307.5256113281718, + "seed_x_corpus": 251.3293672967704, + "seed_main": 2863.619335706942, + "three_way_residual": 96.3403926377547, + "full_two_factor_residual": 55.68029507290823, + "mean_squares_F": 174.0009221028382 + }, + "SS_seed_main": 8.578199438795802e-06, + "SS_seed_x_set": 7.987854303384251e-05, + "SS_seed_x_corpus": 9.773906663871888e-05, + "SS_three_way_residual": 0.00025497817795751303, + "interaction_over_corpus_main": 1.8958649364186109, + "per_corpus_main": { + "Github": -0.007589008462216176, + "Pile-CC": -0.0010784467841897794, + "StackExchange": 0.01918634524790659, + "USPTO_Backgrounds": 0.008553659006332356, + "Wikipedia_en": -0.019072549007833006 + }, + "per_set_main": { + "multihop": 0.03897000478447547, + "multilingual": -0.012101488549794467, + "order-ops": -0.09410337736164885, + "poetry": -0.09941347534590889, + "typo": 0.1666483364728767 + }, + "interaction_cells": { + "Github|multihop": 0.010825253338331273, + "Github|multilingual": 0.004990506389932273, + "Github|order-ops": 0.007610653651700841, + "Github|poetry": 0.005839737254594052, + "Github|typo": -0.029266150634558397, + "Pile-CC|multihop": -0.0014643195147315546, + "Pile-CC|multilingual": 0.002317266508138613, + "Pile-CC|order-ops": -0.0007397352292069226, + "Pile-CC|poetry": 0.005281556517861441, + "Pile-CC|typo": -0.005394768282061466, + "StackExchange|multihop": -0.00889970465031055, + "StackExchange|multilingual": -0.011572131405451477, + "StackExchange|order-ops": -0.015160371535679423, + "StackExchange|poetry": -0.021421525094746846, + "StackExchange|typo": 0.05705373268618831, + "USPTO_Backgrounds|multihop": -0.0034681293355034865, + "USPTO_Backgrounds|multilingual": -0.004112035073339945, + "USPTO_Backgrounds|order-ops": -0.006800412184425758, + "USPTO_Backgrounds|poetry": -0.005686797731156865, + "USPTO_Backgrounds|typo": 0.020067374324426082, + "Wikipedia_en|multihop": 0.003006900162214332, + "Wikipedia_en|multilingual": 0.008376393580720537, + "Wikipedia_en|order-ops": 0.015089865297611277, + "Wikipedia_en|poetry": 0.015987029053448218, + "Wikipedia_en|typo": -0.04246018809399431 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.03229682343981307, + "null2_within_cell_mean": 0.013098282533570349, + "null2_max": 0.03720710495762924, + "n_perm": 200, + "n_null_ge_observed": 1, + "p_value": 0.009950248756218905, + "observed": 0.03229682343981307, + "fires": null + }, + "C4_F_test": { + "F": 174.0009221028382, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.09491243854697261, + "SS_total": 0.8596303241674714, + "frac_set_main": 0.9545745992458663, + "frac_corpus_main": 0.012429662709787993, + "frac_interaction": 0.0324770315100478, + "frac_residual_seed": 0.0005187065342977965, + "SS_set": 0.8205812721917581, + "SS_corpus": 0.010684914984507384, + "SS_interaction": 0.02791824112497957, + "SS_residual_seed": 0.00044589586622620044, + "interaction_over_noise_floor": 62.611571982631496, + "candidate_noise_floors": { + "seed_x_set": 342.42184740411517, + "seed_x_corpus": 342.58620888129474, + "seed_main": 3951.24783027885, + "three_way_residual": 101.2242450319376, + "full_two_factor_residual": 62.611571982631496, + "mean_squares_F": 187.8347159478945 + }, + "SS_seed_main": 7.065677052965141e-06, + "SS_seed_x_set": 8.153171690599335e-05, + "SS_seed_x_corpus": 8.149260069792585e-05, + "SS_three_way_residual": 0.0002758058715693162, + "interaction_over_corpus_main": 2.612865068693545, + "per_corpus_main": { + "Github": -0.008748259294467667, + "Pile-CC": 0.0005376297429884047, + "StackExchange": 0.015246730101191347, + "USPTO_Backgrounds": 0.007874253428807224, + "Wikipedia_en": -0.01491035397851928 + }, + "per_set_main": { + "multihop": 0.05372371402169977, + "multilingual": 0.002652220687429832, + "order-ops": -0.07934966812442455, + "poetry": -0.08465976610868459, + "typo": 0.181402045710101, + "association": -0.07376854618612144 + }, + "interaction_cells": { + "Github|multihop": 0.011984504170582763, + "Github|multilingual": 0.006149757222183763, + "Github|order-ops": 0.008769904483952332, + "Github|poetry": 0.006998988086845542, + "Github|typo": -0.028106899802306906, + "Github|association": -0.005796254161257466, + "Pile-CC|multihop": -0.0030803960419097387, + "Pile-CC|multilingual": 0.0007011899809604288, + "Pile-CC|order-ops": -0.0023558117563851066, + "Pile-CC|poetry": 0.003665479990683257, + "Pile-CC|typo": -0.00701084480923965, + "Pile-CC|association": 0.008080382635890851, + "StackExchange|multihop": -0.004960089503595305, + "StackExchange|multilingual": -0.007632516258736233, + "StackExchange|order-ops": -0.011220756388964179, + "StackExchange|poetry": -0.0174819099480316, + "StackExchange|typo": 0.06099334783290354, + "StackExchange|association": -0.01969807573357628, + "USPTO_Backgrounds|multihop": -0.0027887237579783536, + "USPTO_Backgrounds|multilingual": -0.0034326294958148124, + "USPTO_Backgrounds|order-ops": -0.006121006606900625, + "USPTO_Backgrounds|poetry": -0.005007392153631732, + "USPTO_Backgrounds|typo": 0.02074677990195123, + "USPTO_Backgrounds|association": -0.00339702788762572, + "Wikipedia_en|multihop": -0.0011552948670993934, + "Wikipedia_en|multilingual": 0.004214198551406811, + "Wikipedia_en|order-ops": 0.010927670268297551, + "Wikipedia_en|poetry": 0.011824834024134492, + "Wikipedia_en|typo": -0.046622383123308034, + "Wikipedia_en|association": 0.0208109751465686 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999998, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.0324770315100478, + "null2_within_cell_mean": 0.012799147632187868, + "null2_max": 0.03199854571937211, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.0324770315100478, + "fires": null + }, + "C4_F_test": { + "F": 187.8347159478945, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + } + }, + "ladder1b_dir": "results/ladder1b", + "reproduces_the_prose": { + "410m": { + "claimed": { + "interaction": 6.8, + "corpus_main": 1.9, + "set_main": 91.3, + "noise_floor": 4614 + }, + "recomputed": { + "interaction": 4.4645987524086, + "corpus_main": 2.553416835551286, + "set_main": 92.93385922614715, + "noise_floor": 92.77052482119166 + }, + "interaction_matches_to_0p1pp": false, + "corpus_main_matches_to_0p1pp": false + }, + "1b": { + "claimed": { + "interaction": 9.2, + "corpus_main": 2.2, + "set_main": 88.6 + }, + "recomputed": { + "interaction": 3.229682343981307, + "corpus_main": 1.7035403113063254, + "set_main": 95.00877329965921, + "noise_floor": 55.68029507290823 + }, + "interaction_matches_to_0p1pp": false, + "corpus_main_matches_to_0p1pp": false + } + }, + "noise_floor_4614_audit": { + "claimed": 4614, + "claimed_where": "PLAN.tex:52 and :270, HANDOFF.md:63 \u2014 described as 'the seed x set noise floor'. In no results file.", + "candidates_computed": { + "seed_x_set": 395.530142033832, + "seed_x_corpus": 605.3364578998096, + "seed_main": 7192.797219195682, + "three_way_residual": 154.7980032825852, + "full_two_factor_residual": 92.77052482119166, + "mean_squares_F": 289.90789006622396 + }, + "any_candidate_within_2pct": {}, + "reproduces": false, + "reading": "The six VARIANCE FRACTIONS reproduce exactly at both scales, so the decomposition specified in this file is the one that was used. The 4614x figure does NOT follow from it under any of six natural denominators. Cite a named, stored ratio from candidate_noise_floors instead, or drop the multiplier \u2014 do not restate 4614x." + }, + "VERDICT": "DOES NOT REPRODUCE under this decomposition. The numbers in the paper and specs are NOT recoverable from the stored ladders by the decomposition specified in this file's docstring. Either a different decomposition was used and was never written down, or the numbers are wrong. STOP: this is the paper's headline claim and it must not be cited until the discrepancy is resolved by the operator.", + "provenance": { + "schema_version": 1, + "experiment": "E51", + "utc": "2026-08-20T05:28:45+00:00", + "script": { + "path": "experiments/t51_interaction_variance.py", + "sha256": "849f3e2ab42163fc1e5c553f4b1100cec316d6cff89a8b20f0ef26fa008b542a", + "n_lines": 323 + }, + "git": { + "commit": "a4fc687ed7027ca729606455391b9463310911e8", + "commit_short": "a4fc687ed702", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t52_factorial.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/30_repo_health.sh", + "results/d2_device_crossvalidation.json", + "results/r4c/", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "src/provenance.py", + "tests/test_provenance_hash.py", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "89e80dd41d08b1238f8e123d87f968acdb67e1b2db7309e2e6fcbf92aa3f57d4", + "describe": "a4fc687-dirty" + }, + "argv": [ + "experiments/t51_interaction_variance.py", + "--agg", + "best1L", + "--out", + "results/r4c/e51_interaction_best1L.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "51de6746017fd69ad8e9f2350d2e180a3a56a8eed50169fee955fee15ec8bef0" + } +} \ No newline at end of file diff --git a/results/r4c/e51_interaction_mean.json b/results/r4c/e51_interaction_mean.json new file mode 100644 index 0000000000000000000000000000000000000000..bb1bdcec82c68bccd9864842d0e84700dd02b22a --- /dev/null +++ b/results/r4c/e51_interaction_mean.json @@ -0,0 +1,755 @@ +{ + "experiment": "E51 \u2014 the corpus x concept-set interaction decomposition, STORED", + "why": "the programme's headline interaction numbers (6.8% / 1.9% / 4614x at 410M, 9.2% / 2.2% at 1B) appear in six documents including the paper abstract and in NO results JSON; no committed script computed them. This stores the decomposition and reports whether it reproduces them.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json" + ], + "aggregation": "mean", + "n_min": 75, + "decomposition": "balanced two-factor (eval set, corpus) with seed as the replication unit, after averaging over the NESTED N axis (SPINE A1.1)", + "scales": { + "410m": { + "admitted5": { + "grand_mean": 0.04601106279231949, + "SS_total": 0.1897257352090189, + "frac_set_main": 0.9418015768908394, + "frac_corpus_main": 0.016504107475753968, + "frac_interaction": 0.04145089404552656, + "frac_residual_seed": 0.00024342158787985454, + "SS_set": 0.17868399659662784, + "SS_corpus": 0.003131253924806087, + "SS_interaction": 0.00786430134785867, + "SS_residual_seed": 4.6183339726252204e-05, + "interaction_over_noise_floor": 170.28437948562498, + "candidate_noise_floors": { + "seed_x_set": 1425.799358201714, + "seed_x_corpus": 1103.4667770744825, + "seed_main": 4429.654476407887, + "three_way_residual": 247.5748648756233, + "full_two_factor_residual": 170.28437948562498, + "mean_squares_F": 532.1386858925781 + }, + "SS_seed_main": 1.7753757973095952e-06, + "SS_seed_x_set": 5.515713906462618e-06, + "SS_seed_x_corpus": 7.126903601673039e-06, + "SS_three_way_residual": 3.176534642080705e-05, + "interaction_over_corpus_main": 2.511550176610379, + "per_corpus_main": { + "Github": -0.011125136374680537, + "Pile-CC": 0.004636211610227245, + "StackExchange": 0.0030412937451674213, + "USPTO_Backgrounds": 0.006637805575603978, + "Wikipedia_en": -0.0031901745563181136 + }, + "per_set_main": { + "multihop": -0.01839724240842619, + "multilingual": -0.007031875358323746, + "order-ops": -0.027768511666120148, + "poetry": -0.04173227817580496, + "typo": 0.09492990760867502 + }, + "interaction_cells": { + "Github|multihop": 0.0013589329678202408, + "Github|multilingual": 0.004643485227168405, + "Github|order-ops": 0.015624129515586851, + "Github|poetry": 0.011433115916479573, + "Github|typo": -0.03305966362705504, + "Pile-CC|multihop": -0.0008280199372608377, + "Pile-CC|multilingual": -0.00024093187616053974, + "Pile-CC|order-ops": -0.008567148441609301, + "Pile-CC|poetry": -0.005305982352460915, + "Pile-CC|typo": 0.014942082607491622, + "StackExchange|multihop": -0.0005763162459663587, + "StackExchange|multilingual": 0.002431621973454541, + "StackExchange|order-ops": -0.010991756056705722, + "StackExchange|poetry": -0.0033847523125142692, + "StackExchange|typo": 0.012521202641731823, + "USPTO_Backgrounds|multihop": -0.003738449814415956, + "USPTO_Backgrounds|multilingual": -0.005565673703346603, + "USPTO_Backgrounds|order-ops": -0.0003234321202164811, + "USPTO_Backgrounds|poetry": -0.006539259309614166, + "USPTO_Backgrounds|typo": 0.016166814947593228, + "Wikipedia_en|multihop": 0.0037838530298229187, + "Wikipedia_en|multilingual": -0.0012685016211157751, + "Wikipedia_en|order-ops": 0.004258207102944667, + "Wikipedia_en|poetry": 0.0037968780581097775, + "Wikipedia_en|typo": -0.010570436569761567 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999998, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.04145089404552656, + "null2_within_cell_mean": 0.01570868860357089, + "null2_max": 0.04085835880996856, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.04145089404552656, + "fires": null + }, + "C4_F_test": { + "F": 532.1386858925781, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03855929267938615, + "SS_total": 0.21471950308050708, + "frac_set_main": 0.9485491009983655, + "frac_corpus_main": 0.012553001193726798, + "frac_interaction": 0.03868244955823542, + "frac_residual_seed": 0.00021544824967214605, + "SS_set": 0.20367199161383076, + "SS_corpus": 0.00269537417848603, + "SS_interaction": 0.00830587634708109, + "SS_residual_seed": 4.6260941109168224e-05, + "interaction_over_noise_floor": 179.5440418620232, + "candidate_noise_floors": { + "seed_x_set": 1439.6570737770473, + "seed_x_corpus": 1321.3432273347746, + "seed_main": 5433.228715257993, + "three_way_residual": 254.18154334457762, + "full_two_factor_residual": 179.5440418620232, + "mean_squares_F": 538.6321255860696 + }, + "SS_seed_main": 1.5287183334940267e-06, + "SS_seed_x_set": 5.7693436154833775e-06, + "SS_seed_x_corpus": 6.285934021726149e-06, + "SS_three_way_residual": 3.267694513846486e-05, + "interaction_over_corpus_main": 3.0815299832494625, + "per_corpus_main": { + "Github": -0.009400472959960684, + "Pile-CC": 0.0039040938509135384, + "StackExchange": 0.0024977877378193, + "USPTO_Backgrounds": 0.005706280981475811, + "Wikipedia_en": -0.0027076896102479486 + }, + "per_set_main": { + "multihop": -0.01094547229549285, + "multilingual": 0.0004198947546095941, + "order-ops": -0.020316741553186808, + "poetry": -0.03428050806287162, + "typo": 0.10238167772160836, + "association": -0.03725885056466668 + }, + "interaction_cells": { + "Github|multihop": -0.0003657304468996092, + "Github|multilingual": 0.002918821812448555, + "Github|order-ops": 0.013899466100866995, + "Github|poetry": 0.009708452501759723, + "Github|typo": -0.03478432704177489, + "Github|association": 0.00862331707359923, + "Pile-CC|multihop": -9.590217794713113e-05, + "Pile-CC|multilingual": 0.0004911858831531668, + "Pile-CC|order-ops": -0.007835030682295595, + "Pile-CC|poetry": -0.004573864593147209, + "Pile-CC|typo": 0.01567420036680533, + "Pile-CC|association": -0.0036605887965685605, + "StackExchange|multihop": -3.281023861823745e-05, + "StackExchange|multilingual": 0.0029751279808026623, + "StackExchange|order-ops": -0.0104482500493576, + "StackExchange|poetry": -0.002841246305166148, + "StackExchange|typo": 0.013064708649079944, + "StackExchange|association": -0.002717530036740627, + "USPTO_Backgrounds|multihop": -0.00280692522028779, + "USPTO_Backgrounds|multilingual": -0.004634149109218437, + "USPTO_Backgrounds|order-ops": 0.0006080924739116852, + "USPTO_Backgrounds|poetry": -0.005607734715486, + "USPTO_Backgrounds|typo": 0.017098339541721394, + "USPTO_Backgrounds|association": -0.004657622970640873, + "Wikipedia_en|multihop": 0.0033013680837527537, + "Wikipedia_en|multilingual": -0.00175098656718594, + "Wikipedia_en|order-ops": 0.0037757221568745017, + "Wikipedia_en|poetry": 0.0033143931120396125, + "Wikipedia_en|typo": -0.011052921515831732, + "Wikipedia_en|association": 0.002412424730350811 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.03868244955823542, + "null2_within_cell_mean": 0.014335816830223799, + "null2_max": 0.040952251466010725, + "n_perm": 200, + "n_null_ge_observed": 1, + "p_value": 0.009950248756218905, + "observed": 0.03868244955823542, + "fires": null + }, + "C4_F_test": { + "F": 538.6321255860696, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + }, + "1b": { + "admitted5": { + "grand_mean": 0.053087449426807105, + "SS_total": 0.422970010370717, + "frac_set_main": 0.8908620460630291, + "frac_corpus_main": 0.02634040417272565, + "frac_interaction": 0.08227329209779771, + "frac_residual_seed": 0.0005242576664476654, + "SS_set": 0.3768079288621576, + "SS_corpus": 0.011141201026106646, + "SS_interaction": 0.03479913521183853, + "SS_residual_seed": 0.00022174527061429692, + "interaction_over_noise_floor": 156.932930814872, + "candidate_noise_floors": { + "seed_x_set": 516.813748276614, + "seed_x_corpus": 873.4244354547992, + "seed_main": 2727.89636659163, + "three_way_residual": 341.7968777303598, + "full_two_factor_residual": 156.932930814872, + "mean_squares_F": 490.415408796475 + }, + "SS_seed_main": 1.2756765850059878e-05, + "SS_seed_x_set": 6.733399668232704e-05, + "SS_seed_x_corpus": 3.9842181875434176e-05, + "SS_three_way_residual": 0.00010181232620647642, + "interaction_over_corpus_main": 3.123463541344903, + "per_corpus_main": { + "Github": -0.013213875120771781, + "Pile-CC": 0.002508724327698088, + "StackExchange": 0.016107360578053405, + "USPTO_Backgrounds": 0.009294884886226755, + "Wikipedia_en": -0.01469709467120646 + }, + "per_set_main": { + "multihop": -0.009875860533543994, + "multilingual": -0.028356028312063287, + "order-ops": -0.0507545970761705, + "poetry": -0.04953761677457286, + "typo": 0.13852410269635065 + }, + "interaction_cells": { + "Github|multihop": 0.01195790359484298, + "Github|multilingual": 0.013154787644965658, + "Github|order-ops": 0.013430325348661946, + "Github|poetry": 0.01249850960548169, + "Github|typo": -0.05104152619395227, + "Pile-CC|multihop": -0.004329754438617134, + "Pile-CC|multilingual": -0.0025543827861214882, + "Pile-CC|order-ops": -0.003266300051306778, + "Pile-CC|poetry": -0.0026571964097785797, + "Pile-CC|typo": 0.012807633685823959, + "StackExchange|multihop": -0.010056094690669495, + "StackExchange|multilingual": -0.013443728840821192, + "StackExchange|order-ops": -0.015109284577580782, + "StackExchange|poetry": -0.01644479715265333, + "StackExchange|typo": 0.05505390526172484, + "USPTO_Backgrounds|multihop": -0.006659656063864192, + "USPTO_Backgrounds|multilingual": -0.008971247159975715, + "USPTO_Backgrounds|order-ops": -0.008922109536527265, + "USPTO_Backgrounds|poetry": -0.007661691932894649, + "USPTO_Backgrounds|typo": 0.032214704693261785, + "Wikipedia_en|multihop": 0.009087601598307848, + "Wikipedia_en|multilingual": 0.011814571141952744, + "Wikipedia_en|order-ops": 0.013867368816752872, + "Wikipedia_en|poetry": 0.014265175889844876, + "Wikipedia_en|typo": -0.049034717446858354 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.08227329209779771, + "null2_within_cell_mean": 0.02861348390718644, + "null2_max": 0.07967134400164823, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.08227329209779771, + "fires": null + }, + "C4_F_test": { + "F": 490.415408796475, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.04534566801596647, + "SS_total": 0.4500390958860032, + "frac_set_main": 0.8972081832205824, + "frac_corpus_main": 0.021231869865458452, + "frac_interaction": 0.08106453533784189, + "frac_residual_seed": 0.0004954115761171526, + "SS_set": 0.40377875959811443, + "SS_corpus": 0.009555171518220199, + "SS_interaction": 0.03648221019186132, + "SS_residual_seed": 0.00022295457780722322, + "interaction_over_noise_floor": 163.6306845576659, + "candidate_noise_floors": { + "seed_x_set": 525.4971264207749, + "seed_x_corpus": 1058.2720747851324, + "seed_main": 3420.1565332771074, + "three_way_residual": 336.5822080516736, + "full_two_factor_residual": 163.6306845576659, + "mean_squares_F": 490.8920536729977 + }, + "SS_seed_main": 1.0666824701413589e-05, + "SS_seed_x_set": 6.942418589488036e-05, + "SS_seed_x_corpus": 3.44733750999416e-05, + "SS_three_way_residual": 0.00010839019211098766, + "interaction_over_corpus_main": 3.8180591653740095, + "per_corpus_main": { + "Github": -0.011791794175570386, + "Pile-CC": 0.0025228940764449523, + "StackExchange": 0.013335096315131523, + "USPTO_Backgrounds": 0.007950380447991415, + "Wikipedia_en": -0.012016576663997497 + }, + "per_set_main": { + "multihop": -0.0021340791227033584, + "multilingual": -0.02061424690122265, + "order-ops": -0.043012815665329864, + "poetry": -0.04179583536373222, + "typo": 0.14626588410719127, + "association": -0.03870890705420318 + }, + "interaction_cells": { + "Github|multihop": 0.010535822649641584, + "Github|multilingual": 0.01173270669976427, + "Github|order-ops": 0.012008244403460551, + "Github|poetry": 0.011076428660280295, + "Github|typo": -0.05246360713915364, + "Github|association": 0.0071104047260069775, + "Pile-CC|multihop": -0.004343924187363998, + "Pile-CC|multilingual": -0.0025685525348683527, + "Pile-CC|order-ops": -0.0032804698000536425, + "Pile-CC|poetry": -0.002671366158525444, + "Pile-CC|typo": 0.012793463937077108, + "Pile-CC|association": 7.084874373431554e-05, + "StackExchange|multihop": -0.007283830427747613, + "StackExchange|multilingual": -0.010671464577899317, + "StackExchange|order-ops": -0.0123370203146589, + "StackExchange|poetry": -0.01367253288973145, + "StackExchange|typo": 0.05782616952464674, + "StackExchange|association": -0.013861321314609432, + "USPTO_Backgrounds|multihop": -0.005315151625628851, + "USPTO_Backgrounds|multilingual": -0.007626742721740382, + "USPTO_Backgrounds|order-ops": -0.007577605098291917, + "USPTO_Backgrounds|poetry": -0.0063171874946593085, + "USPTO_Backgrounds|typo": 0.03355920913149715, + "USPTO_Backgrounds|association": -0.00672252219117668, + "Wikipedia_en|multihop": 0.006407083591098886, + "Wikipedia_en|multilingual": 0.009134053134743775, + "Wikipedia_en|order-ops": 0.01118685080954391, + "Wikipedia_en|poetry": 0.011584657882635914, + "Wikipedia_en|typo": -0.051715235454067296, + "Wikipedia_en|association": 0.013402590036044812 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999998, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.08106453533784189, + "null2_within_cell_mean": 0.028655556457478505, + "null2_max": 0.07696448765237092, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.08106453533784189, + "fires": null + }, + "C4_F_test": { + "F": 490.8920536729977, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + } + }, + "ladder1b_dir": "results/ladder1b", + "reproduces_the_prose": { + "410m": { + "claimed": { + "interaction": 6.8, + "corpus_main": 1.9, + "set_main": 91.3, + "noise_floor": 4614 + }, + "recomputed": { + "interaction": 4.145089404552656, + "corpus_main": 1.6504107475753969, + "set_main": 94.18015768908394, + "noise_floor": 170.28437948562498 + }, + "interaction_matches_to_0p1pp": false, + "corpus_main_matches_to_0p1pp": false + }, + "1b": { + "claimed": { + "interaction": 9.2, + "corpus_main": 2.2, + "set_main": 88.6 + }, + "recomputed": { + "interaction": 8.227329209779771, + "corpus_main": 2.634040417272565, + "set_main": 89.08620460630291, + "noise_floor": 156.932930814872 + }, + "interaction_matches_to_0p1pp": false, + "corpus_main_matches_to_0p1pp": false + } + }, + "noise_floor_4614_audit": { + "claimed": 4614, + "claimed_where": "PLAN.tex:52 and :270, HANDOFF.md:63 \u2014 described as 'the seed x set noise floor'. In no results file.", + "candidates_computed": { + "seed_x_set": 1425.799358201714, + "seed_x_corpus": 1103.4667770744825, + "seed_main": 4429.654476407887, + "three_way_residual": 247.5748648756233, + "full_two_factor_residual": 170.28437948562498, + "mean_squares_F": 532.1386858925781 + }, + "any_candidate_within_2pct": {}, + "reproduces": false, + "reading": "The six VARIANCE FRACTIONS reproduce exactly at both scales, so the decomposition specified in this file is the one that was used. The 4614x figure does NOT follow from it under any of six natural denominators. Cite a named, stored ratio from candidate_noise_floors instead, or drop the multiplier \u2014 do not restate 4614x." + }, + "VERDICT": "DOES NOT REPRODUCE under this decomposition. The numbers in the paper and specs are NOT recoverable from the stored ladders by the decomposition specified in this file's docstring. Either a different decomposition was used and was never written down, or the numbers are wrong. STOP: this is the paper's headline claim and it must not be cited until the discrepancy is resolved by the operator.", + "provenance": { + "schema_version": 1, + "experiment": "E51", + "utc": "2026-08-20T05:28:39+00:00", + "script": { + "path": "experiments/t51_interaction_variance.py", + "sha256": "849f3e2ab42163fc1e5c553f4b1100cec316d6cff89a8b20f0ef26fa008b542a", + "n_lines": 323 + }, + "git": { + "commit": "a4fc687ed7027ca729606455391b9463310911e8", + "commit_short": "a4fc687ed702", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t52_factorial.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/30_repo_health.sh", + "results/d2_device_crossvalidation.json", + "results/r4c/", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "src/provenance.py", + "tests/test_provenance_hash.py", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "89e80dd41d08b1238f8e123d87f968acdb67e1b2db7309e2e6fcbf92aa3f57d4", + "describe": "a4fc687-dirty" + }, + "argv": [ + "experiments/t51_interaction_variance.py", + "--agg", + "mean", + "--out", + "results/r4c/e51_interaction_mean.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "b8519129761464c7b891b0fa3b775d31006f8f8c0234df1d672790cefa033d4b" + } +} \ No newline at end of file diff --git a/results/r4c/e51_interaction_min.json b/results/r4c/e51_interaction_min.json new file mode 100644 index 0000000000000000000000000000000000000000..c3fb4d49eb6c8926e38b0f262ef79cf03d44ac51 --- /dev/null +++ b/results/r4c/e51_interaction_min.json @@ -0,0 +1,755 @@ +{ + "experiment": "E51 \u2014 the corpus x concept-set interaction decomposition, STORED", + "why": "the programme's headline interaction numbers (6.8% / 1.9% / 4614x at 410M, 9.2% / 2.2% at 1B) appear in six documents including the paper abstract and in NO results JSON; no committed script computed them. This stores the decomposition and reports whether it reproduces them.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json" + ], + "aggregation": "min", + "n_min": 75, + "decomposition": "balanced two-factor (eval set, corpus) with seed as the replication unit, after averaging over the NESTED N axis (SPINE A1.1)", + "scales": { + "410m": { + "admitted5": { + "grand_mean": 0.13779479668770642, + "SS_total": 0.8563522821953781, + "frac_set_main": 0.9606544098391524, + "frac_corpus_main": 0.01911884671540871, + "frac_interaction": 0.019989092146925354, + "frac_residual_seed": 0.00023765129851367858, + "SS_set": 0.8226585962668123, + "SS_corpus": 0.016372468017683856, + "SS_interaction": 0.017117704679033237, + "SS_residual_seed": 0.0002035132318488837, + "interaction_over_noise_floor": 84.11101589573194, + "candidate_noise_floors": { + "seed_x_set": 469.67847751637584, + "seed_x_corpus": 449.5093717335184, + "seed_main": 1862.5014836471103, + "three_way_residual": 142.89035471753772, + "full_two_factor_residual": 84.11101589573194, + "mean_squares_F": 262.8469246741623 + }, + "SS_seed_main": 9.190706600412321e-06, + "SS_seed_x_set": 3.644558032454534e-05, + "SS_seed_x_corpus": 3.8080862725996926e-05, + "SS_three_way_residual": 0.000119796082197928, + "interaction_over_corpus_main": 1.0455176739722107, + "per_corpus_main": { + "Github": -0.02491172251662553, + "Pile-CC": 0.007144224316667336, + "StackExchange": 0.00759470230110626, + "USPTO_Backgrounds": 0.017545162269596404, + "Wikipedia_en": -0.007372366370744482 + }, + "per_set_main": { + "multihop": -0.035357705384298596, + "multilingual": 0.010359489503566466, + "order-ops": -0.0525312769147747, + "poetry": -0.11570446393898184, + "typo": 0.19323395673448865 + }, + "interaction_cells": { + "Github|multihop": -0.0007140622977716282, + "Github|multilingual": -0.007043800930184946, + "Github|order-ops": 0.01871437518859534, + "Github|poetry": 0.019862900353469742, + "Github|typo": -0.030819412314108466, + "Pile-CC|multihop": 0.00029693832063451553, + "Pile-CC|multilingual": 0.01679345156196352, + "Pile-CC|order-ops": -0.017564069802477905, + "Pile-CC|poetry": -0.0076386171158822624, + "Pile-CC|typo": 0.008112297035762162, + "StackExchange|multihop": -0.003136607356741178, + "StackExchange|multilingual": 0.015389516496508548, + "StackExchange|order-ops": -0.023655994755644644, + "StackExchange|poetry": -0.012643524762031844, + "StackExchange|typo": 0.024046610377909117, + "USPTO_Backgrounds|multihop": -0.001525378258925597, + "USPTO_Backgrounds|multilingual": -0.018665904331704536, + "USPTO_Backgrounds|order-ops": 0.021578042653114055, + "USPTO_Backgrounds|poetry": -0.014530214980450179, + "USPTO_Backgrounds|typo": 0.013143454917966257, + "Wikipedia_en|multihop": 0.005079109592803943, + "Wikipedia_en|multilingual": -0.006473262796582502, + "Wikipedia_en|order-ops": 0.0009276467164132385, + "Wikipedia_en|poetry": 0.01494945650489457, + "Wikipedia_en|typo": -0.014482950017529167 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.019989092146925354, + "null2_within_cell_mean": 0.01063909554222114, + "null2_max": 0.02194005756607954, + "n_perm": 200, + "n_null_ge_observed": 2, + "p_value": 0.014925373134328358, + "observed": 0.019989092146925354, + "fires": null + }, + "C4_F_test": { + "F": 262.8469246741623, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.11657869872105847, + "SS_total": 1.0593780266758217, + "frac_set_main": 0.9677507332320924, + "frac_corpus_main": 0.014601149361442571, + "frac_interaction": 0.01745174927769506, + "frac_residual_seed": 0.00019636812877023966, + "SS_set": 1.0252138620854936, + "SS_corpus": 0.015468136797723965, + "SS_interaction": 0.01848799971184579, + "SS_residual_seed": 0.00020802808075864014, + "interaction_over_noise_floor": 88.87261587197005, + "candidate_noise_floors": { + "seed_x_set": 506.4390508613355, + "seed_x_corpus": 539.5889612170749, + "seed_main": 1733.3656909381066, + "three_way_residual": 146.04267087516274, + "full_two_factor_residual": 88.87261587197005, + "mean_squares_F": 266.61784761591014 + }, + "SS_seed_main": 1.0665954569482673e-05, + "SS_seed_x_set": 3.6505873076734475e-05, + "SS_seed_x_corpus": 3.42631170032556e-05, + "SS_three_way_residual": 0.00012659313610916722, + "interaction_over_corpus_main": 1.1952312003450976, + "per_corpus_main": { + "Github": -0.021646806332978455, + "Pile-CC": 0.005823709916369879, + "StackExchange": 0.006569505960094632, + "USPTO_Backgrounds": 0.0162644117125553, + "Wikipedia_en": -0.007010821256041341 + }, + "per_set_main": { + "multihop": -0.01414160741765065, + "multilingual": 0.03157558747021441, + "order-ops": -0.03131517894812676, + "poetry": -0.0944883659723339, + "typo": 0.2144500547011366, + "association": -0.1060804898332397 + }, + "interaction_cells": { + "Github|multihop": -0.0039789784814187185, + "Github|multilingual": -0.010308717113832022, + "Github|order-ops": 0.015449459004948235, + "Github|poetry": 0.016597984169822666, + "Github|typo": -0.03408432849775554, + "Github|association": 0.016324580918235382, + "Pile-CC|multihop": 0.0016174527209319861, + "Pile-CC|multilingual": 0.018113965962260975, + "Pile-CC|order-ops": -0.016243555402180462, + "Pile-CC|poetry": -0.006318102715584806, + "Pile-CC|typo": 0.009432811436059618, + "Pile-CC|association": -0.006602572001487256, + "StackExchange|multihop": -0.00211141101572955, + "StackExchange|multilingual": 0.016414712837520162, + "StackExchange|order-ops": -0.022630798414633016, + "StackExchange|poetry": -0.011618328421020216, + "StackExchange|typo": 0.02507180671892073, + "StackExchange|association": -0.005125981705058194, + "USPTO_Backgrounds|multihop": -0.00024462770188449223, + "USPTO_Backgrounds|multilingual": -0.01738515377466343, + "USPTO_Backgrounds|order-ops": 0.02285879321015513, + "USPTO_Backgrounds|poetry": -0.013249464423409074, + "USPTO_Backgrounds|typo": 0.014424205475007362, + "USPTO_Backgrounds|association": -0.006403752785205635, + "Wikipedia_en|multihop": 0.004717564478100789, + "Wikipedia_en|multilingual": -0.0068348079112856425, + "Wikipedia_en|order-ops": 0.0005661016017100978, + "Wikipedia_en|poetry": 0.014587911390191444, + "Wikipedia_en|typo": -0.014844495132232308, + "Wikipedia_en|association": 0.0018077255735156478 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.01745174927769506, + "null2_within_cell_mean": 0.00892228958337106, + "null2_max": 0.01919770492897262, + "n_perm": 200, + "n_null_ge_observed": 1, + "p_value": 0.009950248756218905, + "observed": 0.01745174927769506, + "fires": null + }, + "C4_F_test": { + "F": 266.61784761591014, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + }, + "1b": { + "admitted5": { + "grand_mean": 0.1480757641122632, + "SS_total": 1.4953053189811738, + "frac_set_main": 0.9414618752120758, + "frac_corpus_main": 0.019252567551293127, + "frac_interaction": 0.03896419437299789, + "frac_residual_seed": 0.0003213628636333453, + "SS_set": 1.407772949622607, + "SS_corpus": 0.028788466663492964, + "SS_interaction": 0.05826336709576006, + "SS_residual_seed": 0.0004805355993139628, + "interaction_over_noise_floor": 121.24672382012868, + "candidate_noise_floors": { + "seed_x_set": 318.41012782690046, + "seed_x_corpus": 540.3982094946863, + "seed_main": 4356.578213665738, + "three_way_residual": 330.3582684348384, + "full_two_factor_residual": 121.24672382012868, + "mean_squares_F": 378.89601193790213 + }, + "SS_seed_main": 1.33736534129008e-05, + "SS_seed_x_set": 0.00018298214159643246, + "SS_seed_x_corpus": 0.00010781561832012872, + "SS_three_way_residual": 0.00017636418598450257, + "interaction_over_corpus_main": 2.0238440545235643, + "per_corpus_main": { + "Github": -0.016227103566218704, + "Pile-CC": 0.0014382544833989597, + "StackExchange": 0.027996306129775606, + "USPTO_Backgrounds": 0.013180507967425953, + "Wikipedia_en": -0.02638796501438176 + }, + "per_set_main": { + "multihop": 0.023006067699531008, + "multilingual": -0.012536726958960992, + "order-ops": -0.1295692707771169, + "poetry": -0.12657430624145835, + "typo": 0.24567423627800533 + }, + "interaction_cells": { + "Github|multihop": 0.014585864881408367, + "Github|multilingual": 0.018227343931350698, + "Github|order-ops": 0.019257406706788716, + "Github|poetry": 0.01270426584540732, + "Github|typo": -0.06477488136495513, + "Pile-CC|multihop": -0.004119715291474563, + "Pile-CC|multilingual": -0.00010274600503701325, + "Pile-CC|order-ops": -0.006091934271334198, + "Pile-CC|poetry": -0.004839615188539065, + "Pile-CC|typo": 0.015154010756384728, + "StackExchange|multihop": -0.009873327310862295, + "StackExchange|multilingual": -0.01575313390942204, + "StackExchange|order-ops": -0.023775526725554053, + "StackExchange|poetry": -0.028603691671575837, + "StackExchange|typo": 0.07800567961741411, + "USPTO_Backgrounds|multihop": -0.005922071723977018, + "USPTO_Backgrounds|multilingual": -0.014171564374562545, + "USPTO_Backgrounds|order-ops": -0.01079955578826014, + "USPTO_Backgrounds|poetry": -0.0006683603501213553, + "USPTO_Backgrounds|typo": 0.031561552236921, + "Wikipedia_en|multihop": 0.005329249444905482, + "Wikipedia_en|multilingual": 0.011800100357670845, + "Wikipedia_en|order-ops": 0.02140961007835962, + "Wikipedia_en|poetry": 0.021407401364828826, + "Wikipedia_en|typo": -0.059946361245764884 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.03896419437299789, + "null2_within_cell_mean": 0.015364855631216263, + "null2_max": 0.04007583284759099, + "n_perm": 200, + "n_null_ge_observed": 2, + "p_value": 0.014925373134328358, + "observed": 0.03896419437299789, + "fires": null + }, + "C4_F_test": { + "F": 378.89601193790213, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.12882262603121086, + "SS_total": 1.6641059108385379, + "frac_set_main": 0.9462020632537814, + "frac_corpus_main": 0.01532602196740708, + "frac_interaction": 0.03817204128614127, + "frac_residual_seed": 0.0002998734926703771, + "SS_set": 1.5745804463082378, + "SS_corpus": 0.0255041237456034, + "SS_interaction": 0.06352231953304038, + "SS_residual_seed": 0.0004990212516565714, + "interaction_over_noise_floor": 127.2938162897253, + "candidate_noise_floors": { + "seed_x_set": 340.590688890299, + "seed_x_corpus": 619.2745270867264, + "seed_main": 5996.682977333629, + "three_way_residual": 318.6525837721436, + "full_two_factor_residual": 127.2938162897253, + "mean_squares_F": 381.88144886917587 + }, + "SS_seed_main": 1.0592909408942109e-05, + "SS_seed_x_set": 0.0001865063303404055, + "SS_seed_x_corpus": 0.000102575379342455, + "SS_three_way_residual": 0.00019934663256477087, + "interaction_over_corpus_main": 2.4906685744885024, + "per_corpus_main": { + "Github": -0.01696461529012712, + "Pile-CC": 0.0029324998088653964, + "StackExchange": 0.022691883881472863, + "USPTO_Backgrounds": 0.012523611700014453, + "Wikipedia_en": -0.02118338010022565 + }, + "per_set_main": { + "multihop": 0.04225920578058334, + "multilingual": 0.0067164111220913425, + "order-ops": -0.11031613269606458, + "poetry": -0.10732116816040603, + "typo": 0.26492737435905767, + "association": -0.09626569040526178 + }, + "interaction_cells": { + "Github|multihop": 0.015323376605316796, + "Github|multilingual": 0.018964855655259127, + "Github|order-ops": 0.019994918430697117, + "Github|poetry": 0.01344177756931575, + "Github|typo": -0.06403736964104671, + "Github|association": -0.0036875586195420085, + "Pile-CC|multihop": -0.0056139606169409995, + "Pile-CC|multilingual": -0.00159699133050345, + "Pile-CC|order-ops": -0.007586179596800649, + "Pile-CC|poetry": -0.006333860514005474, + "Pile-CC|typo": 0.013659765430918291, + "Pile-CC|association": 0.00747122662733235, + "StackExchange|multihop": -0.004568905062559553, + "StackExchange|multilingual": -0.010448711661119298, + "StackExchange|order-ops": -0.01847110447725131, + "StackExchange|poetry": -0.023299269423273095, + "StackExchange|typo": 0.08331010186571686, + "StackExchange|association": -0.02652211124151352, + "USPTO_Backgrounds|multihop": -0.0052651754565655184, + "USPTO_Backgrounds|multilingual": -0.013514668107151045, + "USPTO_Backgrounds|order-ops": -0.01014265952084864, + "USPTO_Backgrounds|poetry": -1.1464082709827794e-05, + "USPTO_Backgrounds|typo": 0.0322184485043325, + "USPTO_Backgrounds|association": -0.0032844813370574433, + "Wikipedia_en|multihop": 0.00012466453074938544, + "Wikipedia_en|multilingual": 0.006595515443514749, + "Wikipedia_en|order-ops": 0.016205025164203524, + "Wikipedia_en|poetry": 0.01620281645067273, + "Wikipedia_en|typo": -0.065150946159921, + "Wikipedia_en|association": 0.026022924570780648 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.03817204128614127, + "null2_within_cell_mean": 0.015031071721915602, + "null2_max": 0.03698554883536268, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.03817204128614127, + "fires": null + }, + "C4_F_test": { + "F": 381.88144886917587, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + } + }, + "ladder1b_dir": "results/ladder1b", + "reproduces_the_prose": { + "410m": { + "claimed": { + "interaction": 6.8, + "corpus_main": 1.9, + "set_main": 91.3, + "noise_floor": 4614 + }, + "recomputed": { + "interaction": 1.9989092146925354, + "corpus_main": 1.911884671540871, + "set_main": 96.06544098391524, + "noise_floor": 84.11101589573194 + }, + "interaction_matches_to_0p1pp": false, + "corpus_main_matches_to_0p1pp": true + }, + "1b": { + "claimed": { + "interaction": 9.2, + "corpus_main": 2.2, + "set_main": 88.6 + }, + "recomputed": { + "interaction": 3.896419437299789, + "corpus_main": 1.9252567551293127, + "set_main": 94.14618752120758, + "noise_floor": 121.24672382012868 + }, + "interaction_matches_to_0p1pp": false, + "corpus_main_matches_to_0p1pp": false + } + }, + "noise_floor_4614_audit": { + "claimed": 4614, + "claimed_where": "PLAN.tex:52 and :270, HANDOFF.md:63 \u2014 described as 'the seed x set noise floor'. In no results file.", + "candidates_computed": { + "seed_x_set": 469.67847751637584, + "seed_x_corpus": 449.5093717335184, + "seed_main": 1862.5014836471103, + "three_way_residual": 142.89035471753772, + "full_two_factor_residual": 84.11101589573194, + "mean_squares_F": 262.8469246741623 + }, + "any_candidate_within_2pct": {}, + "reproduces": false, + "reading": "The six VARIANCE FRACTIONS reproduce exactly at both scales, so the decomposition specified in this file is the one that was used. The 4614x figure does NOT follow from it under any of six natural denominators. Cite a named, stored ratio from candidate_noise_floors instead, or drop the multiplier \u2014 do not restate 4614x." + }, + "VERDICT": "DOES NOT REPRODUCE under this decomposition. The numbers in the paper and specs are NOT recoverable from the stored ladders by the decomposition specified in this file's docstring. Either a different decomposition was used and was never written down, or the numbers are wrong. STOP: this is the paper's headline claim and it must not be cited until the discrepancy is resolved by the operator.", + "provenance": { + "schema_version": 1, + "experiment": "E51", + "utc": "2026-08-20T05:28:32+00:00", + "script": { + "path": "experiments/t51_interaction_variance.py", + "sha256": "849f3e2ab42163fc1e5c553f4b1100cec316d6cff89a8b20f0ef26fa008b542a", + "n_lines": 323 + }, + "git": { + "commit": "a4fc687ed7027ca729606455391b9463310911e8", + "commit_short": "a4fc687ed702", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t52_factorial.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/30_repo_health.sh", + "results/d2_device_crossvalidation.json", + "results/r4c/", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "src/provenance.py", + "tests/test_provenance_hash.py", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "89e80dd41d08b1238f8e123d87f968acdb67e1b2db7309e2e6fcbf92aa3f57d4", + "describe": "a4fc687-dirty" + }, + "argv": [ + "experiments/t51_interaction_variance.py", + "--agg", + "min", + "--out", + "results/r4c/e51_interaction_min.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "0f17e5271defad582744e98ce0ad8489d16a7945192d5bd0863a660db693efb2" + } +} \ No newline at end of file diff --git a/results/r4c/e51_interaction_persist.json b/results/r4c/e51_interaction_persist.json new file mode 100644 index 0000000000000000000000000000000000000000..07674f346486fe38b57684d960bffa2bb97a4f27 --- /dev/null +++ b/results/r4c/e51_interaction_persist.json @@ -0,0 +1,754 @@ +{ + "experiment": "E51 \u2014 the corpus x concept-set interaction decomposition, STORED", + "why": "the programme's headline interaction numbers (6.8% / 1.9% / 4614x at 410M, 9.2% / 2.2% at 1B) appear in six documents including the paper abstract and in NO results JSON; no committed script computed them. This stores the decomposition and reports whether it reproduces them.", + "recomputes_not_remeasures": true, + "inputs": [ + "results/ladder410/*.json", + "results/ladder1b/*.json" + ], + "aggregation": "persist", + "n_min": 75, + "decomposition": "balanced two-factor (eval set, corpus) with seed as the replication unit, after averaging over the NESTED N axis (SPINE A1.1)", + "scales": { + "410m": { + "admitted5": { + "grand_mean": 0.041563127900002834, + "SS_total": 0.22925593623758608, + "frac_set_main": 0.9127829399393488, + "frac_corpus_main": 0.01906450114831279, + "frac_interaction": 0.06780626428099203, + "frac_residual_seed": 0.00034629463134648467, + "SS_set": 0.2092609074774917, + "SS_corpus": 0.004370650059658984, + "SS_interaction": 0.015544988600512018, + "SS_residual_seed": 7.939009992338807e-05, + "interaction_over_noise_floor": 195.8051270311163, + "candidate_noise_floors": { + "seed_x_set": 2768.1445461585845, + "seed_x_corpus": 2547.878926111827, + "seed_main": 10268.19304676178, + "three_way_residual": 234.96272012579493, + "full_two_factor_residual": 195.8051270311163, + "mean_squares_F": 611.8910219722385 + }, + "SS_seed_main": 1.5138971900625057e-06, + "SS_seed_x_set": 5.615670837017576e-06, + "SS_seed_x_corpus": 6.10114885805596e-06, + "SS_three_way_residual": 6.615938303825178e-05, + "interaction_over_corpus_main": 3.5566765557352586, + "per_corpus_main": { + "Github": -0.01347907410647357, + "Pile-CC": 0.004152620384147217, + "StackExchange": 0.004112475200761065, + "USPTO_Backgrounds": 0.00817213694672702, + "Wikipedia_en": -0.0029581584251617415 + }, + "per_set_main": { + "multihop": -0.026766659081874818, + "multilingual": -0.007186749365197628, + "order-ops": -0.03100104450900741, + "poetry": -0.038628393975836214, + "typo": 0.10358284693191608 + }, + "interaction_cells": { + "Github|multihop": 0.0023481492099414816, + "Github|multilingual": 0.007358160156824216, + "Github|order-ops": 0.02190276691766114, + "Github|poetry": 0.014917517935270082, + "Github|typo": -0.046526594219696946, + "Pile-CC|multihop": -0.0020486935037057966, + "Pile-CC|multilingual": -0.002015691274651746, + "Pile-CC|order-ops": -0.009327498430404486, + "Pile-CC|poetry": -0.005629628390659461, + "Pile-CC|typo": 0.01902151159942149, + "StackExchange|multihop": -0.005006295451394596, + "StackExchange|multilingual": 0.0005318301469256631, + "StackExchange|order-ops": -0.012138998990460316, + "StackExchange|poetry": -0.002743446891852852, + "StackExchange|typo": 0.0193569111867821, + "USPTO_Backgrounds|multihop": -0.003962081805792314, + "USPTO_Backgrounds|multilingual": -0.0031076970647595306, + "USPTO_Backgrounds|order-ops": -0.010028111813667362, + "USPTO_Backgrounds|poetry": -0.007273591605950647, + "USPTO_Backgrounds|typo": 0.024371482290169853, + "Wikipedia_en|multihop": 0.008668921550951246, + "Wikipedia_en|multilingual": -0.0027666019643385886, + "Wikipedia_en|order-ops": 0.009591842316871033, + "Wikipedia_en|poetry": 0.0007291489531928777, + "Wikipedia_en|typo": -0.016223310856676575 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 1.0, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06780626428099203, + "null2_within_cell_mean": 0.023461171497061152, + "null2_max": 0.058449129594287226, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.06780626428099203, + "fires": null + }, + "C4_F_test": { + "F": 611.8910219722385, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03464607279448547, + "SS_total": 0.25078694938130414, + "frac_set_main": 0.9202689818852297, + "frac_corpus_main": 0.014505336726722001, + "frac_interaction": 0.06490812752157134, + "frac_residual_seed": 0.00031755386647712333, + "SS_set": 0.23079145057723538, + "SS_corpus": 0.003637749147443202, + "SS_interaction": 0.016278111291187544, + "SS_residual_seed": 7.963836543803573e-05, + "interaction_over_noise_floor": 204.40036911421873, + "candidate_noise_floors": { + "seed_x_set": 2664.38297317969, + "seed_x_corpus": 3357.1772819678595, + "seed_main": 15217.493238347859, + "three_way_residual": 240.763440450699, + "full_two_factor_residual": 204.40036911421873, + "mean_squares_F": 613.2011073426562 + }, + "SS_seed_main": 1.0696972908893393e-06, + "SS_seed_x_set": 6.109523839120302e-06, + "SS_seed_x_corpus": 4.848749387951859e-06, + "SS_three_way_residual": 6.761039492007427e-05, + "interaction_over_corpus_main": 4.474775645986651, + "per_corpus_main": { + "Github": -0.011242694633211085, + "Pile-CC": 0.003450384108972908, + "StackExchange": 0.003416929789484449, + "USPTO_Backgrounds": 0.006799981244456077, + "Wikipedia_en": -0.0024246005097023557 + }, + "per_set_main": { + "multihop": -0.019849603976357454, + "multilingual": -0.0002696942596802643, + "order-ops": -0.024083989403490047, + "poetry": -0.03171133887031885, + "typo": 0.11049990203743343, + "association": -0.03458527552758682 + }, + "interaction_cells": { + "Github|multihop": 0.0001117697366790002, + "Github|multilingual": 0.005121780683561734, + "Github|order-ops": 0.01966638744439865, + "Github|poetry": 0.0126811384620076, + "Github|typo": -0.04876297369295941, + "Github|association": 0.011181897366312435, + "Pile-CC|multihop": -0.0013464572285314877, + "Pile-CC|multilingual": -0.001313454999477437, + "Pile-CC|order-ops": -0.008625262155230178, + "Pile-CC|poetry": -0.004927392115485152, + "Pile-CC|typo": 0.019723747874595812, + "Pile-CC|association": -0.003511181375871558, + "StackExchange|multihop": -0.0043107500401179805, + "StackExchange|multilingual": 0.0012273755582022788, + "StackExchange|order-ops": -0.011443453579183704, + "StackExchange|poetry": -0.0020479014805762363, + "StackExchange|typo": 0.02005245659805873, + "StackExchange|association": -0.0034777270563830995, + "USPTO_Backgrounds|multihop": -0.0025899261035213714, + "USPTO_Backgrounds|multilingual": -0.001735541362488588, + "USPTO_Backgrounds|order-ops": -0.00865595611139642, + "USPTO_Backgrounds|poetry": -0.005901435903679704, + "USPTO_Backgrounds|typo": 0.025743637992440817, + "USPTO_Backgrounds|association": -0.006860778511354727, + "Wikipedia_en|multihop": 0.00813536363549186, + "Wikipedia_en|multilingual": -0.0033001598797979745, + "Wikipedia_en|order-ops": 0.009058284401411647, + "Wikipedia_en|poetry": 0.00019559103773349185, + "Wikipedia_en|typo": -0.016756868772135947, + "Wikipedia_en|association": 0.002667789577296943 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 1.0000000000000002, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.06490812752157134, + "null2_within_cell_mean": 0.022186420523225118, + "null2_max": 0.062092652683997974, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.06490812752157134, + "fires": null + }, + "C4_F_test": { + "F": 613.2011073426562, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + }, + "1b": { + "admitted5": { + "grand_mean": 0.04294701039968502, + "SS_total": 0.5877733083330603, + "frac_set_main": 0.8857950497345805, + "frac_corpus_main": 0.022117202251311613, + "frac_interaction": 0.09158325214755474, + "frac_residual_seed": 0.0005044958665530746, + "SS_set": 0.520646686887542, + "SS_corpus": 0.012999901138324835, + "SS_interaction": 0.053830191102669095, + "SS_residual_seed": 0.00029652920452425476, + "interaction_over_noise_floor": 181.5341972438537, + "candidate_noise_floors": { + "seed_x_set": 701.2113462326924, + "seed_x_corpus": 1300.55822013104, + "seed_main": 3666.99058665031, + "three_way_residual": 328.85037669047756, + "full_two_factor_residual": 181.5341972438537, + "mean_squares_F": 567.2943663870428 + }, + "SS_seed_main": 1.4679664381642555e-05, + "SS_seed_x_set": 7.676742738387734e-05, + "SS_seed_x_corpus": 4.139006641105643e-05, + "SS_three_way_residual": 0.00016369204634767822, + "interaction_over_corpus_main": 4.1408154208167804, + "per_corpus_main": { + "Github": -0.015690614938115084, + "Pile-CC": 0.0030669355334802736, + "StackExchange": 0.015805140123480846, + "USPTO_Backgrounds": 0.011754592504973217, + "Wikipedia_en": -0.014936053223819253 + }, + "per_set_main": { + "multihop": -0.04065852216755351, + "multilingual": -0.04060835923999548, + "order-ops": -0.04294701039968502, + "poetry": -0.04241251089654508, + "typo": 0.1666264027037791 + }, + "interaction_cells": { + "Github|multihop": 0.01617605183583995, + "Github|multilingual": 0.01740079140379315, + "Github|order-ops": 0.015690614938115084, + "Github|poetry": 0.015399069754584209, + "Github|typo": -0.06466652793233241, + "Pile-CC|multihop": -0.005239843551867775, + "Pile-CC|multilingual": -0.003502033432679519, + "Pile-CC|order-ops": -0.0030669355334802736, + "Pile-CC|poetry": -0.003601435036620214, + "Pile-CC|typo": 0.01541024755464776, + "StackExchange|multihop": -0.01531970322575598, + "StackExchange|multilingual": -0.015847441318134474, + "StackExchange|order-ops": -0.015805140123480846, + "StackExchange|poetry": -0.016339639626620786, + "StackExchange|typo": 0.06331192429399204, + "USPTO_Backgrounds|multihop": -0.009882193042320159, + "USPTO_Backgrounds|multilingual": -0.012975283813263697, + "USPTO_Backgrounds|order-ops": -0.011754592504973217, + "USPTO_Backgrounds|poetry": -0.012289092008113157, + "USPTO_Backgrounds|typo": 0.046901161368670195, + "Wikipedia_en|multihop": 0.014265687984103964, + "Wikipedia_en|multilingual": 0.014923967160284518, + "Wikipedia_en|order-ops": 0.014936053223819253, + "Wikipedia_en|poetry": 0.01683109691676994, + "Wikipedia_en|typo": -0.06095680528497768 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 75, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.09158325214755474, + "null2_within_cell_mean": 0.029931755338755018, + "null2_max": 0.08481827845754085, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.09158325214755474, + "fires": null + }, + "C4_F_test": { + "F": 567.2943663870428, + "df_interaction": 16, + "df_error_seed": 50, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (16,50); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + }, + "all6": { + "grand_mean": 0.03645773986613171, + "SS_total": 0.6067938204847383, + "frac_set_main": 0.8892583495369545, + "frac_corpus_main": 0.018617391676965456, + "frac_interaction": 0.0916251460361479, + "frac_residual_seed": 0.000499112749932075, + "SS_set": 0.5395964713134814, + "SS_corpus": 0.011296918223126638, + "SS_interaction": 0.055597572415746255, + "SS_residual_seed": 0.0003028585323839276, + "interaction_over_noise_floor": 183.5760477940451, + "candidate_noise_floors": { + "seed_x_set": 696.3305263109522, + "seed_x_corpus": 1715.3652591502535, + "seed_main": 4592.651780597178, + "three_way_residual": 311.4751704961482, + "full_two_factor_residual": 183.5760477940451, + "mean_squares_F": 550.7281433821353 + }, + "SS_seed_main": 1.2105767010386526e-05, + "SS_seed_x_set": 7.984365228147229e-05, + "SS_seed_x_corpus": 3.241150659847677e-05, + "SS_three_way_residual": 0.00017849760649359299, + "interaction_over_corpus_main": 4.921481356032917, + "per_corpus_main": { + "Github": -0.013744076981490095, + "Pile-CC": 0.002771445592894911, + "StackExchange": 0.013257216481447578, + "USPTO_Backgrounds": 0.01009742612987461, + "Wikipedia_en": -0.012382011222727005 + }, + "per_set_main": { + "multihop": -0.0341692516340002, + "multilingual": -0.034119088706442174, + "order-ops": -0.03645773986613171, + "poetry": -0.03592324036299177, + "typo": 0.1731156732373324, + "association": -0.03244635266776655 + }, + "interaction_cells": { + "Github|multihop": 0.014229513879214964, + "Github|multilingual": 0.015454253447168165, + "Github|order-ops": 0.013744076981490091, + "Github|poetry": 0.013452531797959216, + "Github|typo": -0.06661306588895739, + "Github|association": 0.009732689783124929, + "Pile-CC|multihop": -0.004944353611282412, + "Pile-CC|multilingual": -0.0032065434920941494, + "Pile-CC|order-ops": -0.002771445592894911, + "Pile-CC|poetry": -0.0033059450960348513, + "Pile-CC|typo": 0.01570573749523313, + "Pile-CC|association": -0.0014774497029268058, + "StackExchange|multihop": -0.012771779583722713, + "StackExchange|multilingual": -0.0132995176761012, + "StackExchange|order-ops": -0.013257216481447578, + "StackExchange|poetry": -0.013791715984587519, + "StackExchange|typo": 0.06585984793602533, + "StackExchange|association": -0.012739618210166344, + "USPTO_Backgrounds|multihop": -0.008225026667221552, + "USPTO_Backgrounds|multilingual": -0.011318117438165083, + "USPTO_Backgrounds|order-ops": -0.01009742612987461, + "USPTO_Backgrounds|poetry": -0.010631925633014551, + "USPTO_Backgrounds|typo": 0.04855832774376881, + "USPTO_Backgrounds|association": -0.008285831875493012, + "Wikipedia_en|multihop": 0.01171164598301172, + "Wikipedia_en|multilingual": 0.012369925159192274, + "Wikipedia_en|order-ops": 0.012382011222727002, + "Wikipedia_en|poetry": 0.014277054915677698, + "Wikipedia_en|typo": -0.06351084728606993, + "Wikipedia_en|association": 0.012770210005461219 + }, + "corpora": [ + "Github", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_cells": 90, + "C1_fractions_sum_to_one": { + "sum": 0.9999999999999999, + "fires": true + }, + "C3_permutation_null": { + "STATUS": "MIS-SPECIFIED \u2014 DIAGNOSTIC ONLY, DO NOT CITE. Shuffling corpus labels within each (seed,set) cell destroys the corpus MAIN effect as well as the interaction, and the destroyed main-effect variance is re-absorbed as apparent interaction in the null. The null mean is therefore inflated (~2.3-3.0% rather than ~0) and the test is badly under-powered: at 410M with 2000 draws the null MAX exceeds the observed value. A correct null must preserve both main effects. The F test below is correctly specified for this design and is what should be cited.", + "null1_wholerow_mean": 0.0916251460361479, + "null2_within_cell_mean": 0.030780725304871703, + "null2_max": 0.08516371461203105, + "n_perm": 200, + "n_null_ge_observed": 0, + "p_value": 0.004975124378109453, + "observed": 0.0916251460361479, + "fires": null + }, + "C4_F_test": { + "F": 550.7281433821353, + "df_interaction": 20, + "df_error_seed": 60, + "F_crit_0.001": 3.3, + "note": "F = MS_interaction / MS_seed-residual. df (20,60); the 0.1% critical value is ~3.3. The error term is seed replication, so this asks exactly the question the prose asks: is the interaction larger than the seed-level scatter?", + "fires": true + } + } + } + }, + "ladder1b_dir": "results/ladder1b", + "reproduces_the_prose": { + "410m": { + "claimed": { + "interaction": 6.8, + "corpus_main": 1.9, + "set_main": 91.3, + "noise_floor": 4614 + }, + "recomputed": { + "interaction": 6.780626428099203, + "corpus_main": 1.9064501148312791, + "set_main": 91.27829399393488, + "noise_floor": 195.8051270311163 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + }, + "1b": { + "claimed": { + "interaction": 9.2, + "corpus_main": 2.2, + "set_main": 88.6 + }, + "recomputed": { + "interaction": 9.158325214755473, + "corpus_main": 2.2117202251311614, + "set_main": 88.57950497345804, + "noise_floor": 181.5341972438537 + }, + "interaction_matches_to_0p1pp": true, + "corpus_main_matches_to_0p1pp": true + } + }, + "noise_floor_4614_audit": { + "claimed": 4614, + "claimed_where": "PLAN.tex:52 and :270, HANDOFF.md:63 \u2014 described as 'the seed x set noise floor'. In no results file.", + "candidates_computed": { + "seed_x_set": 2768.1445461585845, + "seed_x_corpus": 2547.878926111827, + "seed_main": 10268.19304676178, + "three_way_residual": 234.96272012579493, + "full_two_factor_residual": 195.8051270311163, + "mean_squares_F": 611.8910219722385 + }, + "any_candidate_within_2pct": {}, + "reproduces": false, + "reading": "The six VARIANCE FRACTIONS reproduce exactly at both scales, so the decomposition specified in this file is the one that was used. The 4614x figure does NOT follow from it under any of six natural denominators. Cite a named, stored ratio from candidate_noise_floors instead, or drop the multiplier \u2014 do not restate 4614x." + }, + "VERDICT": "REPRODUCES \u2014 the headline interaction numbers now have a results file and match the prose to within 0.1 percentage points. || BUT the '4614x noise floor' does NOT reproduce under any natural denominator (nearest candidates: seed x set 2768x, seed x corpus 2548x, seed main 10268x, three-way residual 235x). The multiplier is unsupported and must not be restated until the operator names the intended denominator.", + "provenance": { + "schema_version": 1, + "experiment": "E51", + "utc": "2026-08-20T05:28:26+00:00", + "script": { + "path": "experiments/t51_interaction_variance.py", + "sha256": "849f3e2ab42163fc1e5c553f4b1100cec316d6cff89a8b20f0ef26fa008b542a", + "n_lines": 323 + }, + "git": { + "commit": "a4fc687ed7027ca729606455391b9463310911e8", + "commit_short": "a4fc687ed702", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t52_factorial.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/30_repo_health.sh", + "results/d2_device_crossvalidation.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "src/provenance.py", + "tests/test_provenance_hash.py", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "89e80dd41d08b1238f8e123d87f968acdb67e1b2db7309e2e6fcbf92aa3f57d4", + "describe": "a4fc687-dirty" + }, + "argv": [ + "experiments/t51_interaction_variance.py", + "--agg", + "persist", + "--out", + "results/r4c/e51_interaction_persist.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410/ladder_Github_s0.json", + "sha256": "6309f7636b8ef9484191be4b3f492f66600cd8a93b06483c234ff8caf877503b", + "bytes": 15784, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s1.json", + "sha256": "aabd00d0ed32678f1c298e9435e061f01bb0390c0883f4af14ebbdc75370be11", + "bytes": 15766, + "exists": true + }, + { + "path": "results/ladder410/ladder_Github_s2.json", + "sha256": "1feb27ab51615306337e4ce0f1cbb5d03bc20f947eb3c6c8dc103c264a16abcb", + "bytes": 15769, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s0.json", + "sha256": "dda406c8efc0ae463fcadbab98b043610e0e5848bf1102ea38847546bacc0909", + "bytes": 10904, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s1.json", + "sha256": "8c7335a765473ae8a698453c8faf97475b860baa4347221eece1f232dd89ead9", + "bytes": 10917, + "exists": true + }, + { + "path": "results/ladder410/ladder_Pile-CC_s2.json", + "sha256": "2720e604dc095ac3ac8e72cb5c6e1caf498b04763582eac0fe965ed14840f81b", + "bytes": 10907, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s0.json", + "sha256": "8658d3f32c1f20100d104e6cd524c4c0d3e04044af0cecca08959318ffb38907", + "bytes": 15777, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s1.json", + "sha256": "14615d4d76b66c2ed591c4c628b1a8e7d1ce7ef8c3466290127d450c773b236f", + "bytes": 15789, + "exists": true + }, + { + "path": "results/ladder410/ladder_StackExchange_s2.json", + "sha256": "4190fb2dd294b529b8c947c3641d6d33799f8cc6fb5bb29749d51fcede8253bf", + "bytes": 15780, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s0.json", + "sha256": "60246f23acb932386192ae742899a4cedd768fa082a5ff28e33afc8c1935304b", + "bytes": 10913, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s1.json", + "sha256": "7d3e4e6053174f5fefcc38c29e73116e42d010d760c039194aa602a51f1c87e8", + "bytes": 10901, + "exists": true + }, + { + "path": "results/ladder410/ladder_USPTO_Backgrounds_s2.json", + "sha256": "b950a4d857dfcf81da969a37e40a276cfe7d3c1e2c902f93bc6f00c8e434278b", + "bytes": 10910, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s0.json", + "sha256": "48fceb1cf8b910d9266ab8d9ab3cde75543e6f116fd5414086258c9fd6db6076", + "bytes": 15636, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s1.json", + "sha256": "c50a33e631f489b01c6880874d5c072a7e621ef3e8d33d5a602b68708bcdcd96", + "bytes": 15555, + "exists": true + }, + { + "path": "results/ladder410/ladder_Wikipedia_en_s2.json", + "sha256": "83765d399663a6e9ff97f01a30e7eb60e296c2e5521cdd6c9c46f7c1370378ac", + "bytes": 10911, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s0.json", + "sha256": "70f08c0c04fae13c225fca1ba9e69d01df797dddb66100d5624c448278937f19", + "bytes": 12878, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s1.json", + "sha256": "afbee942d86a66672f2153a9aeb21907a7d342ed3bc9fb85441249c2c5fed8bf", + "bytes": 12927, + "exists": true + }, + { + "path": "results/ladder1b/tv_Github_s2.json", + "sha256": "7b5d8b49c9e50247d8160ee64a2d07279e4570401c1eca6291801783e04985a4", + "bytes": 12914, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s0.json", + "sha256": "cfae7085a49f4e0983907dd8aa66678c9eae2edac83a98f3dfb2b483d4e83892", + "bytes": 12843, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s1.json", + "sha256": "a3b969ef0bcb1fba73e95ccaba713f70f1dc15077db63e896d5371031a189588", + "bytes": 12894, + "exists": true + }, + { + "path": "results/ladder1b/tv_Pile-CC_s2.json", + "sha256": "6bbc136053598f58d2c2cd0e3041f0ae74dc230d37f1e0ac835d267497978112", + "bytes": 12909, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s0.json", + "sha256": "808c4f56ea2edb5b9aa625b201f0d7e8139dfbf0c74017fe25c26112c6a53cab", + "bytes": 12972, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s1.json", + "sha256": "fd8eb1c2f96410e15c9cc57d63871cc2197ae999adb7cf2a496c9660f2e6b0d9", + "bytes": 12962, + "exists": true + }, + { + "path": "results/ladder1b/tv_StackExchange_s2.json", + "sha256": "764a95eb1b6cd2d47c42fad54c320969cc2684569b10c803e0ff243c615dcf90", + "bytes": 12958, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s0.json", + "sha256": "74a7317e4c90f91eaba17f918aaff4db64cc01ce7e9558f5294f56ae10a7fa86", + "bytes": 12974, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s1.json", + "sha256": "88af3b2e98111782662e46ba0e3c7a7364c7b42b22f289265942145562ad0b1e", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_USPTO_Backgrounds_s2.json", + "sha256": "9a80132d9f3a75434f17bcde380b595762dacf036b9d7d9667b10c8e8a459876", + "bytes": 12976, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s0.json", + "sha256": "2afe54c57a5d16c03963f461a2cc75ad66b375fe213caa0b1432105285c76075", + "bytes": 13034, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s1.json", + "sha256": "07fdb704221ff92bc376e19648b68e44e5b95e4d20e6110d99bbfc6f2db88b8a", + "bytes": 13083, + "exists": true + }, + { + "path": "results/ladder1b/tv_Wikipedia_en_s2.json", + "sha256": "90b2485e1df2168f90dec9bbab4a569915189ada97324d17854eccd970c5586f", + "bytes": 13105, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "ea19ebc11ede6c463d8b42b4f6d65a2ecb18da77d464eaf3018f696c1ad76017" + } +} \ No newline at end of file diff --git a/results/r4e_control_power.json b/results/r4e_control_power.json new file mode 100644 index 0000000000000000000000000000000000000000..e83a87b6981cf99383e01856e8ab3698a5587079 --- /dev/null +++ b/results/r4e_control_power.json @@ -0,0 +1,1597 @@ +{ + "experiment": "R4e \u2014 the control-power audit: which controls could not have failed?", + "prereg": "specs/experiments/R4_corrections.md", + "prereg_sha256": "78dd3d8e4a0ebdd3a4d2b77fac02abf8fdac02f9b645e1f5708cddc6a48c88c0", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "Produce the enumeration as a results file, mark each in place, and add the power question to the experiment-spec template. Include E47's bottom-k control, which was computed, stored, never gated, and would have failed 3 of 5.", + "counting_rule": "one row per (results file, control base name); repeated per-set or per-corpus instances of the SAME control collapse to one row. Smoke, re-run and pooled duplicates of a file are excluded, since they are the same controls counted twice.", + "n_results_files_with_controls": 46, + "n_control_rows": 106, + "n_powerless": 36, + "counts": { + "NONE": 36, + "class:ARITHMETIC_IDENTITY": 2, + "HAS_POWER": 70, + "class:FIRED_TRUE": 61, + "class:FIRED_FALSE": 9, + "class:NO_GATE": 31, + "class:SELF_COMPARISON": 1, + "class:GATE_NEVER_EVALUATED": 2, + "class:DECLARED_NEVER_CODED": 6 + }, + "SLATE_COMPARISON": { + "slate_says": "27 of 97 control fields could not have failed: 8 compare a quantity to itself, 6 are recorders with no gate, 4 are declared in a docstring and never coded, 9 are arithmetic identities", + "recomputed_n_control_rows": 106, + "recomputed_n_powerless": 36, + "note": "the totals depend entirely on the counting rule, which the slate does not state. This file states its rule above so the number is checkable. What does NOT depend on the rule, and is the finding: a large minority of this repository's controls carry no gate at all, and several named in specs have no corresponding key in the results file they belong to." + }, + "powerless_controls": [ + { + "file": "results/e34a_dact_vs_floor_410m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "NONE", + "example_path": "control_C1_max_dact_self", + "hand_audited_class": "ARITHMETIC_IDENTITY", + "why": "C1 fires 'at exactly 0.0' -- D_act of an operator against itself. Definitional." + }, + { + "file": "results/e35_containment.json", + "control": "M1_C2_random_containment", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.M1_C2_random_containment" + }, + { + "file": "results/e35_containment.json", + "control": "M1_C1_containment", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.M1_C1_containment" + }, + { + "file": "results/e35_containment_shard0.json", + "control": "M1_C2_random_containment", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.M1_C2_random_containment" + }, + { + "file": "results/e36_qladder_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_capability" + }, + { + "file": "results/e45_disagreement_geometry.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "NONE", + "example_path": "control_C2_random_delta_r", + "hand_audited_class": "SELF_COMPARISON", + "why": "recorded in the audit as structurally incapable of failing; rebuilt properly it does NOT fire (r = +0.349 against a 0.3 threshold). The stored version is decoration." + }, + { + "file": "results/e48_crossover_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C2_shuf_floor" + }, + { + "file": "results/e48_crossover_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_norm_matched_random" + }, + { + "file": "results/e48b_exposure_growth.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C2_index_matches_bitmaps" + }, + { + "file": "results/e48c_exposure_vs_read.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_github_leverage" + }, + { + "file": "results/e50_concept_energy.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "control_C1_random_direction_r" + }, + { + "file": "results/e50_concept_energy.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "control_C2_dose_r" + }, + { + "file": "results/e51_interaction_variance.json", + "control": "C3", + "n_instances": 4, + "mechanical_class": "GATE_NEVER_EVALUATED", + "power": "NONE", + "example_path": "scales.410m.admitted5.C3_permutation_null" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C4", + "n_instances": 4, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "by_aggregation.persist.C4_permutation" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C6", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "by_aggregation.persist.C6_link_artifact_null" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C5", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "NONE", + "example_path": "controls.C5_self_distance", + "hand_audited_class": "ARITHMETIC_IDENTITY", + "why": "C5_self_distance is stored as {'value': 0.0, 'fires': True}. The distance of an operator from itself is zero by definition; no measurement can make it anything else." + }, + { + "file": "results/e54_aggregation_audit.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_derangement" + }, + { + "file": "results/e56_predictor_registry.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C2_controls_tagged_and_excluded" + }, + { + "file": "results/e56_predictor_registry.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_duplicates_tagged" + }, + { + "file": "results/e59_read_dose_410m.json", + "control": "C4", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C4_cycling" + }, + { + "file": "results/e62_interaction_b613.json", + "control": "C3", + "n_instances": 4, + "mechanical_class": "GATE_NEVER_EVALUATED", + "power": "NONE", + "example_path": "scales.410m.admitted5.C3_permutation_null" + }, + { + "file": "results/t21_conditioning_160m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C1_interpretation" + }, + { + "file": "results/t21_conditioning_160m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_interpretation" + }, + { + "file": "results/t21_conditioning_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C1_interpretation" + }, + { + "file": "results/t21_conditioning_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_interpretation" + }, + { + "file": "results/t21_conditioning_70m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C1_interpretation" + }, + { + "file": "results/t21_conditioning_70m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_interpretation" + }, + { + "file": "results/t22_bootstrap_ci_160m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t22_bootstrap_ci_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t22_bootstrap_ci_persist_160m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t22_bootstrap_ci_persist_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t23_pq_ladder_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_410m_LENGTHMATCHED.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_70m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_70m_LENGTHMATCHED.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_70m_wiki_pile.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + } + ], + "declared_in_a_spec_but_absent_from_the_results_file": [ + { + "spec": "specs/experiments/E57_grid_variance_ci.md", + "results_files_checked": [ + "results/e57_factorial_cells_410m.json", + "results/e57_grid_variance_ci.json" + ], + "control_declared_but_absent_from_all_of_them": "C3" + }, + { + "spec": "specs/experiments/E65_training_axis_floor.md", + "results_files_checked": [ + "results/e65_ckpt_geometry_410m.json", + "results/e65_ckpt_readout_1b.json", + "results/e65_ckpt_readout_410m.json", + "results/e65_ckpt_readout_410m_trainval.json" + ], + "control_declared_but_absent_from_all_of_them": "C2" + }, + { + "spec": "specs/experiments/E65_training_axis_floor.md", + "results_files_checked": [ + "results/e65_ckpt_geometry_410m.json", + "results/e65_ckpt_readout_1b.json", + "results/e65_ckpt_readout_410m.json", + "results/e65_ckpt_readout_410m_trainval.json" + ], + "control_declared_but_absent_from_all_of_them": "C3" + }, + { + "spec": "specs/experiments/E65_training_axis_floor.md", + "results_files_checked": [ + "results/e65_ckpt_geometry_410m.json", + "results/e65_ckpt_readout_1b.json", + "results/e65_ckpt_readout_410m.json", + "results/e65_ckpt_readout_410m_trainval.json" + ], + "control_declared_but_absent_from_all_of_them": "C4" + }, + { + "spec": "specs/experiments/E65_training_axis_floor.md", + "results_files_checked": [ + "results/e65_ckpt_geometry_410m.json", + "results/e65_ckpt_readout_1b.json", + "results/e65_ckpt_readout_410m.json", + "results/e65_ckpt_readout_410m_trainval.json" + ], + "control_declared_but_absent_from_all_of_them": "C5" + }, + { + "spec": "specs/archive/prereg/PREREG_E36_QLADDER.md", + "results_files_checked": [ + "results/e36_qladder_410m.json" + ], + "control_declared_but_absent_from_all_of_them": "C4" + } + ], + "all_control_rows": [ + { + "file": "results/e34a_dact_vs_floor_410m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "NONE", + "example_path": "control_C1_max_dact_self", + "hand_audited_class": "ARITHMETIC_IDENTITY", + "why": "C1 fires 'at exactly 0.0' -- D_act of an operator against itself. Definitional." + }, + { + "file": "results/e34a_dact_vs_floor_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C2_seed_floor" + }, + { + "file": "results/e34b_jeval_dact_410m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C1_max_dact_self" + }, + { + "file": "results/e34b_jeval_dact_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C2_seed_floor" + }, + { + "file": "results/e34b_jeval_dact_410m.json", + "control": "C3", + "n_instances": 3, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "control_C3_skipfirst_only_dact" + }, + { + "file": "results/e35_containment.json", + "control": "M1_C2_random_containment", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.M1_C2_random_containment" + }, + { + "file": "results/e35_containment.json", + "control": "M1_C2_fires", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.M1_C2_fires" + }, + { + "file": "results/e35_containment.json", + "control": "M1_C1_containment", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.M1_C1_containment" + }, + { + "file": "results/e35_containment.json", + "control": "M1_C1_fires", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "controls.M1_C1_fires" + }, + { + "file": "results/e35_containment_shard0.json", + "control": "M1_C2_random_containment", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.M1_C2_random_containment" + }, + { + "file": "results/e35_containment_shard0.json", + "control": "M1_C2_fires", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.M1_C2_fires" + }, + { + "file": "results/e36_qladder_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_Q0_reproduces_e33" + }, + { + "file": "results/e36_qladder_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "controls.C2_shuf_below_everything" + }, + { + "file": "results/e36_qladder_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_capability" + }, + { + "file": "results/e36_qladder_410m.json", + "control": "C5", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C5_self_distance" + }, + { + "file": "results/e38_jgeometry.json", + "control": "E38_C1_fires", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.E38_C1_fires" + }, + { + "file": "results/e38_jgeometry.json", + "control": "E38_C2_fires", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.E38_C2_fires" + }, + { + "file": "results/e44_dread_410m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C1_dread_self" + }, + { + "file": "results/e44_dread_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C2_r_dread_vs_dact" + }, + { + "file": "results/e45_disagreement_geometry.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C1_zero_delta" + }, + { + "file": "results/e45_disagreement_geometry.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "NONE", + "example_path": "control_C2_random_delta_r", + "hand_audited_class": "SELF_COMPARISON", + "why": "recorded in the audit as structurally incapable of failing; rebuilt properly it does NOT fire (r = +0.349 against a 0.3 threshold). The stored version is decoration." + }, + { + "file": "results/e46_disagreement_ablation.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "control_C1_random_inert" + }, + { + "file": "results/e48_competence_gate_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "controls.C1_pilecc_lowest_or_second" + }, + { + "file": "results/e48_competence_gate_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C2_far_below_uniform_null" + }, + { + "file": "results/e48_competence_gate_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C3_seed_spread_small" + }, + { + "file": "results/e48_crossover_410m.json", + "control": "C0", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C0_reproduces_e33_v2" + }, + { + "file": "results/e48_crossover_410m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_identity_equals_logit" + }, + { + "file": "results/e48_crossover_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C2_shuf_floor" + }, + { + "file": "results/e48_crossover_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_norm_matched_random" + }, + { + "file": "results/e48b_exposure_growth.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_random_floor" + }, + { + "file": "results/e48b_exposure_growth.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C2_index_matches_bitmaps" + }, + { + "file": "results/e48b_exposure_growth.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C3_reproduces_stored_5shard" + }, + { + "file": "results/e48c_exposure_vs_read.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_containment_spans_orders" + }, + { + "file": "results/e48c_exposure_vs_read.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C2_seed_sd_small_vs_spread" + }, + { + "file": "results/e48c_exposure_vs_read.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_github_leverage" + }, + { + "file": "results/e50_concept_energy.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "control_C1_random_direction_r" + }, + { + "file": "results/e50_concept_energy.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "control_C2_dose_r" + }, + { + "file": "results/e51_interaction_variance.json", + "control": "C1", + "n_instances": 4, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "scales.410m.admitted5.C1_fractions_sum_to_one" + }, + { + "file": "results/e51_interaction_variance.json", + "control": "C3", + "n_instances": 4, + "mechanical_class": "GATE_NEVER_EVALUATED", + "power": "NONE", + "example_path": "scales.410m.admitted5.C3_permutation_null" + }, + { + "file": "results/e51_interaction_variance.json", + "control": "C4", + "n_instances": 4, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "scales.410m.admitted5.C4_F_test" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C4", + "n_instances": 4, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "by_aggregation.persist.C4_permutation" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C6", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "by_aggregation.persist.C6_link_artifact_null" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_Q0_vs_e48" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C2_shuf_floor_excl_degenerate" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C3_capability" + }, + { + "file": "results/e52_factorial_410m.json", + "control": "C5", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "NONE", + "example_path": "controls.C5_self_distance", + "hand_audited_class": "ARITHMETIC_IDENTITY", + "why": "C5_self_distance is stored as {'value': 0.0, 'fires': True}. The distance of an operator from itself is zero by definition; no measurement can make it anything else." + }, + { + "file": "results/e53_ladder_summary.json", + "control": "C2", + "n_instances": 16, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "grid_shape.per_cell.Github|s0.C2_reconstruction_matches_fitted_blocks" + }, + { + "file": "results/e53_ladder_summary.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C1_agrees_with_e54" + }, + { + "file": "results/e54_aggregation_audit.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_derangement" + }, + { + "file": "results/e54_aggregation_audit.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_persist_column_reproduces_stored_headlines" + }, + { + "file": "results/e55_matrix_robustness.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_agrees_with_e54" + }, + { + "file": "results/e55_matrix_robustness.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C2_e36_diagonal_leakage" + }, + { + "file": "results/e56_predictor_registry.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_e31_rows_reproduce_exactly" + }, + { + "file": "results/e56_predictor_registry.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C2_controls_tagged_and_excluded" + }, + { + "file": "results/e56_predictor_registry.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C3_duplicates_tagged" + }, + { + "file": "results/e57_factorial_cells_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "control_C1_reproduces_stored_e52_matrix" + }, + { + "file": "results/e57_grid_variance_ci.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "by_aggregation.persist.controls.C1_draws_average_to_stored_matrix" + }, + { + "file": "results/e57_grid_variance_ci.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "by_aggregation.persist.controls.C2_bootstrap_non_degenerate" + }, + { + "file": "results/e58_algebra_audit.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_zero_delta_by_orientation" + }, + { + "file": "results/e58_algebra_audit.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C3_pre_norm_reproduces_stored_e45" + }, + { + "file": "results/e59_read_dose_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_row_ordering_vs_e52" + }, + { + "file": "results/e59_read_dose_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C2_logit_arm_moves_with_dose" + }, + { + "file": "results/e59_read_dose_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C3_prefix_fit_overlap" + }, + { + "file": "results/e59_read_dose_410m.json", + "control": "C4", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "controls.C4_cycling" + }, + { + "file": "results/e60_fitter_determinism.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1_repeat_run_bit_identical" + }, + { + "file": "results/e60_fitter_determinism.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C2_prompts_long_enough" + }, + { + "file": "results/e61_randomized_null_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_competence_destroyed" + }, + { + "file": "results/e61_randomized_null_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C2_operator_well_formed" + }, + { + "file": "results/e61_randomized_null_410m.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C3_real_arm_reproduces_e48" + }, + { + "file": "results/e62_band_adjudication.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C2_band" + }, + { + "file": "results/e62_band_adjudication.json", + "control": "C3", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C3_prompt_pool" + }, + { + "file": "results/e62_band_adjudication.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_shared_layer_dispersion" + }, + { + "file": "results/e62_interaction_b613.json", + "control": "C1", + "n_instances": 4, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "scales.410m.admitted5.C1_fractions_sum_to_one" + }, + { + "file": "results/e62_interaction_b613.json", + "control": "C3", + "n_instances": 4, + "mechanical_class": "GATE_NEVER_EVALUATED", + "power": "NONE", + "example_path": "scales.410m.admitted5.C3_permutation_null" + }, + { + "file": "results/e62_interaction_b613.json", + "control": "C4", + "n_instances": 4, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "scales.410m.admitted5.C4_F_test" + }, + { + "file": "results/e65_ckpt_geometry_410m.json", + "control": "C1B", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C1b_final_checkpoint_anchor" + }, + { + "file": "results/e65_ckpt_geometry_410m.json", + "control": "C2B", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "controls.C2b_token_set_identity" + }, + { + "file": "results/e65_ckpt_readout_1b.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "controls.C1_final_checkpoint_reproduces_e48" + }, + { + "file": "results/e65_ckpt_readout_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "controls.C1_final_checkpoint_reproduces_e48" + }, + { + "file": "results/e65_ckpt_readout_410m_trainval.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "controls.C1_final_checkpoint_reproduces_e48" + }, + { + "file": "results/e66_fitter_equivalence_cuda.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_FALSE", + "power": "HAS_POWER", + "example_path": "controls.C1_trainval_reproduces_stored" + }, + { + "file": "results/r4_corrections.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "R4i_1b_corrected_band.C1_reproduces_e53_on_the_stored_band" + }, + { + "file": "results/t21_conditioning_160m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C1_interpretation" + }, + { + "file": "results/t21_conditioning_160m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_interpretation" + }, + { + "file": "results/t21_conditioning_410m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C1_interpretation" + }, + { + "file": "results/t21_conditioning_410m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_interpretation" + }, + { + "file": "results/t21_conditioning_70m.json", + "control": "C1", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C1_interpretation" + }, + { + "file": "results/t21_conditioning_70m.json", + "control": "C2", + "n_instances": 1, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_interpretation" + }, + { + "file": "results/t22_bootstrap_ci_160m.json", + "control": "C1", + "n_instances": 13, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_test_all_sets_fired" + }, + { + "file": "results/t22_bootstrap_ci_160m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t22_bootstrap_ci_410m.json", + "control": "C1", + "n_instances": 13, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_test_all_sets_fired" + }, + { + "file": "results/t22_bootstrap_ci_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t22_bootstrap_ci_persist_160m.json", + "control": "C1", + "n_instances": 13, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_test_all_sets_fired" + }, + { + "file": "results/t22_bootstrap_ci_persist_160m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t22_bootstrap_ci_persist_410m.json", + "control": "C1", + "n_instances": 13, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_test_all_sets_fired" + }, + { + "file": "results/t22_bootstrap_ci_persist_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_width_ratio" + }, + { + "file": "results/t23_pq_ladder_410m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_distance_max" + }, + { + "file": "results/t23_pq_ladder_410m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_410m_LENGTHMATCHED.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_distance_max" + }, + { + "file": "results/t23_pq_ladder_410m_LENGTHMATCHED.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_70m.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_distance_max" + }, + { + "file": "results/t23_pq_ladder_70m.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_70m_LENGTHMATCHED.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_distance_max" + }, + { + "file": "results/t23_pq_ladder_70m_LENGTHMATCHED.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + }, + { + "file": "results/t23_pq_ladder_70m_wiki_pile.json", + "control": "C1", + "n_instances": 2, + "mechanical_class": "FIRED_TRUE", + "power": "HAS_POWER", + "example_path": "C1_self_distance_max" + }, + { + "file": "results/t23_pq_ladder_70m_wiki_pile.json", + "control": "C2", + "n_instances": 2, + "mechanical_class": "NO_GATE", + "power": "NONE", + "example_path": "C2_shuffle_floor_median" + } + ], + "REQUIRED_CHANGE_TO_THE_SPEC_TEMPLATE": "specs/experiments/README.md's CONTROLS row becomes: 'each control, the number it actually produced, AND the number that would have made it fail. A control you cannot state a failing value for did not have power, and must be labelled a RECORDER.'", + "provenance": { + "schema_version": 1, + "experiment": "R4e", + "utc": "2026-08-20T05:40:29+00:00", + "script": { + "path": "tools/r4e_control_power.py", + "sha256": "0da2f8420f04763361858970051546c18b226d9057a170a80d3101e3f472a170", + "n_lines": 225 + }, + "git": { + "commit": "a4fc687ed7027ca729606455391b9463310911e8", + "commit_short": "a4fc687ed702", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t52_factorial.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/30_repo_health.sh", + "results/d2_device_crossvalidation.json", + "results/r4_corrections.json", + "results/r4c/", + "results/r4e_control_power.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "src/provenance.py", + "tests/test_provenance_hash.py", + "tmp/", + "tools/make_paper_tables.py", + "tools/r4_recomputes.py", + "tools/r4e_control_power.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "92b8cb115b61279d058e9407d236219c4e51fe050a6dab82b0b6d8c1277a81a5", + "describe": "a4fc687-dirty" + }, + "argv": [ + "tools/r4e_control_power.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e34a_dact_vs_floor_410m.json", + "sha256": "941eab8e5b4759ec292e16d8f68aed101523404ac9f5a767a18e7b59133fdaa2", + "bytes": 5113, + "exists": true + }, + { + "path": "results/e34b_jeval_dact_410m.json", + "sha256": "cfc18da9384bee3c5c4de07fa38049117c33570f15eca9beb3295a8dc19ffe0f", + "bytes": 4514, + "exists": true + }, + { + "path": "results/e35_containment.json", + "sha256": "996e73d840cecfb944a8fdd257716cfb94f67063d6dc17ee4bfad56290d7f823", + "bytes": 6598, + "exists": true + }, + { + "path": "results/e35_containment_shard0.json", + "sha256": "e1684b2d6fe1c650d573b7fd5568ca7deaea2a88d6942050ac43fff89eceaf1c", + "bytes": 3723, + "exists": true + }, + { + "path": "results/e36_qladder_410m.json", + "sha256": "d0cc7b6381ac515f6d53401705e246d8c20c67884cda25e4b1d35d5312f45153", + "bytes": 80656, + "exists": true + }, + { + "path": "results/e38_jgeometry.json", + "sha256": "f1de05f2c3fc57350b0bec1e14cbae7220a95c1dcac81dd8d5c3700970094e65", + "bytes": 12626, + "exists": true + }, + { + "path": "results/e44_dread_410m.json", + "sha256": "3ea05b7a931bf3f93a19689ad2f500fea992fa1ce2ba571c20f84fabc35964fa", + "bytes": 3140, + "exists": true + }, + { + "path": "results/e45_disagreement_geometry.json", + "sha256": "dfd1a336e310f2f0aaf8b966f0b4ae09395a4833cbd14b4481e516c043be8739", + "bytes": 4406, + "exists": true + }, + { + "path": "results/e46_disagreement_ablation.json", + "sha256": "c11823b902ebe9079c962bf5a37d6fc9c41e7f140fc67b084c71fd2e83ea997f", + "bytes": 15834, + "exists": true + }, + { + "path": "results/e48_competence_gate_410m.json", + "sha256": "37b9f18a33810e338c921e998af5eaab98164cc9b0a9848fc549a1d0b0950f8a", + "bytes": 19534, + "exists": true + }, + { + "path": "results/e48_crossover_410m.json", + "sha256": "7335aab33c2ea0b6bc3e9deec90181823ab2fb68202626a66c8658ad2cae4051", + "bytes": 116036, + "exists": true + }, + { + "path": "results/e48b_exposure_growth.json", + "sha256": "30ba5ecd395f1cdff653201c634a135a6d0d41e4b12c359a60c307cd85ad7d95", + "bytes": 32715, + "exists": true + }, + { + "path": "results/e48c_exposure_vs_read.json", + "sha256": "ff344efc91604138b698251c630e0d4f320d5b5b27ac9f87e2914d848ab1c374", + "bytes": 9875, + "exists": true + }, + { + "path": "results/e50_concept_energy.json", + "sha256": "96bea599c1a779cc1953a399e08ec1011f29a373890907135fd7c29ddf443892", + "bytes": 909, + "exists": true + }, + { + "path": "results/e51_interaction_variance.json", + "sha256": "2559e4017660a6a53949cb2a739fb3e7884e712851f2a04c8e04273dd217f62b", + "bytes": 29478, + "exists": true + }, + { + "path": "results/e52_factorial_410m.json", + "sha256": "6e81f3cae37d0c1906f8912af4bf8e75db0a0f259b2869f209701b74b6682590", + "bytes": 41190, + "exists": true + }, + { + "path": "results/e53_ladder_summary.json", + "sha256": "511c9d9b65199907f1785c2a3f5a147bce9c73352f990bd918a2201ce9cc9bf7", + "bytes": 33794, + "exists": true + }, + { + "path": "results/e54_aggregation_audit.json", + "sha256": "886885b41660dd01808aa626d5f87dddf6d5402f2c9b2f8241ac4470df82276b", + "bytes": 69079, + "exists": true + }, + { + "path": "results/e55_matrix_robustness.json", + "sha256": "9d5668d4d0c58c18d692c8b93038bacf712c7c0e6d695bbaa3c3215f671cc495", + "bytes": 89986, + "exists": true + }, + { + "path": "results/e56_predictor_registry.json", + "sha256": "a28bdfac4ef4fe3fc83f5e28e04b6cfe400f4a06b5bc05a4f49d9dfc29d1ee18", + "bytes": 18048, + "exists": true + }, + { + "path": "results/e57_factorial_cells_410m.json", + "sha256": "4c3b428670ba12195b4144d580ee0303426331207f60c34f6185a7cabecf8375", + "bytes": 82248, + "exists": true + }, + { + "path": "results/e57_grid_variance_ci.json", + "sha256": "7f2a2d4765420bba7e290c003d95d893efbb4b57169afd345de8119f2a3d1956", + "bytes": 6880, + "exists": true + }, + { + "path": "results/e58_algebra_audit.json", + "sha256": "56931b57eaf55dec1f85f7c8bca9bc69c10c9a3b22d541e9f5ff0a2b62c16fef", + "bytes": 9382, + "exists": true + }, + { + "path": "results/e59_read_dose_410m.json", + "sha256": "7b95f7db997008ea6917f166d71fdd89d0a02e12f5cd644bbfb6ef14ad6af6ba", + "bytes": 40401, + "exists": true + }, + { + "path": "results/e60_fitter_determinism.json", + "sha256": "3a069ee10debc3775c2b86eb7f9b27f46907e2305a237419409e769a39b13161", + "bytes": 4432, + "exists": true + }, + { + "path": "results/e61_randomized_null_410m.json", + "sha256": "db4b5e64cdb856a1398b91c48579fb8648ccb9082e98da937bb489d7d2520c10", + "bytes": 6583, + "exists": true + }, + { + "path": "results/e62_band_adjudication.json", + "sha256": "39820894aca707c687b10995830e39a7df50bfd0e11bf830596c0591fa9072e3", + "bytes": 8434, + "exists": true + }, + { + "path": "results/e62_interaction_b613.json", + "sha256": "d6f9aa7cf1f3ea5580eddf44db14e055a69ddef19008a6c6f518480d7624c436", + "bytes": 28056, + "exists": true + }, + { + "path": "results/e65_ckpt_geometry_410m.json", + "sha256": "209935237e5fc4aecdb511820f1bbbcc69cbafcfc76a178ecd1c6c4410f2a1c2", + "bytes": 7520, + "exists": true + }, + { + "path": "results/e65_ckpt_readout_1b.json", + "sha256": "0030f71f099283fe4ee40ea6516f6b37489b126c4d65eb56fcee060d0b4b8ce0", + "bytes": 7938, + "exists": true + }, + { + "path": "results/e65_ckpt_readout_410m.json", + "sha256": "2ee38de47ec874cfdc109f301a14ebf2fb9ac6bc2c19e1c12d03ea0ef6316be8", + "bytes": 7984, + "exists": true + }, + { + "path": "results/e65_ckpt_readout_410m_trainval.json", + "sha256": "9d5fd510e99eb3bb7c560163d4e07c8f9168852b3a028e07cb7c24ddb7e8d5bf", + "bytes": 8554, + "exists": true + }, + { + "path": "results/e66_fitter_equivalence_cuda.json", + "sha256": "d6e9bd3cd0c9e9426fc69e9d8d711f9a46e934214aaa52f67e877d947ec55cd2", + "bytes": 3208, + "exists": true + }, + { + "path": "results/r4_corrections.json", + "sha256": "b36c8a2d68e0f5e205038d73514a61ebdac7cfb56248508d6523b0f9bde4bfa2", + "bytes": 66242, + "exists": true + }, + { + "path": "results/t21_conditioning_160m.json", + "sha256": "b95933a3c7f46e9ab672cc583ebd3d7ee9d3eba5c48928c6a7d3be886e3e3f61", + "bytes": 3575, + "exists": true + }, + { + "path": "results/t21_conditioning_410m.json", + "sha256": "8028a313e841eb9c22ba11a9df5bb2578a42a8e13f1344de5445c1ebd585aa45", + "bytes": 3772, + "exists": true + }, + { + "path": "results/t21_conditioning_70m.json", + "sha256": "25645afbfed6b68c0fde2d82b14173aeffefbad7d9bf499a85706027adaa63d0", + "bytes": 3435, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_160m.json", + "sha256": "bee341836fa3252d2c7b3bbb955134363025531c90c658ae7d1fffb8152fe668", + "bytes": 8994, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_410m.json", + "sha256": "5439e0b15e315208d59f52f6dc990d36f1670ea4a1d101ebdc3b346a5930633a", + "bytes": 9262, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_persist_160m.json", + "sha256": "bd3d3c61ed3b4ebe87796c06a9bb31345cff80c5573e9c8b4bfb683578daca5e", + "bytes": 9212, + "exists": true + }, + { + "path": "results/t22_bootstrap_ci_persist_410m.json", + "sha256": "27d2b7110d23db75dc1d61fe9d3987119eab68a5d18add13b73dbf2c57e6de79", + "bytes": 9415, + "exists": true + }, + { + "path": "results/t23_pq_ladder_410m.json", + "sha256": "f8dd8356f214b92af92c5bf59ce4da8313977fc0b456eee3eaea956f1f8fc471", + "bytes": 6171, + "exists": true + }, + { + "path": "results/t23_pq_ladder_410m_LENGTHMATCHED.json", + "sha256": "2956d53bb5b10207ed922413b3fb898104678c84636dbc459542dedfb5ba809e", + "bytes": 6181, + "exists": true + }, + { + "path": "results/t23_pq_ladder_70m.json", + "sha256": "c883427cddfff055c40b289269b6e0fdd7ab332febb9d6e55cacb0fcc0688343", + "bytes": 2664, + "exists": true + }, + { + "path": "results/t23_pq_ladder_70m_LENGTHMATCHED.json", + "sha256": "46102213cb5ebe3610729a1b94c41d95adac7f3d696395d3ec248053f9b5d553", + "bytes": 2686, + "exists": true + }, + { + "path": "results/t23_pq_ladder_70m_wiki_pile.json", + "sha256": "2b8b9467090688c8da660ed40954a7ae6db66d6efdc1292ee526da696987a7e8", + "bytes": 1899, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "56db754c22f378604b4440ee13ed21dd61fba710386a1a0fdbc445d1d861ee3b" + } +} \ No newline at end of file diff --git a/results/r4g_e28_provenance.json b/results/r4g_e28_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d1e8552cf4be2ac3c603cf93d9e8b774d9076878 --- /dev/null +++ b/results/r4g_e28_provenance.json @@ -0,0 +1,139 @@ +{ + "experiment": "R4g \u2014 E28 provenance and artifact-ledger repair", + "prereg": "specs/experiments/R4_corrections.md", + "prereg_sha256": "78dd3d8e4a0ebdd3a4d2b77fac02abf8fdac02f9b645e1f5708cddc6a48c88c0", + "status": "PRE-REGISTERED", + "dry_run": false, + "findings_closed": [ + "M-3 corpus default leak", + "M-4 null agreement gate", + "F-11 operators absent from ARTIFACTS.md" + ], + "n_sidecars": 166, + "n_corpus_backfilled": 166, + "corpus_recorded_to_true": { + "wikitext -> Github": 36, + "wikitext -> Pile-CC": 30, + "wikitext -> StackExchange": 36, + "wikitext -> USPTO_Backgrounds": 30, + "wikitext -> Wikipedia_en": 34 + }, + "anchor_agreement_gate_states_before": { + "None": 166 + }, + "n_operators_added_to_ARTIFACTS_md": 166, + "artifacts_md_stale_line_removed": false, + "hash_verification": { + "ran": true, + "n_checked": 166, + "n_mismatched": 0, + "mismatches": [], + "missing_pt": [] + }, + "H3_defect": "repro/30_repo_health.sh H3 tested `os.path.basename(p) not in led` -- a SUBSTRING match of the filename against the ledger text. A file named in the ledger with the WRONG hash passed. Fixed to compare the hash.", + "provenance": { + "schema_version": 1, + "experiment": "R4g", + "utc": "2026-08-20T05:42:49+00:00", + "script": { + "path": "tools/r4g_e28_provenance.py", + "sha256": "5a60a33e6ec0291925cc547efad8a64e2128ddfd384790f413ce2a47875f30f8", + "n_lines": 154 + }, + "git": { + "commit": "a4fc687ed7027ca729606455391b9463310911e8", + "commit_short": "a4fc687ed702", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "README.md", + "RTIFACTS.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t52_factorial.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "repro/30_repo_health.sh", + "results/d2_device_crossvalidation.json", + "results/e28_Github_410m_n100_s0_provenance.json", + "results/e28_Github_410m_n100_s1_provenance.json", + "results/e28_Github_410m_n100_s2_provenance.json", + "results/e28_Github_410m_n200_s0_provenance.json", + "results/e28_Github_410m_n200_s1_provenance.json", + "results/e28_Github_410m_n200_s2_provenance.json", + "results/e28_Github_410m_n25_s0_provenance.json", + "results/e28_Github_410m_n25_s1_provenance.json", + "results/e28_Github_410m_n25_s2_provenance.json", + "results/e28_Github_410m_n400_s0_provenance.json", + "results/e28_Github_410m_n400_s1_provenance.json", + "results/e28_Github_410m_n400_s2_provenance.json", + "results/e28_Github_410m_n50_s0_provenance.json", + "results/e28_Github_410m_n50_s1_provenance.json", + "results/e28_Github_410m_n50_s2_provenance.json", + "results/e28_Github_410m_n800_s0_provenance.json", + "results/e28_Github_410m_n800_s1_provenance.json", + "results/e28_Github_410m_n800_s2_provenance.json", + "results/e28_Github_70m_n100_s0_provenance.json", + "results/e28_Github_70m_n100_s1_provenance.json", + "results/e28_Github_70m_n100_s2_provenance.json", + "results/e28_Github_70m_n200_s0_provenance.json", + "results/e28_Github_70m_n200_s1_provenance.json", + "results/e28_Github_70m_n200_s2_provenance.json", + "results/e28_Github_70m_n25_s0_provenance.json", + "results/e28_Github_70m_n25_s1_provenance.json", + "results/e28_Github_70m_n25_s2_provenance.json", + "results/e28_Github_70m_n400_s0_provenance.json", + "results/e28_Github_70m_n400_s1_provenance.json", + "results/e28_Github_70m_n400_s2_provenance.json", + "results/e28_Github_70m_n50_s0_provenance.json", + "results/e28_Github_70m_n50_s1_provenance.json", + "results/e28_Github_70m_n50_s2_provenance.json", + "results/e28_Github_70m_n800_s0_provenance.json", + "results/e28_Github_70m_n800_s1_provenance.json", + "results/e28_Github_70m_n800_s2_provenance.json", + "results/e28_Pile-CC_410m_n100_s0_provenance.json", + "results/e28_Pile-CC_410m_n100_s1_provenance.json", + "results/e28_Pile-CC_410m_n100_s2_provenance.json", + "results/e28_Pile-CC_410m_n200_s0_provenance.json", + "results/e28_Pile-CC_410m_n200_s1_provenance.json", + "results/e28_Pile-CC_410m_n200_s2_provenance.json", + "results/e28_Pile-CC_410m_n25_s0_provenance.json" + ], + "dirty_diff_sha256": "a2c9d667571dd519842e57bc21763a77586571bbfbc3965d1d6bae4e5d03816f", + "describe": "a4fc687-dirty" + }, + "argv": [ + "tools/r4g_e28_provenance.py", + "--verify-hashes" + ], + "cwd": ".", + "inputs": [], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "3354827742fb3744881fdc4abb18d1b2f9dfc012e0d8dab0883e3ef38c72e2f1" + } +} \ No newline at end of file diff --git a/results/r5_corpus_axis_uncertainty.json b/results/r5_corpus_axis_uncertainty.json new file mode 100644 index 0000000000000000000000000000000000000000..646b3c3549e6006dd88912eafe810c206eecbeee --- /dev/null +++ b/results/r5_corpus_axis_uncertainty.json @@ -0,0 +1,1031 @@ +{ + "experiment": "R5 \u2014 corpus-axis uncertainty, stated as such", + "prereg": "specs/experiments/R5_corpus_axis_uncertainty.md", + "prereg_sha256": "d75f281093c237e17821563ad98d4da9f75858234806aac749d5e2b1d9bd0bfd", + "status": "PRE-REGISTERED", + "recomputes_not_remeasures": true, + "readout_convention": "UNSTRIPPED (legacy)", + "decision_rule_verbatim": "Descriptive; no branch. The deliverable is that the paper stops printing one interval whose unit is not the unit the claim is about.", + "PRIMARY": { + "what": "the range of fit_pct over all 28 leave-two-out splits, beside the seed bootstrap", + "persist_l2o_fit_pct_range": [ + 78.22601673224129, + 94.57279616379387 + ], + "min_l2o_fit_pct_range": [ + 42.0114078749595, + 81.17827115221291 + ], + "CAPTION": "the leave-two-out range is CORPUS-axis uncertainty on a panel of eight chosen corpora; the bootstrap interval is SEED-axis uncertainty with those eight held fixed. They are not comparable and must never be printed as though one bounded the other." + }, + "declared_bias": "n = 8 corpora, chosen not sampled. Leave-k-out on a chosen panel measures sensitivity to THESE corpora, not sampling error over a population of corpora. No resampling of this panel can produce the latter.", + "controls": { + "C1_full_panel_recovered": { + "required": "the full-panel fit_pct must match the stored E57 point estimate to <= 1e-9", + "stored": 91.21993064640641, + "recomputed": 91.21993064640641, + "fires": true + }, + "C2_leave_k_out_counts_reproduce": { + "required": "8/8 LOO and 28/28 L2O under persist; 27/28 under min", + "observed": { + "persist_loo": "8/8", + "persist_l2o": "28/28", + "min_loo": "8/8", + "min_l2o": "27/28" + }, + "fires": true + } + }, + "by_aggregation": { + "persist": { + "full_panel": { + "fit_pct": 91.21993064640641, + "read_pct": 7.133265187892407, + "resid_pct": 1.6468041657011878, + "SS_row": 0.00289945925135949, + "SS_col": 0.00022673347364850273, + "SS_total": 0.003178537004811584 + }, + "leave_one_corpus_out": { + "per_split": { + "drop_Wikipedia_en": { + "fit_pct": 92.62271892860798, + "read_pct": 6.044474474909023, + "resid_pct": 1.332806596483, + "SS_row": 0.0024885033026064893, + "SS_col": 0.0001623974643297355, + "SS_total": 0.0026867094071429554 + }, + "drop_USPTO_Backgrounds": { + "fit_pct": 90.93746718003844, + "read_pct": 6.993844859630081, + "resid_pct": 2.0686879603314856, + "SS_row": 0.0020403358046330276, + "SS_col": 0.0001569187324175249, + "SS_total": 0.002243669048527117 + }, + "drop_Pile-CC": { + "fit_pct": 90.84291026481041, + "read_pct": 7.2900814225666135, + "resid_pct": 1.867008312622964, + "SS_row": 0.0020915322832429457, + "SS_col": 0.00016784403536083202, + "SS_total": 0.0023023616010826296 + }, + "drop_StackExchange": { + "fit_pct": 93.33692623435269, + "read_pct": 5.156511202775666, + "resid_pct": 1.5065625628716504, + "SS_row": 0.0023076398735639276, + "SS_col": 0.0001274883515033158, + "SS_total": 0.002472376118075549 + }, + "drop_Github": { + "fit_pct": 81.04881620398683, + "read_pct": 15.567785798307998, + "resid_pct": 3.383397997705165, + "SS_row": 0.0008595795701408636, + "SS_col": 0.00016510729275644095, + "SS_total": 0.0010605701728911624 + }, + "drop_OOD_News_2024": { + "fit_pct": 92.43636904450099, + "read_pct": 6.2773435795102195, + "resid_pct": 1.2862873759887863, + "SS_row": 0.0024427297841783774, + "SS_col": 0.00016588550897978752, + "SS_total": 0.0026426068109646227 + }, + "drop_OOD_arXiv_2023": { + "fit_pct": 91.1983253417951, + "read_pct": 7.343503025106673, + "resid_pct": 1.4581716330982242, + "SS_row": 0.0025882890861403246, + "SS_col": 0.00020841510699551552, + "SS_total": 0.0028380883930048907 + }, + "drop_OOD_CommonPile": { + "fit_pct": 91.10865204439243, + "read_pct": 7.5957558760329, + "resid_pct": 1.2955920795746683, + "SS_row": 0.0025575773760413197, + "SS_col": 0.00021322599935963315, + "SS_total": 0.002807172884958442 + } + }, + "fit_pct": { + "n": 8, + "min": 81.04881620398683, + "max": 93.33692623435269, + "range": 12.288110030365857, + "median": 91.15348869309376, + "mean": 90.4415231553106, + "sd": 3.653221362340598, + "values_sorted": [ + 81.04881620398683, + 90.84291026481041, + 90.93746718003844, + 91.10865204439243, + 91.1983253417951, + 92.43636904450099, + 92.62271892860798, + 93.33692623435269 + ] + }, + "read_pct": { + "n": 8, + "min": 5.156511202775666, + "max": 15.567785798307998, + "range": 10.411274595532333, + "median": 7.141963141098348, + "mean": 7.7836625298548965, + "sd": 3.039012063696094, + "values_sorted": [ + 5.156511202775666, + 6.044474474909023, + 6.2773435795102195, + 6.993844859630081, + 7.2900814225666135, + 7.343503025106673, + 7.5957558760329, + 15.567785798307998 + ] + } + }, + "leave_two_corpora_out": { + "per_split": { + "drop_Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 92.90422389516985, + "read_pct": 5.441867087941937, + "resid_pct": 1.6539090168882127, + "SS_row": 0.0017211943343555555, + "SS_col": 0.00010081899839829131, + "SS_total": 0.0018526545534654011 + }, + "drop_Wikipedia_en+Pile-CC": { + "fit_pct": 92.78940527673387, + "read_pct": 5.805123560121626, + "resid_pct": 1.4054711631444974, + "SS_row": 0.001781297246852905, + "SS_col": 0.00011144214778018857, + "SS_total": 0.0019197205128542293 + }, + "drop_Wikipedia_en+StackExchange": { + "fit_pct": 94.49208659572241, + "read_pct": 4.277500631331475, + "resid_pct": 1.230412772946112, + "SS_row": 0.0019330119745323854, + "SS_col": 8.750425817994251e-05, + "SS_total": 0.0020456866222064054 + }, + "drop_Wikipedia_en+Github": { + "fit_pct": 82.09091532665984, + "read_pct": 14.917080642721988, + "resid_pct": 2.992004030618184, + "SS_row": 0.0006407085118244404, + "SS_col": 0.00011642580060573012, + "SS_total": 0.0007804864999676329 + }, + "drop_Wikipedia_en+OOD_News_2024": { + "fit_pct": 93.9502366304117, + "read_pct": 5.198166807781096, + "resid_pct": 0.8515965618072017, + "SS_row": 0.00202186125486693, + "SS_col": 0.00011186743580362448, + "SS_total": 0.002152055521499829 + }, + "drop_Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 92.50118296926041, + "read_pct": 6.285402740424156, + "resid_pct": 1.2134142903154328, + "SS_row": 0.002182121513343729, + "SS_col": 0.00014827391498838517, + "SS_total": 0.002359020115525314 + }, + "drop_Wikipedia_en+OOD_CommonPile": { + "fit_pct": 92.64058010271503, + "read_pct": 6.302906156312658, + "resid_pct": 1.0565137409723027, + "SS_row": 0.002151280366481146, + "SS_col": 0.00014636478151167342, + "SS_total": 0.002322179291295368 + }, + "drop_USPTO_Backgrounds+Pile-CC": { + "fit_pct": 90.14600839627873, + "read_pct": 7.155713139154008, + "resid_pct": 2.6982784645672604, + "SS_row": 0.0012274882666364572, + "SS_col": 9.74369700221853e-05, + "SS_total": 0.0013616667986456608 + }, + "drop_USPTO_Backgrounds+StackExchange": { + "fit_pct": 91.91641433529908, + "read_pct": 6.028536093227016, + "resid_pct": 2.0550495714739028, + "SS_row": 0.0014579572407595874, + "SS_col": 9.56232672027267e-05, + "SS_total": 0.0015861772364630649 + }, + "drop_USPTO_Backgrounds+Github": { + "fit_pct": 78.67789669144574, + "read_pct": 16.686938451159083, + "resid_pct": 4.63516485739518, + "SS_row": 0.0005220570970444435, + "SS_col": 0.00011072404083876044, + "SS_total": 0.0006635371800695381 + }, + "drop_USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 92.8584738297847, + "read_pct": 5.603648447513692, + "resid_pct": 1.537877722701608, + "SS_row": 0.0016922531784223482, + "SS_col": 0.00010212091050999871, + "SS_total": 0.0018224003783697247 + }, + "drop_USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 90.83454709308151, + "read_pct": 7.262154654564543, + "resid_pct": 1.9032982523539506, + "SS_row": 0.001790386123741724, + "SS_col": 0.00014314004239681012, + "SS_total": 0.0019710409541724796 + }, + "drop_USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 90.88325953331656, + "read_pct": 7.4767248885812, + "resid_pct": 1.6400155781022396, + "SS_row": 0.001783486192421091, + "SS_col": 0.0001467226821725886, + "SS_total": 0.0019623924159182355 + }, + "drop_Pile-CC+StackExchange": { + "fit_pct": 92.30547577664414, + "read_pct": 5.759735551025383, + "resid_pct": 1.934788672330485, + "SS_row": 0.001477096540267815, + "SS_col": 9.216880562821185e-05, + "SS_total": 0.0016002263439300335 + }, + "drop_Pile-CC+Github": { + "fit_pct": 78.22601673224129, + "read_pct": 17.505815035390924, + "resid_pct": 4.268168232367786, + "SS_row": 0.0005395605936948367, + "SS_col": 0.00012074560802371178, + "SS_total": 0.0006897457089521647 + }, + "drop_Pile-CC+OOD_News_2024": { + "fit_pct": 92.69565552535016, + "read_pct": 6.039823374018541, + "resid_pct": 1.2645211006312875, + "SS_row": 0.0017605394987247212, + "SS_col": 0.00011471247012620215, + "SS_total": 0.0018992686213252501 + }, + "drop_Pile-CC+OOD_arXiv_2023": { + "fit_pct": 90.71979950869466, + "read_pct": 7.634466926636535, + "resid_pct": 1.6457335646687892, + "SS_row": 0.0018332855113547715, + "SS_col": 0.00015427897415247864, + "SS_total": 0.0020208218286230533 + }, + "drop_Pile-CC+OOD_CommonPile": { + "fit_pct": 90.62675022807012, + "read_pct": 7.8865495143733275, + "resid_pct": 1.4867002575565618, + "SS_row": 0.0018149944424635188, + "SS_col": 0.00015794501626482704, + "SS_total": 0.002002713810101242 + }, + "drop_StackExchange+Github": { + "fit_pct": 89.32817149258182, + "read_pct": 7.622681009904747, + "resid_pct": 3.049147497513449, + "SS_row": 0.0006863786026612127, + "SS_col": 5.857105381973562e-05, + "SS_total": 0.000768378655011664 + }, + "drop_StackExchange+OOD_News_2024": { + "fit_pct": 94.57279616379387, + "read_pct": 4.319917957104046, + "resid_pct": 1.1072858791020976, + "SS_row": 0.0019068957767913943, + "SS_col": 8.710362432575191e-05, + "SS_total": 0.002016325893007093 + }, + "drop_StackExchange+OOD_arXiv_2023": { + "fit_pct": 93.87631217339211, + "read_pct": 4.839335951027906, + "resid_pct": 1.2843518755799832, + "SS_row": 0.0020301025164910255, + "SS_col": 0.00010465204549345105, + "SS_total": 0.0021625290443252295 + }, + "drop_StackExchange+OOD_CommonPile": { + "fit_pct": 93.76170317509077, + "read_pct": 5.2406725818332065, + "resid_pct": 0.9976242430760142, + "SS_row": 0.0020183906687347942, + "SS_col": 0.00011281497966513646, + "SS_total": 0.0021526813191155964 + }, + "drop_Github+OOD_News_2024": { + "fit_pct": 80.24195935005882, + "read_pct": 16.67283677372139, + "resid_pct": 3.085203876219787, + "SS_row": 0.0005707159589394428, + "SS_col": 0.00011858451743487034, + "SS_total": 0.000711243797586835 + }, + "drop_Github+OOD_arXiv_2023": { + "fit_pct": 79.78959397518167, + "read_pct": 17.096919581278616, + "resid_pct": 3.1134864435397085, + "SS_row": 0.0006924657713719666, + "SS_col": 0.00014837814075876956, + "SS_total": 0.0008678647639031176 + }, + "drop_Github+OOD_CommonPile": { + "fit_pct": 78.33153134124592, + "read_pct": 19.179427920928127, + "resid_pct": 2.4890407378259503, + "SS_row": 0.0006341048448405616, + "SS_col": 0.00015526018651351216, + "SS_total": 0.0008095141687938252 + }, + "drop_OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 92.42390828541038, + "read_pct": 6.452502495109725, + "resid_pct": 1.1235892194799069, + "SS_row": 0.00213941040480709, + "SS_col": 0.00014936125544975012, + "SS_total": 0.002314780281959585 + }, + "drop_OOD_News_2024+OOD_CommonPile": { + "fit_pct": 92.21951857698689, + "read_pct": 6.671225766742414, + "resid_pct": 1.1092556562707094, + "SS_row": 0.0021163641531827472, + "SS_col": 0.0001530992927352586, + "SS_total": 0.002294919975553722 + }, + "drop_OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 91.13337272113088, + "read_pct": 7.871049745782995, + "resid_pct": 0.9955775330861203, + "SS_row": 0.0022454280269441603, + "SS_col": 0.0001939341777104541, + "SS_total": 0.0024638921614535156 + } + }, + "fit_pct": { + "n": 28, + "min": 78.22601673224129, + "max": 94.57279616379387, + "range": 16.346779431552577, + "median": 92.06796645614298, + "mean": 89.53349270363404, + "sd": 5.390424775410946, + "values_sorted": [ + 78.22601673224129, + 78.33153134124592, + 78.67789669144574, + 79.78959397518167, + 80.24195935005882, + 82.09091532665984, + 89.32817149258182, + 90.14600839627873, + 90.62675022807012, + 90.71979950869466, + 90.83454709308151, + 90.88325953331656, + 91.13337272113088, + 91.91641433529908, + 92.21951857698689, + 92.30547577664414, + 92.42390828541038, + 92.50118296926041, + 92.64058010271503, + 92.69565552535016, + 92.78940527673387, + 92.8584738297847, + 92.90422389516985, + 93.76170317509077, + 93.87631217339211, + 93.9502366304117, + 94.49208659572241, + 94.57279616379387 + ] + }, + "read_pct": { + "n": 28, + "min": 4.277500631331475, + "max": 19.179427920928127, + "range": 14.901927289596653, + "median": 6.56186413092607, + "mean": 8.54409726734687, + "sd": 4.562481059045396, + "values_sorted": [ + 4.277500631331475, + 4.319917957104046, + 4.839335951027906, + 5.198166807781096, + 5.2406725818332065, + 5.441867087941937, + 5.603648447513692, + 5.759735551025383, + 5.805123560121626, + 6.028536093227016, + 6.039823374018541, + 6.285402740424156, + 6.302906156312658, + 6.452502495109725, + 6.671225766742414, + 7.155713139154008, + 7.262154654564543, + 7.4767248885812, + 7.622681009904747, + 7.634466926636535, + 7.871049745782995, + 7.8865495143733275, + 14.917080642721988, + 16.67283677372139, + 16.686938451159083, + 17.096919581278616, + 17.505815035390924, + 19.179427920928127 + ] + } + }, + "n_loo_preserving_ordering": 8, + "n_l2o_preserving_ordering": 28, + "l2o_splits_that_REVERSE": {}, + "corpus_random_effect_on_the_fit_axis": { + "MS_fit": 0.0004142084644799272, + "df_fit": 7, + "MS_residual": 1.0682506082365496e-06, + "df_residual": 49, + "variance_component": 5.164252673396133e-05, + "CAPTION_THAT_MUST_TRAVEL_WITH_IT": "estimated on 8 corpora that were CHOSEN, not sampled, on 7 degrees of freedom. This is a description of one panel dressed in the notation of an inference about a population of corpora. It is reported so the weakness is visible, not because it licenses a population claim." + }, + "seed_bootstrap_for_comparison": { + "fit_pct": { + "point": 91.21993064640641, + "ci_lo": 87.60889894657397, + "ci_hi": 92.17106192216394 + }, + "read_pct": { + "point": 7.133265187892407, + "ci_lo": 6.310765498639046, + "ci_hi": 9.75205779444106 + }, + "UNIT": "seed factors resampled; the eight corpora HELD FIXED" + } + }, + "min": { + "full_panel": { + "fit_pct": 73.41700738536099, + "read_pct": 22.67267017800786, + "resid_pct": 3.910322436631158, + "SS_row": 0.009152380847464421, + "SS_col": 0.0028264419878745047, + "SS_total": 0.012466295172485285 + }, + "leave_one_corpus_out": { + "per_split": { + "drop_Wikipedia_en": { + "fit_pct": 72.21409286150445, + "read_pct": 23.71515132910453, + "resid_pct": 4.070755809391016, + "SS_row": 0.0071385143231368, + "SS_col": 0.0023442923774289245, + "SS_total": 0.009885209438035003 + }, + "drop_USPTO_Backgrounds": { + "fit_pct": 69.97297180223329, + "read_pct": 24.704987971694027, + "resid_pct": 5.322040226072688, + "SS_row": 0.005290266294461959, + "SS_col": 0.001867806408753525, + "SS_total": 0.007560442494016115 + }, + "drop_Pile-CC": { + "fit_pct": 75.22745446862925, + "read_pct": 20.50190993884228, + "resid_pct": 4.270635592528474, + "SS_row": 0.007679731316003274, + "SS_col": 0.002092974711258694, + "SS_total": 0.010208681617966769 + }, + "drop_StackExchange": { + "fit_pct": 74.77416486260654, + "read_pct": 22.702208386947053, + "resid_pct": 2.5236267504464065, + "SS_row": 0.007417306082931756, + "SS_col": 0.002251970699691434, + "SS_total": 0.009919610732611528 + }, + "drop_Github": { + "fit_pct": 58.596571037346294, + "read_pct": 34.84817420406899, + "resid_pct": 6.555254758584715, + "SS_row": 0.0029121274949246135, + "SS_col": 0.001731881651964116, + "SS_total": 0.004969791650553375 + }, + "drop_OOD_News_2024": { + "fit_pct": 75.8043220820044, + "read_pct": 20.54338347833448, + "resid_pct": 3.6522944396611114, + "SS_row": 0.007855734051337857, + "SS_col": 0.002128946644306913, + "SS_total": 0.010363174335679168 + }, + "drop_OOD_arXiv_2023": { + "fit_pct": 74.1434371562919, + "read_pct": 22.097545670856885, + "resid_pct": 3.759017172851229, + "SS_row": 0.007821070837958752, + "SS_col": 0.0023309746171126114, + "SS_total": 0.010548567935247184 + }, + "drop_OOD_CommonPile": { + "fit_pct": 77.20909139763098, + "read_pct": 20.313039456100714, + "resid_pct": 2.4778691462682962, + "SS_row": 0.008436368390983178, + "SS_col": 0.0022195350429612785, + "SS_total": 0.01092665156171237 + } + }, + "fit_pct": { + "n": 8, + "min": 58.596571037346294, + "max": 77.20909139763098, + "range": 18.61252036028469, + "median": 74.45880100944922, + "mean": 72.24276320853089, + "sd": 5.563827266245453, + "values_sorted": [ + 58.596571037346294, + 69.97297180223329, + 72.21409286150445, + 74.1434371562919, + 74.77416486260654, + 75.22745446862925, + 75.8043220820044, + 77.20909139763098 + ] + }, + "read_pct": { + "n": 8, + "min": 20.313039456100714, + "max": 34.84817420406899, + "range": 14.535134747968275, + "median": 22.39987702890197, + "mean": 23.67830005449362, + "sd": 4.479330432376851, + "values_sorted": [ + 20.313039456100714, + 20.50190993884228, + 20.54338347833448, + 22.097545670856885, + 22.702208386947053, + 23.71515132910453, + 24.704987971694027, + 34.84817420406899 + ] + } + }, + "leave_two_corpora_out": { + "per_split": { + "drop_Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 69.70911059376193, + "read_pct": 24.806620056599378, + "resid_pct": 5.484269349638686, + "SS_row": 0.004067999610340125, + "SS_col": 0.0014476346042081366, + "SS_total": 0.005835678544296559 + }, + "drop_Wikipedia_en+Pile-CC": { + "fit_pct": 74.27637442756014, + "read_pct": 21.35911579521616, + "resid_pct": 4.364509777223703, + "SS_row": 0.005858390808857898, + "SS_col": 0.0016846547589915292, + "SS_total": 0.007887286979214956 + }, + "drop_Wikipedia_en+StackExchange": { + "fit_pct": 73.4016665053439, + "read_pct": 24.123595386290692, + "resid_pct": 2.4747381083654205, + "SS_row": 0.005696459960214043, + "SS_col": 0.0018721522515351143, + "SS_total": 0.007760668430871827 + }, + "drop_Wikipedia_en+Github": { + "fit_pct": 47.1372449404656, + "read_pct": 43.845072157538624, + "resid_pct": 9.017682901995776, + "SS_row": 0.0015293056720319259, + "SS_col": 0.001422495473077824, + "SS_total": 0.003244367959908223 + }, + "drop_Wikipedia_en+OOD_News_2024": { + "fit_pct": 74.58834495659279, + "read_pct": 21.867082182884847, + "resid_pct": 3.5445728605223614, + "SS_row": 0.005830279995148275, + "SS_col": 0.0017092645221895085, + "SS_total": 0.00781660995232063 + }, + "drop_Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 73.1737398117515, + "read_pct": 22.973417055456252, + "resid_pct": 3.852843132792238, + "SS_row": 0.005922873244675989, + "SS_col": 0.0018595282620048979, + "SS_total": 0.008094260673177717 + }, + "drop_Wikipedia_en+OOD_CommonPile": { + "fit_pct": 76.65092905025313, + "read_pct": 20.941908114529095, + "resid_pct": 2.40716283521778, + "SS_row": 0.006524133582243283, + "SS_col": 0.001782467710426286, + "SS_total": 0.008511486635688388 + }, + "drop_USPTO_Backgrounds+Pile-CC": { + "fit_pct": 72.42964843924064, + "read_pct": 21.054112387579057, + "resid_pct": 6.516239173180304, + "SS_row": 0.004030064808796127, + "SS_col": 0.0011714738265614939, + "SS_total": 0.005564109305565447 + }, + "drop_USPTO_Backgrounds+StackExchange": { + "fit_pct": 69.88418071540721, + "read_pct": 26.35581157270119, + "resid_pct": 3.7600077118916078, + "SS_row": 0.00386509174056336, + "SS_col": 0.0014576636455156016, + "SS_total": 0.005530710528471906 + }, + "drop_USPTO_Backgrounds+Github": { + "fit_pct": 42.0114078749595, + "read_pct": 47.61343085472334, + "resid_pct": 10.375161270317161, + "SS_row": 0.0010091119318196653, + "SS_col": 0.0011436722457237635, + "SS_total": 0.002401995036260465 + }, + "drop_USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 74.93567784036671, + "read_pct": 19.95982228631515, + "resid_pct": 5.104499873318141, + "SS_row": 0.004446290013155272, + "SS_col": 0.0011843111459544351, + "SS_total": 0.00593347540356821 + }, + "drop_USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 70.82400576139757, + "read_pct": 24.290616152385706, + "resid_pct": 4.8853780862167255, + "SS_row": 0.00418883721289548, + "SS_col": 0.0014366518212209274, + "SS_total": 0.005914431368097784 + }, + "drop_USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 72.08814955375458, + "read_pct": 24.241553046524665, + "resid_pct": 3.670297399720758, + "SS_row": 0.004530780056367145, + "SS_col": 0.0015235950119188446, + "SS_total": 0.00628505528913409 + }, + "drop_Pile-CC+StackExchange": { + "fit_pct": 75.2683107770375, + "read_pct": 21.959603367863302, + "resid_pct": 2.7720858550991916, + "SS_row": 0.0058326455645225156, + "SS_col": 0.0017016800544607997, + "SS_total": 0.007749138388132806 + }, + "drop_Pile-CC+Github": { + "fit_pct": 61.13057575432661, + "read_pct": 32.15027784254386, + "resid_pct": 6.71914640312953, + "SS_row": 0.0025062498718950517, + "SS_col": 0.0013181068349182516, + "SS_total": 0.004099830307450798 + }, + "drop_Pile-CC+OOD_News_2024": { + "fit_pct": 78.70428343626931, + "read_pct": 17.49805926335739, + "resid_pct": 3.7976573003732943, + "SS_row": 0.006481387596668556, + "SS_col": 0.0014409851576519452, + "SS_total": 0.008235114168743878 + }, + "drop_Pile-CC+OOD_arXiv_2023": { + "fit_pct": 76.32740589735464, + "read_pct": 19.434271366595024, + "resid_pct": 4.238322736050335, + "SS_row": 0.006354478333607439, + "SS_col": 0.0016179595634961254, + "SS_total": 0.008325290580624426 + }, + "drop_Pile-CC+OOD_CommonPile": { + "fit_pct": 79.0793245863189, + "read_pct": 18.16182035962983, + "resid_pct": 2.758855054051259, + "SS_row": 0.007016026259087557, + "SS_col": 0.0016113416398353864, + "SS_total": 0.008872137307431381 + }, + "drop_StackExchange+Github": { + "fit_pct": 64.19576519470036, + "read_pct": 33.101664859298566, + "resid_pct": 2.7025699460010824, + "SS_row": 0.0021025909615042726, + "SS_col": 0.0010841721589082403, + "SS_total": 0.003275279849266211 + }, + "drop_StackExchange+OOD_News_2024": { + "fit_pct": 75.68119691496648, + "read_pct": 21.501457192553758, + "resid_pct": 2.8173458924797576, + "SS_row": 0.006201778136442006, + "SS_col": 0.0017619603356465053, + "SS_total": 0.008194608950767745 + }, + "drop_StackExchange+OOD_arXiv_2023": { + "fit_pct": 75.66308318285165, + "read_pct": 21.88171045120262, + "resid_pct": 2.4552063659457226, + "SS_row": 0.006192287226352423, + "SS_col": 0.0017908051115267483, + "SS_total": 0.0081840270920335 + }, + "drop_StackExchange+OOD_CommonPile": { + "fit_pct": 81.17827115221291, + "read_pct": 17.532211187475802, + "resid_pct": 1.2895176603112795, + "SS_row": 0.006853207838537141, + "SS_col": 0.0014800991131186758, + "SS_total": 0.008442170227654968 + }, + "drop_Github+OOD_News_2024": { + "fit_pct": 60.09234275665318, + "read_pct": 33.53263046705697, + "resid_pct": 6.37502677628985, + "SS_row": 0.002251606600654257, + "SS_col": 0.001256437819418618, + "SS_total": 0.0037469109995798395 + }, + "drop_Github+OOD_arXiv_2023": { + "fit_pct": 59.36821556956008, + "read_pct": 34.438925881511416, + "resid_pct": 6.192858548928506, + "SS_row": 0.0025118638569241857, + "SS_col": 0.001457107854146347, + "SS_total": 0.004230990998847026 + }, + "drop_Github+OOD_CommonPile": { + "fit_pct": 67.95813838086735, + "read_pct": 28.340415884299983, + "resid_pct": 3.7014457348326655, + "SS_row": 0.0026044529176717747, + "SS_col": 0.0010861286167703074, + "SS_total": 0.003832437114559073 + }, + "drop_OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 76.94110075840524, + "read_pct": 19.531209121719506, + "resid_pct": 3.527690119875254, + "SS_row": 0.006538872835323175, + "SS_col": 0.0016598682824676939, + "SS_total": 0.008498543393413632 + }, + "drop_OOD_News_2024+OOD_CommonPile": { + "fit_pct": 79.70567974665822, + "read_pct": 18.245451187320302, + "resid_pct": 2.048869066021478, + "SS_row": 0.0071568625721961845, + "SS_col": 0.0016382795696669266, + "SS_total": 0.008979112398192986 + }, + "drop_OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 77.40134945672342, + "read_pct": 20.26441408833755, + "resid_pct": 2.3342364549390195, + "SS_row": 0.007058370273471752, + "SS_col": 0.001847948892550211, + "SS_total": 0.009119182447094441 + } + }, + "fit_pct": { + "n": 28, + "min": 42.0114078749595, + "max": 81.17827115221291, + "range": 39.16686327725341, + "median": 73.83902046645201, + "mean": 70.70734014413432, + "sd": 9.205683277035039, + "values_sorted": [ + 42.0114078749595, + 47.1372449404656, + 59.36821556956008, + 60.09234275665318, + 61.13057575432661, + 64.19576519470036, + 67.95813838086735, + 69.70911059376193, + 69.88418071540721, + 70.82400576139757, + 72.08814955375458, + 72.42964843924064, + 73.1737398117515, + 73.4016665053439, + 74.27637442756014, + 74.58834495659279, + 74.93567784036671, + 75.2683107770375, + 75.66308318285165, + 75.68119691496648, + 76.32740589735464, + 76.65092905025313, + 76.94110075840524, + 77.40134945672342, + 78.70428343626931, + 79.0793245863189, + 79.70567974665822, + 81.17827115221291 + ] + }, + "read_pct": { + "n": 28, + "min": 17.49805926335739, + "max": 47.61343085472334, + "range": 30.11537159136595, + "median": 21.92065690953296, + "mean": 25.035938556053928, + "sd": 7.472761632017309, + "values_sorted": [ + 17.49805926335739, + 17.532211187475802, + 18.16182035962983, + 18.245451187320302, + 19.434271366595024, + 19.531209121719506, + 19.95982228631515, + 20.26441408833755, + 20.941908114529095, + 21.054112387579057, + 21.35911579521616, + 21.501457192553758, + 21.867082182884847, + 21.88171045120262, + 21.959603367863302, + 22.973417055456252, + 24.123595386290692, + 24.241553046524665, + 24.290616152385706, + 24.806620056599378, + 26.35581157270119, + 28.340415884299983, + 32.15027784254386, + 33.101664859298566, + 33.53263046705697, + 34.438925881511416, + 43.845072157538624, + 47.61343085472334 + ] + } + }, + "n_loo_preserving_ordering": 8, + "n_l2o_preserving_ordering": 27, + "l2o_splits_that_REVERSE": { + "drop_USPTO_Backgrounds+Github": { + "fit_pct": 42.0114078749595, + "read_pct": 47.61343085472334, + "resid_pct": 10.375161270317161, + "SS_row": 0.0010091119318196653, + "SS_col": 0.0011436722457237635, + "SS_total": 0.002401995036260465 + } + }, + "corpus_random_effect_on_the_fit_axis": { + "MS_fit": 0.001307482978209203, + "df_fit": 7, + "MS_residual": 9.948415043803247e-06, + "df_residual": 49, + "variance_component": 0.00016219182039567498, + "CAPTION_THAT_MUST_TRAVEL_WITH_IT": "estimated on 8 corpora that were CHOSEN, not sampled, on 7 degrees of freedom. This is a description of one panel dressed in the notation of an inference about a population of corpora. It is reported so the weakness is visible, not because it licenses a population claim." + }, + "seed_bootstrap_for_comparison": { + "fit_pct": { + "point": 73.41700738536099, + "ci_lo": 68.87079349603536, + "ci_hi": 75.69475005442473 + }, + "read_pct": { + "point": 22.67267017800786, + "ci_lo": 20.29090047208274, + "ci_hi": 26.822293731330955 + }, + "UNIT": "seed factors resampled; the eight corpora HELD FIXED" + } + } + }, + "provenance": { + "schema_version": 1, + "experiment": "R5", + "utc": "2026-08-20T05:45:59+00:00", + "script": { + "path": "tools/r5_corpus_axis_uncertainty.py", + "sha256": "9ac8949d55148e6b16a4bc48bb8ea8e95d923bfe972319966788794199768f1f", + "n_lines": 190 + }, + "git": { + "commit": "33308f9076a119f5d3e8e4f6a0f1d3907ea22ad3", + "commit_short": "33308f9076a1", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/t57_grid_variance_ci.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/r5_corpus_axis_uncertainty.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "51113a70b10d0655b1e037907807bdab894646a318dccce734ad5fad2314884a", + "describe": "33308f9-dirty" + }, + "argv": [ + "tools/r5_corpus_axis_uncertainty.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e57_factorial_cells_410m.json", + "sha256": "4c3b428670ba12195b4144d580ee0303426331207f60c34f6185a7cabecf8375", + "bytes": 82248, + "exists": true + }, + { + "path": "results/e57_grid_variance_ci.json", + "sha256": "7f2a2d4765420bba7e290c003d95d893efbb4b57169afd345de8119f2a3d1956", + "bytes": 6880, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + } + }, + "payload_sha256": "4d5d34a83852139193afe4e4b4566b58e10fa05a32823f2bffb99afdb579850a" + } +} \ No newline at end of file diff --git a/results/r5_corpus_axis_uncertainty_rstrip.json b/results/r5_corpus_axis_uncertainty_rstrip.json new file mode 100644 index 0000000000000000000000000000000000000000..4773ca43378ea23cf92683f44e8dc333b948088b --- /dev/null +++ b/results/r5_corpus_axis_uncertainty_rstrip.json @@ -0,0 +1,1260 @@ +{ + "experiment": "R5 \u2014 corpus-axis uncertainty, stated as such", + "prereg": "docs/experiments/preregs/R5_corpus_axis_uncertainty.md", + "prereg_sha256": "f9381134f218d41f48f19fec7451f481cca92d2d0fbf499fe49ac8282af27762", + "status": "PRE-REGISTERED", + "recomputes_not_remeasures": true, + "readout_convention": "STRIPPED (anchor rule)", + "decision_rule_verbatim": "Descriptive; no branch. The deliverable is that the paper stops printing one interval whose unit is not the unit the claim is about.", + "PRIMARY": { + "what": "the range of fit_pct over all 28 leave-two-out splits, beside the seed bootstrap", + "persist_l2o_fit_pct_range": [ + 42.08107123390732, + 70.3751011932439 + ], + "min_l2o_fit_pct_range": [ + 21.009005511431113, + 63.40419715517061 + ], + "CAPTION": "the leave-two-out range is CORPUS-axis uncertainty on a panel of eight chosen corpora; the bootstrap interval is SEED-axis uncertainty with those eight held fixed. They are not comparable and must never be printed as though one bounded the other." + }, + "declared_bias": "n = 8 corpora, chosen not sampled. Leave-k-out on a chosen panel measures sensitivity to THESE corpora, not sampling error over a population of corpora. No resampling of this panel can produce the latter.", + "leave_k_out_ordering_collapse": { + "what": "how often the fit axis still beats the read axis when corpora are dropped. Published, at the unstripped readout: 8/8 LOO and 28/28 L2O under persist, 27/28 under min.", + "persist": { + "loo": "6/8", + "l2o": "16/28", + "reversing_splits": [ + "drop_Github+OOD_News_2024", + "drop_Github+OOD_arXiv_2023", + "drop_OOD_News_2024+OOD_arXiv_2023", + "drop_Pile-CC+Github", + "drop_Pile-CC+OOD_CommonPile", + "drop_Pile-CC+OOD_News_2024", + "drop_Pile-CC+OOD_arXiv_2023", + "drop_Pile-CC+StackExchange", + "drop_USPTO_Backgrounds+Github", + "drop_USPTO_Backgrounds+OOD_arXiv_2023", + "drop_USPTO_Backgrounds+Pile-CC", + "drop_USPTO_Backgrounds+StackExchange" + ] + }, + "min": { + "loo": "6/8", + "l2o": "20/28", + "reversing_splits": [ + "drop_Github+OOD_CommonPile", + "drop_Github+OOD_News_2024", + "drop_Github+OOD_arXiv_2023", + "drop_Pile-CC+Github", + "drop_StackExchange+Github", + "drop_USPTO_Backgrounds+Github", + "drop_Wikipedia_en+Github", + "drop_Wikipedia_en+StackExchange" + ] + } + }, + "controls": { + "C1_full_panel_recovered": { + "required": "the full-panel fit_pct must match the stored E57 point estimate to <= 1e-9", + "stored": 50.697184677169346, + "recomputed": 50.697184677169346, + "fires": true + }, + "C2_leave_k_out_counts_reproduce": { + "applicable": false, + "fires": null, + "required_on_the_unstripped_arm": "8/8 LOO and 28/28 L2O under persist; 27/28 under min", + "observed_here": { + "persist_loo": "6/8", + "persist_l2o": "16/28", + "min_loo": "6/8", + "min_l2o": "20/28" + }, + "why": "this arm is scored at the corrected readout; the stored counts describe the unstripped one. The change in these counts IS the finding, not a control failure -- see leave_k_out_ordering_collapse below." + } + }, + "by_aggregation": { + "persist": { + "full_panel": { + "fit_pct": 50.697184677169346, + "read_pct": 48.13211438274321, + "resid_pct": 1.1707009400874475, + "SS_row": 0.002451851883005098, + "SS_col": 0.0023277982009026798, + "SS_total": 0.004836268322625911 + }, + "leave_one_corpus_out": { + "per_split": { + "drop_Wikipedia_en": { + "fit_pct": 52.373250427828836, + "read_pct": 46.61005338540753, + "resid_pct": 1.016696186763633, + "SS_row": 0.002116650644054738, + "SS_col": 0.0018837326060867608, + "SS_total": 0.004041472749474497 + }, + "drop_USPTO_Backgrounds": { + "fit_pct": 48.90013785418412, + "read_pct": 49.70701457618453, + "resid_pct": 1.3928475696313505, + "SS_row": 0.00155709127122614, + "SS_col": 0.0015827840556622249, + "SS_total": 0.003184226751812537 + }, + "drop_Pile-CC": { + "fit_pct": 47.4662599964337, + "read_pct": 51.23363164594214, + "resid_pct": 1.3001083576241557, + "SS_row": 0.0017854077451573856, + "SS_col": 0.0019271146022475398, + "SS_total": 0.003761424947513306 + }, + "drop_StackExchange": { + "fit_pct": 54.30293560630481, + "read_pct": 44.71750309530783, + "resid_pct": 0.9795612983873632, + "SS_row": 0.0016821625513670025, + "SS_col": 0.0013852309871960446, + "SS_total": 0.003097737778971375 + }, + "drop_Github": { + "fit_pct": 50.83451434240633, + "read_pct": 47.96562247603154, + "resid_pct": 1.199863181562129, + "SS_row": 0.0014594142098569613, + "SS_col": 0.0013770508468841456, + "SS_total": 0.0028709120736883148 + }, + "drop_OOD_News_2024": { + "fit_pct": 50.49955454200501, + "read_pct": 48.12259484369172, + "resid_pct": 1.3778506143032678, + "SS_row": 0.0018665700757229543, + "SS_col": 0.0017787126305571638, + "SS_total": 0.0036962109718618615 + }, + "drop_OOD_arXiv_2023": { + "fit_pct": 50.57242596342314, + "read_pct": 48.44478039653153, + "resid_pct": 0.9827936400453328, + "SS_row": 0.0021019570009871476, + "SS_col": 0.0020135250262548763, + "SS_total": 0.004156330175869757 + }, + "drop_OOD_CommonPile": { + "fit_pct": 50.85297785477784, + "read_pct": 48.11818900866957, + "resid_pct": 1.0288331365525942, + "SS_row": 0.0021639551019274087, + "SS_col": 0.0020475811839017967, + "SS_total": 0.004255316390924187 + } + }, + "fit_pct": { + "n": 8, + "min": 47.4662599964337, + "max": 54.30293560630481, + "range": 6.836675609871108, + "median": 50.70347015291473, + "mean": 50.72525707342047, + "sd": 1.9225743361037082, + "values_sorted": [ + 47.4662599964337, + 48.90013785418412, + 50.49955454200501, + 50.57242596342314, + 50.83451434240633, + 50.85297785477784, + 52.373250427828836, + 54.30293560630481 + ] + }, + "read_pct": { + "n": 8, + "min": 44.71750309530783, + "max": 51.23363164594214, + "range": 6.516128550634306, + "median": 48.120391926180645, + "mean": 48.1149236784708, + "sd": 1.8096739983852268, + "values_sorted": [ + 44.71750309530783, + 46.61005338540753, + 47.96562247603154, + 48.11818900866957, + 48.12259484369172, + 48.44478039653153, + 49.70701457618453, + 51.23363164594214 + ] + } + }, + "leave_two_corpora_out": { + "per_split": { + "drop_Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 52.30067209055684, + "read_pct": 46.47792330335211, + "resid_pct": 1.2214046060910648, + "SS_row": 0.001324378155708848, + "SS_col": 0.0011769322244863602, + "SS_total": 0.0025322392672425553 + }, + "drop_Wikipedia_en+Pile-CC": { + "fit_pct": 50.30922645008709, + "read_pct": 48.56307298661112, + "resid_pct": 1.127700563301794, + "SS_row": 0.0015320454739175871, + "SS_col": 0.0014788706052254992, + "SS_total": 0.0030452574647268004 + }, + "drop_Wikipedia_en+StackExchange": { + "fit_pct": 55.489003458453865, + "read_pct": 43.88542177383025, + "resid_pct": 0.625574767715879, + "SS_row": 0.0014373986894285133, + "SS_col": 0.001136817095480035, + "SS_total": 0.002590420803835003 + }, + "drop_Wikipedia_en+Github": { + "fit_pct": 50.74751292554641, + "read_pct": 48.37431796644749, + "resid_pct": 0.8781691080060962, + "SS_row": 0.0011481634680889563, + "SS_col": 0.0010944698859288232, + "SS_total": 0.0022625019471860033 + }, + "drop_Wikipedia_en+OOD_News_2024": { + "fit_pct": 53.07363642980562, + "read_pct": 45.68002694795497, + "resid_pct": 1.246336622239413, + "SS_row": 0.0015525756399087111, + "SS_col": 0.001336288632936775, + "SS_total": 0.0029253236528499873 + }, + "drop_Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 52.185386350617975, + "read_pct": 47.016680600124765, + "resid_pct": 0.7979330492572599, + "SS_row": 0.001778467012746059, + "SS_col": 0.0016023186057172742, + "SS_total": 0.00340797900929021 + }, + "drop_Wikipedia_en+OOD_CommonPile": { + "fit_pct": 52.8489532185628, + "read_pct": 46.25686799605305, + "resid_pct": 0.894178785384148, + "SS_row": 0.0018377208002726096, + "SS_col": 0.0016084937031818625, + "SS_total": 0.003477307852574692 + }, + "drop_USPTO_Backgrounds+Pile-CC": { + "fit_pct": 42.08107123390732, + "read_pct": 56.08737719584901, + "resid_pct": 1.831551570243672, + "SS_row": 0.0008927460589612373, + "SS_col": 0.0011898885527590996, + "SS_total": 0.0021214908099627854 + }, + "drop_USPTO_Backgrounds+StackExchange": { + "fit_pct": 46.59915642774981, + "read_pct": 52.21732878700266, + "resid_pct": 1.1835147852475376, + "SS_row": 0.0008345315317930185, + "SS_col": 0.0009351458420995555, + "SS_total": 0.001790872616088936 + }, + "drop_USPTO_Backgrounds+Github": { + "fit_pct": 49.077827715914886, + "read_pct": 49.432990888192485, + "resid_pct": 1.4891813958926392, + "SS_row": 0.0008833536059144145, + "SS_col": 0.0008897462007687642, + "SS_total": 0.0017999036367861937 + }, + "drop_USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 52.3829095816429, + "read_pct": 45.86575505849258, + "resid_pct": 1.7513353598645232, + "SS_row": 0.0011710132639116933, + "SS_col": 0.0010253231056039243, + "SS_total": 0.002235487248157869 + }, + "drop_USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 49.03669211778151, + "read_pct": 49.84208112785427, + "resid_pct": 1.121226754364218, + "SS_row": 0.0013372222016183137, + "SS_col": 0.0013591850220838926, + "SS_total": 0.0027269828854002472 + }, + "drop_USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 49.564449013565095, + "read_pct": 49.26922160720974, + "resid_pct": 1.16632937922517, + "SS_row": 0.0013743227673695137, + "SS_col": 0.0013661367034833892, + "SS_total": 0.0027727994454117325 + }, + "drop_Pile-CC+StackExchange": { + "fit_pct": 45.29050803527647, + "read_pct": 53.588997206697115, + "resid_pct": 1.1204947580264173, + "SS_row": 0.0009808707963172088, + "SS_col": 0.001160593789829693, + "SS_total": 0.0021657314940102135 + }, + "drop_Pile-CC+Github": { + "fit_pct": 47.510177227994554, + "read_pct": 51.26384953198994, + "resid_pct": 1.2259732400155092, + "SS_row": 0.0010482680773145135, + "SS_col": 0.0011310893816867363, + "SS_total": 0.002206407423580057 + }, + "drop_Pile-CC+OOD_News_2024": { + "fit_pct": 48.709639615780446, + "read_pct": 49.739598768691835, + "resid_pct": 1.550761615527722, + "SS_row": 0.0013709475483775788, + "SS_col": 0.0013999360604411222, + "SS_total": 0.0028145302638072347 + }, + "drop_Pile-CC+OOD_arXiv_2023": { + "fit_pct": 48.06804052585788, + "read_pct": 50.89878687805069, + "resid_pct": 1.033172596091423, + "SS_row": 0.0015524586256840994, + "SS_col": 0.0016438835421880595, + "SS_total": 0.00322971065327484 + }, + "drop_Pile-CC+OOD_CommonPile": { + "fit_pct": 47.939009855314474, + "read_pct": 50.925820744916464, + "resid_pct": 1.1351693997690666, + "SS_row": 0.0015733137439308065, + "SS_col": 0.0016713381010736786, + "SS_total": 0.0032819070495599535 + }, + "drop_StackExchange+Github": { + "fit_pct": 70.3751011932439, + "read_pct": 28.25276809650362, + "resid_pct": 1.3721307102524773, + "SS_row": 0.0010134908653474635, + "SS_col": 0.0004068757543660278, + "SS_total": 0.0014401270451668776 + }, + "drop_StackExchange+OOD_News_2024": { + "fit_pct": 53.89267941625402, + "read_pct": 45.005943565729666, + "resid_pct": 1.1013770180163052, + "SS_row": 0.0012667960799823523, + "SS_col": 0.0010579053315315048, + "SS_total": 0.0023505902725635994 + }, + "drop_StackExchange+OOD_arXiv_2023": { + "fit_pct": 55.79032934066861, + "read_pct": 43.382788884458606, + "resid_pct": 0.8268817748727869, + "SS_row": 0.0014209067842753738, + "SS_col": 0.0011049029424133304, + "SS_total": 0.002546869324966679 + }, + "drop_StackExchange+OOD_CommonPile": { + "fit_pct": 55.344126996122945, + "read_pct": 43.8330744033383, + "resid_pct": 0.822798600538758, + "SS_row": 0.0014662405438463527, + "SS_col": 0.0011612764414209791, + "SS_total": 0.0026493155162589667 + }, + "drop_Github+OOD_News_2024": { + "fit_pct": 45.368746755934815, + "read_pct": 53.22972039891428, + "resid_pct": 1.4015328451509066, + "SS_row": 0.0008948664328086211, + "SS_col": 0.0010499185765263758, + "SS_total": 0.001972429253165476 + }, + "drop_Github+OOD_arXiv_2023": { + "fit_pct": 49.11347777791474, + "read_pct": 49.76969929872892, + "resid_pct": 1.1168229233563398, + "SS_row": 0.0011149170888759177, + "SS_col": 0.0011298138671279533, + "SS_total": 0.002270083771948383 + }, + "drop_Github+OOD_CommonPile": { + "fit_pct": 49.59262915478606, + "read_pct": 49.29390919821961, + "resid_pct": 1.1134616469943321, + "SS_row": 0.0011878062792057384, + "SS_col": 0.0011806515579058, + "SS_total": 0.0023951266537985236 + }, + "drop_OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 49.4010775978473, + "read_pct": 49.41528089410974, + "resid_pct": 1.1836415080429687, + "SS_row": 0.0015138221310629437, + "SS_col": 0.001514257369832251, + "SS_total": 0.0030643504244711258 + }, + "drop_OOD_News_2024+OOD_CommonPile": { + "fit_pct": 50.24811436118757, + "read_pct": 48.55664572481285, + "resid_pct": 1.1952399139995844, + "SS_row": 0.0015912378558355616, + "SS_col": 0.001537673081109667, + "SS_total": 0.00316676133237083 + }, + "drop_OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 50.61140520804757, + "read_pct": 48.63035002217219, + "resid_pct": 0.7582447697802386, + "SS_row": 0.0018068357156482925, + "SS_col": 0.0017361117108553818, + "SS_total": 0.0035700168928741636 + } + }, + "fit_pct": { + "n": 28, + "min": 42.08107123390732, + "max": 70.3751011932439, + "range": 28.294029959336584, + "median": 49.920371757986814, + "mean": 50.81969857415798, + "sd": 4.910367875011766, + "values_sorted": [ + 42.08107123390732, + 45.29050803527647, + 45.368746755934815, + 46.59915642774981, + 47.510177227994554, + 47.939009855314474, + 48.06804052585788, + 48.709639615780446, + 49.03669211778151, + 49.077827715914886, + 49.11347777791474, + 49.4010775978473, + 49.564449013565095, + 49.59262915478606, + 50.24811436118757, + 50.30922645008709, + 50.61140520804757, + 50.74751292554641, + 52.185386350617975, + 52.30067209055684, + 52.3829095816429, + 52.8489532185628, + 53.07363642980562, + 53.89267941625402, + 55.344126996122945, + 55.489003458453865, + 55.79032934066861, + 70.3751011932439 + ] + }, + "read_pct": { + "n": 28, + "min": 28.25276809650362, + "max": 56.08737719584901, + "range": 27.83460909934539, + "median": 48.94978581469096, + "mean": 48.02701070915387, + "sd": 4.848490077950363, + "values_sorted": [ + 28.25276809650362, + 43.382788884458606, + 43.8330744033383, + 43.88542177383025, + 45.005943565729666, + 45.68002694795497, + 45.86575505849258, + 46.25686799605305, + 46.47792330335211, + 47.016680600124765, + 48.37431796644749, + 48.55664572481285, + 48.56307298661112, + 48.63035002217219, + 49.26922160720974, + 49.29390919821961, + 49.41528089410974, + 49.432990888192485, + 49.739598768691835, + 49.76969929872892, + 49.84208112785427, + 50.89878687805069, + 50.925820744916464, + 51.26384953198994, + 52.21732878700266, + 53.22972039891428, + 53.588997206697115, + 56.08737719584901 + ] + } + }, + "n_loo_preserving_ordering": 6, + "n_l2o_preserving_ordering": 16, + "l2o_splits_that_REVERSE": { + "drop_USPTO_Backgrounds+Pile-CC": { + "fit_pct": 42.08107123390732, + "read_pct": 56.08737719584901, + "resid_pct": 1.831551570243672, + "SS_row": 0.0008927460589612373, + "SS_col": 0.0011898885527590996, + "SS_total": 0.0021214908099627854 + }, + "drop_USPTO_Backgrounds+StackExchange": { + "fit_pct": 46.59915642774981, + "read_pct": 52.21732878700266, + "resid_pct": 1.1835147852475376, + "SS_row": 0.0008345315317930185, + "SS_col": 0.0009351458420995555, + "SS_total": 0.001790872616088936 + }, + "drop_USPTO_Backgrounds+Github": { + "fit_pct": 49.077827715914886, + "read_pct": 49.432990888192485, + "resid_pct": 1.4891813958926392, + "SS_row": 0.0008833536059144145, + "SS_col": 0.0008897462007687642, + "SS_total": 0.0017999036367861937 + }, + "drop_USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 49.03669211778151, + "read_pct": 49.84208112785427, + "resid_pct": 1.121226754364218, + "SS_row": 0.0013372222016183137, + "SS_col": 0.0013591850220838926, + "SS_total": 0.0027269828854002472 + }, + "drop_Pile-CC+StackExchange": { + "fit_pct": 45.29050803527647, + "read_pct": 53.588997206697115, + "resid_pct": 1.1204947580264173, + "SS_row": 0.0009808707963172088, + "SS_col": 0.001160593789829693, + "SS_total": 0.0021657314940102135 + }, + "drop_Pile-CC+Github": { + "fit_pct": 47.510177227994554, + "read_pct": 51.26384953198994, + "resid_pct": 1.2259732400155092, + "SS_row": 0.0010482680773145135, + "SS_col": 0.0011310893816867363, + "SS_total": 0.002206407423580057 + }, + "drop_Pile-CC+OOD_News_2024": { + "fit_pct": 48.709639615780446, + "read_pct": 49.739598768691835, + "resid_pct": 1.550761615527722, + "SS_row": 0.0013709475483775788, + "SS_col": 0.0013999360604411222, + "SS_total": 0.0028145302638072347 + }, + "drop_Pile-CC+OOD_arXiv_2023": { + "fit_pct": 48.06804052585788, + "read_pct": 50.89878687805069, + "resid_pct": 1.033172596091423, + "SS_row": 0.0015524586256840994, + "SS_col": 0.0016438835421880595, + "SS_total": 0.00322971065327484 + }, + "drop_Pile-CC+OOD_CommonPile": { + "fit_pct": 47.939009855314474, + "read_pct": 50.925820744916464, + "resid_pct": 1.1351693997690666, + "SS_row": 0.0015733137439308065, + "SS_col": 0.0016713381010736786, + "SS_total": 0.0032819070495599535 + }, + "drop_Github+OOD_News_2024": { + "fit_pct": 45.368746755934815, + "read_pct": 53.22972039891428, + "resid_pct": 1.4015328451509066, + "SS_row": 0.0008948664328086211, + "SS_col": 0.0010499185765263758, + "SS_total": 0.001972429253165476 + }, + "drop_Github+OOD_arXiv_2023": { + "fit_pct": 49.11347777791474, + "read_pct": 49.76969929872892, + "resid_pct": 1.1168229233563398, + "SS_row": 0.0011149170888759177, + "SS_col": 0.0011298138671279533, + "SS_total": 0.002270083771948383 + }, + "drop_OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 49.4010775978473, + "read_pct": 49.41528089410974, + "resid_pct": 1.1836415080429687, + "SS_row": 0.0015138221310629437, + "SS_col": 0.001514257369832251, + "SS_total": 0.0030643504244711258 + } + }, + "corpus_random_effect_on_the_fit_axis": { + "MS_fit": 0.000350264554715014, + "df_fit": 7, + "MS_residual": 1.1554742595537341e-06, + "df_residual": 49, + "variance_component": 4.363863505693253e-05, + "CAPTION_THAT_MUST_TRAVEL_WITH_IT": "estimated on 8 corpora that were CHOSEN, not sampled, on 7 degrees of freedom. This is a description of one panel dressed in the notation of an inference about a population of corpora. It is reported so the weakness is visible, not because it licenses a population claim." + }, + "seed_bootstrap_for_comparison": { + "fit_pct": { + "point": 50.697184677169346, + "ci_lo": 45.18158148522331, + "ci_hi": 54.188775604491234 + }, + "read_pct": { + "point": 48.13211438274321, + "ci_lo": 44.402403827268486, + "ci_hi": 52.95370961200712 + }, + "UNIT": "seed factors resampled; the eight corpora HELD FIXED" + } + }, + "min": { + "full_panel": { + "fit_pct": 53.35344735234028, + "read_pct": 44.60936175509925, + "resid_pct": 2.0371908925604707, + "SS_row": 0.007990259278199689, + "SS_col": 0.006680737315891797, + "SS_total": 0.014976088096862603 + }, + "leave_one_corpus_out": { + "per_split": { + "drop_Wikipedia_en": { + "fit_pct": 46.81110557693417, + "read_pct": 50.85837600168466, + "resid_pct": 2.330518421381178, + "SS_row": 0.005196327505101434, + "SS_col": 0.005645600009339775, + "SS_total": 0.011100629735311972 + }, + "drop_USPTO_Backgrounds": { + "fit_pct": 56.6792018376361, + "read_pct": 41.06419873316167, + "resid_pct": 2.256599429202232, + "SS_row": 0.006165837317809508, + "SS_col": 0.004467161864773255, + "SS_total": 0.010878482967124762 + }, + "drop_Pile-CC": { + "fit_pct": 56.50763481529161, + "read_pct": 41.42528085634039, + "resid_pct": 2.0670843283680003, + "SS_row": 0.006353545275838439, + "SS_col": 0.004657731620610279, + "SS_total": 0.01124369352319644 + }, + "drop_StackExchange": { + "fit_pct": 53.53726007459882, + "read_pct": 45.32738406925986, + "resid_pct": 1.135355856141325, + "SS_row": 0.0057915044206513265, + "SS_col": 0.004903384014196698, + "SS_total": 0.010817707915163092 + }, + "drop_Github": { + "fit_pct": 41.358656075065184, + "read_pct": 56.523659412915094, + "resid_pct": 2.117684512019725, + "SS_row": 0.004075413757126647, + "SS_col": 0.005569748174516142, + "SS_total": 0.009853835070776593 + }, + "drop_OOD_News_2024": { + "fit_pct": 56.05510540136367, + "read_pct": 42.24118054522516, + "resid_pct": 1.7037140534111703, + "SS_row": 0.006614014596598784, + "SS_col": 0.004984091684481735, + "SS_total": 0.01179912971216692 + }, + "drop_OOD_arXiv_2023": { + "fit_pct": 56.00547196651275, + "read_pct": 42.12050880191628, + "resid_pct": 1.874019231570966, + "SS_row": 0.006677236685851394, + "SS_col": 0.00502180584724022, + "SS_total": 0.01192247195031925 + }, + "drop_OOD_CommonPile": { + "fit_pct": 57.55685418677557, + "read_pct": 40.24133072996441, + "resid_pct": 2.2018150832600214, + "SS_row": 0.006968993772626052, + "SS_col": 0.004872427223858639, + "SS_total": 0.012108017144250508 + } + }, + "fit_pct": { + "n": 8, + "min": 41.358656075065184, + "max": 57.55685418677557, + "range": 16.198198111710383, + "median": 56.03028868393821, + "mean": 53.06391124177223, + "sd": 5.466383090829018, + "values_sorted": [ + 41.358656075065184, + 46.81110557693417, + 53.53726007459882, + 56.00547196651275, + 56.05510540136367, + 56.50763481529161, + 56.6792018376361, + 57.55685418677557 + ] + }, + "read_pct": { + "n": 8, + "min": 40.24133072996441, + "max": 56.523659412915094, + "range": 16.282328682950684, + "median": 42.18084467357072, + "mean": 44.97523989380844, + "sd": 5.408722210484158, + "values_sorted": [ + 40.24133072996441, + 41.06419873316167, + 41.42528085634039, + 42.12050880191628, + 42.24118054522516, + 45.32738406925986, + 50.85837600168466, + 56.523659412915094 + ] + } + }, + "leave_two_corpora_out": { + "per_split": { + "drop_Wikipedia_en+USPTO_Backgrounds": { + "fit_pct": 52.32062604200177, + "read_pct": 45.031943464782344, + "resid_pct": 2.647430493215886, + "SS_row": 0.003984779709332063, + "SS_col": 0.003429667956308495, + "SS_total": 0.0076160780380058425 + }, + "drop_Wikipedia_en+Pile-CC": { + "fit_pct": 52.442755923912436, + "read_pct": 45.218004916593706, + "resid_pct": 2.33923915949386, + "SS_row": 0.0042198568575961565, + "SS_col": 0.0036385103103839557, + "SS_total": 0.00804659629962738 + }, + "drop_Wikipedia_en+StackExchange": { + "fit_pct": 47.13001388878387, + "read_pct": 51.79992772308668, + "resid_pct": 1.070058388129444, + "SS_row": 0.0037954345423474828, + "SS_col": 0.004171508105965026, + "SS_total": 0.008053115688240292 + }, + "drop_Wikipedia_en+Github": { + "fit_pct": 21.009005511431113, + "read_pct": 76.38980395789154, + "resid_pct": 2.601190530677337, + "SS_row": 0.0013060967706594532, + "SS_col": 0.004749033751570207, + "SS_total": 0.006216842439061662 + }, + "drop_Wikipedia_en+OOD_News_2024": { + "fit_pct": 48.998113064740714, + "read_pct": 48.966136042934444, + "resid_pct": 2.03575089232484, + "SS_row": 0.00396422376109931, + "SS_col": 0.0039616366396424876, + "SS_total": 0.00809056413225836 + }, + "drop_Wikipedia_en+OOD_arXiv_2023": { + "fit_pct": 49.163994924145854, + "read_pct": 48.57778104549241, + "resid_pct": 2.2582240303617396, + "SS_row": 0.004233565929640177, + "SS_col": 0.004183086404776941, + "SS_total": 0.008611110501032436 + }, + "drop_Wikipedia_en+OOD_CommonPile": { + "fit_pct": 50.1961522600238, + "read_pct": 47.2140468581699, + "resid_pct": 2.5898008818062905, + "SS_row": 0.0043878010797794706, + "SS_col": 0.0041271260138005285, + "SS_total": 0.008741309606859874 + }, + "drop_USPTO_Backgrounds+Pile-CC": { + "fit_pct": 62.81630399104917, + "read_pct": 34.610844679778396, + "resid_pct": 2.572851329172437, + "SS_row": 0.004591717579455467, + "SS_col": 0.002529967760895109, + "SS_total": 0.007309754455005424 + }, + "drop_USPTO_Backgrounds+StackExchange": { + "fit_pct": 55.48827295667549, + "read_pct": 43.2032514036185, + "resid_pct": 1.3084756397060118, + "SS_row": 0.004051664269884037, + "SS_col": 0.0031546317938482486, + "SS_total": 0.007301838846286535 + }, + "drop_USPTO_Backgrounds+Github": { + "fit_pct": 45.23962835179833, + "read_pct": 52.609332663665406, + "resid_pct": 2.1510389845362563, + "SS_row": 0.0031736682259821377, + "SS_col": 0.003690670625461981, + "SS_total": 0.00701523938548443 + }, + "drop_USPTO_Backgrounds+OOD_News_2024": { + "fit_pct": 62.89153318271039, + "read_pct": 35.25419459192547, + "resid_pct": 1.854272225364142, + "SS_row": 0.00495741295352644, + "SS_col": 0.002778905078183072, + "SS_total": 0.007882480681659212 + }, + "drop_USPTO_Backgrounds+OOD_arXiv_2023": { + "fit_pct": 57.2648652589358, + "read_pct": 40.747019920579476, + "resid_pct": 1.9881148204847274, + "SS_row": 0.004867279037701993, + "SS_col": 0.0034633298273117723, + "SS_total": 0.008499590483088558 + }, + "drop_USPTO_Backgrounds+OOD_CommonPile": { + "fit_pct": 59.18141567642879, + "read_pct": 38.42474601345063, + "resid_pct": 2.393838310120584, + "SS_row": 0.005165723905942789, + "SS_col": 0.003353952026877899, + "SS_total": 0.008728625104519478 + }, + "drop_Pile-CC+StackExchange": { + "fit_pct": 53.016947757511794, + "read_pct": 45.766170925816546, + "resid_pct": 1.2168813166716623, + "SS_row": 0.0040097346277011055, + "SS_col": 0.0034613497777705093, + "SS_total": 0.0075631185824592845 + }, + "drop_Pile-CC+Github": { + "fit_pct": 43.67099341643807, + "read_pct": 54.18028651100135, + "resid_pct": 2.148720072560576, + "SS_row": 0.003151604075265619, + "SS_col": 0.003910028108104757, + "SS_total": 0.007216698839919983 + }, + "drop_Pile-CC+OOD_News_2024": { + "fit_pct": 61.7879384237363, + "read_pct": 36.71273480001148, + "resid_pct": 1.4993267762522144, + "SS_row": 0.005221213421600819, + "SS_col": 0.00310230489269495, + "SS_total": 0.008450214645120852 + }, + "drop_Pile-CC+OOD_arXiv_2023": { + "fit_pct": 58.57043175120877, + "read_pct": 39.66601100408638, + "resid_pct": 1.7635572447048484, + "SS_row": 0.005030484883082387, + "SS_col": 0.0034068259830459255, + "SS_total": 0.008588778898626725 + }, + "drop_Pile-CC+OOD_CommonPile": { + "fit_pct": 60.53397420111428, + "read_pct": 37.200777927836896, + "resid_pct": 2.2652478710488317, + "SS_row": 0.0053998878280406455, + "SS_col": 0.003318467531287114, + "SS_total": 0.008920425098970699 + }, + "drop_StackExchange+Github": { + "fit_pct": 42.84227087213813, + "read_pct": 56.17386765237576, + "resid_pct": 0.9838614754861128, + "SS_row": 0.002744695593868939, + "SS_col": 0.0035987860563274103, + "SS_total": 0.006406512862169296 + }, + "drop_StackExchange+OOD_News_2024": { + "fit_pct": 55.292155304881796, + "read_pct": 43.595433444021396, + "resid_pct": 1.1124112510968156, + "SS_row": 0.004789825018717411, + "SS_col": 0.003776566434435356, + "SS_total": 0.008662756935963595 + }, + "drop_StackExchange+OOD_arXiv_2023": { + "fit_pct": 56.67529835096962, + "read_pct": 42.53112995095901, + "resid_pct": 0.7935716980713718, + "SS_row": 0.004422158846283918, + "SS_col": 0.0033185429636448764, + "SS_total": 0.007802621203507546 + }, + "drop_StackExchange+OOD_CommonPile": { + "fit_pct": 61.972325663831406, + "read_pct": 36.781390055683254, + "resid_pct": 1.2462842804853407, + "SS_row": 0.004994645515146454, + "SS_col": 0.0029643877797810077, + "SS_total": 0.008059477293525961 + }, + "drop_Github+OOD_News_2024": { + "fit_pct": 41.51713771708313, + "read_pct": 56.74484797143785, + "resid_pct": 1.7380143114790165, + "SS_row": 0.0029996975187752076, + "SS_col": 0.004099930511181624, + "SS_total": 0.0072252030937598015 + }, + "drop_Github+OOD_arXiv_2023": { + "fit_pct": 46.003190504868655, + "read_pct": 51.97958476412894, + "resid_pct": 2.0172247310024174, + "SS_row": 0.0035766981963146665, + "SS_col": 0.004041356371823152, + "SS_total": 0.007774891604390209 + }, + "drop_Github+OOD_CommonPile": { + "fit_pct": 46.56314258467692, + "read_pct": 51.21295811286591, + "resid_pct": 2.2238993024571663, + "SS_row": 0.0034611778164965, + "SS_col": 0.003806812528064806, + "SS_total": 0.007433299438933296 + }, + "drop_OOD_News_2024+OOD_arXiv_2023": { + "fit_pct": 58.85391680517784, + "read_pct": 39.71228389826037, + "resid_pct": 1.4337992965617947, + "SS_row": 0.005407399066490273, + "SS_col": 0.0036486979718018944, + "SS_total": 0.009187832110461239 + }, + "drop_OOD_News_2024+OOD_CommonPile": { + "fit_pct": 59.80536612060682, + "read_pct": 38.411537342785614, + "resid_pct": 1.7830965366075628, + "SS_row": 0.005624108740959004, + "SS_col": 0.0036122287503026797, + "SS_total": 0.00940402025065295 + }, + "drop_OOD_arXiv_2023+OOD_CommonPile": { + "fit_pct": 63.40419715517061, + "read_pct": 34.532472053862335, + "resid_pct": 2.063330790967069, + "SS_row": 0.005740589262780428, + "SS_col": 0.003126555451913031, + "SS_total": 0.009053957814072384 + } + }, + "fit_pct": { + "n": 28, + "min": 21.009005511431113, + "max": 63.40419715517061, + "range": 42.395191643739494, + "median": 54.154551531196795, + "mean": 52.66614170221613, + "sd": 9.01725618646078, + "values_sorted": [ + 21.009005511431113, + 41.51713771708313, + 42.84227087213813, + 43.67099341643807, + 45.23962835179833, + 46.003190504868655, + 46.56314258467692, + 47.13001388878387, + 48.998113064740714, + 49.163994924145854, + 50.1961522600238, + 52.32062604200177, + 52.442755923912436, + 53.016947757511794, + 55.292155304881796, + 55.48827295667549, + 56.67529835096962, + 57.2648652589358, + 58.57043175120877, + 58.85391680517784, + 59.18141567642879, + 59.80536612060682, + 60.53397420111428, + 61.7879384237363, + 61.972325663831406, + 62.81630399104917, + 62.89153318271039, + 63.40419715517061 + ] + }, + "read_pct": { + "n": 28, + "min": 34.532472053862335, + "max": 76.38980395789154, + "range": 41.85733190402921, + "median": 44.31368845440187, + "mean": 45.473161417753644, + "sd": 8.931533858887793, + "values_sorted": [ + 34.532472053862335, + 34.610844679778396, + 35.25419459192547, + 36.71273480001148, + 36.781390055683254, + 37.200777927836896, + 38.411537342785614, + 38.42474601345063, + 39.66601100408638, + 39.71228389826037, + 40.747019920579476, + 42.53112995095901, + 43.2032514036185, + 43.595433444021396, + 45.031943464782344, + 45.218004916593706, + 45.766170925816546, + 47.2140468581699, + 48.57778104549241, + 48.966136042934444, + 51.21295811286591, + 51.79992772308668, + 51.97958476412894, + 52.609332663665406, + 54.18028651100135, + 56.17386765237576, + 56.74484797143785, + 76.38980395789154 + ] + } + }, + "n_loo_preserving_ordering": 6, + "n_l2o_preserving_ordering": 20, + "l2o_splits_that_REVERSE": { + "drop_Wikipedia_en+StackExchange": { + "fit_pct": 47.13001388878387, + "read_pct": 51.79992772308668, + "resid_pct": 1.070058388129444, + "SS_row": 0.0037954345423474828, + "SS_col": 0.004171508105965026, + "SS_total": 0.008053115688240292 + }, + "drop_Wikipedia_en+Github": { + "fit_pct": 21.009005511431113, + "read_pct": 76.38980395789154, + "resid_pct": 2.601190530677337, + "SS_row": 0.0013060967706594532, + "SS_col": 0.004749033751570207, + "SS_total": 0.006216842439061662 + }, + "drop_USPTO_Backgrounds+Github": { + "fit_pct": 45.23962835179833, + "read_pct": 52.609332663665406, + "resid_pct": 2.1510389845362563, + "SS_row": 0.0031736682259821377, + "SS_col": 0.003690670625461981, + "SS_total": 0.00701523938548443 + }, + "drop_Pile-CC+Github": { + "fit_pct": 43.67099341643807, + "read_pct": 54.18028651100135, + "resid_pct": 2.148720072560576, + "SS_row": 0.003151604075265619, + "SS_col": 0.003910028108104757, + "SS_total": 0.007216698839919983 + }, + "drop_StackExchange+Github": { + "fit_pct": 42.84227087213813, + "read_pct": 56.17386765237576, + "resid_pct": 0.9838614754861128, + "SS_row": 0.002744695593868939, + "SS_col": 0.0035987860563274103, + "SS_total": 0.006406512862169296 + }, + "drop_Github+OOD_News_2024": { + "fit_pct": 41.51713771708313, + "read_pct": 56.74484797143785, + "resid_pct": 1.7380143114790165, + "SS_row": 0.0029996975187752076, + "SS_col": 0.004099930511181624, + "SS_total": 0.0072252030937598015 + }, + "drop_Github+OOD_arXiv_2023": { + "fit_pct": 46.003190504868655, + "read_pct": 51.97958476412894, + "resid_pct": 2.0172247310024174, + "SS_row": 0.0035766981963146665, + "SS_col": 0.004041356371823152, + "SS_total": 0.007774891604390209 + }, + "drop_Github+OOD_CommonPile": { + "fit_pct": 46.56314258467692, + "read_pct": 51.21295811286591, + "resid_pct": 2.2238993024571663, + "SS_row": 0.0034611778164965, + "SS_col": 0.003806812528064806, + "SS_total": 0.007433299438933296 + } + }, + "corpus_random_effect_on_the_fit_axis": { + "MS_fit": 0.001141465611171384, + "df_fit": 7, + "MS_residual": 6.226357199410564e-06, + "df_residual": 49, + "variance_component": 0.0001419049067464967, + "CAPTION_THAT_MUST_TRAVEL_WITH_IT": "estimated on 8 corpora that were CHOSEN, not sampled, on 7 degrees of freedom. This is a description of one panel dressed in the notation of an inference about a population of corpora. It is reported so the weakness is visible, not because it licenses a population claim." + }, + "seed_bootstrap_for_comparison": { + "fit_pct": { + "point": 53.35344735234028, + "ci_lo": 49.05154716547634, + "ci_hi": 56.958779332004895 + }, + "read_pct": { + "point": 44.60936175509925, + "ci_lo": 40.95862929938772, + "ci_hi": 48.29168997564349 + }, + "UNIT": "seed factors resampled; the eight corpora HELD FIXED" + } + } + }, + "provenance": { + "schema_version": 1, + "experiment": "R5", + "utc": "2026-08-31T23:20:38+00:00", + "script": { + "path": "tools/r5_corpus_axis_uncertainty.py", + "sha256": "0866b3077a89c202448b77cbfdd8e430c9812375d4e20352640ff91a1cf27569", + "n_lines": 216 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "tools/r5_corpus_axis_uncertainty.py", + "--cells", + "results/e57_factorial_cells_410m_rstrip.json", + "--ci", + "results/e57_grid_variance_ci_rstrip.json" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e57_factorial_cells_410m_rstrip.json", + "sha256": "927541002dea5762ced90c315be67a1ae1c7ffa9028c9ef30011320ca181acd3", + "bytes": 84806, + "exists": true + }, + { + "path": "results/e57_grid_variance_ci_rstrip.json", + "sha256": "284e124bcc95a975612f16c253c4289d4b0ea5124408066668516f6ac6bbc88a", + "bytes": 11725, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "2aaf5e5d529f13dc714c17b539a8d6512fffdc9e50378ca916bac124c34ab203" + } +} \ No newline at end of file diff --git a/results/r6/lens_R6_Pile-CC_b0_410m_n200.pt b/results/r6/lens_R6_Pile-CC_b0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d5e4dedc67d631f9ed461c3914f147df2ee0a5b --- /dev/null +++ b/results/r6/lens_R6_Pile-CC_b0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33dae3b0d750d08e1cc7236e477e442e503d99f669a0f3ad0cfa4ef490e272e8 +size 27268171 diff --git a/results/r6/lens_R6_Pile-CC_b1_410m_n200.pt b/results/r6/lens_R6_Pile-CC_b1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5e259af3115b4f1931aac3d5f94482845c1154e --- /dev/null +++ b/results/r6/lens_R6_Pile-CC_b1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10e7a7af7110bb0515adc8cc8b27c638c7cb040aa47c3224280a1b53db0e46d9 +size 27268171 diff --git a/results/r6/lens_R6_Pile-CC_b2_410m_n200.pt b/results/r6/lens_R6_Pile-CC_b2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..70bf700798f2023f154aa6174a953ce33c4f1007 --- /dev/null +++ b/results/r6/lens_R6_Pile-CC_b2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fc8cc67463f391d6e0e5fbe83230263309c13c7babeb27b5331c79588432e75 +size 27268171 diff --git a/results/r6/lens_R6_Pile-CC_b3_410m_n200.pt b/results/r6/lens_R6_Pile-CC_b3_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1f67c2fd6d7fa22666759ba6d0ea29848da575b --- /dev/null +++ b/results/r6/lens_R6_Pile-CC_b3_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ebb1fce54b6c476f46cb5b38067b98af5f7bc85331f042471bbc7e5efab9b66 +size 27268171 diff --git a/results/r6/lens_R6_Wikipedia_en_b0_410m_n200.pt b/results/r6/lens_R6_Wikipedia_en_b0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..a670c33bb103527762525a41b36b67ec80a3e32a --- /dev/null +++ b/results/r6/lens_R6_Wikipedia_en_b0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9bd821d39b93b9353433600c62f0b7c0711dc02d8c3622db934e90f66829886 +size 27268266 diff --git a/results/r6/lens_R6_Wikipedia_en_b1_410m_n200.pt b/results/r6/lens_R6_Wikipedia_en_b1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec1f8dab243df8c06aa2a5148e5d8dd5730b580e --- /dev/null +++ b/results/r6/lens_R6_Wikipedia_en_b1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b24b4903c5ef156ae1a08afda45f6dc3ff89451d1ffe03c0350b9382ce7d0d27 +size 27268266 diff --git a/results/r6/lens_R6_Wikipedia_en_b2_410m_n200.pt b/results/r6/lens_R6_Wikipedia_en_b2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..603eefa0f2f18563bff720d82f47fb10b9c251ad --- /dev/null +++ b/results/r6/lens_R6_Wikipedia_en_b2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51c9fde7e588d29c445bfe2b9067da68ad018609f362de42790d3ded25a4a529 +size 27268266 diff --git a/results/r6/lens_R6_Wikipedia_en_b3_410m_n200.pt b/results/r6/lens_R6_Wikipedia_en_b3_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a1f786b8b5f2f36dbc9e88494f56ec730bddde7 --- /dev/null +++ b/results/r6/lens_R6_Wikipedia_en_b3_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49ad6f5c39d632a4c2b0bc96c10362b3a6f3168ad1ee9b3889a3f0b694bbc55f +size 27268266 diff --git a/results/r6_within_source_410m.json b/results/r6_within_source_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..74660a5d9e399dc1bcf124e04cea8a02ff4370a9 --- /dev/null +++ b/results/r6_within_source_410m.json @@ -0,0 +1,364 @@ +{ + "experiment": "R6 \u2014 within-source resampling: is 'corpus' a factor or a bundle?", + "prereg": "specs/experiments/R6_within_source_resampling.md", + "prereg_sha256": "bc70dec33c9270b782dc4a3edfbf5ad05fd9bb6e1879eda22dbb61e34e07030d", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "ACCEPT (corpus identity is a real factor): between-source share >= 0.70. | REJECT (the effect is sample noise, not source identity): between-source share <= 0.30. STOP and alert -- the paper's factor is not what it is called. | UNCLEAR: anything between. Report the number and stop; do not re-cut it.", + "readout_convention": "STRIPPED \u2014 the anchor rule (R1). NOT comparable to pre-R1 figures and must not be printed beside them.", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0", + "note": "asserted OFF in-process. The rented box's own preflight reported TF32 ENABLED, so the driver-level override alone was not sufficient and the torch flags are set explicitly here." + }, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "sources": [ + "Pile-CC", + "Wikipedia_en" + ], + "n_blocks_per_source": 4, + "n_fit_per_block": 200, + "window_tokens": 128, + "smoke": false, + "device": "cuda", + "n_items": 541, + "operators": { + "Pile-CC|b0": { + "path": "results/r6/lens_R6_Pile-CC_b0_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 0 + }, + "Pile-CC|b1": { + "path": "results/r6/lens_R6_Pile-CC_b1_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 1 + }, + "Pile-CC|b2": { + "path": "results/r6/lens_R6_Pile-CC_b2_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 2 + }, + "Pile-CC|b3": { + "path": "results/r6/lens_R6_Pile-CC_b3_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 3 + }, + "Wikipedia_en|b0": { + "path": "results/r6/lens_R6_Wikipedia_en_b0_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 0 + }, + "Wikipedia_en|b1": { + "path": "results/r6/lens_R6_Wikipedia_en_b1_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 1 + }, + "Wikipedia_en|b2": { + "path": "results/r6/lens_R6_Wikipedia_en_b2_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 2 + }, + "Wikipedia_en|b3": { + "path": "results/r6/lens_R6_Wikipedia_en_b3_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 3 + } + }, + "reads": { + "Pile-CC|b0": { + "min": 0.2621679771691561, + "persist": 0.11054239000659435 + }, + "Pile-CC|b1": { + "min": 0.26131707690656186, + "persist": 0.11126932452898472 + }, + "Pile-CC|b2": { + "min": 0.2626881584525108, + "persist": 0.11166209306102245 + }, + "Pile-CC|b3": { + "min": 0.2631038587540388, + "persist": 0.11242945070844143 + }, + "Wikipedia_en|b0": { + "min": 0.24112006314098836, + "persist": 0.10448124296963215 + }, + "Wikipedia_en|b1": { + "min": 0.244559058919549, + "persist": 0.10538501951377839 + }, + "Wikipedia_en|b2": { + "min": 0.24200886227190493, + "persist": 0.10484272565227001 + }, + "Wikipedia_en|b3": { + "min": 0.2429351445287466, + "persist": 0.10530159547924996 + }, + "__logit__": { + "min": 0.19810852445662022, + "persist": 0.08317913215141744 + } + }, + "own_derangement_floors": { + "Pile-CC|b0": { + "min": 0.27400333285331724, + "persist": 0.0858444731682539 + }, + "Pile-CC|b1": { + "min": 0.27289414703845977, + "persist": 0.0863012796966359 + }, + "Pile-CC|b2": { + "min": 0.274552321434021, + "persist": 0.08471209928393364 + }, + "Pile-CC|b3": { + "min": 0.2762906849384308, + "persist": 0.08559564414899797 + }, + "Wikipedia_en|b0": { + "min": 0.24828002080321313, + "persist": 0.08132097236812115 + }, + "Wikipedia_en|b1": { + "min": 0.24747205674648284, + "persist": 0.08131839893758297 + }, + "Wikipedia_en|b2": { + "min": 0.24484166353940964, + "persist": 0.08248797729611397 + }, + "Wikipedia_en|b3": { + "min": 0.2496955655515194, + "persist": 0.08192630968987942 + } + }, + "PRIMARY_between_source_share_persist": 0.9722598722750339, + "by_aggregation": { + "persist": { + "SS_between_source": 8.380382532599887e-05, + "SS_within_source": 2.3910570462444682e-06, + "SS_total": 8.619488237224316e-05, + "between_source_share": 0.9722598722750339, + "group_means": { + "Pile-CC": 0.11147581457626074, + "Wikipedia_en": 0.10500264590373262 + }, + "per_operator": { + "Pile-CC|b0": 0.11054239000659435, + "Pile-CC|b1": 0.11126932452898472, + "Pile-CC|b2": 0.11166209306102245, + "Pile-CC|b3": 0.11242945070844143, + "Wikipedia_en|b0": 0.10448124296963215, + "Wikipedia_en|b1": 0.10538501951377839, + "Wikipedia_en|b2": 0.10484272565227001, + "Wikipedia_en|b3": 0.10530159547924996 + } + }, + "min": { + "SS_between_source": 0.0007733053322972977, + "SS_within_source": 8.256382724018862e-06, + "SS_total": 0.0007815617150213138, + "between_source_share": 0.9894360450808499, + "group_means": { + "Pile-CC": 0.2623192678205669, + "Wikipedia_en": 0.2426557822152972 + }, + "per_operator": { + "Pile-CC|b0": 0.2621679771691561, + "Pile-CC|b1": 0.26131707690656186, + "Pile-CC|b2": 0.2626881584525108, + "Pile-CC|b3": 0.2631038587540388, + "Wikipedia_en|b0": 0.24112006314098836, + "Wikipedia_en|b1": 0.244559058919549, + "Wikipedia_en|b2": 0.24200886227190493, + "Wikipedia_en|b3": 0.2429351445287466 + } + } + }, + "controls": { + "C1_document_disjointness": { + "required": "0 shared documents between any pair of within-source blocks", + "pairwise_shared_document_counts": { + "Pile-CC|b0xb1": 0, + "Pile-CC|b0xb2": 0, + "Pile-CC|b0xb3": 0, + "Pile-CC|b1xb2": 0, + "Pile-CC|b1xb3": 0, + "Pile-CC|b2xb3": 0, + "Wikipedia_en|b0xb1": 0, + "Wikipedia_en|b0xb2": 0, + "Wikipedia_en|b0xb3": 0, + "Wikipedia_en|b1xb2": 0, + "Wikipedia_en|b1xb3": 0, + "Wikipedia_en|b2xb3": 0 + }, + "max_shared": 0, + "fires": true, + "power": "e58 measured 24-26% overlap when the exclusion step is omitted, so a disjointness check on this corpus family demonstrably can fail" + }, + "C2_shuffled_label_null": { + "required": "the observed between-source share must exceed the shuffled-label null's 95th percentile; chance level for 8 operators in 2 groups is ~1/7 = 0.143", + "null_mean": 0.1402764449833398, + "null_p95": 0.3519234724591613, + "observed": 0.9722598722750339, + "exceeds_null_p95": true, + "n_perm": 2000, + "fires": true + }, + "C3_derangement_floor": { + "required": "every operator clears its OWN layer-deranged floor under persist", + "per_operator": { + "Pile-CC|b0": { + "real": 0.11054239000659435, + "own_derangement": 0.0858444731682539, + "clears": true + }, + "Pile-CC|b1": { + "real": 0.11126932452898472, + "own_derangement": 0.0863012796966359, + "clears": true + }, + "Pile-CC|b2": { + "real": 0.11166209306102245, + "own_derangement": 0.08471209928393364, + "clears": true + }, + "Pile-CC|b3": { + "real": 0.11242945070844143, + "own_derangement": 0.08559564414899797, + "clears": true + }, + "Wikipedia_en|b0": { + "real": 0.10448124296963215, + "own_derangement": 0.08132097236812115, + "clears": true + }, + "Wikipedia_en|b1": { + "real": 0.10538501951377839, + "own_derangement": 0.08131839893758297, + "clears": true + }, + "Wikipedia_en|b2": { + "real": 0.10484272565227001, + "own_derangement": 0.08248797729611397, + "clears": true + }, + "Wikipedia_en|b3": { + "real": 0.10530159547924996, + "own_derangement": 0.08192630968987942, + "clears": true + } + }, + "n_clearing": 8, + "n_operators": 8, + "fires": true + } + }, + "declared_bias": [ + "two sources, not eight: this tests whether SOURCE IDENTITY beats SAMPLING, not whether it beats FORMATTING, which is R7", + "the pools are drawn under the same require_full_window filter, whose selectivity differs by corpus (SUSPICIONS.md S-3). Stated, not solved", + "blocks are contiguous in file order, so any within-file ordering structure (e.g. crawl date) is confounded with block identity" + ], + "VERDICT": "ACCEPT \u2014 corpus identity is a real factor. The between-source share is 0.9723 >= 0.70: holding the source fixed and varying only the sample leaves most of the fit-axis variance between sources, so the effect is not a sampling artifact.", + "provenance": { + "schema_version": 1, + "experiment": "R6", + "utc": "2026-08-20T14:46:21+00:00", + "script": { + "path": "experiments/r6_within_source.py", + "sha256": "89bed5f390572e15a62c1ca0755d0430908a19096836a34c7d72f6bfd10aee89", + "n_lines": 326 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "r6_within_source.py", + "--device", + "cuda", + "--lens-dir", + "/workspace/results/r6", + "--out", + "/workspace/results/r6_within_source_410m.json" + ], + "cwd": "experiments", + "inputs": [ + { + "path": "corpora/Pile-CC.jsonl", + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "bytes": 13512530, + "exists": true + }, + { + "path": "corpora/Wikipedia_en.jsonl", + "sha256": "b30f4cef11add560f7df8ed07a613e3fcc6a4dda865975bec15f2952253bda77", + "bytes": 10329196, + "exists": true + } + ], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.1", + "numpy": "2.4.6", + "platform": "Linux-5.4.0-216-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "461f40c40324d6690adbab7c6aaf42f0c8d8ff2edfbe59632bfde2d42fd30190" + } +} \ No newline at end of file diff --git a/results/r6_within_source_410m_cpu.json b/results/r6_within_source_410m_cpu.json new file mode 100644 index 0000000000000000000000000000000000000000..465a7658910e658086752a8ab689c932e334bf67 --- /dev/null +++ b/results/r6_within_source_410m_cpu.json @@ -0,0 +1,401 @@ +{ + "experiment": "R6 \u2014 within-source resampling: is 'corpus' a factor or a bundle?", + "prereg": "specs/experiments/R6_within_source_resampling.md", + "prereg_sha256": "bc70dec33c9270b782dc4a3edfbf5ad05fd9bb6e1879eda22dbb61e34e07030d", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "ACCEPT (corpus identity is a real factor): between-source share >= 0.70. | REJECT (the effect is sample noise, not source identity): between-source share <= 0.30. STOP and alert -- the paper's factor is not what it is called. | UNCLEAR: anything between. Report the number and stop; do not re-cut it.", + "readout_convention": "STRIPPED \u2014 the anchor rule (R1). NOT comparable to pre-R1 figures and must not be printed beside them.", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0", + "note": "asserted OFF in-process. The rented box's own preflight reported TF32 ENABLED, so the driver-level override alone was not sufficient and the torch flags are set explicitly here." + }, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "sources": [ + "Pile-CC", + "Wikipedia_en" + ], + "n_blocks_per_source": 4, + "n_fit_per_block": 200, + "window_tokens": 128, + "smoke": false, + "device": "cpu", + "n_items": 541, + "operators": { + "Pile-CC|b0": { + "path": "results/r6/lens_R6_Pile-CC_b0_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 0 + }, + "Pile-CC|b1": { + "path": "results/r6/lens_R6_Pile-CC_b1_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 1 + }, + "Pile-CC|b2": { + "path": "results/r6/lens_R6_Pile-CC_b2_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 2 + }, + "Pile-CC|b3": { + "path": "results/r6/lens_R6_Pile-CC_b3_410m_n200.pt", + "n_prompts": 200, + "source": "Pile-CC", + "block": 3 + }, + "Wikipedia_en|b0": { + "path": "results/r6/lens_R6_Wikipedia_en_b0_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 0 + }, + "Wikipedia_en|b1": { + "path": "results/r6/lens_R6_Wikipedia_en_b1_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 1 + }, + "Wikipedia_en|b2": { + "path": "results/r6/lens_R6_Wikipedia_en_b2_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 2 + }, + "Wikipedia_en|b3": { + "path": "results/r6/lens_R6_Wikipedia_en_b3_410m_n200.pt", + "n_prompts": 200, + "source": "Wikipedia_en", + "block": 3 + } + }, + "reads": { + "Pile-CC|b0": { + "min": 0.26216798909008504, + "persist": 0.11054239969234914 + }, + "Pile-CC|b1": { + "min": 0.2610573437064886, + "persist": 0.11126932527404279 + }, + "Pile-CC|b2": { + "min": 0.2626881465315819, + "persist": 0.11166208784561604 + }, + "Pile-CC|b3": { + "min": 0.2631038587540388, + "persist": 0.11242945070844143 + }, + "Wikipedia_en|b0": { + "min": 0.24112006239593028, + "persist": 0.10448124296963215 + }, + "Wikipedia_en|b1": { + "min": 0.24455907382071018, + "persist": 0.10538500796537846 + }, + "Wikipedia_en|b2": { + "min": 0.2420088652521372, + "persist": 0.10484272453468294 + }, + "Wikipedia_en|b3": { + "min": 0.24293515011668204, + "persist": 0.10530158318579197 + }, + "__logit__": { + "min": 0.19810852520167826, + "persist": 0.08317913140635938 + } + }, + "own_derangement_floors": { + "Pile-CC|b0": { + "min": 0.2740033358335495, + "persist": 0.08584446720778942 + }, + "Pile-CC|b1": { + "min": 0.2728941708803177, + "persist": 0.08630126442294568 + }, + "Pile-CC|b2": { + "min": 0.274552334100008, + "persist": 0.08471209779381753 + }, + "Pile-CC|b3": { + "min": 0.2762907147407532, + "persist": 0.08559564228635282 + }, + "Wikipedia_en|b0": { + "min": 0.24820750281214715, + "persist": 0.0813209842890501 + }, + "Wikipedia_en|b1": { + "min": 0.24747206643223763, + "persist": 0.08131839707493782 + }, + "Wikipedia_en|b2": { + "min": 0.24484166949987413, + "persist": 0.08248799033463002 + }, + "Wikipedia_en|b3": { + "min": 0.24969557151198388, + "persist": 0.08192631043493748 + } + }, + "PRIMARY_between_source_share_persist": 0.9722603429765596, + "by_aggregation": { + "persist": { + "SS_between_source": 8.380402065301544e-05, + "SS_within_source": 2.391020889510875e-06, + "SS_total": 8.61950415425264e-05, + "between_source_share": 0.9722603429765596, + "group_means": { + "Pile-CC": 0.11147581588011235, + "Wikipedia_en": 0.10500263966387138 + }, + "per_operator": { + "Pile-CC|b0": 0.11054239969234914, + "Pile-CC|b1": 0.11126932527404279, + "Pile-CC|b2": 0.11166208784561604, + "Pile-CC|b3": 0.11242945070844143, + "Wikipedia_en|b0": 0.10448124296963215, + "Wikipedia_en|b1": 0.10538500796537846, + "Wikipedia_en|b2": 0.10484272453468294, + "Wikipedia_en|b3": 0.10530158318579197 + } + }, + "min": { + "SS_between_source": 0.0007682060595606002, + "SS_within_source": 8.827629106760979e-06, + "SS_total": 0.0007770336886673623, + "between_source_share": 0.9886393225473908, + "group_means": { + "Pile-CC": 0.2622543345205486, + "Wikipedia_en": 0.24265578789636494 + }, + "per_operator": { + "Pile-CC|b0": 0.26216798909008504, + "Pile-CC|b1": 0.2610573437064886, + "Pile-CC|b2": 0.2626881465315819, + "Pile-CC|b3": 0.2631038587540388, + "Wikipedia_en|b0": 0.24112006239593028, + "Wikipedia_en|b1": 0.24455907382071018, + "Wikipedia_en|b2": 0.2420088652521372, + "Wikipedia_en|b3": 0.24293515011668204 + } + } + }, + "controls": { + "C1_document_disjointness": { + "required": "0 shared documents between any pair of within-source blocks", + "pairwise_shared_document_counts": { + "Pile-CC|b0xb1": 0, + "Pile-CC|b0xb2": 0, + "Pile-CC|b0xb3": 0, + "Pile-CC|b1xb2": 0, + "Pile-CC|b1xb3": 0, + "Pile-CC|b2xb3": 0, + "Wikipedia_en|b0xb1": 0, + "Wikipedia_en|b0xb2": 0, + "Wikipedia_en|b0xb3": 0, + "Wikipedia_en|b1xb2": 0, + "Wikipedia_en|b1xb3": 0, + "Wikipedia_en|b2xb3": 0 + }, + "max_shared": 0, + "fires": true, + "power": "e58 measured 24-26% overlap when the exclusion step is omitted, so a disjointness check on this corpus family demonstrably can fail" + }, + "C2_shuffled_label_null": { + "required": "the observed between-source share must exceed the shuffled-label null's 95th percentile; chance level for 8 operators in 2 groups is ~1/7 = 0.143", + "null_mean": 0.14027642234851745, + "null_p95": 0.3519222671414726, + "observed": 0.9722603429765596, + "exceeds_null_p95": true, + "n_perm": 2000, + "fires": true + }, + "C3_derangement_floor": { + "required": "every operator clears its OWN layer-deranged floor under persist", + "per_operator": { + "Pile-CC|b0": { + "real": 0.11054239969234914, + "own_derangement": 0.08584446720778942, + "clears": true + }, + "Pile-CC|b1": { + "real": 0.11126932527404279, + "own_derangement": 0.08630126442294568, + "clears": true + }, + "Pile-CC|b2": { + "real": 0.11166208784561604, + "own_derangement": 0.08471209779381753, + "clears": true + }, + "Pile-CC|b3": { + "real": 0.11242945070844143, + "own_derangement": 0.08559564228635282, + "clears": true + }, + "Wikipedia_en|b0": { + "real": 0.10448124296963215, + "own_derangement": 0.0813209842890501, + "clears": true + }, + "Wikipedia_en|b1": { + "real": 0.10538500796537846, + "own_derangement": 0.08131839707493782, + "clears": true + }, + "Wikipedia_en|b2": { + "real": 0.10484272453468294, + "own_derangement": 0.08248799033463002, + "clears": true + }, + "Wikipedia_en|b3": { + "real": 0.10530158318579197, + "own_derangement": 0.08192631043493748, + "clears": true + } + }, + "n_clearing": 8, + "n_operators": 8, + "fires": true + } + }, + "declared_bias": [ + "two sources, not eight: this tests whether SOURCE IDENTITY beats SAMPLING, not whether it beats FORMATTING, which is R7", + "the pools are drawn under the same require_full_window filter, whose selectivity differs by corpus (SUSPICIONS.md S-3). Stated, not solved", + "blocks are contiguous in file order, so any within-file ordering structure (e.g. crawl date) is confounded with block identity" + ], + "VERDICT": "ACCEPT \u2014 corpus identity is a real factor. The between-source share is 0.9723 >= 0.70: holding the source fixed and varying only the sample leaves most of the fit-axis variance between sources, so the effect is not a sampling artifact.", + "provenance": { + "schema_version": 1, + "experiment": "R6", + "utc": "2026-08-20T15:02:37+00:00", + "script": { + "path": "experiments/r6_within_source.py", + "sha256": "89bed5f390572e15a62c1ca0755d0430908a19096836a34c7d72f6bfd10aee89", + "n_lines": 326 + }, + "git": { + "commit": "13c0a89e3f96566367a026a224e415910871fd2e", + "commit_short": "13c0a89e3f96", + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "EADME.md", + "experiments/d2_device_crossvalidation.py", + "experiments/e28_ladder_410m.py", + "figures/f0_grid_single.png", + "figures/fB_band_width.png", + "figures/make_figures.py", + "paper/.gitignore", + "paper/CONTEXT.md", + "paper/PAPER_HANDOFF.md", + "paper/build.sh", + "paper/checklist.tex", + "paper/neurips_2026.tex", + "paper/paper.tex", + "paper/paper_5pp.tex", + "paper/paper_9pp.tex", + "paper/reference/", + "paper/refs.bib", + "paper/tables_auto.tex", + "results/d2_device_crossvalidation.json", + "results/e52_factorial_410m_rstrip_w8.json", + "results/r6_within_source_410m.json", + "review/", + "specs/AUDIT_READINESS.md", + "specs/DISTRIBUTION_RESULTS.md", + "specs/EXPERIMENT_VALIDITY_AUDIT.md", + "specs/HANDOFF.md", + "specs/IMPLEMENTATION_RULES.md", + "specs/POSTREVIEW_PAPER.md", + "specs/POSTREVIEW_RELEASE.md", + "specs/README.md", + "specs/VALIDITY_AGENT_BRIEF.md", + "specs/archive/PAPER_HANDOFF.md", + "specs/archive/PAPER_REWRITE.md", + "tmp/", + "tools/make_paper_tables.py", + "tools/readout_exposure.py" + ], + "dirty_diff_sha256": "2d46c81e42b07a89c101e398f615af898c3ec2845b95740f4ba7ce92ca722e32", + "describe": "13c0a89-dirty" + }, + "argv": [ + "r6_within_source.py", + "--device", + "cpu", + "--lens-dir", + "../results/r6", + "--out", + "../results/r6_within_source_410m_cpu.json" + ], + "cwd": "experiments", + "inputs": [ + { + "path": "corpora/Pile-CC.jsonl", + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "bytes": 13512530, + "exists": true + }, + { + "path": "corpora/Wikipedia_en.jsonl", + "sha256": "b30f4cef11add560f7df8ed07a613e3fcc6a4dda865975bec15f2952253bda77", + "bytes": 10329196, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "2a4a53d520beb3bca0afc7185a96e9435376f7777a6c930cb0028c29545f0a5d" + } +} \ No newline at end of file diff --git a/results/r7/lens_R7_matched_Github_s0_410m_n200.pt b/results/r7/lens_R7_matched_Github_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..712b1d4353a8b93db84133d2a2938c50aaef0a0a --- /dev/null +++ b/results/r7/lens_R7_matched_Github_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf143643e1ccded5f111e45a44abc3ad3987234b91ef0df2c638bbe0de926f9 +size 27268304 diff --git a/results/r7/lens_R7_matched_Github_s1_410m_n200.pt b/results/r7/lens_R7_matched_Github_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f0e939c09c581c4bc994146dc4060e2701c1b0e --- /dev/null +++ b/results/r7/lens_R7_matched_Github_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b81e8e18dc6b6bbc2a0296ceaf2fa0bf1f612b19af2e6fd4f4cf09584fa5d97 +size 27268304 diff --git a/results/r7/lens_R7_matched_Github_s2_410m_n200.pt b/results/r7/lens_R7_matched_Github_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc1a460eec9b1b1c5ddcb6c199e26b6ba3f51ec8 --- /dev/null +++ b/results/r7/lens_R7_matched_Github_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c15445f647b3d0d233d75d290bdffe12cacc29bbd1cc0d1268d4c3289cbd7bfd +size 27268304 diff --git a/results/r7/lens_R7_matched_Pile-CC_s0_410m_n200.pt b/results/r7/lens_R7_matched_Pile-CC_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5100c572607dee3f47786eb0a11411e4d0a173e --- /dev/null +++ b/results/r7/lens_R7_matched_Pile-CC_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:257d2fe462d5fad6c0b9544b4eb04ef67f58de7a155f76a133267502901fb307 +size 27268323 diff --git a/results/r7/lens_R7_matched_Pile-CC_s1_410m_n200.pt b/results/r7/lens_R7_matched_Pile-CC_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bd2d75096fb01ff9d89d62f52e3ba985bac57f2 --- /dev/null +++ b/results/r7/lens_R7_matched_Pile-CC_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9eb00a20a026ee835e8ef29c6c11c7ba417a5efc910f9a98efd221adef5275b +size 27268323 diff --git a/results/r7/lens_R7_matched_Pile-CC_s2_410m_n200.pt b/results/r7/lens_R7_matched_Pile-CC_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..02a72cbe76747b96fa37e09190a8401c9d38ea9d --- /dev/null +++ b/results/r7/lens_R7_matched_Pile-CC_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42fd87602328b1931e2d2827d30f71ec250ea93455134d9f6a6a46be1fd31d08 +size 27268323 diff --git a/results/r7/lens_R7_matched_StackExchange_s0_410m_n200.pt b/results/r7/lens_R7_matched_StackExchange_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9e79a5919698ae32433d8114967aa9b4320319c --- /dev/null +++ b/results/r7/lens_R7_matched_StackExchange_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9452d30c8a399af5eb90367fb6877358ca1527ab2401d3a25895419d353ef404 +size 27268437 diff --git a/results/r7/lens_R7_matched_StackExchange_s1_410m_n200.pt b/results/r7/lens_R7_matched_StackExchange_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..005bf81400c4bc30fffdfb4ca5c5ffbb782a831f --- /dev/null +++ b/results/r7/lens_R7_matched_StackExchange_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be0042ae80e91ea9ab3928877389f37bc6d19c170d7076d642e2001de6853d32 +size 27268437 diff --git a/results/r7/lens_R7_matched_StackExchange_s2_410m_n200.pt b/results/r7/lens_R7_matched_StackExchange_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6df6bfa747c9575d4bf04afe60f621e562668a2 --- /dev/null +++ b/results/r7/lens_R7_matched_StackExchange_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2cacc90d6fbf7a38d84d3d372793d070aea548b2e5a659f632e74a04bb2c4e +size 27268437 diff --git a/results/r7/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt b/results/r7/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ff3f993cef884f3a2c473e40bd4d722208b64a5 --- /dev/null +++ b/results/r7/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7cb96c7c790c8b260720756a2067e1b44e4192f4e39dc9ccd58069c764ee96a +size 27268513 diff --git a/results/r7/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt b/results/r7/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..1184b978fbbacb54173fa7f328d2f0bbc0cb60dc --- /dev/null +++ b/results/r7/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9df86067524293ae11dc6a849f6b79bc7dd0a5863aaadf184a0bee17ea6e5ca1 +size 27268513 diff --git a/results/r7/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt b/results/r7/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed390aed052f4922fbb9ac54f638de08ac81baec --- /dev/null +++ b/results/r7/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc23ef148a56c0463a7f88af418c6ea04a403a0a6f33b452dff7ed1cd1a5c7ff +size 27268513 diff --git a/results/r7/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt b/results/r7/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5d8ee46104f9df6beb85c4a37e896bee247b731 --- /dev/null +++ b/results/r7/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2f6153d750dceb248d1050adccb792c74b8137d129eeb27a236d5a7288f19a1 +size 27268418 diff --git a/results/r7/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt b/results/r7/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab1c901011c8966cd082667ba352a3f38ebf7339 --- /dev/null +++ b/results/r7/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da38a7badcbf45eed064963864befa803da85551dc40e4d679a996ecbab86301 +size 27268418 diff --git a/results/r7/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt b/results/r7/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..90db7ddb65b3b4950b042ded14691674bd9dbe94 --- /dev/null +++ b/results/r7/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e87a8aaec3fc4abdda5ea33e417a1303deb3e8ce9135d51d796d8f6800ffd88 +size 27268418 diff --git a/results/r7/lens_R7_unmatched_Github_s0_410m_n200.pt b/results/r7/lens_R7_unmatched_Github_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f83cb71b7c5d4f876e0af8f78bb201095e1d92b --- /dev/null +++ b/results/r7/lens_R7_unmatched_Github_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49a278ac0cf1b26aaaa8834e6638e968f90204077b22b3a5f67b3ff98e48d495 +size 27268342 diff --git a/results/r7/lens_R7_unmatched_Github_s1_410m_n200.pt b/results/r7/lens_R7_unmatched_Github_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..ced78e56b8d7799a03710e1ff48271dfd895c46d --- /dev/null +++ b/results/r7/lens_R7_unmatched_Github_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4714d69a2f8c6f5e9e8d32e21f5f3af476e6054ddb24279e38a8128968a9e07 +size 27268342 diff --git a/results/r7/lens_R7_unmatched_Github_s2_410m_n200.pt b/results/r7/lens_R7_unmatched_Github_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e10f06e049319743d422bda5af3c1945df3d41a --- /dev/null +++ b/results/r7/lens_R7_unmatched_Github_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:958112fcc4d90dce85e1beefa815354694f7a2ffd7d9e1d058e3c2b948a9c44e +size 27268342 diff --git a/results/r7/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt b/results/r7/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5a50d1e9651c0395915da35de4d8e806dfbfe89 --- /dev/null +++ b/results/r7/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d90dab194cdc620eeed5533f99e0c4908fc98b5622d2252db05979807f64220 +size 27268361 diff --git a/results/r7/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt b/results/r7/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1638a4481a95065ecf44a50a3e5600dfebcec7e --- /dev/null +++ b/results/r7/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d996d6da417511ca26a0c3a8c87ad5cb58c29a24d64dc60928c930951ac7985 +size 27268361 diff --git a/results/r7/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt b/results/r7/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..07f20106a22abeff366fd4d387e084c825c6ebdd --- /dev/null +++ b/results/r7/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37cf8cb50602c1d319df7061e2c5b584977628a60ba6e0556c6d86432cc0364f +size 27268361 diff --git a/results/r7/lens_R7_unmatched_StackExchange_s0_410m_n200.pt b/results/r7/lens_R7_unmatched_StackExchange_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd4b4b8e4f7b1d0d741a73e0a1fd112077c59d31 --- /dev/null +++ b/results/r7/lens_R7_unmatched_StackExchange_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efe02b31dc3c5cb83a10f732921bd60b1a60f6baa7c69a0827615e20dc74d065 +size 27268475 diff --git a/results/r7/lens_R7_unmatched_StackExchange_s1_410m_n200.pt b/results/r7/lens_R7_unmatched_StackExchange_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..c45f968e2d04bd42fc6b8b240f8491fab4f88932 --- /dev/null +++ b/results/r7/lens_R7_unmatched_StackExchange_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77279ef0d58e134f86899f2062055c8d850493222bd0cf7b4c112494b2108df0 +size 27268475 diff --git a/results/r7/lens_R7_unmatched_StackExchange_s2_410m_n200.pt b/results/r7/lens_R7_unmatched_StackExchange_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..9358bc79190ff992e0f5923fcd357c099647c19e --- /dev/null +++ b/results/r7/lens_R7_unmatched_StackExchange_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a308cb623d4fee979c3fbd19f0cdec9e3589dae1d1f3f804c690a72eaf7f7c82 +size 27268475 diff --git a/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s0_410m_n200.pt b/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..431b11977143f427a7b5032bd80fd0c69c3baf4c --- /dev/null +++ b/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:961a6e747fa37130ca039fab945b659e593c43b1907e1832498860a575907efb +size 27268551 diff --git a/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s1_410m_n200.pt b/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..28583f46b65a62ea03c3420d81fc3bb33797de48 --- /dev/null +++ b/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3267710f269fea1caa1d646e1935f3ca60af9bc2d79b63737ba7228d263ca97e +size 27268551 diff --git a/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s2_410m_n200.pt b/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc537143c59697828e53ffa9f817f17b4df3e030 --- /dev/null +++ b/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906b3c272fe52e2abb9818e69e5293f731d988e238df48e0949401add764f413 +size 27268551 diff --git a/results/r7/lens_R7_unmatched_Wikipedia_en_s0_410m_n200.pt b/results/r7/lens_R7_unmatched_Wikipedia_en_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..04e91091957544def952588ab8b23bd16a495ea5 --- /dev/null +++ b/results/r7/lens_R7_unmatched_Wikipedia_en_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1067b44c08b7f59ac25b19a6f59933c5fd89b36f8fc78e2df578a63fcf1c7876 +size 27268456 diff --git a/results/r7/lens_R7_unmatched_Wikipedia_en_s1_410m_n200.pt b/results/r7/lens_R7_unmatched_Wikipedia_en_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdcb13a1d2c2de01379af589f4ebb092324b567b --- /dev/null +++ b/results/r7/lens_R7_unmatched_Wikipedia_en_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cee4952b3a420dcfbd807048cd685917811f53bb6973fc9d5839e3a17aa8e1af +size 27268456 diff --git a/results/r7/lens_R7_unmatched_Wikipedia_en_s2_410m_n200.pt b/results/r7/lens_R7_unmatched_Wikipedia_en_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..0aeae757694c5f22ae828bcece8908664e76512f --- /dev/null +++ b/results/r7/lens_R7_unmatched_Wikipedia_en_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4538e53b11faa1a6942130a73981a805a92cd35aba5693a88270ec59c7a74b0 +size 27268456 diff --git a/results/r7_box_sha256.txt b/results/r7_box_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..d5ee773f5f057a7afb7b7854c504129eb0894234 --- /dev/null +++ b/results/r7_box_sha256.txt @@ -0,0 +1,30 @@ +cbf143643e1ccded5f111e45a44abc3ad3987234b91ef0df2c638bbe0de926f9 /workspace/results/r7/lens_R7_matched_Github_s0_410m_n200.pt +2b81e8e18dc6b6bbc2a0296ceaf2fa0bf1f612b19af2e6fd4f4cf09584fa5d97 /workspace/results/r7/lens_R7_matched_Github_s1_410m_n200.pt +c15445f647b3d0d233d75d290bdffe12cacc29bbd1cc0d1268d4c3289cbd7bfd /workspace/results/r7/lens_R7_matched_Github_s2_410m_n200.pt +257d2fe462d5fad6c0b9544b4eb04ef67f58de7a155f76a133267502901fb307 /workspace/results/r7/lens_R7_matched_Pile-CC_s0_410m_n200.pt +e9eb00a20a026ee835e8ef29c6c11c7ba417a5efc910f9a98efd221adef5275b /workspace/results/r7/lens_R7_matched_Pile-CC_s1_410m_n200.pt +42fd87602328b1931e2d2827d30f71ec250ea93455134d9f6a6a46be1fd31d08 /workspace/results/r7/lens_R7_matched_Pile-CC_s2_410m_n200.pt +9452d30c8a399af5eb90367fb6877358ca1527ab2401d3a25895419d353ef404 /workspace/results/r7/lens_R7_matched_StackExchange_s0_410m_n200.pt +be0042ae80e91ea9ab3928877389f37bc6d19c170d7076d642e2001de6853d32 /workspace/results/r7/lens_R7_matched_StackExchange_s1_410m_n200.pt +1c2cacc90d6fbf7a38d84d3d372793d070aea548b2e5a659f632e74a04bb2c4e /workspace/results/r7/lens_R7_matched_StackExchange_s2_410m_n200.pt +d7cb96c7c790c8b260720756a2067e1b44e4192f4e39dc9ccd58069c764ee96a /workspace/results/r7/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt +9df86067524293ae11dc6a849f6b79bc7dd0a5863aaadf184a0bee17ea6e5ca1 /workspace/results/r7/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt +dc23ef148a56c0463a7f88af418c6ea04a403a0a6f33b452dff7ed1cd1a5c7ff /workspace/results/r7/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt +c2f6153d750dceb248d1050adccb792c74b8137d129eeb27a236d5a7288f19a1 /workspace/results/r7/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt +da38a7badcbf45eed064963864befa803da85551dc40e4d679a996ecbab86301 /workspace/results/r7/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt +2e87a8aaec3fc4abdda5ea33e417a1303deb3e8ce9135d51d796d8f6800ffd88 /workspace/results/r7/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt +49a278ac0cf1b26aaaa8834e6638e968f90204077b22b3a5f67b3ff98e48d495 /workspace/results/r7/lens_R7_unmatched_Github_s0_410m_n200.pt +d4714d69a2f8c6f5e9e8d32e21f5f3af476e6054ddb24279e38a8128968a9e07 /workspace/results/r7/lens_R7_unmatched_Github_s1_410m_n200.pt +958112fcc4d90dce85e1beefa815354694f7a2ffd7d9e1d058e3c2b948a9c44e /workspace/results/r7/lens_R7_unmatched_Github_s2_410m_n200.pt +3d90dab194cdc620eeed5533f99e0c4908fc98b5622d2252db05979807f64220 /workspace/results/r7/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt +3d996d6da417511ca26a0c3a8c87ad5cb58c29a24d64dc60928c930951ac7985 /workspace/results/r7/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt +37cf8cb50602c1d319df7061e2c5b584977628a60ba6e0556c6d86432cc0364f /workspace/results/r7/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt +efe02b31dc3c5cb83a10f732921bd60b1a60f6baa7c69a0827615e20dc74d065 /workspace/results/r7/lens_R7_unmatched_StackExchange_s0_410m_n200.pt +77279ef0d58e134f86899f2062055c8d850493222bd0cf7b4c112494b2108df0 /workspace/results/r7/lens_R7_unmatched_StackExchange_s1_410m_n200.pt +a308cb623d4fee979c3fbd19f0cdec9e3589dae1d1f3f804c690a72eaf7f7c82 /workspace/results/r7/lens_R7_unmatched_StackExchange_s2_410m_n200.pt +961a6e747fa37130ca039fab945b659e593c43b1907e1832498860a575907efb /workspace/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s0_410m_n200.pt +3267710f269fea1caa1d646e1935f3ca60af9bc2d79b63737ba7228d263ca97e /workspace/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s1_410m_n200.pt +906b3c272fe52e2abb9818e69e5293f731d988e238df48e0949401add764f413 /workspace/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s2_410m_n200.pt +1067b44c08b7f59ac25b19a6f59933c5fd89b36f8fc78e2df578a63fcf1c7876 /workspace/results/r7/lens_R7_unmatched_Wikipedia_en_s0_410m_n200.pt +cee4952b3a420dcfbd807048cd685917811f53bb6973fc9d5839e3a17aa8e1af /workspace/results/r7/lens_R7_unmatched_Wikipedia_en_s1_410m_n200.pt +b4538e53b11faa1a6942130a73981a805a92cd35aba5693a88270ec59c7a74b0 /workspace/results/r7/lens_R7_unmatched_Wikipedia_en_s2_410m_n200.pt diff --git a/results/r7_box_sha256_remote.txt b/results/r7_box_sha256_remote.txt new file mode 100644 index 0000000000000000000000000000000000000000..d5ee773f5f057a7afb7b7854c504129eb0894234 --- /dev/null +++ b/results/r7_box_sha256_remote.txt @@ -0,0 +1,30 @@ +cbf143643e1ccded5f111e45a44abc3ad3987234b91ef0df2c638bbe0de926f9 /workspace/results/r7/lens_R7_matched_Github_s0_410m_n200.pt +2b81e8e18dc6b6bbc2a0296ceaf2fa0bf1f612b19af2e6fd4f4cf09584fa5d97 /workspace/results/r7/lens_R7_matched_Github_s1_410m_n200.pt +c15445f647b3d0d233d75d290bdffe12cacc29bbd1cc0d1268d4c3289cbd7bfd /workspace/results/r7/lens_R7_matched_Github_s2_410m_n200.pt +257d2fe462d5fad6c0b9544b4eb04ef67f58de7a155f76a133267502901fb307 /workspace/results/r7/lens_R7_matched_Pile-CC_s0_410m_n200.pt +e9eb00a20a026ee835e8ef29c6c11c7ba417a5efc910f9a98efd221adef5275b /workspace/results/r7/lens_R7_matched_Pile-CC_s1_410m_n200.pt +42fd87602328b1931e2d2827d30f71ec250ea93455134d9f6a6a46be1fd31d08 /workspace/results/r7/lens_R7_matched_Pile-CC_s2_410m_n200.pt +9452d30c8a399af5eb90367fb6877358ca1527ab2401d3a25895419d353ef404 /workspace/results/r7/lens_R7_matched_StackExchange_s0_410m_n200.pt +be0042ae80e91ea9ab3928877389f37bc6d19c170d7076d642e2001de6853d32 /workspace/results/r7/lens_R7_matched_StackExchange_s1_410m_n200.pt +1c2cacc90d6fbf7a38d84d3d372793d070aea548b2e5a659f632e74a04bb2c4e /workspace/results/r7/lens_R7_matched_StackExchange_s2_410m_n200.pt +d7cb96c7c790c8b260720756a2067e1b44e4192f4e39dc9ccd58069c764ee96a /workspace/results/r7/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt +9df86067524293ae11dc6a849f6b79bc7dd0a5863aaadf184a0bee17ea6e5ca1 /workspace/results/r7/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt +dc23ef148a56c0463a7f88af418c6ea04a403a0a6f33b452dff7ed1cd1a5c7ff /workspace/results/r7/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt +c2f6153d750dceb248d1050adccb792c74b8137d129eeb27a236d5a7288f19a1 /workspace/results/r7/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt +da38a7badcbf45eed064963864befa803da85551dc40e4d679a996ecbab86301 /workspace/results/r7/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt +2e87a8aaec3fc4abdda5ea33e417a1303deb3e8ce9135d51d796d8f6800ffd88 /workspace/results/r7/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt +49a278ac0cf1b26aaaa8834e6638e968f90204077b22b3a5f67b3ff98e48d495 /workspace/results/r7/lens_R7_unmatched_Github_s0_410m_n200.pt +d4714d69a2f8c6f5e9e8d32e21f5f3af476e6054ddb24279e38a8128968a9e07 /workspace/results/r7/lens_R7_unmatched_Github_s1_410m_n200.pt +958112fcc4d90dce85e1beefa815354694f7a2ffd7d9e1d058e3c2b948a9c44e /workspace/results/r7/lens_R7_unmatched_Github_s2_410m_n200.pt +3d90dab194cdc620eeed5533f99e0c4908fc98b5622d2252db05979807f64220 /workspace/results/r7/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt +3d996d6da417511ca26a0c3a8c87ad5cb58c29a24d64dc60928c930951ac7985 /workspace/results/r7/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt +37cf8cb50602c1d319df7061e2c5b584977628a60ba6e0556c6d86432cc0364f /workspace/results/r7/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt +efe02b31dc3c5cb83a10f732921bd60b1a60f6baa7c69a0827615e20dc74d065 /workspace/results/r7/lens_R7_unmatched_StackExchange_s0_410m_n200.pt +77279ef0d58e134f86899f2062055c8d850493222bd0cf7b4c112494b2108df0 /workspace/results/r7/lens_R7_unmatched_StackExchange_s1_410m_n200.pt +a308cb623d4fee979c3fbd19f0cdec9e3589dae1d1f3f804c690a72eaf7f7c82 /workspace/results/r7/lens_R7_unmatched_StackExchange_s2_410m_n200.pt +961a6e747fa37130ca039fab945b659e593c43b1907e1832498860a575907efb /workspace/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s0_410m_n200.pt +3267710f269fea1caa1d646e1935f3ca60af9bc2d79b63737ba7228d263ca97e /workspace/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s1_410m_n200.pt +906b3c272fe52e2abb9818e69e5293f731d988e238df48e0949401add764f413 /workspace/results/r7/lens_R7_unmatched_USPTO_Backgrounds_s2_410m_n200.pt +1067b44c08b7f59ac25b19a6f59933c5fd89b36f8fc78e2df578a63fcf1c7876 /workspace/results/r7/lens_R7_unmatched_Wikipedia_en_s0_410m_n200.pt +cee4952b3a420dcfbd807048cd685917811f53bb6973fc9d5839e3a17aa8e1af /workspace/results/r7/lens_R7_unmatched_Wikipedia_en_s1_410m_n200.pt +b4538e53b11faa1a6942130a73981a805a92cd35aba5693a88270ec59c7a74b0 /workspace/results/r7/lens_R7_unmatched_Wikipedia_en_s2_410m_n200.pt diff --git a/results/r7_matched_pools_410m.json b/results/r7_matched_pools_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..b68d3a4549ec8a27cda46bcca7c08dfc2d5ec7c1 --- /dev/null +++ b/results/r7_matched_pools_410m.json @@ -0,0 +1,577 @@ +{ + "experiment": "R7 \u2014 length- and format-matched fitting pools", + "prereg": "specs/experiments/R7_length_matched_pools.md", + "prereg_sha256": "93164c3e37653398c512820eda09b574ecb0f59496f2bdbec73a6bd05f4faa65", + "status": "PRE-REGISTERED", + "decision_rule_verbatim": "ACCEPT if the matched spread stays above 20 seed SDs. UNCLEAR between 5 and 20. REJECT below 5, in which case the corpus effect is substantially a length effect and the paper says so.", + "readout_convention": "STRIPPED \u2014 the anchor rule (R1). Not comparable to pre-R1 figures.", + "DESIGN_FINDING_length_is_matched_by_construction": "fitting truncates to max_seq_len=128 and filters to documents of >= 128 tokens, so EVERY fitting prompt is already exactly 128 tokens. Fitting-side document length is matched by construction and cannot be the corpus effect; the P-ladder's length effect is a property of the READ prompts. What this experiment therefore matches is the lexical composition of the fitted window, via type-token ratio.", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false + }, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "K": [ + 1, + 2, + 5, + 10, + 20, + 50, + 100 + ], + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "corpora": [ + "Pile-CC", + "StackExchange", + "Wikipedia_en", + "Github", + "USPTO_Backgrounds" + ], + "seeds": [ + 0, + 1, + 2 + ], + "n_fit": 200, + "window_tokens": 128, + "matching_target_ttr": 0.625, + "per_corpus_median_ttr_prelim": { + "Pile-CC": 0.6953125, + "StackExchange": 0.625, + "Wikipedia_en": 0.6484375, + "Github": 0.5546875, + "USPTO_Backgrounds": 0.6171875 + }, + "operators": { + "matched|Pile-CC|s0": { + "path": "results/r7/lens_R7_matched_Pile-CC_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 517.5 + }, + "matched|Pile-CC|s1": { + "path": "results/r7/lens_R7_matched_Pile-CC_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.640625, + "median_doc_tokens": 617.0 + }, + "matched|Pile-CC|s2": { + "path": "results/r7/lens_R7_matched_Pile-CC_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.65625, + "median_doc_tokens": 567.0 + }, + "matched|StackExchange|s0": { + "path": "results/r7/lens_R7_matched_StackExchange_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 445.0 + }, + "matched|StackExchange|s1": { + "path": "results/r7/lens_R7_matched_StackExchange_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 455.0 + }, + "matched|StackExchange|s2": { + "path": "results/r7/lens_R7_matched_StackExchange_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.640625, + "median_doc_tokens": 474.5 + }, + "matched|Wikipedia_en|s0": { + "path": "results/r7/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 539.5 + }, + "matched|Wikipedia_en|s1": { + "path": "results/r7/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6328125, + "median_doc_tokens": 495.0 + }, + "matched|Wikipedia_en|s2": { + "path": "results/r7/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.640625, + "median_doc_tokens": 464.5 + }, + "matched|Github|s0": { + "path": "results/r7/lens_R7_matched_Github_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 945.0 + }, + "matched|Github|s1": { + "path": "results/r7/lens_R7_matched_Github_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 872.0 + }, + "matched|Github|s2": { + "path": "results/r7/lens_R7_matched_Github_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 1062.5 + }, + "matched|USPTO_Backgrounds|s0": { + "path": "results/r7/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 587.5 + }, + "matched|USPTO_Backgrounds|s1": { + "path": "results/r7/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6171875, + "median_doc_tokens": 586.0 + }, + "matched|USPTO_Backgrounds|s2": { + "path": "results/r7/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.609375, + "median_doc_tokens": 521.5 + }, + "unmatched|Pile-CC|s0": { + "path": "results/r7/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6953125, + "median_doc_tokens": 545.0 + }, + "unmatched|Pile-CC|s1": { + "path": "results/r7/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6953125, + "median_doc_tokens": 611.0 + }, + "unmatched|Pile-CC|s2": { + "path": "results/r7/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6875, + "median_doc_tokens": 597.0 + }, + "unmatched|StackExchange|s0": { + "path": "results/r7/lens_R7_unmatched_StackExchange_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6171875, + "median_doc_tokens": 467.0 + }, + "unmatched|StackExchange|s1": { + "path": "results/r7/lens_R7_unmatched_StackExchange_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 472.0 + }, + "unmatched|StackExchange|s2": { + "path": "results/r7/lens_R7_unmatched_StackExchange_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.625, + "median_doc_tokens": 519.0 + }, + "unmatched|Wikipedia_en|s0": { + "path": "results/r7/lens_R7_unmatched_Wikipedia_en_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6484375, + "median_doc_tokens": 503.5 + }, + "unmatched|Wikipedia_en|s1": { + "path": "results/r7/lens_R7_unmatched_Wikipedia_en_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6484375, + "median_doc_tokens": 519.5 + }, + "unmatched|Wikipedia_en|s2": { + "path": "results/r7/lens_R7_unmatched_Wikipedia_en_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.640625, + "median_doc_tokens": 455.5 + }, + "unmatched|Github|s0": { + "path": "results/r7/lens_R7_unmatched_Github_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.55078125, + "median_doc_tokens": 849.5 + }, + "unmatched|Github|s1": { + "path": "results/r7/lens_R7_unmatched_Github_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.5546875, + "median_doc_tokens": 928.5 + }, + "unmatched|Github|s2": { + "path": "results/r7/lens_R7_unmatched_Github_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.5546875, + "median_doc_tokens": 1028.5 + }, + "unmatched|USPTO_Backgrounds|s0": { + "path": "results/r7/lens_R7_unmatched_USPTO_Backgrounds_s0_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.6171875, + "median_doc_tokens": 638.5 + }, + "unmatched|USPTO_Backgrounds|s1": { + "path": "results/r7/lens_R7_unmatched_USPTO_Backgrounds_s1_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.61328125, + "median_doc_tokens": 567.0 + }, + "unmatched|USPTO_Backgrounds|s2": { + "path": "results/r7/lens_R7_unmatched_USPTO_Backgrounds_s2_410m_n200.pt", + "n_prompts": 200, + "median_ttr": 0.609375, + "median_doc_tokens": 596.0 + } + }, + "reads": { + "matched|Pile-CC|s0": { + "min": 0.2629059787839651, + "persist": 0.11104488547425717 + }, + "matched|Pile-CC|s1": { + "min": 0.26485057137906554, + "persist": 0.11187964479904622 + }, + "matched|Pile-CC|s2": { + "min": 0.2678849395364523, + "persist": 0.11355064909439534 + }, + "matched|StackExchange|s0": { + "min": 0.2629284989088774, + "persist": 0.11739591336809099 + }, + "matched|StackExchange|s1": { + "min": 0.26507909297943116, + "persist": 0.11282895365729928 + }, + "matched|StackExchange|s2": { + "min": 0.265835801884532, + "persist": 0.11469835797324776 + }, + "matched|Wikipedia_en|s0": { + "min": 0.24423853494226933, + "persist": 0.10404843266587704 + }, + "matched|Wikipedia_en|s1": { + "min": 0.24238815866410732, + "persist": 0.10449281248729676 + }, + "matched|Wikipedia_en|s2": { + "min": 0.24118152372539042, + "persist": 0.10616923421621323 + }, + "matched|Github|s0": { + "min": 0.23123155497014522, + "persist": 0.09138620859012007 + }, + "matched|Github|s1": { + "min": 0.2281100694090128, + "persist": 0.0875131414271891 + }, + "matched|Github|s2": { + "min": 0.23235553316771984, + "persist": 0.09170255502685905 + }, + "matched|USPTO_Backgrounds|s0": { + "min": 0.2628229308873415, + "persist": 0.12083235094323755 + }, + "matched|USPTO_Backgrounds|s1": { + "min": 0.2603642299771309, + "persist": 0.11865631840191782 + }, + "matched|USPTO_Backgrounds|s2": { + "min": 0.26331988610327245, + "persist": 0.11954337037168443 + }, + "unmatched|Pile-CC|s0": { + "min": 0.2621679771691561, + "persist": 0.11054239000659435 + }, + "unmatched|Pile-CC|s1": { + "min": 0.26131707690656186, + "persist": 0.11126932452898472 + }, + "unmatched|Pile-CC|s2": { + "min": 0.2626881584525108, + "persist": 0.11166209306102245 + }, + "unmatched|StackExchange|s0": { + "min": 0.26657002456486223, + "persist": 0.11692522251978517 + }, + "unmatched|StackExchange|s1": { + "min": 0.26630163192749023, + "persist": 0.11690782881341874 + }, + "unmatched|StackExchange|s2": { + "min": 0.26447896286845207, + "persist": 0.11550639802590013 + }, + "unmatched|Wikipedia_en|s0": { + "min": 0.24112006314098836, + "persist": 0.10448124296963215 + }, + "unmatched|Wikipedia_en|s1": { + "min": 0.244559058919549, + "persist": 0.10538501951377839 + }, + "unmatched|Wikipedia_en|s2": { + "min": 0.24200886227190493, + "persist": 0.10484272565227001 + }, + "unmatched|Github|s0": { + "min": 0.2362893268465996, + "persist": 0.0990156196989119 + }, + "unmatched|Github|s1": { + "min": 0.24347587414085864, + "persist": 0.10213937694206834 + }, + "unmatched|Github|s2": { + "min": 0.23865904659032822, + "persist": 0.09923102268949151 + }, + "unmatched|USPTO_Backgrounds|s0": { + "min": 0.2593545038253069, + "persist": 0.12008050135336816 + }, + "unmatched|USPTO_Backgrounds|s1": { + "min": 0.26404815055429937, + "persist": 0.12104237610474229 + }, + "unmatched|USPTO_Backgrounds|s2": { + "min": 0.2643235493451357, + "persist": 0.12123281219974161 + }, + "__logit__": { + "min": 0.19810852445662022, + "persist": 0.08317913215141744 + } + }, + "PRIMARY_matched_spread_over_seed_sd_persist": 22.23725814088611, + "by_aggregation": { + "persist": { + "matched": { + "per_corpus_mean": { + "Pile-CC": 0.11215839312256624, + "StackExchange": 0.11497440833287935, + "Wikipedia_en": 0.10490349312312901, + "Github": 0.09020063501472274, + "USPTO_Backgrounds": 0.11967734657227994 + }, + "per_corpus_seed_sd": { + "Pile-CC": 0.0010417895823696538, + "StackExchange": 0.0018746436284955115, + "Wikipedia_en": 0.0009132154701795106, + "Github": 0.001904728345335221, + "USPTO_Backgrounds": 0.0008933986127759952 + }, + "pooled_seed_sd": 0.0013255551278311783, + "spread": 0.029476711557557198, + "spread_over_seed_sd": 22.23725814088611 + }, + "unmatched": { + "per_corpus_mean": { + "Pile-CC": 0.11115793586553384, + "StackExchange": 0.11644648311970135, + "Wikipedia_en": 0.10490299604522685, + "Github": 0.10012867311015725, + "USPTO_Backgrounds": 0.12078522988595068 + }, + "per_corpus_seed_sd": { + "Pile-CC": 0.00046385292804432027, + "StackExchange": 0.0006647784708435966, + "Wikipedia_en": 0.00037141836306929114, + "Github": 0.0014244992188681901, + "USPTO_Backgrounds": 0.0005043465772968607 + }, + "pooled_seed_sd": 0.0006857791116244517, + "spread": 0.020656556775793436, + "spread_over_seed_sd": 30.12129769724663 + } + }, + "min": { + "matched": { + "per_corpus_mean": { + "Pile-CC": 0.26521382989982767, + "StackExchange": 0.26461446459094684, + "Wikipedia_en": 0.24260273911058902, + "Github": 0.23056571918229263, + "USPTO_Backgrounds": 0.26216901565591494 + }, + "per_corpus_seed_sd": { + "Pile-CC": 0.0020488175627878186, + "StackExchange": 0.0012315335433519994, + "Wikipedia_en": 0.0012572093358016057, + "Github": 0.0017960129182743854, + "USPTO_Backgrounds": 0.0012922021594725486 + }, + "pooled_seed_sd": 0.0015251551039376714, + "spread": 0.03464811071753504, + "spread_over_seed_sd": 22.717762034877605 + }, + "unmatched": { + "per_corpus_mean": { + "Pile-CC": 0.2620577375094096, + "StackExchange": 0.26578353978693486, + "Wikipedia_en": 0.24256266144414743, + "Github": 0.23947474919259548, + "USPTO_Backgrounds": 0.26257540124158063 + }, + "per_corpus_seed_sd": { + "Pile-CC": 0.0005651434852296106, + "StackExchange": 0.0009289597602075775, + "Wikipedia_en": 0.001457553460110884, + "Github": 0.0029900550223103148, + "USPTO_Backgrounds": 0.002280291830463482 + }, + "pooled_seed_sd": 0.0016444007116643738, + "spread": 0.026308790594339382, + "spread_over_seed_sd": 15.99901435685408 + } + } + }, + "controls": { + "C1_pools_are_matched": { + "required": "post-matching spread in the matched statistic below 10% of pre-matching", + "statistic": "median type-token ratio over the fitted 128-token window", + "pre_matching_median_ttr": { + "Pile-CC": 0.6953125, + "StackExchange": 0.625, + "Wikipedia_en": 0.6484375, + "Github": 0.5546875, + "USPTO_Backgrounds": 0.609375 + }, + "post_matching_median_ttr": { + "Pile-CC": 0.640625, + "StackExchange": 0.625, + "Wikipedia_en": 0.625, + "Github": 0.625, + "USPTO_Backgrounds": 0.625 + }, + "pre_matching_spread": 0.140625, + "post_matching_spread": 0.015625, + "ratio": 0.1111111111111111, + "fires": false + }, + "C2_unmatched_arm_same_code_path": { + "required": "the unmatched arm must reproduce a spread from the SAME code path, so any movement is attributable to matching and not to the pipeline", + "unmatched_spread_over_seed_sd_persist": 30.12129769724663, + "matched_spread_over_seed_sd_persist": 22.23725814088611, + "note": "the published 58x is UNSTRIPPED; this arm is stripped, so the comparison that matters is matched-vs-unmatched WITHIN this run, both at the corrected readout", + "fires": true + }, + "C3_derangement_floor": { + "required": "every operator clears its own layer-deranged floor under persist", + "n_clearing": 30, + "n_operators": 30, + "fires": true + } + }, + "declared_bias": [ + "matching on one statistic leaves markup, code-ness and surprisal free", + "Github may not be matchable against English prose; if its matched pool falls short the run ABORTS and reports the shortfall rather than padding", + "document length is matched by construction (see DESIGN_FINDING), so this is a lexical-composition control, not a length control" + ], + "VERDICT": "ACCEPT \u2014 the corpus effect is NOT substantially a lexical-composition effect. With the fitting pools matched on window type-token ratio the between-corpus spread is 22.2 seed SDs, above the pre-registered 20.", + "provenance": { + "schema_version": 1, + "experiment": "R7", + "utc": "2026-08-20T17:11:20+00:00", + "script": { + "path": "experiments/r7_matched_pools.py", + "sha256": "aae13463200e4eb35ffb008fcd5968de32d05f5af0363503ba2831297745664d", + "n_lines": 254 + }, + "git": { + "commit": null, + "commit_short": null, + "branch": null, + "dirty": false, + "dirty_files": [], + "dirty_diff_sha256": null, + "describe": null + }, + "argv": [ + "r7_matched_pools.py", + "--device", + "cuda", + "--lens-dir", + "/workspace/results/r7", + "--out", + "/workspace/results/r7_matched_pools_410m.json" + ], + "cwd": "experiments", + "inputs": [ + { + "path": "corpora/Pile-CC.jsonl", + "sha256": "abbb9f19a6eb6f4c12fb382735832ceec271fece15cfebcf0fe838765d4b1e74", + "bytes": 13512530, + "exists": true + }, + { + "path": "corpora/StackExchange.jsonl", + "sha256": "bc25e2d5775e9719ebcbdaa3bd115d827f72ebf11f47fedb377f8b0a69a28971", + "bytes": 5980156, + "exists": true + }, + { + "path": "corpora/Wikipedia_en.jsonl", + "sha256": "b30f4cef11add560f7df8ed07a613e3fcc6a4dda865975bec15f2952253bda77", + "bytes": 10329196, + "exists": true + }, + { + "path": "corpora/Github.jsonl", + "sha256": "f2bb459d10cad6c13a8b66c86191bc43df4393ba5b722a5315ea878fad2d1a26", + "bytes": 17486378, + "exists": true + }, + { + "path": "corpora/USPTO_Backgrounds.jsonl", + "sha256": "b09ac74bdc8b169ee5c9bea6be481262f743947a92107164f30eca2346e3ccb9", + "bytes": 10221054, + "exists": true + } + ], + "env": { + "python": "3.11.9", + "torch": "2.11.0+cu128", + "transformers": "5.15.1", + "numpy": "2.4.6", + "platform": "Linux-5.15.0-181-generic-x86_64-with-glibc2.35", + "machine": "x86_64", + "hostname": "", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": false, + "NVIDIA_TF32_OVERRIDE": "0" + } + }, + "payload_sha256": "bb4dad27226cb58effcc42b6b4b6f59b50915f21250c191f7f50770db35143ad" + } +} \ No newline at end of file diff --git a/results/r7_verified_snapshot/MANIFEST.json b/results/r7_verified_snapshot/MANIFEST.json new file mode 100644 index 0000000000000000000000000000000000000000..b4597dc041a14ee13859de8f9f4cec834abcf3e1 --- /dev/null +++ b/results/r7_verified_snapshot/MANIFEST.json @@ -0,0 +1,34 @@ +{ + "stamped": "2026-08-20T12:31:20", + "verified_saved": [ + "lens_R7_matched_Github_s0_410m_n200.pt", + "lens_R7_matched_Github_s1_410m_n200.pt", + "lens_R7_matched_Github_s2_410m_n200.pt", + "lens_R7_matched_Pile-CC_s1_410m_n200.pt", + "lens_R7_matched_Pile-CC_s2_410m_n200.pt", + "lens_R7_matched_StackExchange_s0_410m_n200.pt", + "lens_R7_matched_StackExchange_s1_410m_n200.pt", + "lens_R7_matched_StackExchange_s2_410m_n200.pt", + "lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt", + "lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt", + "lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt", + "lens_R7_matched_Wikipedia_en_s0_410m_n200.pt", + "lens_R7_matched_Wikipedia_en_s1_410m_n200.pt", + "lens_R7_matched_Wikipedia_en_s2_410m_n200.pt", + "lens_R7_unmatched_Pile-CC_s0_410m_n200.pt", + "lens_R7_unmatched_Pile-CC_s1_410m_n200.pt", + "lens_R7_unmatched_Pile-CC_s2_410m_n200.pt" + ], + "skipped": [ + [ + "lens_R7_matched_Pile-CC_s0_410m_n200.pt", + "does not match box hash" + ], + [ + "lens_R7_unmatched_StackExchange_s0_410m_n200.pt", + "does not match box hash" + ] + ], + "source": "results/r7", + "against": "results/r7_box_sha256.txt" +} \ No newline at end of file diff --git a/results/r7_verified_snapshot/lens_R7_matched_Github_s0_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Github_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..712b1d4353a8b93db84133d2a2938c50aaef0a0a --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Github_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf143643e1ccded5f111e45a44abc3ad3987234b91ef0df2c638bbe0de926f9 +size 27268304 diff --git a/results/r7_verified_snapshot/lens_R7_matched_Github_s1_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Github_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f0e939c09c581c4bc994146dc4060e2701c1b0e --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Github_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b81e8e18dc6b6bbc2a0296ceaf2fa0bf1f612b19af2e6fd4f4cf09584fa5d97 +size 27268304 diff --git a/results/r7_verified_snapshot/lens_R7_matched_Github_s2_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Github_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc1a460eec9b1b1c5ddcb6c199e26b6ba3f51ec8 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Github_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c15445f647b3d0d233d75d290bdffe12cacc29bbd1cc0d1268d4c3289cbd7bfd +size 27268304 diff --git a/results/r7_verified_snapshot/lens_R7_matched_Pile-CC_s1_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Pile-CC_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bd2d75096fb01ff9d89d62f52e3ba985bac57f2 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Pile-CC_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9eb00a20a026ee835e8ef29c6c11c7ba417a5efc910f9a98efd221adef5275b +size 27268323 diff --git a/results/r7_verified_snapshot/lens_R7_matched_Pile-CC_s2_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Pile-CC_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..02a72cbe76747b96fa37e09190a8401c9d38ea9d --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Pile-CC_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42fd87602328b1931e2d2827d30f71ec250ea93455134d9f6a6a46be1fd31d08 +size 27268323 diff --git a/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s0_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9e79a5919698ae32433d8114967aa9b4320319c --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9452d30c8a399af5eb90367fb6877358ca1527ab2401d3a25895419d353ef404 +size 27268437 diff --git a/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s1_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..005bf81400c4bc30fffdfb4ca5c5ffbb782a831f --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be0042ae80e91ea9ab3928877389f37bc6d19c170d7076d642e2001de6853d32 +size 27268437 diff --git a/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s2_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6df6bfa747c9575d4bf04afe60f621e562668a2 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_StackExchange_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2cacc90d6fbf7a38d84d3d372793d070aea548b2e5a659f632e74a04bb2c4e +size 27268437 diff --git a/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ff3f993cef884f3a2c473e40bd4d722208b64a5 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7cb96c7c790c8b260720756a2067e1b44e4192f4e39dc9ccd58069c764ee96a +size 27268513 diff --git a/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..1184b978fbbacb54173fa7f328d2f0bbc0cb60dc --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9df86067524293ae11dc6a849f6b79bc7dd0a5863aaadf184a0bee17ea6e5ca1 +size 27268513 diff --git a/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed390aed052f4922fbb9ac54f638de08ac81baec --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_USPTO_Backgrounds_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc23ef148a56c0463a7f88af418c6ea04a403a0a6f33b452dff7ed1cd1a5c7ff +size 27268513 diff --git a/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5d8ee46104f9df6beb85c4a37e896bee247b731 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2f6153d750dceb248d1050adccb792c74b8137d129eeb27a236d5a7288f19a1 +size 27268418 diff --git a/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab1c901011c8966cd082667ba352a3f38ebf7339 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da38a7badcbf45eed064963864befa803da85551dc40e4d679a996ecbab86301 +size 27268418 diff --git a/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..90db7ddb65b3b4950b042ded14691674bd9dbe94 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_matched_Wikipedia_en_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e87a8aaec3fc4abdda5ea33e417a1303deb3e8ce9135d51d796d8f6800ffd88 +size 27268418 diff --git a/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5a50d1e9651c0395915da35de4d8e806dfbfe89 --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s0_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d90dab194cdc620eeed5533f99e0c4908fc98b5622d2252db05979807f64220 +size 27268361 diff --git a/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1638a4481a95065ecf44a50a3e5600dfebcec7e --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s1_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d996d6da417511ca26a0c3a8c87ad5cb58c29a24d64dc60928c930951ac7985 +size 27268361 diff --git a/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt b/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt new file mode 100644 index 0000000000000000000000000000000000000000..07f20106a22abeff366fd4d387e084c825c6ebdd --- /dev/null +++ b/results/r7_verified_snapshot/lens_R7_unmatched_Pile-CC_s2_410m_n200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37cf8cb50602c1d319df7061e2c5b584977628a60ba6e0556c6d86432cc0364f +size 27268361 diff --git a/results/r8_c1_unstripped/ladder_Github_s0.json b/results/r8_c1_unstripped/ladder_Github_s0.json new file mode 100644 index 0000000000000000000000000000000000000000..364e83409936ce284c003b382eac4ddaedbc902f --- /dev/null +++ b/results/r8_c1_unstripped/ladder_Github_s0.json @@ -0,0 +1,661 @@ +{ + "corpus": "Github", + "seed": 0, + "model": "EleutherAI/pythia-410m-deduped", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "device": "cpu", + "rstrip": false, + "readout_convention": "UNSTRIPPED (LEGACY) \u2014 readout at the trailing-space token", + "fitted_N": [ + 25, + 50, + 100, + 200, + 400, + 800 + ], + "ladder_N": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "n_items": 541, + "by_N": { + "25": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.016856929237422134, + "persist": 0.006934812824640956 + }, + "multilingual": { + "min": 0.11131254530378751, + "best1L": 0.07360406113522393, + "mean": 0.03054052504843899, + "persist": 0.026468455392335142 + }, + "order-ops": { + "min": 0.08311688367809568, + "best1L": 0.08311688367809568, + "mean": 0.021678321861795018, + "persist": 0.01558441509093557 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.24702381023338862, + "best1L": 0.15476190910807677, + "mean": 0.08447802292981318, + "persist": 0.07886904730860676 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.00047778308258524964, + "persist": 0.0 + } + }, + "50": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06657420311655317, + "mean": 0.017603755213453302, + "persist": 0.005547850259712764 + }, + "multilingual": { + "min": 0.11675126690949712, + "best1L": 0.07831762065844876, + "mean": 0.032437106394874196, + "persist": 0.029006526406322206 + }, + "order-ops": { + "min": 0.07532467506825924, + "best1L": 0.07532467506825924, + "mean": 0.018581418486844217, + "persist": 0.012987012575779642 + }, + "poetry": { + "min": 0.018950437727783407, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095628363745, + "best1L": 0.15922619349190167, + "mean": 0.09741300317857947, + "persist": 0.08779761926936251 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0011944577147785043, + "persist": 0.0 + } + }, + "75": { + "multihop": { + "min": 0.07766990257161004, + "best1L": 0.06518724028553281, + "mean": 0.018670649966225028, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12327773869037628, + "best1L": 0.08665699937513896, + "mean": 0.034277904379580705, + "persist": 0.031544597287263186 + }, + "order-ops": { + "min": 0.0753246761326279, + "best1L": 0.07402597394372736, + "mean": 0.0212787209810423, + "persist": 0.015584415756165981 + }, + "poetry": { + "min": 0.020408162980207374, + "best1L": 0.011661807341235024, + "mean": 0.004597443355513471, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.28422618763787405, + "best1L": 0.16220238006540708, + "mean": 0.10382326080330781, + "persist": 0.0952380955485361 + }, + "association": { + "min": 0.010869565419852734, + "best1L": 0.004658385179936886, + "mean": 0.0013139034771094365, + "persist": 0.0 + } + }, + "100": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823690840175, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.12146482989192009, + "best1L": 0.08375634465898786, + "mean": 0.03352485144776957, + "persist": 0.030094271525740623 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.021478521737403104, + "persist": 0.018181818138275827 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004373177952532258, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.15922618817005837, + "mean": 0.09993131937725204, + "persist": 0.09077380983425039 + }, + "association": { + "min": 0.00776397529989481, + "best1L": 0.003105590119957924, + "mean": 0.0009555661568551191, + "persist": 0.0 + } + }, + "150": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017070308011690422, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.10913705639541149, + "best1L": 0.07142857102943319, + "mean": 0.030373179553342716, + "persist": 0.025018128167305673 + }, + "order-ops": { + "min": 0.08181818202137947, + "best1L": 0.08181818202137947, + "mean": 0.024675324891826937, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928603359631, + "best1L": 0.15922619242753303, + "mean": 0.08928571282220739, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "175": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017176997398824563, + "persist": 0.002773925129856382 + }, + "multilingual": { + "min": 0.11167512647807598, + "best1L": 0.07287889798837048, + "mean": 0.03101467008569411, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023876123629244312, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004709576323096242, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.26488095415490015, + "best1L": 0.16071428837520735, + "mean": 0.09191849787852593, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.000597228853231562, + "persist": 0.0 + } + }, + "200": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01717699766491673, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11167512674416814, + "best1L": 0.07251631641494376, + "mean": 0.031014670551355396, + "persist": 0.025743291713297367 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023676323671159998, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709091109889, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2633928582072258, + "best1L": 0.16071428837520735, + "mean": 0.09100274728345019, + "persist": 0.07886904890515975 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + }, + "300": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01771044486667961, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.113850615386452, + "best1L": 0.0746918051902737, + "mean": 0.03187928767874837, + "persist": 0.027918781686042036 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07792207838169166, + "mean": 0.023076922865584493, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807474281107, + "mean": 0.004821709024586848, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2752976140805653, + "best1L": 0.16220237793666975, + "mean": 0.09581043757498264, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.001552795059978962, + "best1L": 0.001552795059978962, + "mean": 0.00011944577064631241, + "persist": 0.0 + } + }, + "350": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11421319682683263, + "best1L": 0.07722987633730684, + "mean": 0.03226976083325488, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.023076922998630574, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.278273805975914, + "best1L": 0.16220237793666975, + "mean": 0.09764194036168712, + "persist": 0.08184523973613977 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "375": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07759245777768749, + "mean": 0.032409214946840494, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2812499978712627, + "best1L": 0.16220237793666975, + "mean": 0.09855769308550018, + "persist": 0.08184523893786329 + }, + "association": { + "min": 0.003105590119957924, + "best1L": 0.001552795059978962, + "mean": 0.00023889154129262482, + "persist": 0.0 + } + }, + "400": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11493835970759392, + "best1L": 0.07650471345654555, + "mean": 0.0322697607002088, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.07922077950622354, + "best1L": 0.07922077950622354, + "mean": 0.022977022820019295, + "persist": 0.019480519528899874 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004597443555082593, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27827380171843935, + "best1L": 0.16071428837520735, + "mean": 0.09775641188025475, + "persist": 0.08333333488553762 + }, + "association": { + "min": 0.004658385046890804, + "best1L": 0.001552795059978962, + "mean": 0.0003583373036235571, + "persist": 0.0 + } + }, + "600": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06657420311655317, + "mean": 0.018137202681308345, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11747643191899572, + "best1L": 0.08085569100720542, + "mean": 0.033078596993748634, + "persist": 0.02864394456680332 + }, + "order-ops": { + "min": 0.07792207838169166, + "best1L": 0.07662337619279112, + "mean": 0.023476524245260016, + "persist": 0.020779220653431758 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2782738038471767, + "best1L": 0.16071428837520735, + "mean": 0.09878662947033133, + "persist": 0.08779761953545469 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "700": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.01792382390704006, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07759245804377965, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.02417582429812423, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529761833803995, + "best1L": 0.16220237793666975, + "mean": 0.09592490643262863, + "persist": 0.08333333382116896 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "750": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07868020236492157, + "mean": 0.03249288756134255, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.0792207805705922, + "best1L": 0.07792207838169166, + "mean": 0.023976024739178165, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27678571215697695, + "best1L": 0.16220237793666975, + "mean": 0.09661172330379486, + "persist": 0.08482142870447465 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "775": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.11638868546911649, + "best1L": 0.07868020236492157, + "mean": 0.03254866899390306, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023776224381955608, + "persist": 0.022077922310147966 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.27529762046677725, + "best1L": 0.16220237793666975, + "mean": 0.0974130045090403, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.004658385179936886, + "best1L": 0.003105590119957924, + "mean": 0.0003583373119389372, + "persist": 0.0 + } + }, + "800": { + "multihop": { + "min": 0.07628293974058968, + "best1L": 0.06518724028553281, + "mean": 0.017923823640947894, + "persist": 0.004160887694784573 + }, + "multilingual": { + "min": 0.1156635225883552, + "best1L": 0.07831762092454093, + "mean": 0.03246499631287796, + "persist": 0.028281363126422678 + }, + "order-ops": { + "min": 0.08051948169512409, + "best1L": 0.07922077950622354, + "mean": 0.023876124427520802, + "persist": 0.02337662343467985 + }, + "poetry": { + "min": 0.017492711410990784, + "best1L": 0.011661807341235024, + "mean": 0.004485310720545905, + "persist": 0.004373177752963134 + }, + "typo": { + "min": 0.2738095223903656, + "best1L": 0.16220237793666975, + "mean": 0.09695512801408768, + "persist": 0.086309524652149 + }, + "association": { + "min": 0.006211180106869766, + "best1L": 0.003105590119957924, + "mean": 0.0004777830742698695, + "persist": 0.0 + } + } + }, + "runtime_s": 237.9 +} \ No newline at end of file diff --git a/results/r8_ladder_flatness.json b/results/r8_ladder_flatness.json new file mode 100644 index 0000000000000000000000000000000000000000..ef7939afd1d0b958dfdd391783fea6f728858d04 --- /dev/null +++ b/results/r8_ladder_flatness.json @@ -0,0 +1,1187 @@ +{ + "experiment": "R8 \u2014 the 410M ladder at the corrected readout: does S2 survive?", + "prereg": "docs/experiments/preregs/R8_ladder_rstrip_S2.md", + "prereg_sha256": "ad39ef2bcc321d16fefa87b046c303e402a40dfcb9bb6ce81716d687b28267f4", + "status": "PRE-REGISTERED", + "promoted_from": "POSTREVIEW_EXPERIMENTS.md \u00a75 Tier C item S-1, which promotes itself: 'If R1 returns QUALIFIED or REJECT this becomes Tier A immediately.' R1 returned QUALIFIED.", + "decision_rule_verbatim": "FLAT STANDS \u2014 the largest per-corpus range over N is below 25% of the between-corpus spread, under both aggregations. | OVERTURNED \u2014 the largest range over N is at or above the between-corpus spread. STOP and alert. | UNCLEAR \u2014 anything between 25% and 100%. Report and stop; do not re-cut.", + "DISAGREEMENT_the_published_range_does_not_reproduce": "the programme's stated range over N -- 1.2-3.7 seed SD under persist, 1.2-2.3 under min -- does NOT reproduce under the estimands defined above, on either the full reconstructed N grid or the fitted-N-only subset, and no script in the tree computes it. Recomputed on the SAME cells with e53's own seed-SD convention: persist 3.75-7.54 over all reconstructed N and 2.49-6.27 over fitted N only; min 1.53-9.87 and 1.27-7.02. Reported as a disagreement, NOT adopted (R4's standing rule). It does not contaminate this experiment's verdict: the registered rule is a RELATIVE comparison between the N axis and the corpus axis computed with identical estimands on both conventions, so the stripped-vs-unstripped contrast is valid whether or not the published absolute number can be located.", + "note_on_the_strict_reading": "the strict reading of 'flat' -- that N moves the read no more than seed noise -- ALREADY FAILS at the unstripped readout (published 1.2-3.9 seed SD, all above 1). 'Flat' has always meant small RELATIVE TO THE CORPUS AXIS, and the rule is written on that comparison. Both readings are reported.", + "device": "cpu (R3 established the CPU numbers are the ones to print)", + "n_cells": { + "unstripped": 15, + "stripped": 15 + }, + "admitted_sets": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo" + ], + "asymptote_n_min": 75, + "by_aggregation": { + "persist": { + "unstripped": { + "per_corpus": { + "Pile-CC": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.04533529427079927, + "50": 0.04592010579293682, + "75": 0.0459495924590599, + "100": 0.04576752880322082, + "150": 0.045718906216678165, + "175": 0.04526161145241488, + "200": 0.04546590782701969, + "300": 0.045813309179530255, + "350": 0.045863591631253565, + "375": 0.045764385341178805, + "400": 0.04583690164699441 + }, + "asymptote_mean": 0.04571574828415005, + "asymptote_per_seed": [ + 0.04550649982004885, + 0.04579123274556228, + 0.045849512286839034 + ], + "seed_sd": 0.00018354240934833726, + "range_over_N": 0.000687981006645022, + "argmax_N": 75, + "argmin_N": 175, + "range_over_N_in_seed_sd": 3.7483490005807454 + }, + "StackExchange": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.044386870227754116, + "50": 0.045147582365288624, + "75": 0.04529293298366524, + "100": 0.04539694303557986, + "150": 0.04594243833174308, + "175": 0.045657445587927385, + "200": 0.045729962124356205, + "300": 0.04617513137913885, + "350": 0.04582916813946906, + "375": 0.04582916813946906, + "400": 0.0457057896114531, + "600": 0.045628237085683004, + "700": 0.04560406488322076, + "750": 0.04555572069117001, + "775": 0.04555572069117001, + "800": 0.04553154859514463 + }, + "asymptote_mean": 0.04567387651994216, + "asymptote_per_seed": [ + 0.046019043400883676, + 0.04552208540238896, + 0.04548050075655385 + ], + "seed_sd": 0.0002996455447132135, + "range_over_N": 0.0017882611513847374, + "argmax_N": 300, + "argmin_N": 25, + "range_over_N_in_seed_sd": 5.967921709285738 + }, + "Wikipedia_en": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.039633920549282005, + "50": 0.03904116153717041, + "75": 0.03754242879471609, + "100": 0.03835085562119881, + "150": 0.03785870068130039, + "175": 0.03818781351049741, + "200": 0.03820955348866326, + "300": 0.03929003203908602, + "350": 0.0390407570859506, + "375": 0.03899385027942204, + "400": 0.03900539613373223 + }, + "asymptote_mean": 0.0384977097371741, + "asymptote_per_seed": [ + 0.03849767729462612, + 0.038935919759410714, + 0.03805953215748545 + ], + "seed_sd": 0.00043819380186336355, + "range_over_N": 0.0020914917545659176, + "argmax_N": 25, + "argmin_N": 75, + "range_over_N_in_seed_sd": 4.772983428045112 + }, + "Github": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.026286449815545763, + "50": 0.02871567802947192, + "75": 0.02909637482038566, + "100": 0.028561258963530973, + "150": 0.028028438807952975, + "175": 0.028137753389420966, + "200": 0.027896880509243126, + "300": 0.027730462717868033, + "350": 0.028112143916743142, + "375": 0.028051173349931128, + "400": 0.02803962750449067, + "600": 0.02776784048903556, + "700": 0.02760600383792605, + "750": 0.027983460664039568, + "775": 0.028088551156577612, + "800": 0.028076782982264247 + }, + "asymptote_mean": 0.028084053793529264, + "asymptote_per_seed": [ + 0.028301037585704913, + 0.0282971920464568, + 0.02765393174842608 + ], + "seed_sd": 0.0003725015802729375, + "range_over_N": 0.002809925004839897, + "argmax_N": 75, + "argmin_N": 25, + "range_over_N_in_seed_sd": 7.543390830130232 + }, + "USPTO_Backgrounds": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.04794271270788851, + "50": 0.04956653222796463, + "75": 0.04951468558893317, + "100": 0.0492529208550141, + "150": 0.04858364301423232, + "175": 0.04951962594475065, + "200": 0.049615370109677315, + "300": 0.050262588954397606, + "350": 0.050082464400856265, + "375": 0.05030765310816822, + "400": 0.05047843164453904 + }, + "asymptote_mean": 0.049735264846729854, + "asymptote_per_seed": [ + 0.05013739873492529, + 0.049171620719726125, + 0.04989677508553815 + ], + "seed_sd": 0.0005027384624140496, + "range_over_N": 0.0025357189366505273, + "argmax_N": 400, + "argmin_N": 25, + "range_over_N_in_seed_sd": 5.043813287080745 + } + }, + "pooled_seed_sd": 0.0003593243597223803, + "between_corpus_spread": 0.02165121105320059, + "between_corpus_spread_in_seed_sd": 60.25533885297579, + "range_over_N_in_seed_sd_min": 3.7483490005807454, + "range_over_N_in_seed_sd_max": 7.543390830130232, + "largest_range_over_N_abs": 0.002809925004839897, + "N_axis_as_fraction_of_corpus_axis": 0.12978142413999147, + "grand_mean_admitted_read": 0.041541330636305086 + }, + "stripped": { + "per_corpus": { + "Pile-CC": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.11140980166161345, + "50": 0.11171722078607196, + "75": 0.11184505554182189, + "100": 0.11113762687891722, + "150": 0.1108233797053496, + "175": 0.11071240506356671, + "200": 0.11109574589700927, + "300": 0.11135068345105364, + "350": 0.11140096593825591, + "375": 0.11121517966190975, + "400": 0.11155332014674232 + }, + "asymptote_mean": 0.11123715136495847, + "asymptote_per_seed": [ + 0.11094552632125597, + 0.1112170891924983, + 0.11154883858112116 + ], + "seed_sd": 0.00030215606695676016, + "range_over_N": 0.0011326504782551844, + "argmax_N": 75, + "argmin_N": 175, + "range_over_N_in_seed_sd": 3.7485610984513893 + }, + "StackExchange": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.11559250369845403, + "50": 0.11587556014280943, + "75": 0.11493782970522132, + "100": 0.11490755103351105, + "150": 0.11645313599812132, + "175": 0.11608156284228677, + "200": 0.11614819526494968, + "300": 0.11716873303410552, + "350": 0.1170201028209357, + "375": 0.11676036217028186, + "400": 0.11672356364627678, + "600": 0.11686687982153325, + "700": 0.11701586762709276, + "750": 0.11688094343103113, + "775": 0.11670189930924348, + "800": 0.11677019133099488 + }, + "asymptote_mean": 0.11645977271682753, + "asymptote_per_seed": [ + 0.1166914964189791, + 0.11629787676644568, + 0.11638994496505783 + ], + "seed_sd": 0.0002058908543453051, + "range_over_N": 0.0022611820005944694, + "argmax_N": 300, + "argmin_N": 100, + "range_over_N_in_seed_sd": 10.98243051049844 + }, + "Wikipedia_en": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.10608478283420915, + "50": 0.10540544429705256, + "75": 0.10338134654753264, + "100": 0.10401072965136596, + "150": 0.1035244595702915, + "175": 0.10376699191651174, + "200": 0.10422163241143738, + "300": 0.10532628296031839, + "350": 0.10489796395635322, + "375": 0.10494352126760143, + "400": 0.10466527061625605 + }, + "asymptote_mean": 0.10430424432196314, + "asymptote_per_seed": [ + 0.10454646102965824, + 0.10439181266618626, + 0.10397445927004492 + ], + "seed_sd": 0.0002958845457820791, + "range_over_N": 0.0027034362866765055, + "argmax_N": 25, + "argmin_N": 75, + "range_over_N_in_seed_sd": 9.136794486953718 + }, + "Github": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.0991229980563124, + "50": 0.10184138774694432, + "75": 0.10179506748737324, + "100": 0.10125406717083284, + "150": 0.10017822992411397, + "175": 0.10081879361754373, + "200": 0.10075108146383649, + "300": 0.10050985165649937, + "350": 0.10088564858195327, + "375": 0.10091714199100221, + "400": 0.10116533665429978, + "600": 0.09975035602138156, + "700": 0.10003907173162414, + "750": 0.10032406495440574, + "775": 0.10059643150085494, + "800": 0.10068889623951344 + }, + "asymptote_mean": 0.1006910027853739, + "asymptote_per_seed": [ + 0.09991459704517405, + 0.10191356187534271, + 0.10024484943560495 + ], + "seed_sd": 0.0010715664732975508, + "range_over_N": 0.0027183896906319144, + "argmax_N": 50, + "argmin_N": 25, + "range_over_N_in_seed_sd": 2.536837198971488 + }, + "USPTO_Backgrounds": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.11834790558509883, + "50": 0.11880435993274052, + "75": 0.119253069765511, + "100": 0.11924516103629555, + "150": 0.11928617633169605, + "175": 0.11961021457044851, + "200": 0.12005227912394772, + "300": 0.1203472932889348, + "350": 0.12025963287090971, + "375": 0.12012084909670409, + "400": 0.12055136768058652 + }, + "asymptote_mean": 0.119858449307226, + "asymptote_per_seed": [ + 0.12001419902912208, + 0.11976263592877086, + 0.11979851296378506 + ], + "seed_sd": 0.00013607083564593327, + "range_over_N": 0.0022034620954876905, + "argmax_N": 400, + "argmin_N": 25, + "range_over_N_in_seed_sd": 16.193492786516483 + } + }, + "pooled_seed_sd": 0.0004023137552055257, + "between_corpus_spread": 0.019167446521852094, + "between_corpus_spread_in_seed_sd": 47.64303052989135, + "range_over_N_in_seed_sd_min": 2.536837198971488, + "range_over_N_in_seed_sd_max": 16.193492786516483, + "largest_range_over_N_abs": 0.0027183896906319144, + "N_axis_as_fraction_of_corpus_axis": 0.14182325681893931, + "grand_mean_admitted_read": 0.11051012409926982 + } + }, + "min": { + "unstripped": { + "per_corpus": { + "Pile-CC": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.14498046482247964, + "50": 0.14558872278070167, + "75": 0.1463522210628504, + "100": 0.14653975673552072, + "150": 0.1441910287720107, + "175": 0.1443764969617838, + "200": 0.14452077530856644, + "300": 0.14418798155550447, + "350": 0.1445165145166573, + "375": 0.14495462362787553, + "400": 0.144860134832561 + }, + "asymptote_mean": 0.1449443925970367, + "asymptote_per_seed": [ + 0.14479848204683216, + 0.1454267738209594, + 0.14460792192331856 + ], + "seed_sd": 0.00042848223307768936, + "range_over_N": 0.0023517751800162523, + "argmax_N": 100, + "argmin_N": 300, + "range_over_N_in_seed_sd": 5.488617726630092 + }, + "StackExchange": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.14666887025038403, + "50": 0.14551860642220293, + "75": 0.14544100905990318, + "100": 0.1451335997276363, + "150": 0.14634832455111402, + "175": 0.14604226470525777, + "200": 0.14556079080239648, + "300": 0.14552399167524915, + "350": 0.1458800637118873, + "375": 0.14546014671879154, + "400": 0.14515601756672064, + "600": 0.1452305003823269, + "700": 0.14470378605737572, + "750": 0.14497471542230675, + "775": 0.14487550930962675, + "800": 0.14512226615278495 + }, + "asymptote_mean": 0.14538949898881268, + "asymptote_per_seed": [ + 0.1467122168660316, + 0.14415640338343017, + 0.14529987671697625 + ], + "seed_sd": 0.0012802615956001497, + "range_over_N": 0.0019650841930083096, + "argmax_N": 25, + "argmin_N": 700, + "range_over_N_in_seed_sd": 1.5349083341730132 + }, + "Wikipedia_en": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.13082604543971163, + "50": 0.1295923725480125, + "75": 0.12996750015410638, + "100": 0.1300698409034383, + "150": 0.13060871564029228, + "175": 0.1304364996563111, + "200": 0.1304755835572169, + "300": 0.13099318303700005, + "350": 0.13103487109321924, + "375": 0.1310609740249458, + "400": 0.13103428384555238 + }, + "asymptote_mean": 0.13063127243467582, + "asymptote_per_seed": [ + 0.130278628825077, + 0.1313029898389701, + 0.1303121986399804 + ], + "seed_sd": 0.0005819664393605851, + "range_over_N": 0.0014686014769332845, + "argmax_N": 375, + "argmin_N": 50, + "range_over_N_in_seed_sd": 2.5235157521228513 + }, + "Github": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.11039294185382979, + "50": 0.11233418318664744, + "75": 0.11374441755137274, + "100": 0.1131990646056476, + "150": 0.11251308188019764, + "175": 0.11257432017120576, + "200": 0.1126159229360166, + "300": 0.11377179957039299, + "350": 0.1136853156079139, + "375": 0.11384777862223841, + "400": 0.11352454951653877, + "600": 0.11241270611506132, + "700": 0.11214043575205973, + "750": 0.11206791918015196, + "775": 0.11205277520098857, + "800": 0.11228798598583255 + }, + "asymptote_mean": 0.11288843376397276, + "asymptote_per_seed": [ + 0.11265755031181841, + 0.11329100217138018, + 0.11271674880871967 + ], + "seed_sd": 0.00034988870729241053, + "range_over_N": 0.0034548367684086173, + "argmax_N": 375, + "argmin_N": 25, + "range_over_N_in_seed_sd": 9.874101954143166 + }, + "USPTO_Backgrounds": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.15301355724888188, + "50": 0.1550550834408828, + "75": 0.15543101529280345, + "100": 0.15492235880699895, + "150": 0.15547383686616306, + "175": 0.15519450467434667, + "200": 0.1550266497280626, + "300": 0.1555748948383899, + "350": 0.15522601664775892, + "375": 0.15540267906728245, + "400": 0.155666866242176 + }, + "asymptote_mean": 0.15532431357377577, + "asymptote_per_seed": [ + 0.1562625119492175, + 0.15341835618906077, + 0.15629207258304906 + ], + "seed_sd": 0.0016506736873259887, + "range_over_N": 0.0026533089932941067, + "argmax_N": 400, + "argmin_N": 25, + "range_over_N_in_seed_sd": 1.6074097586133687 + } + }, + "pooled_seed_sd": 0.0008582545325313646, + "between_corpus_spread": 0.04243587980980301, + "between_corpus_spread_in_seed_sd": 49.444399302665154, + "range_over_N_in_seed_sd_min": 1.5349083341730132, + "range_over_N_in_seed_sd_max": 9.874101954143166, + "largest_range_over_N_abs": 0.0034548367684086173, + "N_axis_as_fraction_of_corpus_axis": 0.08141310569954352, + "grand_mean_admitted_read": 0.13783558227165474 + }, + "stripped": { + "per_corpus": { + "Pile-CC": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.26404472844054305, + "50": 0.2638368241577631, + "75": 0.26409451774365844, + "100": 0.2643362818285823, + "150": 0.2619438216888479, + "175": 0.2620632932388357, + "200": 0.2623990206580077, + "300": 0.2623331439193516, + "350": 0.2629714218722213, + "375": 0.2631079659664205, + "400": 0.26348820185909666 + }, + "asymptote_mean": 0.26297085208611354, + "asymptote_per_seed": [ + 0.2618689411718931, + 0.2648016188500656, + 0.262241996236382 + ], + "seed_sd": 0.0015964249836051675, + "range_over_N": 0.0023924601397344314, + "argmax_N": 100, + "argmin_N": 150, + "range_over_N_in_seed_sd": 1.4986361177658327 + }, + "StackExchange": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.26562379392839613, + "50": 0.2647398108527774, + "75": 0.26520457413224946, + "100": 0.2652369648218155, + "150": 0.2650046367020834, + "175": 0.26492826065846853, + "200": 0.26430597802003225, + "300": 0.2643939961989721, + "350": 0.2649892245020185, + "375": 0.26447684310731434, + "400": 0.2644324549606868, + "600": 0.26538911866290227, + "700": 0.2643076193474588, + "750": 0.26485594163338344, + "775": 0.26486748705307644, + "800": 0.26520082439695086 + }, + "asymptote_mean": 0.26482813744267236, + "asymptote_per_seed": [ + 0.2660208510501044, + 0.26433823129200207, + 0.26412532998591054 + ], + "seed_sd": 0.001038391088920308, + "range_over_N": 0.001317815908363884, + "argmax_N": 25, + "argmin_N": 200, + "range_over_N_in_seed_sd": 1.269094007474693 + }, + "Wikipedia_en": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.24237711208739451, + "50": 0.24029758608057386, + "75": 0.24172537178688105, + "100": 0.24146796410814636, + "150": 0.2417010490649513, + "175": 0.2418503462115214, + "200": 0.24160844892086017, + "300": 0.24190224872990734, + "350": 0.24181912128946612, + "375": 0.2417103005218364, + "400": 0.24130723383277655 + }, + "asymptote_mean": 0.24167689827403852, + "asymptote_per_seed": [ + 0.24169260509726073, + 0.24214427703252386, + 0.24119381269233095 + ], + "seed_sd": 0.0004754268016386835, + "range_over_N": 0.0020795260068206534, + "argmax_N": 25, + "argmin_N": 50, + "range_over_N_in_seed_sd": 4.374019301505553 + }, + "Github": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400, + 600, + 700, + 750, + 775, + 800 + ], + "seed_mean_curve": { + "25": 0.23424001009691328, + "50": 0.24093831881348576, + "75": 0.242072430758604, + "100": 0.2403166172848571, + "150": 0.2386540600143018, + "175": 0.2403273320535109, + "200": 0.24049733920643726, + "300": 0.24051590704669557, + "350": 0.2410325521958016, + "375": 0.241287478848937, + "400": 0.2408776698101844, + "600": 0.23984588651607433, + "700": 0.23940109185697067, + "750": 0.23962425569161064, + "775": 0.2394300678035333, + "800": 0.23983843859639906 + }, + "asymptote_mean": 0.24026579483456553, + "asymptote_per_seed": [ + 0.23969338652202668, + 0.24081224109977484, + 0.2402917568818951 + ], + "seed_sd": 0.0005598789266110654, + "range_over_N": 0.007832420661690709, + "argmax_N": 75, + "argmin_N": 25, + "range_over_N_in_seed_sd": 13.989490029746566 + }, + "USPTO_Backgrounds": { + "n_values": [ + 25, + 50, + 75, + 100, + 150, + 175, + 200, + 300, + 350, + 375, + 400 + ], + "seed_mean_curve": { + "25": 0.26270444586518266, + "50": 0.26220700178472767, + "75": 0.26030243096784467, + "100": 0.26096931962917247, + "150": 0.261633845746872, + "175": 0.2613803464564539, + "200": 0.26112591090301673, + "300": 0.26286505758762363, + "350": 0.2618682941510564, + "375": 0.262044957280159, + "400": 0.26221079599289665 + }, + "asymptote_mean": 0.2616001065238995, + "asymptote_per_seed": [ + 0.26057245060210193, + 0.2604688920465017, + 0.26375897692309486 + ], + "seed_sd": 0.0018703534810299764, + "range_over_N": 0.0025626266197789627, + "argmax_N": 300, + "argmin_N": 75, + "range_over_N_in_seed_sd": 1.370129574847938 + } + }, + "pooled_seed_sd": 0.00110809505636104, + "between_corpus_spread": 0.02456234260810683, + "between_corpus_spread_in_seed_sd": 22.166277583413308, + "range_over_N_in_seed_sd_min": 1.269094007474693, + "range_over_N_in_seed_sd_max": 13.989490029746566, + "largest_range_over_N_abs": 0.007832420661690709, + "N_axis_as_fraction_of_corpus_axis": 0.3188792203845251, + "grand_mean_admitted_read": 0.2542683578322579 + } + } + }, + "controls": { + "C4_N_grid_identical_across_seeds": { + "required": "every seed of a corpus reconstructs the SAME N grid", + "ragged_corpora": { + "Wikipedia_en": { + "0": 16, + "1": 16, + "2": 11 + } + }, + "fires": false, + "why_it_matters": "t53_ladder_summary.py:136 averages each seed over ITS OWN N grid. Where the grids differ, the resulting 'seed SD' conflates seed variation with N-grid variation, and it is the denominator of the programme's 58.4x headline. Measured: Wikipedia_en seed 2 is missing its N=800 block (results/lenses/e28/e28_Wikipedia_en_410m_n800_s2.pt does not exist), and it is the ONLY corpus affected. Under `min` that corpus's seed SD is 1.121e-04 on the ragged grid against 5.820e-04 on the intersected one -- a 5.2x difference in one of the five terms of the pooled denominator, which moves spread_over_seed_sd from 55.5 to 49.4. This tool uses the INTERSECTED grid, which is the clean estimand.", + "convention_used_here": "intersected N grid across seeds" + }, + "C1_recovery_without_the_flag": { + "required": "the same code without --rstrip reproduces results/ladder410_cpu at max_abs_diff = 0.0 over every (N, eval set, aggregation) value", + "checked": true, + "c1_dir": "results/r8_c1_unstripped", + "cells_compared": [ + "Github|s0" + ], + "n_values_compared": 384, + "max_abs_diff": 0.0, + "worst_at": null, + "fires": true + }, + "C2_the_convention_bites_here_too": { + "required": "the stripped ladder's admitted mean must be materially above the unstripped one", + "grand_mean_ratio_stripped_over_unstripped": { + "persist": 2.660245168042099, + "min": 1.844722194673436 + }, + "grid_moved_by_for_comparison": 2.55, + "fires": true + }, + "C3_seed_sd_does_not_collapse": { + "required": "pooled seed SD non-zero and within an order of magnitude of the unstripped 3.7e-04", + "stripped_pooled_seed_sd": { + "persist": 0.0004023137552055257, + "min": 0.00110809505636104 + }, + "unstripped_pooled_seed_sd": { + "persist": 0.0003593243597223803, + "min": 0.0008582545325313646 + }, + "fires": true + } + }, + "VERDICT": "UNCLEAR \u2014 the largest range over N is {'persist': 0.14182325681893931, 'min': 0.3188792203845251} of the between-corpus spread, between 25% and 100% under at least one aggregation. Report and stop; do not re-cut (CLAUDE.md \u00a72.9).", + "provenance": { + "schema_version": 1, + "experiment": "R8", + "utc": "2026-08-31T23:20:40+00:00", + "script": { + "path": "tools/r8_ladder_flatness.py", + "sha256": "01d8abe5a9ee9d324f094524bd4f19ecd175cc517f35e2eb0cf89cb90d1d4a2c", + "n_lines": 242 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "tools/r8_ladder_flatness.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/ladder410_cpu_rstrip/ladder_Github_s0.json", + "sha256": "ae3657a11bf48ec3406a7d882011881b198279e59e28cd65108b4c9ac34eaa6f", + "bytes": 15830, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Github_s1.json", + "sha256": "bcdb1e83c24ecaea61a0cb9049d041373cf76dbde1ccaa011ff92590414ad4b9", + "bytes": 15812, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Github_s2.json", + "sha256": "65ce1da93e9b79e893ab314357fbbca83d457af670d47ca25b02643bc4b3d955", + "bytes": 15799, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Pile-CC_s0.json", + "sha256": "620b0528817227111dd87e42a8d3e7cbf85269a978f80d280905dd0b8fc770ba", + "bytes": 10985, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Pile-CC_s1.json", + "sha256": "f474a277cf4aa3e3ec2dc1f1711b2bd9a311cf4a61ca3f0dc9fea5470e21c9ee", + "bytes": 10960, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Pile-CC_s2.json", + "sha256": "71f21d033d1f47e9aeb7c508ce379a2601451b02b40f05ef5e1318e52ea49df6", + "bytes": 10963, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_StackExchange_s0.json", + "sha256": "690b1cbfe102013199a97a92d9e325be3deec66fe7853ef36e75e42b03315a0b", + "bytes": 15799, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_StackExchange_s1.json", + "sha256": "1b78f145b4323034f2dcb02d56f69a645ac1cfc0ce85b398c2df0e45d29b859f", + "bytes": 15786, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_StackExchange_s2.json", + "sha256": "4e0271679b54629b88730bb52a010536e3fb040630f6295996898d085b7afff3", + "bytes": 15818, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s0.json", + "sha256": "b0a0a61246ef5f7c4aea7184d24ad67c7c35b0bee0ba2ded68fd9d7c513689b1", + "bytes": 10972, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s1.json", + "sha256": "855e5de3a33d599516dd154a060b0acb817102742b5aa0bc68008d12f6571b49", + "bytes": 10948, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_USPTO_Backgrounds_s2.json", + "sha256": "d0479790f3d402d8beda7439e8d7f9f8c97c41e5cdfa71de9304fb95dd722fb9", + "bytes": 10962, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s0.json", + "sha256": "790b4ec9a5b8a2e2645c2b98b3a5df80e03a689d66bf60553f4aee37a82b1155", + "bytes": 15702, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s1.json", + "sha256": "96b16e44559f67b86aa25d5293438fe50fd52b5b018d2397afae8459825c4b69", + "bytes": 15603, + "exists": true + }, + { + "path": "results/ladder410_cpu_rstrip/ladder_Wikipedia_en_s2.json", + "sha256": "c3c6721bedf7e108262c383100cbbfad346558da4d09c39e3edfcd9ae5db0ce4", + "bytes": 10982, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "ede0ae7e973380e0a4a2371bdec3f512c64e462a37be336350c0fbe1f0879c85" + } +} \ No newline at end of file diff --git a/results/r9_permutation_calibrated_min.json b/results/r9_permutation_calibrated_min.json new file mode 100644 index 0000000000000000000000000000000000000000..677515e71395755b9f4502a8a014a44efc5f52d0 --- /dev/null +++ b/results/r9_permutation_calibrated_min.json @@ -0,0 +1,723 @@ +{ + "experiment": "R9 \u2014 permutation-calibrated min: the published statistic against its own null", + "prereg": "docs/experiments/preregs/R9_permutation_calibrated_min.md", + "prereg_sha256": "09da59598844066dcf3f67b39eafb84bbda37f194d48b48aa4af4d45839091c9", + "status": "PRE-REGISTERED", + "recomputes_not_remeasures": true, + "source_file": "results/e48_crossover_410m_rstrip.json", + "readout_convention": "STRIPPED \u2014 the anchor rule (R1)", + "PRIMARY_AGGREGATION": "min \u2014 the source's own operational definition of recovery", + "SECONDARY_AGGREGATION": "persist \u2014 DEMOTED, and explicitly labelled: it was selected with knowledge of the derangement-control outcome, which is outcome-dependent metric selection. It is reported as a robustness arm, never as the adjudicator.", + "decision_rule_verbatim": "Descriptive; no accept/reject branch, and none may be invented. The deliverable is that the paper stops reading a raw min score as specificity and instead reports where the real operator sits in its own null.", + "declared_bias": [ + "15 draws floors the empirical p at 1/16 = 0.0625 \u2014 NO single corpus can reach p < 0.05 on this null whatever the effect. Per-corpus values are ranks with a floor, not significance tests; the inferential statement is the across-corpora count.", + "a derangement preserves entries, norms and spectra but destroys layer correspondence. It is a null for CORRESPONDENCE, not for 'the Jacobians carry no layer-specific information'. This measures a failure of procedure-plus-aggregation to distinguish correct from mismatched operators and must not drift into the stronger claim." + ], + "by_aggregation": { + "min": { + "per_corpus": { + "Wikipedia_en": { + "real": 0.24160844944417476, + "n_draws": 15, + "null_min": 0.24522592797875403, + "null_max": 0.27249130234122276, + "null_mean": 0.2583880298336347, + "null_sd": 0.010042110713700085, + "n_null_ge_real": 15, + "beats_every_draw": false, + "empirical_p_one_sided": 1.0, + "p_floor": 0.0625, + "z_vs_null": -1.6709216685460513, + "rank_of_real_among_null_plus_real": 1, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "USPTO_Backgrounds": { + "real": 0.26112590581178663, + "n_draws": 15, + "null_min": 0.25584742724895476, + "null_max": 0.27396334931254385, + "null_mean": 0.2651418822258711, + "null_sd": 0.0056958831201224585, + "n_null_ge_real": 11, + "beats_every_draw": false, + "empirical_p_one_sided": 0.75, + "p_floor": 0.0625, + "z_vs_null": -0.7050665067014444, + "rank_of_real_among_null_plus_real": 5, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "Pile-CC": { + "real": 0.26239902824163436, + "n_draws": 15, + "null_min": 0.26529838889837265, + "null_max": 0.2866306558251381, + "null_mean": 0.2748033052186171, + "null_sd": 0.006241391753991934, + "n_null_ge_real": 15, + "beats_every_draw": false, + "empirical_p_one_sided": 1.0, + "p_floor": 0.0625, + "z_vs_null": -1.9874216306081265, + "rank_of_real_among_null_plus_real": 1, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "StackExchange": { + "real": 0.26430597975850106, + "n_draws": 15, + "null_min": 0.2517006419599056, + "null_max": 0.2754390150308609, + "null_mean": 0.26492709482709564, + "null_sd": 0.006832857811436386, + "n_null_ge_real": 10, + "beats_every_draw": false, + "empirical_p_one_sided": 0.6875, + "p_floor": 0.0625, + "z_vs_null": -0.09090121377251528, + "rank_of_real_among_null_plus_real": 6, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "Github": { + "real": 0.24049734957516195, + "n_draws": 15, + "null_min": 0.20614932514727116, + "null_max": 0.250272324681282, + "null_mean": 0.232407046854496, + "null_sd": 0.01363968726090833, + "n_null_ge_real": 5, + "beats_every_draw": false, + "empirical_p_one_sided": 0.375, + "p_floor": 0.0625, + "z_vs_null": 0.593144297659444, + "rank_of_real_among_null_plus_real": 11, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "OOD_News_2024": { + "real": 0.25006329715251924, + "n_draws": 15, + "null_min": 0.2447837095707655, + "null_max": 0.27084239572286606, + "null_mean": 0.2571750570833683, + "null_sd": 0.009493997580076404, + "n_null_ge_real": 10, + "beats_every_draw": false, + "empirical_p_one_sided": 0.6875, + "p_floor": 0.0625, + "z_vs_null": -0.7490796022292463, + "rank_of_real_among_null_plus_real": 6, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "OOD_arXiv_2023": { + "real": 0.2692730328068137, + "n_draws": 15, + "null_min": 0.25658972561359406, + "null_max": 0.29188896268606185, + "null_mean": 0.2703329712400834, + "null_sd": 0.01202479445851545, + "n_null_ge_real": 7, + "beats_every_draw": false, + "empirical_p_one_sided": 0.5, + "p_floor": 0.0625, + "z_vs_null": -0.0881460749226426, + "rank_of_real_among_null_plus_real": 9, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "OOD_CommonPile": { + "real": 0.2528811445459723, + "n_draws": 15, + "null_min": 0.24596437588334083, + "null_max": 0.2828520558774471, + "null_mean": 0.2607669298350811, + "null_sd": 0.011161853302325888, + "n_null_ge_real": 11, + "beats_every_draw": false, + "empirical_p_one_sided": 0.75, + "p_floor": 0.0625, + "z_vs_null": -0.7064942600047919, + "rank_of_real_among_null_plus_real": 5, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + } + }, + "n_corpora": 8, + "n_beating_every_own_derangement": 0, + "corpora_beating_every_draw": [], + "corpora_NOT_beating_every_draw": [ + "Github", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "empirical_p_by_corpus": { + "Wikipedia_en": 1.0, + "USPTO_Backgrounds": 0.75, + "Pile-CC": 1.0, + "StackExchange": 0.6875, + "Github": 0.375, + "OOD_News_2024": 0.6875, + "OOD_arXiv_2023": 0.5, + "OOD_CommonPile": 0.75 + }, + "z_by_corpus": { + "Wikipedia_en": -1.6709216685460513, + "USPTO_Backgrounds": -0.7050665067014444, + "Pile-CC": -1.9874216306081265, + "StackExchange": -0.09090121377251528, + "Github": 0.593144297659444, + "OOD_News_2024": -0.7490796022292463, + "OOD_arXiv_2023": -0.0881460749226426, + "OOD_CommonPile": -0.7064942600047919 + }, + "median_z": -0.7057803833531182 + }, + "persist": { + "per_corpus": { + "Wikipedia_en": { + "real": 0.10422163642942905, + "n_draws": 15, + "null_min": 0.07532001119107008, + "null_max": 0.08325023017823696, + "null_mean": 0.07876515122130513, + "null_sd": 0.0021518106347064208, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 11.830262755252644, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "USPTO_Backgrounds": { + "real": 0.1200522749684751, + "n_draws": 15, + "null_min": 0.08238022662699222, + "null_max": 0.0909794308245182, + "null_mean": 0.08645985628012567, + "null_sd": 0.002699549498615797, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 12.443712814146973, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "Pile-CC": { + "real": 0.11109574835281819, + "n_draws": 15, + "null_min": 0.07991705778986216, + "null_max": 0.08584446720778942, + "null_mean": 0.08255375813382368, + "null_sd": 0.001788323825895816, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 15.96019121687713, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "StackExchange": { + "real": 0.11614818507805466, + "n_draws": 15, + "null_min": 0.0757306307554245, + "null_max": 0.08735615264158696, + "null_mean": 0.08374283629314352, + "null_sd": 0.0037725259116690156, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 8.589828020710554, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "Github": { + "real": 0.1007510763593018, + "n_draws": 15, + "null_min": 0.0770264295861125, + "null_max": 0.08669814679306001, + "null_mean": 0.08166873932505647, + "null_sd": 0.002662612940796284, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 7.166770934621293, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "OOD_News_2024": { + "real": 0.09791990842204541, + "n_draws": 15, + "null_min": 0.06849358156323433, + "null_max": 0.07370346884708852, + "null_mean": 0.07102687652378033, + "null_sd": 0.0016048114342165146, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 16.75775192329343, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "OOD_arXiv_2023": { + "real": 0.10589559227228165, + "n_draws": 15, + "null_min": 0.07451336067169904, + "null_max": 0.08164319396018982, + "null_mean": 0.07916459808126092, + "null_sd": 0.002277032019581107, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 11.73940197641063, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + }, + "OOD_CommonPile": { + "real": 0.10663878545165062, + "n_draws": 15, + "null_min": 0.07607217859476804, + "null_max": 0.08328658873215318, + "null_mean": 0.07937348173310359, + "null_sd": 0.001626432258332929, + "n_null_ge_real": 0, + "beats_every_draw": true, + "empirical_p_one_sided": 0.0625, + "p_floor": 0.0625, + "z_vs_null": 16.76387293651787, + "rank_of_real_among_null_plus_real": 16, + "draw_keys": [ + "s0|d0", + "s0|d1", + "s0|d2", + "s0|d3", + "s0|d4", + "s1|d0", + "s1|d1", + "s1|d2", + "s1|d3", + "s1|d4", + "s2|d0", + "s2|d1", + "s2|d2", + "s2|d3", + "s2|d4" + ] + } + }, + "n_corpora": 8, + "n_beating_every_own_derangement": 8, + "corpora_beating_every_draw": [ + "Github", + "OOD_CommonPile", + "OOD_News_2024", + "OOD_arXiv_2023", + "Pile-CC", + "StackExchange", + "USPTO_Backgrounds", + "Wikipedia_en" + ], + "corpora_NOT_beating_every_draw": [], + "empirical_p_by_corpus": { + "Wikipedia_en": 0.0625, + "USPTO_Backgrounds": 0.0625, + "Pile-CC": 0.0625, + "StackExchange": 0.0625, + "Github": 0.0625, + "OOD_News_2024": 0.0625, + "OOD_arXiv_2023": 0.0625, + "OOD_CommonPile": 0.0625 + }, + "z_by_corpus": { + "Wikipedia_en": 11.830262755252644, + "USPTO_Backgrounds": 12.443712814146973, + "Pile-CC": 15.96019121687713, + "StackExchange": 8.589828020710554, + "Github": 7.166770934621293, + "OOD_News_2024": 16.75775192329343, + "OOD_arXiv_2023": 11.73940197641063, + "OOD_CommonPile": 16.76387293651787 + }, + "median_z": 12.136987784699809 + } + }, + "controls": { + "C1_null_is_the_corpus_own_derangement": { + "required": "per-corpus draw keys of the form s{seed}|d{draw}, read from that corpus's rung", + "n_corpora_checked": 16, + "malformed": [], + "fires": true, + "why": "R4d found E36 building its derangement from Pile-CC's operator and comparing it against every corpus's. That must not recur here." + }, + "C2_aggregations_disagree": { + "required": "under persist the real operator beats its own null on 8 of 8; under min it does not. If they agreed the whole framing would be wrong.", + "persist_n_beating": 8, + "min_n_beating": 0, + "fires": true + } + }, + "VERDICT": "CALIBRATED. Under the SOURCE'S OWN statistic (`min`), the real operator exceeds every one of its 15 own layer-derangement draws on 0 of 8 corpora (median z = -0.71). Under `persist` it does so on 8 of 8 (median z = +12.14). The published statistic, read against its own null rather than as a raw score, does not certify layer-to-derivative correspondence \u2014 and that statement is now made WITHOUT selecting the metric on the control it has to pass.", + "provenance": { + "schema_version": 1, + "experiment": "R9", + "utc": "2026-08-31T23:20:42+00:00", + "script": { + "path": "tools/r9_permutation_calibrated_min.py", + "sha256": "4c566271e27637424217f24886abcae15e97919e7c10067f3923c75348a7d110", + "n_lines": 145 + }, + "git": { + "commit": "9987d82887ed339bf69b5e7fa1e4fc98d5bbb4ab", + "commit_short": "9987d82887ed", + "git_repository_present": true, + "branch": "pythia", + "dirty": true, + "dirty_files": [ + "CONTEXT.md", + "NEELNANDA_REP.md", + "docs/experiments/COVERAGE.md", + "docs/experiments/METRICS.md", + "docs/experiments/descriptions/D1.md", + "docs/experiments/descriptions/E33.md", + "docs/experiments/descriptions/E34A.md", + "docs/experiments/descriptions/E34B.md", + "docs/experiments/descriptions/E36.md", + "docs/experiments/descriptions/E37.md", + "docs/experiments/descriptions/E44.md", + "docs/experiments/descriptions/E58.md", + "docs/experiments/preregs/E66d_s_quantization_limited.md", + "docs/experiments/preregs/superseded/PREREG_PYTHIA_T7_v2.md", + "docs/reproducibility/ARTIFACTS.md", + "docs/reproducibility/PAYLOAD_REDACTIONS.md", + "docs/reproducibility/SCRIPT_PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/MANIFEST.sha256", + "docs/reproducibility/review-snapshot-2026-08-19/PROVENANCE.md", + "docs/reproducibility/review-snapshot-2026-08-19/REDACTIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/EXTERNAL_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/AGENT_REPORT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/CONTEXT_UNDER_REVIEW.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/FINDINGS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/JUSTIFIABILITY.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/LEDGER.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/PHASE_A_ACCOUNT.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/SUSPICIONS.md", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Github_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Pile-CC_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_StackExchange_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_USPTO_Backgrounds_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s0.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s1.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/data/rstrip_rescore/rs_Wikipedia_en_s2.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/balance_test.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/check_payload.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/coverage2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derange.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/derangement_result.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_base.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/logit_baseline.json", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_anova.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_e52.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/my_rescore.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg2.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/prereg_audit.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rescore_all.py", + "docs/reproducibility/review-snapshot-2026-08-19/review/reproducibility/scripts/rstrip_anova.py", + "docs/validity/AUDIT_READINESS.md", + "docs/validity/audit-2026-08-22/DIGEST.json", + "experiments/d1_min_union_diagnostic.py", + "experiments/e28_eval_410m.sh", + "experiments/e28_ladder_410m.py", + "experiments/exploratory/descriptions/T0.md", + "experiments/exploratory/descriptions/T12.md" + ], + "dirty_diff_sha256": "aff7ed44a7f13d5d00e837c12a7446366431d5a5ef0602ddfcfb051e2a0ddd20", + "describe": "9987d82-dirty" + }, + "argv": [ + "tools/r9_permutation_calibrated_min.py" + ], + "cwd": ".", + "inputs": [ + { + "path": "results/e48_crossover_410m_rstrip.json", + "sha256": "eef75b627898125b090cb0b8b351a398885b1162f27c334bcef895d980064cc6", + "bytes": 111498, + "exists": true + } + ], + "env": { + "python": "3.14.2", + "torch": "2.13.0", + "transformers": "5.14.1", + "numpy": "2.5.1", + "platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "tf32": { + "allow_tf32_matmul": false, + "allow_tf32_cudnn": true, + "NVIDIA_TF32_OVERRIDE": null + }, + "threads": { + "torch_num_threads": 10, + "torch_num_interop_threads": 14, + "OMP_NUM_THREADS": null, + "os_cpu_count": 14 + } + }, + "payload_sha256": "b1575a600d482ac6d93dc1614d376a28a55488f95c3e0daedb4403eceab5279b" + } +} \ No newline at end of file diff --git a/results/t0_smoke_160m.json b/results/t0_smoke_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..777ab9cedf87c66489f90cba3656f054de237285 --- /dev/null +++ b/results/t0_smoke_160m.json @@ -0,0 +1,254 @@ +{ + "experiment": "T0 port gate \u2014 jlens on GPT-NeoX/Pythia", + "status": "SMOKE (tiny-first, not a result)", + "model": "EleutherAI/pythia-160m-deduped", + "n_prompts": 3, + "dim_batch": 64, + "max_seq_len": 128, + "device": "cpu", + "dtype": "float32", + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "n_layers": 12, + "d_model": 768, + "ALL_GATES_PASS": true, + "gates": { + "G1_autodetect_worked": false, + "G1_fallback_reason": "could not locate the text decoder inside GPTNeoXForCausalLM (tried 6 known layouts); pass layout= explicitly", + "G1_layout_path": "gpt_neox", + "G1_layout_lm_head": "lm_head", + "G1_layout_norm": "final_layer_norm", + "G1_layout_embed": "embed_in", + "G1_transformers_version": "5.14.1", + "G1_PASS": true, + "G1_note": "anchor's shipped Pythia layout uses lm_head='embed_out', which transformers 5.x renamed to 'lm_head'; explicit Layout supplied.", + "G2_embed_in_shape": [ + 50304, + 768 + ], + "G2_embed_out_shape": [ + 50304, + 768 + ], + "G2_same_storage": false, + "G2_max_abs_diff": 0.4976806640625, + "G2_PASS": true, + "G2_note": "UNTIED: W_U is an independent matrix. Gemma-2 ties, Pythia does not -- this is the tied-vs-untied control.", + "G3_logit_softcap": null, + "G3_PASS": true, + "G4_bos_token_id": 0, + "G4_has_add_bos_attr": true, + "G4_first_token_id": 0, + "G4_bos_actually_prepended": true, + "G4_PASS": true, + "G4_note": "jlens force_bos=True sets tokenizer.add_bos_token when the attr exists. GPT-NeoX fast tokenizers often ignore it. Recorded so the attention-sink condition is known rather than assumed.", + "G8_skip_first": 16, + "G8_seq_lens": [ + 50, + 49, + 49 + ], + "G8_n_valid_positions": [ + 33, + 32, + 32 + ], + "G8_PASS": true, + "G5_n_source_layers": 11, + "G5_expected_layers": 11, + "G5_d_model": 768, + "G5_all_finite": true, + "G5_all_nondegenerate": true, + "G5_PASS": true, + "G5_fit_secs": 15.84, + "G5_secs_per_prompt": 5.28, + "G6_readout_layer": 5, + "G6_jlens_top8": [ + "\u53f7", + "\u3052", + "\u8a00", + "\u540d", + "\ufffd", + "\u3063\u3066\u3044\u308b", + "\u308b\u3068", + "\u3063\u305f" + ], + "G6_logitlens_top8": [ + " yearly", + " annually", + " \u201d", + " annual", + " regional", + " collabor", + " \u201c", + " syn" + ], + "G6_model_top8": [ + " the", + " \u00ab", + " La", + " Le", + " \"", + " Paris", + " L", + " la" + ], + "G6_jlens_differs_from_logitlens": true, + "G6_jlens_finite": true, + "G6_PASS": true, + "G7_roundtrip_max_abs_diff": 0.0, + "G7_lens_bytes": 12980477, + "G7_PASS": true + }, + "per_layer": [ + { + "layer": 0, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 20.8511, + "fro_over_sqrt_d": 0.7524, + "top_sv": 8.1259, + "stable_rank": 6.5844, + "dist_from_identity": 1.4451 + }, + { + "layer": 1, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 19.8027, + "fro_over_sqrt_d": 0.7146, + "top_sv": 7.2778, + "stable_rank": 7.4037, + "dist_from_identity": 1.4725 + }, + { + "layer": 2, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 19.335, + "fro_over_sqrt_d": 0.6977, + "top_sv": 7.0245, + "stable_rank": 7.5763, + "dist_from_identity": 1.4585 + }, + { + "layer": 3, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 20.3182, + "fro_over_sqrt_d": 0.7332, + "top_sv": 7.9443, + "stable_rank": 6.5413, + "dist_from_identity": 1.3464 + }, + { + "layer": 4, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 23.0794, + "fro_over_sqrt_d": 0.8328, + "top_sv": 9.3146, + "stable_rank": 6.1394, + "dist_from_identity": 1.1954 + }, + { + "layer": 5, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 25.7759, + "fro_over_sqrt_d": 0.9301, + "top_sv": 10.7622, + "stable_rank": 5.7363, + "dist_from_identity": 1.0633 + }, + { + "layer": 6, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 28.3686, + "fro_over_sqrt_d": 1.0237, + "top_sv": 11.4475, + "stable_rank": 6.1412, + "dist_from_identity": 0.9497 + }, + { + "layer": 7, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 28.9073, + "fro_over_sqrt_d": 1.0431, + "top_sv": 12.0901, + "stable_rank": 5.7168, + "dist_from_identity": 0.8463 + }, + { + "layer": 8, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 31.8156, + "fro_over_sqrt_d": 1.148, + "top_sv": 12.2788, + "stable_rank": 6.7138, + "dist_from_identity": 0.7366 + }, + { + "layer": 9, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 33.0032, + "fro_over_sqrt_d": 1.1909, + "top_sv": 12.5741, + "stable_rank": 6.889, + "dist_from_identity": 0.635 + }, + { + "layer": 10, + "shape": [ + 768, + 768 + ], + "finite": true, + "fro_norm": 33.6649, + "fro_over_sqrt_d": 1.2148, + "top_sv": 14.6628, + "stable_rank": 5.2713, + "dist_from_identity": 0.566 + } + ], + "runtime_s": 29.0, + "centered": false, + "env": { + "note": "not recorded at run time; backfilled 2026-08-10", + "device": "cpu", + "dtype": "float32" + } +} \ No newline at end of file diff --git a/results/t0_smoke_1b.json b/results/t0_smoke_1b.json new file mode 100644 index 0000000000000000000000000000000000000000..60e90945b578c51a7422138b79164ba0523a7486 --- /dev/null +++ b/results/t0_smoke_1b.json @@ -0,0 +1,306 @@ +{ + "experiment": "T0 port gate \u2014 jlens on GPT-NeoX/Pythia", + "status": "SMOKE (tiny-first, not a result)", + "model": "EleutherAI/pythia-1b-deduped", + "n_prompts": 3, + "dim_batch": 64, + "max_seq_len": 128, + "device": "cpu", + "dtype": "float32", + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "n_layers": 16, + "d_model": 2048, + "ALL_GATES_PASS": true, + "gates": { + "G1_autodetect_worked": false, + "G1_fallback_reason": "could not locate the text decoder inside GPTNeoXForCausalLM (tried 6 known layouts); pass layout= explicitly", + "G1_layout_path": "gpt_neox", + "G1_layout_lm_head": "lm_head", + "G1_layout_norm": "final_layer_norm", + "G1_layout_embed": "embed_in", + "G1_transformers_version": "5.14.1", + "G1_PASS": true, + "G1_note": "anchor's shipped Pythia layout uses lm_head='embed_out', which transformers 5.x renamed to 'lm_head'; explicit Layout supplied.", + "G2_embed_in_shape": [ + 50304, + 2048 + ], + "G2_embed_out_shape": [ + 50304, + 2048 + ], + "G2_same_storage": false, + "G2_max_abs_diff": 0.4397544860839844, + "G2_PASS": true, + "G2_note": "UNTIED: W_U is an independent matrix. Gemma-2 ties, Pythia does not -- this is the tied-vs-untied control.", + "G3_logit_softcap": null, + "G3_PASS": true, + "G4_bos_token_id": 0, + "G4_has_add_bos_attr": true, + "G4_first_token_id": 0, + "G4_bos_actually_prepended": true, + "G4_PASS": true, + "G4_note": "jlens force_bos=True sets tokenizer.add_bos_token when the attr exists. GPT-NeoX fast tokenizers often ignore it. Recorded so the attention-sink condition is known rather than assumed.", + "G8_skip_first": 16, + "G8_seq_lens": [ + 50, + 49, + 49 + ], + "G8_n_valid_positions": [ + 33, + 32, + 32 + ], + "G8_PASS": true, + "G5_n_source_layers": 15, + "G5_expected_layers": 15, + "G5_d_model": 2048, + "G5_all_finite": true, + "G5_all_nondegenerate": true, + "G5_PASS": true, + "G5_fit_secs": 555.5, + "G5_secs_per_prompt": 185.166, + "G6_readout_layer": 7, + "G6_jlens_top8": [ + " \"_", + " _", + " '", + " Method", + " \"", + " '_", + " after", + " Mor" + ], + "G6_logitlens_top8": [ + "ously", + "zitter", + "igraph", + " \"~", + "aris", + "OUS", + " upon", + "upon" + ], + "G6_model_top8": [ + " the", + " Paris", + " \"", + " Le", + " V", + " Vers", + " '", + " Les" + ], + "G6_jlens_differs_from_logitlens": true, + "G6_jlens_finite": true, + "G6_PASS": true, + "G7_roundtrip_max_abs_diff": 0.0, + "G7_lens_bytes": 125834483, + "G7_PASS": true + }, + "per_layer": [ + { + "layer": 0, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 28.2656, + "fro_over_sqrt_d": 0.6246, + "top_sv": 6.1944, + "stable_rank": 20.8215, + "dist_from_identity": 1.637 + }, + { + "layer": 1, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 26.0963, + "fro_over_sqrt_d": 0.5767, + "top_sv": 5.5234, + "stable_rank": 22.323, + "dist_from_identity": 1.7125 + }, + { + "layer": 2, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 27.0101, + "fro_over_sqrt_d": 0.5968, + "top_sv": 5.5828, + "stable_rank": 23.4076, + "dist_from_identity": 1.6301 + }, + { + "layer": 3, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 28.0204, + "fro_over_sqrt_d": 0.6192, + "top_sv": 5.8949, + "stable_rank": 22.594, + "dist_from_identity": 1.5264 + }, + { + "layer": 4, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 29.0728, + "fro_over_sqrt_d": 0.6424, + "top_sv": 6.0569, + "stable_rank": 23.0397, + "dist_from_identity": 1.4463 + }, + { + "layer": 5, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 32.5011, + "fro_over_sqrt_d": 0.7182, + "top_sv": 6.5746, + "stable_rank": 24.4377, + "dist_from_identity": 1.2807 + }, + { + "layer": 6, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 36.6504, + "fro_over_sqrt_d": 0.8099, + "top_sv": 7.5618, + "stable_rank": 23.4916, + "dist_from_identity": 1.1176 + }, + { + "layer": 7, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 40.2499, + "fro_over_sqrt_d": 0.8894, + "top_sv": 8.2133, + "stable_rank": 24.0155, + "dist_from_identity": 1.0011 + }, + { + "layer": 8, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 45.0699, + "fro_over_sqrt_d": 0.9959, + "top_sv": 9.3158, + "stable_rank": 23.4063, + "dist_from_identity": 0.8855 + }, + { + "layer": 9, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 48.0488, + "fro_over_sqrt_d": 1.0617, + "top_sv": 8.9914, + "stable_rank": 28.5566, + "dist_from_identity": 0.7706 + }, + { + "layer": 10, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 52.6541, + "fro_over_sqrt_d": 1.1635, + "top_sv": 9.0242, + "stable_rank": 34.0446, + "dist_from_identity": 0.6963 + }, + { + "layer": 11, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 53.6588, + "fro_over_sqrt_d": 1.1857, + "top_sv": 7.9312, + "stable_rank": 45.7719, + "dist_from_identity": 0.6014 + }, + { + "layer": 12, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 55.9735, + "fro_over_sqrt_d": 1.2369, + "top_sv": 7.7764, + "stable_rank": 51.809, + "dist_from_identity": 0.5431 + }, + { + "layer": 13, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 53.9825, + "fro_over_sqrt_d": 1.1929, + "top_sv": 7.0081, + "stable_rank": 59.3349, + "dist_from_identity": 0.4592 + }, + { + "layer": 14, + "shape": [ + 2048, + 2048 + ], + "finite": true, + "fro_norm": 47.7475, + "fro_over_sqrt_d": 1.0551, + "top_sv": 5.4725, + "stable_rank": 76.1243, + "dist_from_identity": 0.3219 + } + ], + "runtime_s": 597.3, + "centered": false, + "env": { + "note": "not recorded at run time; backfilled 2026-08-10", + "device": "cpu", + "dtype": "float32" + } +} \ No newline at end of file diff --git a/results/t0_smoke_410m.json b/results/t0_smoke_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..f82618cdf82147411b91d5301769f568c13c838d --- /dev/null +++ b/results/t0_smoke_410m.json @@ -0,0 +1,410 @@ +{ + "experiment": "T0 port gate \u2014 jlens on GPT-NeoX/Pythia", + "status": "SMOKE (tiny-first, not a result)", + "model": "EleutherAI/pythia-410m-deduped", + "n_prompts": 3, + "dim_batch": 64, + "max_seq_len": 128, + "device": "cpu", + "dtype": "float32", + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "n_layers": 24, + "d_model": 1024, + "ALL_GATES_PASS": true, + "gates": { + "G1_autodetect_worked": false, + "G1_fallback_reason": "could not locate the text decoder inside GPTNeoXForCausalLM (tried 6 known layouts); pass layout= explicitly", + "G1_layout_path": "gpt_neox", + "G1_layout_lm_head": "lm_head", + "G1_layout_norm": "final_layer_norm", + "G1_layout_embed": "embed_in", + "G1_transformers_version": "5.14.1", + "G1_PASS": true, + "G1_note": "anchor's shipped Pythia layout uses lm_head='embed_out', which transformers 5.x renamed to 'lm_head'; explicit Layout supplied.", + "G2_embed_in_shape": [ + 50304, + 1024 + ], + "G2_embed_out_shape": [ + 50304, + 1024 + ], + "G2_same_storage": false, + "G2_max_abs_diff": 0.4120140075683594, + "G2_PASS": true, + "G2_note": "UNTIED: W_U is an independent matrix. Gemma-2 ties, Pythia does not -- this is the tied-vs-untied control.", + "G3_logit_softcap": null, + "G3_PASS": true, + "G4_bos_token_id": 0, + "G4_has_add_bos_attr": true, + "G4_first_token_id": 0, + "G4_bos_actually_prepended": true, + "G4_PASS": true, + "G4_note": "jlens force_bos=True sets tokenizer.add_bos_token when the attr exists. GPT-NeoX fast tokenizers often ignore it. Recorded so the attention-sink condition is known rather than assumed.", + "G8_skip_first": 16, + "G8_seq_lens": [ + 50, + 49, + 49 + ], + "G8_n_valid_positions": [ + 33, + 32, + 32 + ], + "G8_PASS": true, + "G5_n_source_layers": 23, + "G5_expected_layers": 23, + "G5_d_model": 1024, + "G5_all_finite": true, + "G5_all_nondegenerate": true, + "G5_PASS": true, + "G5_fit_secs": 93.93, + "G5_secs_per_prompt": 31.308, + "G6_readout_layer": 11, + "G6_jlens_top8": [ + " \u201c", + " \u2018", + " \"", + " xe", + " \"_", + " \\\"", + " '", + " \"*" + ], + "G6_logitlens_top8": [ + "nett", + "nad", + "avir", + "{\"", + " \u200b", + " ecc", + "oxin", + "\"}](#" + ], + "G6_model_top8": [ + " the", + " Paris", + " \"", + " \u201c", + " Vers", + " La", + " Les", + " by" + ], + "G6_jlens_differs_from_logitlens": true, + "G6_jlens_finite": true, + "G6_PASS": true, + "G7_roundtrip_max_abs_diff": 0.0, + "G7_lens_bytes": 48242013, + "G7_PASS": true + }, + "per_layer": [ + { + "layer": 0, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 10.9009, + "fro_over_sqrt_d": 0.3407, + "top_sv": 2.1408, + "stable_rank": 25.9291, + "dist_from_identity": 2.9138 + }, + { + "layer": 1, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 10.0973, + "fro_over_sqrt_d": 0.3155, + "top_sv": 1.9517, + "stable_rank": 26.7649, + "dist_from_identity": 3.118 + }, + { + "layer": 2, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 10.0506, + "fro_over_sqrt_d": 0.3141, + "top_sv": 1.9109, + "stable_rank": 27.6645, + "dist_from_identity": 3.1139 + }, + { + "layer": 3, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 9.8733, + "fro_over_sqrt_d": 0.3085, + "top_sv": 1.8776, + "stable_rank": 27.6501, + "dist_from_identity": 3.1365 + }, + { + "layer": 4, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 10.5389, + "fro_over_sqrt_d": 0.3293, + "top_sv": 1.9963, + "stable_rank": 27.8707, + "dist_from_identity": 2.9198 + }, + { + "layer": 5, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 10.7785, + "fro_over_sqrt_d": 0.3368, + "top_sv": 2.0824, + "stable_rank": 26.7912, + "dist_from_identity": 2.8066 + }, + { + "layer": 6, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 11.1251, + "fro_over_sqrt_d": 0.3477, + "top_sv": 2.1819, + "stable_rank": 25.9984, + "dist_from_identity": 2.6977 + }, + { + "layer": 7, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 12.121, + "fro_over_sqrt_d": 0.3788, + "top_sv": 2.4588, + "stable_rank": 24.3017, + "dist_from_identity": 2.4515 + }, + { + "layer": 8, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 13.4393, + "fro_over_sqrt_d": 0.42, + "top_sv": 2.7522, + "stable_rank": 23.8446, + "dist_from_identity": 2.1824 + }, + { + "layer": 9, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 15.0409, + "fro_over_sqrt_d": 0.47, + "top_sv": 3.2356, + "stable_rank": 21.6093, + "dist_from_identity": 1.923 + }, + { + "layer": 10, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 17.0935, + "fro_over_sqrt_d": 0.5342, + "top_sv": 3.6506, + "stable_rank": 21.924, + "dist_from_identity": 1.6659 + }, + { + "layer": 11, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 20.1455, + "fro_over_sqrt_d": 0.6295, + "top_sv": 4.2316, + "stable_rank": 22.6641, + "dist_from_identity": 1.3855 + }, + { + "layer": 12, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 21.8139, + "fro_over_sqrt_d": 0.6817, + "top_sv": 4.4557, + "stable_rank": 23.9686, + "dist_from_identity": 1.227 + }, + { + "layer": 13, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 23.5371, + "fro_over_sqrt_d": 0.7355, + "top_sv": 4.6225, + "stable_rank": 25.9265, + "dist_from_identity": 1.0927 + }, + { + "layer": 14, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 26.5139, + "fro_over_sqrt_d": 0.8286, + "top_sv": 4.9366, + "stable_rank": 28.8468, + "dist_from_identity": 0.9423 + }, + { + "layer": 15, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 29.0576, + "fro_over_sqrt_d": 0.9081, + "top_sv": 5.3451, + "stable_rank": 29.5538, + "dist_from_identity": 0.8374 + }, + { + "layer": 16, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 32.7586, + "fro_over_sqrt_d": 1.0237, + "top_sv": 5.3953, + "stable_rank": 36.8649, + "dist_from_identity": 0.6996 + }, + { + "layer": 17, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 34.9328, + "fro_over_sqrt_d": 1.0916, + "top_sv": 5.3423, + "stable_rank": 42.7565, + "dist_from_identity": 0.609 + }, + { + "layer": 18, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 36.53, + "fro_over_sqrt_d": 1.1416, + "top_sv": 5.1909, + "stable_rank": 49.5234, + "dist_from_identity": 0.567 + }, + { + "layer": 19, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 36.6732, + "fro_over_sqrt_d": 1.146, + "top_sv": 4.7791, + "stable_rank": 58.8842, + "dist_from_identity": 0.5058 + }, + { + "layer": 20, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 36.6086, + "fro_over_sqrt_d": 1.144, + "top_sv": 4.81, + "stable_rank": 57.9271, + "dist_from_identity": 0.4592 + }, + { + "layer": 21, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 36.6001, + "fro_over_sqrt_d": 1.1438, + "top_sv": 4.588, + "stable_rank": 63.6384, + "dist_from_identity": 0.4143 + }, + { + "layer": 22, + "shape": [ + 1024, + 1024 + ], + "finite": true, + "fro_norm": 34.574, + "fro_over_sqrt_d": 1.0804, + "top_sv": 5.2817, + "stable_rank": 42.8499, + "dist_from_identity": 0.3344 + } + ], + "runtime_s": 114.8, + "centered": false, + "env": { + "note": "not recorded at run time; backfilled 2026-08-10", + "device": "cpu", + "dtype": "float32" + } +} \ No newline at end of file diff --git a/results/t0_smoke_70m.json b/results/t0_smoke_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..0895488438d6fd9b63b0042ecd69eb7cf6ccff1c --- /dev/null +++ b/results/t0_smoke_70m.json @@ -0,0 +1,176 @@ +{ + "experiment": "T0 port gate \u2014 jlens on GPT-NeoX/Pythia", + "status": "SMOKE (tiny-first, not a result)", + "model": "EleutherAI/pythia-70m-deduped", + "n_prompts": 3, + "dim_batch": 64, + "max_seq_len": 128, + "device": "cpu", + "dtype": "float32", + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "n_layers": 6, + "d_model": 512, + "ALL_GATES_PASS": true, + "gates": { + "G1_autodetect_worked": false, + "G1_fallback_reason": "could not locate the text decoder inside GPTNeoXForCausalLM (tried 6 known layouts); pass layout= explicitly", + "G1_layout_path": "gpt_neox", + "G1_layout_lm_head": "lm_head", + "G1_layout_norm": "final_layer_norm", + "G1_layout_embed": "embed_in", + "G1_transformers_version": "5.14.1", + "G1_PASS": true, + "G1_note": "anchor's shipped Pythia layout uses lm_head='embed_out', which transformers 5.x renamed to 'lm_head'; explicit Layout supplied.", + "G2_embed_in_shape": [ + 50304, + 512 + ], + "G2_embed_out_shape": [ + 50304, + 512 + ], + "G2_same_storage": false, + "G2_max_abs_diff": 0.4896240234375, + "G2_PASS": true, + "G2_note": "UNTIED: W_U is an independent matrix. Gemma-2 ties, Pythia does not -- this is the tied-vs-untied control.", + "G3_logit_softcap": null, + "G3_PASS": true, + "G4_bos_token_id": 0, + "G4_has_add_bos_attr": true, + "G4_first_token_id": 0, + "G4_bos_actually_prepended": true, + "G4_PASS": true, + "G4_note": "jlens force_bos=True sets tokenizer.add_bos_token when the attr exists. GPT-NeoX fast tokenizers often ignore it. Recorded so the attention-sink condition is known rather than assumed.", + "G8_skip_first": 16, + "G8_seq_lens": [ + 50, + 49, + 49 + ], + "G8_n_valid_positions": [ + 33, + 32, + 32 + ], + "G8_PASS": true, + "G5_n_source_layers": 5, + "G5_expected_layers": 5, + "G5_d_model": 512, + "G5_all_finite": true, + "G5_all_nondegenerate": true, + "G5_PASS": true, + "G5_fit_secs": 3.09, + "G5_secs_per_prompt": 1.032, + "G6_readout_layer": 2, + "G6_jlens_top8": [ + "quart", + " Notre", + " called", + " Cathedral", + "\u00e8res", + " Pruss", + " Pall", + "oused" + ], + "G6_logitlens_top8": [ + " about", + " _\"", + " \"", + " worth", + " rise", + " descent", + " nest", + " down" + ], + "G6_model_top8": [ + " the", + " \"", + " a", + " in", + " '", + " Paris", + " to", + " into" + ], + "G6_jlens_differs_from_logitlens": true, + "G6_jlens_finite": true, + "G6_PASS": true, + "G7_roundtrip_max_abs_diff": 0.0, + "G7_lens_bytes": 2624199, + "G7_PASS": true + }, + "per_layer": [ + { + "layer": 0, + "shape": [ + 512, + 512 + ], + "finite": true, + "fro_norm": 33.3806, + "fro_over_sqrt_d": 1.4752, + "top_sv": 12.2844, + "stable_rank": 7.3838, + "dist_from_identity": 1.0297 + }, + { + "layer": 1, + "shape": [ + 512, + 512 + ], + "finite": true, + "fro_norm": 28.4982, + "fro_over_sqrt_d": 1.2595, + "top_sv": 11.3852, + "stable_rank": 6.2655, + "dist_from_identity": 1.0292 + }, + { + "layer": 2, + "shape": [ + 512, + 512 + ], + "finite": true, + "fro_norm": 29.1155, + "fro_over_sqrt_d": 1.2867, + "top_sv": 12.2724, + "stable_rank": 5.6284, + "dist_from_identity": 0.9485 + }, + { + "layer": 3, + "shape": [ + 512, + 512 + ], + "finite": true, + "fro_norm": 30.7394, + "fro_over_sqrt_d": 1.3585, + "top_sv": 14.0085, + "stable_rank": 4.8151, + "dist_from_identity": 0.8189 + }, + { + "layer": 4, + "shape": [ + 512, + 512 + ], + "finite": true, + "fro_norm": 33.6002, + "fro_over_sqrt_d": 1.4849, + "top_sv": 18.0057, + "stable_rank": 3.4823, + "dist_from_identity": 0.7303 + } + ], + "runtime_s": 9.9, + "centered": false, + "env": { + "note": "not recorded at run time; backfilled 2026-08-10", + "device": "cpu", + "dtype": "float32" + } +} \ No newline at end of file diff --git a/results/t10_target_layer_160m.json b/results/t10_target_layer_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..82fa72ac228dd32d26156cfbbb9faf7b9bcd2fa3 --- /dev/null +++ b/results/t10_target_layer_160m.json @@ -0,0 +1,53 @@ +{ + "experiment": "T10 target_layer: final vs penultimate", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "why": "the anchor contradicts itself: SS2.1 says final, SSA.7 prose says penultimate is the default used throughout the paper, SSA.7 pseudocode says final. Nanda used penultimate. Our lenses all used the library default (final).", + "model": "EleutherAI/pythia-160m-deduped", + "n_prompts": 50, + "centered": false, + "device": "cuda", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "results": { + "final": { + "target_layer": 11, + "n_source_layers": 11, + "fit_s": 28.7, + "evals": { + "multihop": { + "auc": 0.1095, + "pass@10": 0.1, + "median_rank": 726.0 + }, + "typo": { + "auc": 0.3926, + "pass@10": 0.4, + "median_rank": 33.0 + } + }, + "dispersion_median": 0.2793 + }, + "penultimate": { + "target_layer": 10, + "n_source_layers": 10, + "fit_s": 26.1, + "evals": { + "multihop": { + "auc": 0.0589, + "pass@10": 0.05, + "median_rank": 1043.0 + }, + "typo": { + "auc": 0.2607, + "pass@10": 0.225, + "median_rank": 122.0 + } + }, + "dispersion_median": 0.2323 + } + }, + "runtime_s": 63.2 +} \ No newline at end of file diff --git a/results/t10_target_layer_410m.json b/results/t10_target_layer_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..20c6eb0f524d4edae2b4b6c2a399f3e6fcc3d282 --- /dev/null +++ b/results/t10_target_layer_410m.json @@ -0,0 +1,53 @@ +{ + "experiment": "T10 target_layer: final vs penultimate", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "why": "the anchor contradicts itself: SS2.1 says final, SSA.7 prose says penultimate is the default used throughout the paper, SSA.7 pseudocode says final. Nanda used penultimate. Our lenses all used the library default (final).", + "model": "EleutherAI/pythia-410m-deduped", + "n_prompts": 50, + "centered": false, + "device": "cuda", + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "results": { + "final": { + "target_layer": 23, + "n_source_layers": 23, + "fit_s": 118.9, + "evals": { + "multihop": { + "auc": 0.2077, + "pass@10": 0.175, + "median_rank": 106.0 + }, + "typo": { + "auc": 0.3752, + "pass@10": 0.35, + "median_rank": 22.0 + } + }, + "dispersion_median": 0.4553 + }, + "penultimate": { + "target_layer": 22, + "n_source_layers": 22, + "fit_s": 113.8, + "evals": { + "multihop": { + "auc": 0.1858, + "pass@10": 0.125, + "median_rank": 70.0 + }, + "typo": { + "auc": 0.4402, + "pass@10": 0.4, + "median_rank": 15.0 + } + }, + "dispersion_median": 0.4709 + } + }, + "runtime_s": 257.1 +} \ No newline at end of file diff --git a/results/t10_target_layer_70m.json b/results/t10_target_layer_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..c1a473c27eb1f1b8061f528b02d3164a2201df59 --- /dev/null +++ b/results/t10_target_layer_70m.json @@ -0,0 +1,52 @@ +{ + "experiment": "T10 target_layer: final vs penultimate", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "why": "the anchor contradicts itself: SS2.1 says final, SSA.7 prose says penultimate is the default used throughout the paper, SSA.7 pseudocode says final. Nanda used penultimate. Our lenses all used the library default (final).", + "model": "EleutherAI/pythia-70m-deduped", + "n_prompts": 50, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "results": { + "final": { + "target_layer": 5, + "n_source_layers": 5, + "fit_s": 223.8, + "evals": { + "multihop": { + "auc": 0.0463, + "pass@10": 0.025, + "median_rank": 919.0 + }, + "typo": { + "auc": 0.0988, + "pass@10": 0.125, + "median_rank": 497.0 + } + }, + "dispersion_median": 0.1408 + }, + "penultimate": { + "target_layer": 4, + "n_source_layers": 4, + "fit_s": 172.8, + "evals": { + "multihop": { + "auc": 0.0038, + "pass@10": 0.0, + "median_rank": 1778.0 + }, + "typo": { + "auc": 0.1094, + "pass@10": 0.125, + "median_rank": 935.0 + } + }, + "dispersion_median": 0.0787 + } + }, + "runtime_s": 429.8 +} \ No newline at end of file diff --git a/results/t11_fit_distribution_shortskip_70m.json b/results/t11_fit_distribution_shortskip_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..3ba54a3716b2ffe60667ebdb8fa6edbc4c9458b0 --- /dev/null +++ b/results/t11_fit_distribution_shortskip_70m.json @@ -0,0 +1,61 @@ +{ + "experiment": "T11 fit-distribution (THESIS C2 / direction 2)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prediction_registered_before_reading": "If the averaged Jacobian is a corpus-specific estimator, task-adjacent fitting raises pass@k. If it is a model-internal geometric map, the delta is ~0.", + "model": "EleutherAI/pythia-70m-deduped", + "n_prompts": 40, + "centered": false, + "skip_first_override": 2, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "results": { + "typo": { + "logit_lens_floor": { + "auc": 0.0968 + }, + "wikitext": { + "corpus": "wikitext", + "auc": 0.0934, + "pass@10": 0.1333, + "median_rank": 473.0, + "dispersion_median": 0.1143, + "fit_s": 335.8 + }, + "TASK": { + "corpus": "eval:typo", + "auc": 0.1117, + "pass@10": 0.1, + "median_rank": 500.0, + "dispersion_median": 0.562, + "fit_s": 24.7 + }, + "task_minus_wikitext_auc": 0.0183 + }, + "multilingual": { + "logit_lens_floor": { + "auc": 0.0615 + }, + "wikitext": { + "corpus": "wikitext", + "auc": 0.0348, + "pass@10": 0.025, + "median_rank": 1772.0, + "dispersion_median": 0.1143, + "fit_s": 296.7 + }, + "TASK": { + "corpus": "eval:multilingual", + "auc": 0.0106, + "pass@10": 0.0083, + "median_rank": 2746.0, + "dispersion_median": 0.5907, + "fit_s": 25.2 + }, + "task_minus_wikitext_auc": -0.0242 + } + }, + "runtime_s": 695.1 +} \ No newline at end of file diff --git a/results/t11_multihop_backfill_70m.json b/results/t11_multihop_backfill_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..01b05e21a827e1ed944fb95c7ab6af4b39623298 --- /dev/null +++ b/results/t11_multihop_backfill_70m.json @@ -0,0 +1,84 @@ +{ + "experiment": "T11 fit-distribution (THESIS C2 / direction 2)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prediction_registered_before_reading": "If the averaged Jacobian is a corpus-specific estimator, task-adjacent fitting raises pass@k. If it is a model-internal geometric map, the delta is ~0.", + "model": "EleutherAI/pythia-70m-deduped", + "n_prompts": 40, + "centered": false, + "device": "cpu", + "skip_first_override": 2, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "results": { + "multihop": { + "logit_lens_floor": { + "auc": 0.0513 + }, + "wikitext": { + "corpus": "wikitext", + "auc": 0.0401, + "pass@10": 0.025, + "median_rank": 911.0, + "dispersion_median": 0.1143, + "fit_s": 195.3 + }, + "TASK": { + "corpus": "eval:multihop", + "auc": 0.0469, + "pass@10": 0.05, + "median_rank": 984.0, + "dispersion_median": 0.2448, + "fit_s": 33.0 + }, + "task_minus_wikitext_auc": 0.0068 + }, + "multilingual": { + "logit_lens_floor": { + "auc": 0.0505 + }, + "wikitext": { + "corpus": "wikitext", + "auc": 0.028, + "pass@10": 0.0187, + "median_rank": 1919.0, + "dispersion_median": 0.1143, + "fit_s": 173.2 + }, + "TASK": { + "corpus": "eval:multilingual", + "auc": 0.0089, + "pass@10": 0.0063, + "median_rank": 2608.0, + "dispersion_median": 0.5907, + "fit_s": 22.8 + }, + "task_minus_wikitext_auc": -0.0191 + }, + "typo": { + "logit_lens_floor": { + "auc": 0.0913 + }, + "wikitext": { + "corpus": "wikitext", + "auc": 0.095, + "pass@10": 0.125, + "median_rank": 516.0, + "dispersion_median": 0.1143, + "fit_s": 180.0 + }, + "TASK": { + "corpus": "eval:typo", + "auc": 0.1088, + "pass@10": 0.1, + "median_rank": 767.0, + "dispersion_median": 0.562, + "fit_s": 19.0 + }, + "task_minus_wikitext_auc": 0.0138 + } + }, + "runtime_s": 640.3 +} \ No newline at end of file diff --git a/results/t11b_refit_feasibility.json b/results/t11b_refit_feasibility.json new file mode 100644 index 0000000000000000000000000000000000000000..bdcf8150854fb207aa92184e90148b958e8b883c --- /dev/null +++ b/results/t11b_refit_feasibility.json @@ -0,0 +1,58 @@ +{ + "experiment": "T11b refit-on-target feasibility", + "finding": "The J-lens fitter requires seq_len > skip_first+1 = 18 tokens. Most of the anchor's own evaluation prompts are shorter, so the refit-on-target cell of the OOD factorial is STRUCTURALLY BLOCKED for most sets by the estimator's own position-skipping rule.", + "skip_first": 16, + "min_seq_len_required": 18, + "tokenizer": "EleutherAI/pythia-410m-deduped", + "by_set": { + "multihop": { + "n": 93, + "fittable": 27, + "pct": 29.0, + "median_len": 15 + }, + "multilingual": { + "n": 107, + "fittable": 3, + "pct": 2.8, + "median_len": 10 + }, + "order-ops": { + "n": 55, + "fittable": 1, + "pct": 1.8, + "median_len": 8 + }, + "poetry": { + "n": 98, + "fittable": 98, + "pct": 100.0, + "median_len": 26 + }, + "typo": { + "n": 96, + "fittable": 0, + "pct": 0.0, + "median_len": 8 + }, + "association": { + "n": 102, + "fittable": 102, + "pct": 100.0, + "median_len": 28 + }, + "probe-swap": { + "n": 90, + "fittable": 24, + "pct": 26.7, + "median_len": 16 + } + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "centered": false, + "env": { + "note": "not recorded at run time; backfilled 2026-08-10", + "device": "cpu", + "dtype": "float32" + } +} \ No newline at end of file diff --git a/results/t12_concept_hessian_410m.json b/results/t12_concept_hessian_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..06dd3e0086c7764affb8d13b0d31458552d0f843 --- /dev/null +++ b/results/t12_concept_hessian_410m.json @@ -0,0 +1,1385 @@ +{ + "experiment": "T12 restricted concept Hessian via finite-difference interaction residual", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration; bounded diagnostic, not a lens", + "limitation": "V^T H_F V is OBSERVABLE-RELATIVE: it measures interaction in the chosen logit, not entanglement in the representation. Direction 5 is therefore kept out of the paper's spine.", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "layer": 15, + "observable": " Paris", + "prompt": "The capital of France is the city of", + "concepts": [ + "France", + "China", + "Japan", + "Germany", + "water", + "music" + ], + "amps": [ + 0.25, + 0.5, + 1.0, + 2.0, + 4.0 + ], + "baseline_logit": 21.18341, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "amp": 0.25, + "pool": "concept", + "a": "France", + "b": "China", + "interaction": -0.00286, + "additive_prediction": 0.16422, + "rel_interaction": 0.01743 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "France", + "b": "Japan", + "interaction": -0.00444, + "additive_prediction": 0.18932, + "rel_interaction": 0.02343 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "France", + "b": "Germany", + "interaction": -0.00481, + "additive_prediction": 0.19333, + "rel_interaction": 0.0249 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "France", + "b": "water", + "interaction": -0.00264, + "additive_prediction": 0.0692, + "rel_interaction": 0.03815 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "France", + "b": "music", + "interaction": -0.00222, + "additive_prediction": 0.11562, + "rel_interaction": 0.01924 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "China", + "b": "Japan", + "interaction": -0.00756, + "additive_prediction": 0.15509, + "rel_interaction": 0.04877 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "China", + "b": "Germany", + "interaction": -0.00426, + "additive_prediction": 0.15911, + "rel_interaction": 0.02677 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "China", + "b": "water", + "interaction": -0.00342, + "additive_prediction": 0.03498, + "rel_interaction": 0.09766 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "China", + "b": "music", + "interaction": -0.0008, + "additive_prediction": 0.0814, + "rel_interaction": 0.00977 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "Japan", + "b": "Germany", + "interaction": -0.00619, + "additive_prediction": 0.1842, + "rel_interaction": 0.0336 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "Japan", + "b": "water", + "interaction": -0.00505, + "additive_prediction": 0.06007, + "rel_interaction": 0.08408 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "Japan", + "b": "music", + "interaction": -0.00223, + "additive_prediction": 0.10649, + "rel_interaction": 0.0209 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "Germany", + "b": "water", + "interaction": -0.00279, + "additive_prediction": 0.06409, + "rel_interaction": 0.04357 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "Germany", + "b": "music", + "interaction": -0.0024, + "additive_prediction": 0.11051, + "rel_interaction": 0.02171 + }, + { + "amp": 0.25, + "pool": "concept", + "a": "water", + "b": "music", + "interaction": -0.00348, + "additive_prediction": -0.01362, + "rel_interaction": 0.25548 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand0", + "b": "rand1", + "interaction": 0.00026, + "additive_prediction": 0.00284, + "rel_interaction": 0.09057 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand0", + "b": "rand2", + "interaction": 0.00058, + "additive_prediction": -0.00677, + "rel_interaction": 0.08539 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand0", + "b": "rand3", + "interaction": 0.00042, + "additive_prediction": -0.00214, + "rel_interaction": 0.19777 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand0", + "b": "rand4", + "interaction": 0.00062, + "additive_prediction": 0.03354, + "rel_interaction": 0.01843 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand0", + "b": "rand5", + "interaction": 0.00079, + "additive_prediction": 0.05151, + "rel_interaction": 0.01537 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand1", + "b": "rand2", + "interaction": 0.00029, + "additive_prediction": -0.00835, + "rel_interaction": 0.03424 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand1", + "b": "rand3", + "interaction": -0.0003, + "additive_prediction": -0.00373, + "rel_interaction": 0.07981 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand1", + "b": "rand4", + "interaction": 0.00011, + "additive_prediction": 0.03195, + "rel_interaction": 0.0034 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand1", + "b": "rand5", + "interaction": 0.00017, + "additive_prediction": 0.04992, + "rel_interaction": 0.0034 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand2", + "b": "rand3", + "interaction": -4e-05, + "additive_prediction": -0.01334, + "rel_interaction": 0.003 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand2", + "b": "rand4", + "interaction": 0.00069, + "additive_prediction": 0.02234, + "rel_interaction": 0.03091 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand2", + "b": "rand5", + "interaction": 0.00011, + "additive_prediction": 0.04031, + "rel_interaction": 0.0027 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand3", + "b": "rand4", + "interaction": 0.00032, + "additive_prediction": 0.02697, + "rel_interaction": 0.01202 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand3", + "b": "rand5", + "interaction": 0.00034, + "additive_prediction": 0.04494, + "rel_interaction": 0.0076 + }, + { + "amp": 0.25, + "pool": "random", + "a": "rand4", + "b": "rand5", + "interaction": 0.00088, + "additive_prediction": 0.08061, + "rel_interaction": 0.01088 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "France", + "b": "China", + "interaction": -0.0117, + "additive_prediction": 0.31523, + "rel_interaction": 0.03711 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "France", + "b": "Japan", + "interaction": -0.01791, + "additive_prediction": 0.3644, + "rel_interaction": 0.04915 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "France", + "b": "Germany", + "interaction": -0.01976, + "additive_prediction": 0.3739, + "rel_interaction": 0.05285 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "France", + "b": "water", + "interaction": -0.00995, + "additive_prediction": 0.12891, + "rel_interaction": 0.07719 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "France", + "b": "music", + "interaction": -0.00847, + "additive_prediction": 0.21944, + "rel_interaction": 0.0386 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "China", + "b": "Japan", + "interaction": -0.02938, + "additive_prediction": 0.29226, + "rel_interaction": 0.10053 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "China", + "b": "Germany", + "interaction": -0.01517, + "additive_prediction": 0.30176, + "rel_interaction": 0.05028 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "China", + "b": "water", + "interaction": -0.01274, + "additive_prediction": 0.05677, + "rel_interaction": 0.22444 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "China", + "b": "music", + "interaction": -0.00146, + "additive_prediction": 0.1473, + "rel_interaction": 0.00989 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "Japan", + "b": "Germany", + "interaction": -0.02297, + "additive_prediction": 0.35093, + "rel_interaction": 0.06547 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "Japan", + "b": "water", + "interaction": -0.01832, + "additive_prediction": 0.10593, + "rel_interaction": 0.17292 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "Japan", + "b": "music", + "interaction": -0.00713, + "additive_prediction": 0.19646, + "rel_interaction": 0.0363 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "Germany", + "b": "water", + "interaction": -0.00993, + "additive_prediction": 0.11543, + "rel_interaction": 0.08604 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "Germany", + "b": "music", + "interaction": -0.00756, + "additive_prediction": 0.20596, + "rel_interaction": 0.03672 + }, + { + "amp": 0.5, + "pool": "concept", + "a": "water", + "b": "music", + "interaction": -0.01123, + "additive_prediction": -0.03903, + "rel_interaction": 0.28775 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand0", + "b": "rand1", + "interaction": 0.00072, + "additive_prediction": 0.003, + "rel_interaction": 0.24023 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand0", + "b": "rand2", + "interaction": 0.00093, + "additive_prediction": -0.01523, + "rel_interaction": 0.06136 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand0", + "b": "rand3", + "interaction": -6e-05, + "additive_prediction": -0.00669, + "rel_interaction": 0.00912 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand0", + "b": "rand4", + "interaction": 0.00011, + "additive_prediction": 0.0659, + "rel_interaction": 0.00174 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand0", + "b": "rand5", + "interaction": 2e-05, + "additive_prediction": 0.10179, + "rel_interaction": 0.00021 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand1", + "b": "rand2", + "interaction": 0.00035, + "additive_prediction": -0.02026, + "rel_interaction": 0.01704 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand1", + "b": "rand3", + "interaction": -0.0001, + "additive_prediction": -0.01173, + "rel_interaction": 0.00878 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand1", + "b": "rand4", + "interaction": -0.00013, + "additive_prediction": 0.06087, + "rel_interaction": 0.00216 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand1", + "b": "rand5", + "interaction": 9e-05, + "additive_prediction": 0.09675, + "rel_interaction": 0.00093 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand2", + "b": "rand3", + "interaction": -0.00137, + "additive_prediction": -0.02996, + "rel_interaction": 0.04565 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand2", + "b": "rand4", + "interaction": 0.00091, + "additive_prediction": 0.04264, + "rel_interaction": 0.02138 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand2", + "b": "rand5", + "interaction": -0.00119, + "additive_prediction": 0.07852, + "rel_interaction": 0.01513 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand3", + "b": "rand4", + "interaction": 0.00115, + "additive_prediction": 0.05118, + "rel_interaction": 0.02255 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand3", + "b": "rand5", + "interaction": 0.00114, + "additive_prediction": 0.08706, + "rel_interaction": 0.01315 + }, + { + "amp": 0.5, + "pool": "random", + "a": "rand4", + "b": "rand5", + "interaction": 0.0006, + "additive_prediction": 0.15966, + "rel_interaction": 0.00379 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "France", + "b": "China", + "interaction": -0.03926, + "additive_prediction": 0.57661, + "rel_interaction": 0.0681 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "France", + "b": "Japan", + "interaction": -0.06239, + "additive_prediction": 0.67055, + "rel_interaction": 0.09304 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "France", + "b": "Germany", + "interaction": -0.07343, + "additive_prediction": 0.69659, + "rel_interaction": 0.10542 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "France", + "b": "water", + "interaction": -0.03288, + "additive_prediction": 0.21876, + "rel_interaction": 0.1503 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "France", + "b": "music", + "interaction": -0.02717, + "additive_prediction": 0.39214, + "rel_interaction": 0.06928 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "China", + "b": "Japan", + "interaction": -0.11046, + "additive_prediction": 0.51518, + "rel_interaction": 0.21441 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "China", + "b": "Germany", + "interaction": -0.05549, + "additive_prediction": 0.54123, + "rel_interaction": 0.10252 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "China", + "b": "water", + "interaction": -0.04216, + "additive_prediction": 0.0634, + "rel_interaction": 0.66501 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "China", + "b": "music", + "interaction": -0.00044, + "additive_prediction": 0.23678, + "rel_interaction": 0.00186 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "Japan", + "b": "Germany", + "interaction": -0.08299, + "additive_prediction": 0.63517, + "rel_interaction": 0.13066 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "Japan", + "b": "water", + "interaction": -0.06042, + "additive_prediction": 0.15734, + "rel_interaction": 0.384 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "Japan", + "b": "music", + "interaction": -0.02045, + "additive_prediction": 0.33072, + "rel_interaction": 0.06184 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "Germany", + "b": "water", + "interaction": -0.03373, + "additive_prediction": 0.18338, + "rel_interaction": 0.18391 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "Germany", + "b": "music", + "interaction": -0.02582, + "additive_prediction": 0.35676, + "rel_interaction": 0.07237 + }, + { + "amp": 1.0, + "pool": "concept", + "a": "water", + "b": "music", + "interaction": -0.03537, + "additive_prediction": -0.12107, + "rel_interaction": 0.29211 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand0", + "b": "rand1", + "interaction": 0.00317, + "additive_prediction": -0.00706, + "rel_interaction": 0.44871 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand0", + "b": "rand2", + "interaction": 0.00399, + "additive_prediction": -0.0432, + "rel_interaction": 0.09232 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand0", + "b": "rand3", + "interaction": 0.0004, + "additive_prediction": -0.02705, + "rel_interaction": 0.01495 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand0", + "b": "rand4", + "interaction": 0.00047, + "additive_prediction": 0.12055, + "rel_interaction": 0.00391 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand0", + "b": "rand5", + "interaction": 0.00167, + "additive_prediction": 0.19296, + "rel_interaction": 0.00864 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand1", + "b": "rand2", + "interaction": 0.00041, + "additive_prediction": -0.05477, + "rel_interaction": 0.00745 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand1", + "b": "rand3", + "interaction": -0.00047, + "additive_prediction": -0.03862, + "rel_interaction": 0.0122 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand1", + "b": "rand4", + "interaction": -0.00108, + "additive_prediction": 0.10899, + "rel_interaction": 0.00989 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand1", + "b": "rand5", + "interaction": 0.00041, + "additive_prediction": 0.18139, + "rel_interaction": 0.00225 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand2", + "b": "rand3", + "interaction": -0.00518, + "additive_prediction": -0.07476, + "rel_interaction": 0.06934 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand2", + "b": "rand4", + "interaction": 0.00322, + "additive_prediction": 0.07284, + "rel_interaction": 0.04415 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand2", + "b": "rand5", + "interaction": -0.00349, + "additive_prediction": 0.14525, + "rel_interaction": 0.02402 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand3", + "b": "rand4", + "interaction": 0.00495, + "additive_prediction": 0.08899, + "rel_interaction": 0.05557 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand3", + "b": "rand5", + "interaction": 0.00477, + "additive_prediction": 0.1614, + "rel_interaction": 0.02958 + }, + { + "amp": 1.0, + "pool": "random", + "a": "rand4", + "b": "rand5", + "interaction": 0.00127, + "additive_prediction": 0.309, + "rel_interaction": 0.0041 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "France", + "b": "China", + "interaction": -0.12052, + "additive_prediction": 0.94392, + "rel_interaction": 0.12768 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "France", + "b": "Japan", + "interaction": -0.21015, + "additive_prediction": 1.12532, + "rel_interaction": 0.18675 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "France", + "b": "Germany", + "interaction": -0.25712, + "additive_prediction": 1.19988, + "rel_interaction": 0.21429 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "France", + "b": "water", + "interaction": -0.12455, + "additive_prediction": 0.2903, + "rel_interaction": 0.42904 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "France", + "b": "music", + "interaction": -0.08545, + "additive_prediction": 0.61936, + "rel_interaction": 0.13796 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "China", + "b": "Japan", + "interaction": -0.40398, + "additive_prediction": 0.76432, + "rel_interaction": 0.52854 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "China", + "b": "Germany", + "interaction": -0.18611, + "additive_prediction": 0.83888, + "rel_interaction": 0.22186 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "China", + "b": "water", + "interaction": -0.17416, + "additive_prediction": -0.0707, + "rel_interaction": 2.46344 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "China", + "b": "music", + "interaction": 0.02509, + "additive_prediction": 0.25836, + "rel_interaction": 0.09712 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "Japan", + "b": "Germany", + "interaction": -0.28558, + "additive_prediction": 1.02028, + "rel_interaction": 0.2799 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "Japan", + "b": "water", + "interaction": -0.22149, + "additive_prediction": 0.1107, + "rel_interaction": 2.00072 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "Japan", + "b": "music", + "interaction": -0.04694, + "additive_prediction": 0.43976, + "rel_interaction": 0.10674 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "Germany", + "b": "water", + "interaction": -0.14696, + "additive_prediction": 0.18526, + "rel_interaction": 0.79323 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "Germany", + "b": "music", + "interaction": -0.07979, + "additive_prediction": 0.51432, + "rel_interaction": 0.15513 + }, + { + "amp": 2.0, + "pool": "concept", + "a": "water", + "b": "music", + "interaction": -0.14374, + "additive_prediction": -0.39526, + "rel_interaction": 0.36367 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand0", + "b": "rand1", + "interaction": 0.01136, + "additive_prediction": -0.06549, + "rel_interaction": 0.17344 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand0", + "b": "rand2", + "interaction": 0.01705, + "additive_prediction": -0.13596, + "rel_interaction": 0.12539 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand0", + "b": "rand3", + "interaction": 0.00034, + "additive_prediction": -0.10753, + "rel_interaction": 0.00312 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand0", + "b": "rand4", + "interaction": 0.00055, + "additive_prediction": 0.19636, + "rel_interaction": 0.00282 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand0", + "b": "rand5", + "interaction": 0.01082, + "additive_prediction": 0.34265, + "rel_interaction": 0.03157 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand1", + "b": "rand2", + "interaction": 0.00257, + "additive_prediction": -0.16591, + "rel_interaction": 0.01549 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand1", + "b": "rand3", + "interaction": -0.00464, + "additive_prediction": -0.13748, + "rel_interaction": 0.03374 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand1", + "b": "rand4", + "interaction": -0.00298, + "additive_prediction": 0.16641, + "rel_interaction": 0.01791 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand1", + "b": "rand5", + "interaction": 0.00466, + "additive_prediction": 0.3127, + "rel_interaction": 0.01491 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand2", + "b": "rand3", + "interaction": -0.01831, + "additive_prediction": -0.20795, + "rel_interaction": 0.08807 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand2", + "b": "rand4", + "interaction": 0.01558, + "additive_prediction": 0.09594, + "rel_interaction": 0.16235 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand2", + "b": "rand5", + "interaction": -0.01048, + "additive_prediction": 0.24223, + "rel_interaction": 0.04328 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand3", + "b": "rand4", + "interaction": 0.01864, + "additive_prediction": 0.12436, + "rel_interaction": 0.14987 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand3", + "b": "rand5", + "interaction": 0.01737, + "additive_prediction": 0.27065, + "rel_interaction": 0.06418 + }, + { + "amp": 2.0, + "pool": "random", + "a": "rand4", + "b": "rand5", + "interaction": 0.00379, + "additive_prediction": 0.57454, + "rel_interaction": 0.00659 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "France", + "b": "China", + "interaction": -0.35484, + "additive_prediction": 1.09709, + "rel_interaction": 0.32343 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "France", + "b": "Japan", + "interaction": -0.66024, + "additive_prediction": 1.47819, + "rel_interaction": 0.44666 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "France", + "b": "Germany", + "interaction": -0.84101, + "additive_prediction": 1.71539, + "rel_interaction": 0.49027 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "France", + "b": "water", + "interaction": -0.20005, + "additive_prediction": 0.10025, + "rel_interaction": 1.99557 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "France", + "b": "music", + "interaction": -0.28994, + "additive_prediction": 0.69112, + "rel_interaction": 0.41953 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "China", + "b": "Japan", + "interaction": -1.37892, + "additive_prediction": 0.5374, + "rel_interaction": 2.56593 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "China", + "b": "Germany", + "interaction": -0.67828, + "additive_prediction": 0.7746, + "rel_interaction": 0.87565 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "China", + "b": "water", + "interaction": -0.25452, + "additive_prediction": -0.84054, + "rel_interaction": 0.30281 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "China", + "b": "music", + "interaction": 0.061, + "additive_prediction": -0.24968, + "rel_interaction": 0.24433 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "Japan", + "b": "Germany", + "interaction": -0.9765, + "additive_prediction": 1.15569, + "rel_interaction": 0.84495 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "Japan", + "b": "water", + "interaction": -0.32909, + "additive_prediction": -0.45945, + "rel_interaction": 0.71628 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "Japan", + "b": "music", + "interaction": -0.15908, + "additive_prediction": 0.13142, + "rel_interaction": 1.21048 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "Germany", + "b": "water", + "interaction": -0.18565, + "additive_prediction": -0.22225, + "rel_interaction": 0.83531 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "Germany", + "b": "music", + "interaction": -0.25876, + "additive_prediction": 0.36862, + "rel_interaction": 0.70196 + }, + { + "amp": 4.0, + "pool": "concept", + "a": "water", + "b": "music", + "interaction": -0.15188, + "additive_prediction": -1.24652, + "rel_interaction": 0.12184 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand0", + "b": "rand1", + "interaction": 0.03751, + "additive_prediction": -0.33702, + "rel_interaction": 0.1113 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand0", + "b": "rand2", + "interaction": 0.07106, + "additive_prediction": -0.46682, + "rel_interaction": 0.15223 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand0", + "b": "rand3", + "interaction": 0.00507, + "additive_prediction": -0.4419, + "rel_interaction": 0.01147 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand0", + "b": "rand4", + "interaction": -0.01073, + "additive_prediction": 0.20663, + "rel_interaction": 0.0519 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand0", + "b": "rand5", + "interaction": 0.06167, + "additive_prediction": 0.50089, + "rel_interaction": 0.12312 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand1", + "b": "rand2", + "interaction": 0.01819, + "additive_prediction": -0.54114, + "rel_interaction": 0.03361 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand1", + "b": "rand3", + "interaction": -0.03657, + "additive_prediction": -0.51622, + "rel_interaction": 0.07085 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand1", + "b": "rand4", + "interaction": -0.01788, + "additive_prediction": 0.13231, + "rel_interaction": 0.13513 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand1", + "b": "rand5", + "interaction": 0.02699, + "additive_prediction": 0.42657, + "rel_interaction": 0.06328 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand2", + "b": "rand3", + "interaction": -0.06115, + "additive_prediction": -0.64601, + "rel_interaction": 0.09466 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand2", + "b": "rand4", + "interaction": 0.06196, + "additive_prediction": 0.00252, + "rel_interaction": 24.60159 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand2", + "b": "rand5", + "interaction": -0.02568, + "additive_prediction": 0.29678, + "rel_interaction": 0.08652 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand3", + "b": "rand4", + "interaction": 0.06753, + "additive_prediction": 0.02744, + "rel_interaction": 2.46098 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand3", + "b": "rand5", + "interaction": 0.06242, + "additive_prediction": 0.3217, + "rel_interaction": 0.19404 + }, + { + "amp": 4.0, + "pool": "random", + "a": "rand4", + "b": "rand5", + "interaction": -0.00206, + "additive_prediction": 0.97023, + "rel_interaction": 0.00212 + } + ], + "runtime_s": 182.2 +} \ No newline at end of file diff --git a/results/t13_transport_controls_160m.json b/results/t13_transport_controls_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..bdedb9400da98a673a3a0b802c20947afd214083 --- /dev/null +++ b/results/t13_transport_controls_160m.json @@ -0,0 +1,300 @@ +{ + "experiment": "T13 / E1 \u2014 transport controls: does a BROKEN J still beat the logit lens?", + "status": "EXPLORATORY \u2014 control on cells already burned per PREREG_PYTHIA_T7_v2 section 0", + "model": "EleutherAI/pythia-160m-deduped", + "lens": "pythia/results/lens_160m_n200_db128.pt", + "seed": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "n_layers": 11, + "derangement_layer_map": { + "0": 9, + "1": 7, + "2": 8, + "3": 10, + "4": 6, + "5": 1, + "6": 5, + "7": 4, + "8": 3, + "9": 2, + "10": 0 + }, + "primary_metric": "normalized_auc_over_logk_paper (A.6 per-item binary)", + "n_win_vs_logit": { + "jlens": 2, + "shuffled": 2, + "spec_random": 1, + "rank1": 0, + "scaled_id": 0 + }, + "n_evals": 6, + "SELF_TEST_scaled_identity_equals_logit": true, + "SELF_TEST_worst_abs_auc_diff": 0.0, + "interpretation": "If shuffled/spec_random/rank1 win on as many sets as jlens, T7 measures 'any learned map into the final basis helps', not J-lens-specific workspace reading.", + "results": { + "multihop": { + "logit": { + "auc_paper": 0.1244, + "auc_readme": 0.1345, + "median_best_rank": 406.0 + }, + "jlens": { + "auc_paper": 0.1093, + "auc_readme": 0.1202, + "median_best_rank": 454.0 + }, + "shuffled": { + "auc_paper": 0.0676, + "auc_readme": 0.0732, + "median_best_rank": 473.0 + }, + "spec_random": { + "auc_paper": 0.0029, + "auc_readme": 0.0032, + "median_best_rank": 1188.0 + }, + "rank1": { + "auc_paper": 0.0044, + "auc_readme": 0.0038, + "median_best_rank": 1537.0 + }, + "scaled_id": { + "auc_paper": 0.1244, + "auc_readme": 0.1345, + "median_best_rank": 406.0 + } + }, + "multilingual": { + "logit": { + "auc_paper": 0.0391, + "auc_readme": 0.0388, + "median_best_rank": 868.0 + }, + "jlens": { + "auc_paper": 0.03, + "auc_readme": 0.0277, + "median_best_rank": 1474.0 + }, + "shuffled": { + "auc_paper": 0.035, + "auc_readme": 0.033, + "median_best_rank": 889.0 + }, + "spec_random": { + "auc_paper": 0.0275, + "auc_readme": 0.0287, + "median_best_rank": 1070.0 + }, + "rank1": { + "auc_paper": 0.0042, + "auc_readme": 0.0046, + "median_best_rank": 1677.0 + }, + "scaled_id": { + "auc_paper": 0.0391, + "auc_readme": 0.0388, + "median_best_rank": 868.0 + } + }, + "order-ops": { + "logit": { + "auc_paper": 0.4493, + "auc_readme": 0.4493, + "median_best_rank": 12.0 + }, + "jlens": { + "auc_paper": 0.4402, + "auc_readme": 0.4402, + "median_best_rank": 11.0 + }, + "shuffled": { + "auc_paper": 0.2169, + "auc_readme": 0.2169, + "median_best_rank": 43.0 + }, + "spec_random": { + "auc_paper": 0.0776, + "auc_readme": 0.0776, + "median_best_rank": 430.0 + }, + "rank1": { + "auc_paper": 0.0533, + "auc_readme": 0.0533, + "median_best_rank": 186.0 + }, + "scaled_id": { + "auc_paper": 0.4493, + "auc_readme": 0.4493, + "median_best_rank": 12.0 + } + }, + "poetry": { + "logit": { + "auc_paper": 0.0383, + "auc_readme": 0.0383, + "median_best_rank": 712.0 + }, + "jlens": { + "auc_paper": 0.0031, + "auc_readme": 0.0031, + "median_best_rank": 1309.0 + }, + "shuffled": { + "auc_paper": 0.0378, + "auc_readme": 0.0378, + "median_best_rank": 941.0 + }, + "spec_random": { + "auc_paper": 0.033, + "auc_readme": 0.033, + "median_best_rank": 600.0 + }, + "rank1": { + "auc_paper": 0.0046, + "auc_readme": 0.0046, + "median_best_rank": 781.0 + }, + "scaled_id": { + "auc_paper": 0.0383, + "auc_readme": 0.0383, + "median_best_rank": 712.0 + } + }, + "typo": { + "logit": { + "auc_paper": 0.2423, + "auc_readme": 0.2423, + "median_best_rank": 80.0 + }, + "jlens": { + "auc_paper": 0.2808, + "auc_readme": 0.2808, + "median_best_rank": 146.0 + }, + "shuffled": { + "auc_paper": 0.2569, + "auc_readme": 0.2569, + "median_best_rank": 120.0 + }, + "spec_random": { + "auc_paper": 0.043, + "auc_readme": 0.043, + "median_best_rank": 645.0 + }, + "rank1": { + "auc_paper": 0.0078, + "auc_readme": 0.0078, + "median_best_rank": 683.0 + }, + "scaled_id": { + "auc_paper": 0.2423, + "auc_readme": 0.2423, + "median_best_rank": 80.0 + } + }, + "association": { + "logit": { + "auc_paper": 0.0033, + "auc_readme": 0.0033, + "median_best_rank": 4974.0 + }, + "jlens": { + "auc_paper": 0.0109, + "auc_readme": 0.0109, + "median_best_rank": 1266.0 + }, + "shuffled": { + "auc_paper": 0.0112, + "auc_readme": 0.0112, + "median_best_rank": 1782.0 + }, + "spec_random": { + "auc_paper": 0.0264, + "auc_readme": 0.0264, + "median_best_rank": 1178.0 + }, + "rank1": { + "auc_paper": 0.0, + "auc_readme": 0.0, + "median_best_rank": 2295.0 + }, + "scaled_id": { + "auc_paper": 0.0033, + "auc_readme": 0.0033, + "median_best_rank": 4974.0 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 107.1, + "advantage_decomposition": { + "multihop": { + "auc_logit": 0.1244, + "auc_shuffled": 0.0676, + "auc_jlens": 0.1093, + "total_advantage_vs_logit": -0.0151, + "generic_component_shuffled_minus_logit": -0.0568, + "layer_specific_component_jlens_minus_shuffled": 0.0417 + }, + "multilingual": { + "auc_logit": 0.0391, + "auc_shuffled": 0.035, + "auc_jlens": 0.03, + "total_advantage_vs_logit": -0.0091, + "generic_component_shuffled_minus_logit": -0.0041, + "layer_specific_component_jlens_minus_shuffled": -0.005 + }, + "order-ops": { + "auc_logit": 0.4493, + "auc_shuffled": 0.2169, + "auc_jlens": 0.4402, + "total_advantage_vs_logit": -0.0091, + "generic_component_shuffled_minus_logit": -0.2324, + "layer_specific_component_jlens_minus_shuffled": 0.2233 + }, + "poetry": { + "auc_logit": 0.0383, + "auc_shuffled": 0.0378, + "auc_jlens": 0.0031, + "total_advantage_vs_logit": -0.0352, + "generic_component_shuffled_minus_logit": -0.0005, + "layer_specific_component_jlens_minus_shuffled": -0.0347 + }, + "typo": { + "auc_logit": 0.2423, + "auc_shuffled": 0.2569, + "auc_jlens": 0.2808, + "total_advantage_vs_logit": 0.0385, + "generic_component_shuffled_minus_logit": 0.0146, + "layer_specific_component_jlens_minus_shuffled": 0.0239 + }, + "association": { + "auc_logit": 0.0033, + "auc_shuffled": 0.0112, + "auc_jlens": 0.0109, + "total_advantage_vs_logit": 0.0076, + "generic_component_shuffled_minus_logit": 0.0079, + "layer_specific_component_jlens_minus_shuffled": -0.0003 + } + }, + "n_win_jlens_vs_shuffled": 3, + "decomposition_note": "advantage vs logit splits into GENERIC (shuffled-logit: what ANY fitted Jacobian buys) and LAYER-SPECIFIC (jlens-shuffled: the workspace claim)" +} \ No newline at end of file diff --git a/results/t13_transport_controls_410m.json b/results/t13_transport_controls_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab4a8899061a6589ff39cf4f731b1e7fcb4df90 --- /dev/null +++ b/results/t13_transport_controls_410m.json @@ -0,0 +1,324 @@ +{ + "experiment": "T13 / E1 \u2014 transport controls: does a BROKEN J still beat the logit lens?", + "status": "EXPLORATORY \u2014 control on cells already burned per PREREG_PYTHIA_T7_v2 section 0", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "seed": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "n_layers": 23, + "derangement_layer_map": { + "0": 17, + "1": 8, + "2": 21, + "3": 18, + "4": 7, + "5": 12, + "6": 19, + "7": 1, + "8": 16, + "9": 14, + "10": 2, + "11": 6, + "12": 15, + "13": 11, + "14": 4, + "15": 22, + "16": 13, + "17": 10, + "18": 0, + "19": 3, + "20": 9, + "21": 20, + "22": 5 + }, + "primary_metric": "normalized_auc_over_logk_paper (A.6 per-item binary)", + "n_win_vs_logit": { + "jlens": 6, + "shuffled": 5, + "spec_random": 2, + "rank1": 0, + "scaled_id": 0 + }, + "n_evals": 6, + "SELF_TEST_scaled_identity_equals_logit": true, + "SELF_TEST_worst_abs_auc_diff": 0.0, + "interpretation": "If shuffled/spec_random/rank1 win on as many sets as jlens, T7 measures 'any learned map into the final basis helps', not J-lens-specific workspace reading.", + "results": { + "multihop": { + "logit": { + "auc_paper": 0.1994, + "auc_readme": 0.2112, + "median_best_rank": 136.0 + }, + "jlens": { + "auc_paper": 0.2602, + "auc_readme": 0.2757, + "median_best_rank": 48.0 + }, + "shuffled": { + "auc_paper": 0.1946, + "auc_readme": 0.2077, + "median_best_rank": 88.0 + }, + "spec_random": { + "auc_paper": 0.0197, + "auc_readme": 0.0186, + "median_best_rank": 556.0 + }, + "rank1": { + "auc_paper": 0.0308, + "auc_readme": 0.0323, + "median_best_rank": 879.0 + }, + "scaled_id": { + "auc_paper": 0.1994, + "auc_readme": 0.2112, + "median_best_rank": 136.0 + } + }, + "multilingual": { + "logit": { + "auc_paper": 0.1153, + "auc_readme": 0.1177, + "median_best_rank": 372.0 + }, + "jlens": { + "auc_paper": 0.2169, + "auc_readme": 0.2141, + "median_best_rank": 213.0 + }, + "shuffled": { + "auc_paper": 0.1195, + "auc_readme": 0.1217, + "median_best_rank": 186.0 + }, + "spec_random": { + "auc_paper": 0.0222, + "auc_readme": 0.0224, + "median_best_rank": 559.0 + }, + "rank1": { + "auc_paper": 0.0064, + "auc_readme": 0.0071, + "median_best_rank": 894.0 + }, + "scaled_id": { + "auc_paper": 0.1153, + "auc_readme": 0.1177, + "median_best_rank": 372.0 + } + }, + "order-ops": { + "logit": { + "auc_paper": 0.5234, + "auc_readme": 0.5234, + "median_best_rank": 9.0 + }, + "jlens": { + "auc_paper": 0.5679, + "auc_readme": 0.5679, + "median_best_rank": 8.0 + }, + "shuffled": { + "auc_paper": 0.5241, + "auc_readme": 0.5241, + "median_best_rank": 10.0 + }, + "spec_random": { + "auc_paper": 0.1082, + "auc_readme": 0.1082, + "median_best_rank": 136.0 + }, + "rank1": { + "auc_paper": 0.0893, + "auc_readme": 0.0893, + "median_best_rank": 104.0 + }, + "scaled_id": { + "auc_paper": 0.5234, + "auc_readme": 0.5234, + "median_best_rank": 9.0 + } + }, + "poetry": { + "logit": { + "auc_paper": 0.0287, + "auc_readme": 0.0287, + "median_best_rank": 414.0 + }, + "jlens": { + "auc_paper": 0.0435, + "auc_readme": 0.0435, + "median_best_rank": 250.0 + }, + "shuffled": { + "auc_paper": 0.0718, + "auc_readme": 0.0718, + "median_best_rank": 200.0 + }, + "spec_random": { + "auc_paper": 0.0621, + "auc_readme": 0.0621, + "median_best_rank": 326.0 + }, + "rank1": { + "auc_paper": 0.0046, + "auc_readme": 0.0046, + "median_best_rank": 547.0 + }, + "scaled_id": { + "auc_paper": 0.0287, + "auc_readme": 0.0287, + "median_best_rank": 414.0 + } + }, + "typo": { + "logit": { + "auc_paper": 0.2034, + "auc_readme": 0.2034, + "median_best_rank": 75.0 + }, + "jlens": { + "auc_paper": 0.3241, + "auc_readme": 0.3241, + "median_best_rank": 38.0 + }, + "shuffled": { + "auc_paper": 0.3278, + "auc_readme": 0.3278, + "median_best_rank": 36.0 + }, + "spec_random": { + "auc_paper": 0.0501, + "auc_readme": 0.0501, + "median_best_rank": 424.0 + }, + "rank1": { + "auc_paper": 0.0076, + "auc_readme": 0.0076, + "median_best_rank": 708.0 + }, + "scaled_id": { + "auc_paper": 0.2034, + "auc_readme": 0.2034, + "median_best_rank": 75.0 + } + }, + "association": { + "logit": { + "auc_paper": 0.0076, + "auc_readme": 0.0076, + "median_best_rank": 1659.0 + }, + "jlens": { + "auc_paper": 0.0115, + "auc_readme": 0.0115, + "median_best_rank": 959.0 + }, + "shuffled": { + "auc_paper": 0.0313, + "auc_readme": 0.0313, + "median_best_rank": 456.0 + }, + "spec_random": { + "auc_paper": 0.0109, + "auc_readme": 0.0109, + "median_best_rank": 999.0 + }, + "rank1": { + "auc_paper": 0.0, + "auc_readme": 0.0, + "median_best_rank": 1938.0 + }, + "scaled_id": { + "auc_paper": 0.0076, + "auc_readme": 0.0076, + "median_best_rank": 1659.0 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 303.1, + "advantage_decomposition": { + "multihop": { + "auc_logit": 0.1994, + "auc_shuffled": 0.1946, + "auc_jlens": 0.2602, + "total_advantage_vs_logit": 0.0608, + "generic_component_shuffled_minus_logit": -0.0048, + "layer_specific_component_jlens_minus_shuffled": 0.0656 + }, + "multilingual": { + "auc_logit": 0.1153, + "auc_shuffled": 0.1195, + "auc_jlens": 0.2169, + "total_advantage_vs_logit": 0.1016, + "generic_component_shuffled_minus_logit": 0.0042, + "layer_specific_component_jlens_minus_shuffled": 0.0974 + }, + "order-ops": { + "auc_logit": 0.5234, + "auc_shuffled": 0.5241, + "auc_jlens": 0.5679, + "total_advantage_vs_logit": 0.0445, + "generic_component_shuffled_minus_logit": 0.0007, + "layer_specific_component_jlens_minus_shuffled": 0.0438 + }, + "poetry": { + "auc_logit": 0.0287, + "auc_shuffled": 0.0718, + "auc_jlens": 0.0435, + "total_advantage_vs_logit": 0.0148, + "generic_component_shuffled_minus_logit": 0.0431, + "layer_specific_component_jlens_minus_shuffled": -0.0283 + }, + "typo": { + "auc_logit": 0.2034, + "auc_shuffled": 0.3278, + "auc_jlens": 0.3241, + "total_advantage_vs_logit": 0.1207, + "generic_component_shuffled_minus_logit": 0.1244, + "layer_specific_component_jlens_minus_shuffled": -0.0037 + }, + "association": { + "auc_logit": 0.0076, + "auc_shuffled": 0.0313, + "auc_jlens": 0.0115, + "total_advantage_vs_logit": 0.0039, + "generic_component_shuffled_minus_logit": 0.0237, + "layer_specific_component_jlens_minus_shuffled": -0.0198 + } + }, + "n_win_jlens_vs_shuffled": 3, + "decomposition_note": "advantage vs logit splits into GENERIC (shuffled-logit: what ANY fitted Jacobian buys) and LAYER-SPECIFIC (jlens-shuffled: the workspace claim)" +} \ No newline at end of file diff --git a/results/t13_transport_controls_70m.json b/results/t13_transport_controls_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..380e77a2b95c9499f158cc9536dca57a9e18332b --- /dev/null +++ b/results/t13_transport_controls_70m.json @@ -0,0 +1,288 @@ +{ + "experiment": "T13 / E1 \u2014 transport controls: does a BROKEN J still beat the logit lens?", + "status": "EXPLORATORY \u2014 control on cells already burned per PREREG_PYTHIA_T7_v2 section 0", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200_db128.pt", + "seed": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "n_layers": 5, + "derangement_layer_map": { + "0": 3, + "1": 4, + "2": 0, + "3": 1, + "4": 2 + }, + "primary_metric": "normalized_auc_over_logk_paper (A.6 per-item binary)", + "n_win_vs_logit": { + "jlens": 3, + "shuffled": 1, + "spec_random": 1, + "rank1": 0, + "scaled_id": 0 + }, + "n_evals": 6, + "SELF_TEST_scaled_identity_equals_logit": true, + "SELF_TEST_worst_abs_auc_diff": 0.0, + "interpretation": "If shuffled/spec_random/rank1 win on as many sets as jlens, T7 measures 'any learned map into the final basis helps', not J-lens-specific workspace reading.", + "results": { + "multihop": { + "logit": { + "auc_paper": 0.0292, + "auc_readme": 0.0323, + "median_best_rank": 837.0 + }, + "jlens": { + "auc_paper": 0.0394, + "auc_readme": 0.0436, + "median_best_rank": 799.0 + }, + "shuffled": { + "auc_paper": 0.0345, + "auc_readme": 0.0374, + "median_best_rank": 887.0 + }, + "spec_random": { + "auc_paper": 0.0015, + "auc_readme": 0.0016, + "median_best_rank": 3036.0 + }, + "rank1": { + "auc_paper": 0.0058, + "auc_readme": 0.0054, + "median_best_rank": 1163.0 + }, + "scaled_id": { + "auc_paper": 0.0292, + "auc_readme": 0.0323, + "median_best_rank": 837.0 + } + }, + "multilingual": { + "logit": { + "auc_paper": 0.0431, + "auc_readme": 0.045, + "median_best_rank": 899.0 + }, + "jlens": { + "auc_paper": 0.0385, + "auc_readme": 0.0356, + "median_best_rank": 1277.0 + }, + "shuffled": { + "auc_paper": 0.0413, + "auc_readme": 0.0413, + "median_best_rank": 945.0 + }, + "spec_random": { + "auc_paper": 0.0083, + "auc_readme": 0.0085, + "median_best_rank": 2221.0 + }, + "rank1": { + "auc_paper": 0.0071, + "auc_readme": 0.0079, + "median_best_rank": 918.0 + }, + "scaled_id": { + "auc_paper": 0.0431, + "auc_readme": 0.045, + "median_best_rank": 899.0 + } + }, + "order-ops": { + "logit": { + "auc_paper": 0.3496, + "auc_readme": 0.3496, + "median_best_rank": 22.0 + }, + "jlens": { + "auc_paper": 0.3524, + "auc_readme": 0.3524, + "median_best_rank": 25.0 + }, + "shuffled": { + "auc_paper": 0.2548, + "auc_readme": 0.2548, + "median_best_rank": 35.0 + }, + "spec_random": { + "auc_paper": 0.0598, + "auc_readme": 0.0598, + "median_best_rank": 1193.0 + }, + "rank1": { + "auc_paper": 0.0301, + "auc_readme": 0.0301, + "median_best_rank": 375.0 + }, + "scaled_id": { + "auc_paper": 0.3496, + "auc_readme": 0.3496, + "median_best_rank": 22.0 + } + }, + "poetry": { + "logit": { + "auc_paper": 0.0319, + "auc_readme": 0.0319, + "median_best_rank": 1339.0 + }, + "jlens": { + "auc_paper": 0.0245, + "auc_readme": 0.0245, + "median_best_rank": 755.0 + }, + "shuffled": { + "auc_paper": 0.0207, + "auc_readme": 0.0207, + "median_best_rank": 822.0 + }, + "spec_random": { + "auc_paper": 0.0061, + "auc_readme": 0.0061, + "median_best_rank": 2107.0 + }, + "rank1": { + "auc_paper": 0.0072, + "auc_readme": 0.0072, + "median_best_rank": 741.0 + }, + "scaled_id": { + "auc_paper": 0.0319, + "auc_readme": 0.0319, + "median_best_rank": 1339.0 + } + }, + "typo": { + "logit": { + "auc_paper": 0.0886, + "auc_readme": 0.0886, + "median_best_rank": 478.0 + }, + "jlens": { + "auc_paper": 0.0826, + "auc_readme": 0.0826, + "median_best_rank": 945.0 + }, + "shuffled": { + "auc_paper": 0.0508, + "auc_readme": 0.0508, + "median_best_rank": 992.0 + }, + "spec_random": { + "auc_paper": 0.0016, + "auc_readme": 0.0016, + "median_best_rank": 1733.0 + }, + "rank1": { + "auc_paper": 0.0151, + "auc_readme": 0.0151, + "median_best_rank": 978.0 + }, + "scaled_id": { + "auc_paper": 0.0886, + "auc_readme": 0.0886, + "median_best_rank": 478.0 + } + }, + "association": { + "logit": { + "auc_paper": 0.0076, + "auc_readme": 0.0076, + "median_best_rank": 2671.0 + }, + "jlens": { + "auc_paper": 0.0446, + "auc_readme": 0.0446, + "median_best_rank": 1083.0 + }, + "shuffled": { + "auc_paper": 0.0033, + "auc_readme": 0.0033, + "median_best_rank": 2489.0 + }, + "spec_random": { + "auc_paper": 0.0109, + "auc_readme": 0.0109, + "median_best_rank": 3042.0 + }, + "rank1": { + "auc_paper": 0.0, + "auc_readme": 0.0, + "median_best_rank": 2718.0 + }, + "scaled_id": { + "auc_paper": 0.0076, + "auc_readme": 0.0076, + "median_best_rank": 2671.0 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 43.8, + "advantage_decomposition": { + "multihop": { + "auc_logit": 0.0292, + "auc_shuffled": 0.0345, + "auc_jlens": 0.0394, + "total_advantage_vs_logit": 0.0102, + "generic_component_shuffled_minus_logit": 0.0053, + "layer_specific_component_jlens_minus_shuffled": 0.0049 + }, + "multilingual": { + "auc_logit": 0.0431, + "auc_shuffled": 0.0413, + "auc_jlens": 0.0385, + "total_advantage_vs_logit": -0.0046, + "generic_component_shuffled_minus_logit": -0.0018, + "layer_specific_component_jlens_minus_shuffled": -0.0028 + }, + "order-ops": { + "auc_logit": 0.3496, + "auc_shuffled": 0.2548, + "auc_jlens": 0.3524, + "total_advantage_vs_logit": 0.0028, + "generic_component_shuffled_minus_logit": -0.0948, + "layer_specific_component_jlens_minus_shuffled": 0.0976 + }, + "poetry": { + "auc_logit": 0.0319, + "auc_shuffled": 0.0207, + "auc_jlens": 0.0245, + "total_advantage_vs_logit": -0.0074, + "generic_component_shuffled_minus_logit": -0.0112, + "layer_specific_component_jlens_minus_shuffled": 0.0038 + }, + "typo": { + "auc_logit": 0.0886, + "auc_shuffled": 0.0508, + "auc_jlens": 0.0826, + "total_advantage_vs_logit": -0.006, + "generic_component_shuffled_minus_logit": -0.0378, + "layer_specific_component_jlens_minus_shuffled": 0.0318 + }, + "association": { + "auc_logit": 0.0076, + "auc_shuffled": 0.0033, + "auc_jlens": 0.0446, + "total_advantage_vs_logit": 0.037, + "generic_component_shuffled_minus_logit": -0.0043, + "layer_specific_component_jlens_minus_shuffled": 0.0413 + } + }, + "n_win_jlens_vs_shuffled": 5, + "decomposition_note": "advantage vs logit splits into GENERIC (shuffled-logit: what ANY fitted Jacobian buys) and LAYER-SPECIFIC (jlens-shuffled: the workspace claim)" +} \ No newline at end of file diff --git a/results/t13b_bootstrap_controls_160m.json b/results/t13b_bootstrap_controls_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..b95868864bf6606115807f6bb5391ef3ccc1869f --- /dev/null +++ b/results/t13b_bootstrap_controls_160m.json @@ -0,0 +1,248 @@ +{ + "experiment": "T13b \u2014 paired item bootstrap on the generic / layer-specific decomposition", + "status": "EXPLORATORY \u2014 burned cells per PREREG_PYTHIA_T7_v2 section 0", + "model": "EleutherAI/pythia-160m-deduped", + "lens": "pythia/results/lens_160m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "seed": 0, + "n_boot": 10000, + "bootstrap": "paired over (item,intermediate) pairs; one index draw applied to all 3 arms", + "metric": "normalized pass@k AUC over log k, PAPER form (flat pool), ks=(1,2,5,10,25,100)", + "derangement_layer_map": { + "0": 9, + "1": 7, + "2": 8, + "3": 10, + "4": 6, + "5": 1, + "6": 5, + "7": 4, + "8": 3, + "9": 2, + "10": 0 + }, + "n_sets_specific_positive": 3, + "n_sets_specific_CI_excludes_zero": 1, + "n_sets_generic_CI_excludes_zero": 0, + "headline": "layer-specific advantage is positive on 3/6 sets and survives a 95% paired bootstrap on 1/6; the generic basis-change component survives on 0/6", + "per_set": { + "multihop": { + "n_pairs": 103, + "auc_logit": 0.1244, + "auc_shuffled": 0.0676, + "auc_jlens": 0.1093, + "total": { + "delta": -0.0151, + "ci95": [ + -0.069, + 0.0352 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.2855 + }, + "generic": { + "delta": -0.0568, + "ci95": [ + -0.1078, + -0.0088 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0099 + }, + "specific": { + "delta": 0.0417, + "ci95": [ + -0.0037, + 0.0895 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.0353 + } + }, + "multilingual": { + "n_pairs": 394, + "auc_logit": 0.0391, + "auc_shuffled": 0.035, + "auc_jlens": 0.03, + "total": { + "delta": -0.009, + "ci95": [ + -0.0272, + 0.0087 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1617 + }, + "generic": { + "delta": -0.0041, + "ci95": [ + -0.0213, + 0.0124 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3241 + }, + "specific": { + "delta": -0.005, + "ci95": [ + -0.0133, + 0.0035 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1268 + } + }, + "order-ops": { + "n_pairs": 110, + "auc_logit": 0.4493, + "auc_shuffled": 0.2169, + "auc_jlens": 0.4402, + "total": { + "delta": -0.0091, + "ci95": [ + -0.0908, + 0.0729 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.4144 + }, + "generic": { + "delta": -0.2324, + "ci95": [ + -0.2951, + -0.1694 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0 + }, + "specific": { + "delta": 0.2233, + "ci95": [ + 0.1713, + 0.2769 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0 + } + }, + "poetry": { + "n_pairs": 98, + "auc_logit": 0.0383, + "auc_shuffled": 0.0378, + "auc_jlens": 0.0031, + "total": { + "delta": -0.0352, + "ci95": [ + -0.0656, + -0.0105 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0002 + }, + "generic": { + "delta": -0.0005, + "ci95": [ + -0.0382, + 0.0345 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.5034 + }, + "specific": { + "delta": -0.0348, + "ci95": [ + -0.0593, + -0.0143 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0001 + } + }, + "typo": { + "n_pairs": 96, + "auc_logit": 0.2423, + "auc_shuffled": 0.2569, + "auc_jlens": 0.2808, + "total": { + "delta": 0.0385, + "ci95": [ + -0.0418, + 0.1207 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1736 + }, + "generic": { + "delta": 0.0145, + "ci95": [ + -0.0633, + 0.0953 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3613 + }, + "specific": { + "delta": 0.0239, + "ci95": [ + -0.0221, + 0.0695 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1593 + } + }, + "association": { + "n_pairs": 92, + "auc_logit": 0.0033, + "auc_shuffled": 0.0112, + "auc_jlens": 0.0109, + "total": { + "delta": 0.0076, + "ci95": [ + -0.0016, + 0.0196 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.0842 + }, + "generic": { + "delta": 0.0079, + "ci95": [ + -0.0016, + 0.0237 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1428 + }, + "specific": { + "delta": -0.0003, + "ci95": [ + -0.0174, + 0.0147 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.521 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 58.1 +} \ No newline at end of file diff --git a/results/t13b_bootstrap_controls_410m.json b/results/t13b_bootstrap_controls_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..94ea217492cca2a9a3f83419f8087aac6d1d91ee --- /dev/null +++ b/results/t13b_bootstrap_controls_410m.json @@ -0,0 +1,272 @@ +{ + "experiment": "T13b \u2014 paired item bootstrap on the generic / layer-specific decomposition", + "status": "EXPLORATORY \u2014 burned cells per PREREG_PYTHIA_T7_v2 section 0", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "seed": 0, + "n_boot": 10000, + "bootstrap": "paired over (item,intermediate) pairs; one index draw applied to all 3 arms", + "metric": "normalized pass@k AUC over log k, PAPER form (flat pool), ks=(1,2,5,10,25,100)", + "derangement_layer_map": { + "0": 17, + "1": 8, + "2": 21, + "3": 18, + "4": 7, + "5": 12, + "6": 19, + "7": 1, + "8": 16, + "9": 14, + "10": 2, + "11": 6, + "12": 15, + "13": 11, + "14": 4, + "15": 22, + "16": 13, + "17": 10, + "18": 0, + "19": 3, + "20": 9, + "21": 20, + "22": 5 + }, + "n_sets_specific_positive": 3, + "n_sets_specific_CI_excludes_zero": 3, + "n_sets_generic_CI_excludes_zero": 3, + "headline": "layer-specific advantage is positive on 3/6 sets and survives a 95% paired bootstrap on 3/6; the generic basis-change component survives on 3/6", + "per_set": { + "multihop": { + "n_pairs": 103, + "auc_logit": 0.1994, + "auc_shuffled": 0.1946, + "auc_jlens": 0.2602, + "total": { + "delta": 0.0608, + "ci95": [ + 0.0045, + 0.1185 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0162 + }, + "generic": { + "delta": -0.0048, + "ci95": [ + -0.0484, + 0.0383 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.4112 + }, + "specific": { + "delta": 0.0655, + "ci95": [ + 0.0292, + 0.1014 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0004 + } + }, + "multilingual": { + "n_pairs": 394, + "auc_logit": 0.1153, + "auc_shuffled": 0.1195, + "auc_jlens": 0.2169, + "total": { + "delta": 0.1017, + "ci95": [ + 0.0756, + 0.128 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0 + }, + "generic": { + "delta": 0.0042, + "ci95": [ + -0.015, + 0.0234 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.338 + }, + "specific": { + "delta": 0.0974, + "ci95": [ + 0.072, + 0.1245 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0 + } + }, + "order-ops": { + "n_pairs": 110, + "auc_logit": 0.5234, + "auc_shuffled": 0.5241, + "auc_jlens": 0.5679, + "total": { + "delta": 0.0446, + "ci95": [ + -0.0009, + 0.0907 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.0273 + }, + "generic": { + "delta": 0.0007, + "ci95": [ + -0.035, + 0.0354 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.4804 + }, + "specific": { + "delta": 0.0438, + "ci95": [ + 0.0125, + 0.0752 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0037 + } + }, + "poetry": { + "n_pairs": 98, + "auc_logit": 0.0287, + "auc_shuffled": 0.0718, + "auc_jlens": 0.0435, + "total": { + "delta": 0.0148, + "ci95": [ + -0.0041, + 0.0348 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.0665 + }, + "generic": { + "delta": 0.0432, + "ci95": [ + 0.0194, + 0.0702 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0 + }, + "specific": { + "delta": -0.0283, + "ci95": [ + -0.057, + -0.0025 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0168 + } + }, + "typo": { + "n_pairs": 96, + "auc_logit": 0.2034, + "auc_shuffled": 0.3278, + "auc_jlens": 0.3241, + "total": { + "delta": 0.1208, + "ci95": [ + 0.0627, + 0.1806 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0 + }, + "generic": { + "delta": 0.1244, + "ci95": [ + 0.0628, + 0.1901 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0002 + }, + "specific": { + "delta": -0.0037, + "ci95": [ + -0.0432, + 0.0341 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.4244 + } + }, + "association": { + "n_pairs": 92, + "auc_logit": 0.0076, + "auc_shuffled": 0.0313, + "auc_jlens": 0.0115, + "total": { + "delta": 0.0038, + "ci95": [ + -0.0098, + 0.0164 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.2695 + }, + "generic": { + "delta": 0.0237, + "ci95": [ + 0.0049, + 0.0455 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0058 + }, + "specific": { + "delta": -0.0199, + "ci95": [ + -0.0419, + -0.0016 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.017 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 155.4 +} \ No newline at end of file diff --git a/results/t13b_bootstrap_controls_70m.json b/results/t13b_bootstrap_controls_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..2e41c0afbaab00954cf70b188beb2d101d12ae93 --- /dev/null +++ b/results/t13b_bootstrap_controls_70m.json @@ -0,0 +1,236 @@ +{ + "experiment": "T13b \u2014 paired item bootstrap on the generic / layer-specific decomposition", + "status": "EXPLORATORY \u2014 burned cells per PREREG_PYTHIA_T7_v2 section 0", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "seed": 0, + "n_boot": 10000, + "bootstrap": "paired over (item,intermediate) pairs; one index draw applied to all 3 arms", + "metric": "normalized pass@k AUC over log k, PAPER form (flat pool), ks=(1,2,5,10,25,100)", + "derangement_layer_map": { + "0": 3, + "1": 4, + "2": 0, + "3": 1, + "4": 2 + }, + "n_sets_specific_positive": 5, + "n_sets_specific_CI_excludes_zero": 2, + "n_sets_generic_CI_excludes_zero": 0, + "headline": "layer-specific advantage is positive on 5/6 sets and survives a 95% paired bootstrap on 2/6; the generic basis-change component survives on 0/6", + "per_set": { + "multihop": { + "n_pairs": 103, + "auc_logit": 0.0292, + "auc_shuffled": 0.0345, + "auc_jlens": 0.0394, + "total": { + "delta": 0.0102, + "ci95": [ + -0.0133, + 0.0338 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1905 + }, + "generic": { + "delta": 0.0053, + "ci95": [ + -0.0228, + 0.0355 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3581 + }, + "specific": { + "delta": 0.0049, + "ci95": [ + -0.024, + 0.0321 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3648 + } + }, + "multilingual": { + "n_pairs": 394, + "auc_logit": 0.0431, + "auc_shuffled": 0.0413, + "auc_jlens": 0.0385, + "total": { + "delta": -0.0046, + "ci95": [ + -0.0233, + 0.014 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.311 + }, + "generic": { + "delta": -0.0018, + "ci95": [ + -0.0184, + 0.0147 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.4234 + }, + "specific": { + "delta": -0.0028, + "ci95": [ + -0.0167, + 0.011 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3486 + } + }, + "order-ops": { + "n_pairs": 110, + "auc_logit": 0.3496, + "auc_shuffled": 0.2548, + "auc_jlens": 0.3524, + "total": { + "delta": 0.0028, + "ci95": [ + -0.0738, + 0.0799 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.4613 + }, + "generic": { + "delta": -0.0948, + "ci95": [ + -0.1541, + -0.0344 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0012 + }, + "specific": { + "delta": 0.0976, + "ci95": [ + 0.0512, + 0.145 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0 + } + }, + "poetry": { + "n_pairs": 98, + "auc_logit": 0.0319, + "auc_shuffled": 0.0207, + "auc_jlens": 0.0245, + "total": { + "delta": -0.0074, + "ci95": [ + -0.0354, + 0.0164 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3107 + }, + "generic": { + "delta": -0.0112, + "ci95": [ + -0.0411, + 0.0151 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.2172 + }, + "specific": { + "delta": 0.0038, + "ci95": [ + -0.0161, + 0.023 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3416 + } + }, + "typo": { + "n_pairs": 96, + "auc_logit": 0.0886, + "auc_shuffled": 0.0508, + "auc_jlens": 0.0826, + "total": { + "delta": -0.006, + "ci95": [ + -0.0498, + 0.036 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3841 + }, + "generic": { + "delta": -0.0378, + "ci95": [ + -0.0826, + 0.0029 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.0342 + }, + "specific": { + "delta": 0.0318, + "ci95": [ + -0.0034, + 0.07 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.04 + } + }, + "association": { + "n_pairs": 92, + "auc_logit": 0.0076, + "auc_shuffled": 0.0033, + "auc_jlens": 0.0446, + "total": { + "delta": 0.037, + "ci95": [ + 0.0084, + 0.071 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0049 + }, + "generic": { + "delta": -0.0044, + "ci95": [ + -0.0163, + 0.0049 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.2598 + }, + "specific": { + "delta": 0.0413, + "ci95": [ + 0.0144, + 0.0748 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0001 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 25.8 +} \ No newline at end of file diff --git a/results/t14_dispersion_ladder.json b/results/t14_dispersion_ladder.json new file mode 100644 index 0000000000000000000000000000000000000000..c67fd28a9ec427f822611bc4973f8e0590a6bcc4 --- /dev/null +++ b/results/t14_dispersion_ladder.json @@ -0,0 +1,542 @@ +{ + "experiment": "T14 \u2014 dispersion ladder under relative-depth normalization", + "status": "EXPLORATORY", + "source": "lens_*_provenance.json jacobian_dispersion blocks; no model or GPU needed", + "definition": "disp(l) = E||J_l(x) - Jbar_l||_F^2 / ||Jbar_l||_F^2", + "cancelled_fraction": "1 - 1/sqrt(1+disp) = fraction of rms per-prompt magnitude lost to averaging", + "relative_depths": [ + 0.0, + 0.1, + 0.25, + 0.5, + 0.75, + 0.9, + 1.0 + ], + "models": [ + { + "file": "lens_70m_n200_db128_pen_provenance.json", + "model": "EleutherAI/pythia-70m-deduped", + "n_layers": 6, + "d_model": 512, + "n_source": 4, + "target_penultimate": true, + "dispersion": [ + 0.229049, + 0.174573, + 0.121218, + 0.057114 + ], + "heavy_tail": [ + 1.167, + 1.149, + 1.1271, + 1.0642 + ], + "dispersion_at_rel": { + "0.0": 0.229, + "0.1": 0.2127, + "0.25": 0.1882, + "0.5": 0.1479, + "0.75": 0.1052, + "0.9": 0.0763, + "1.0": 0.0571 + }, + "cancelled_fraction_at_rel": { + "0.0": 0.098, + "0.1": 0.0919, + "0.25": 0.0826, + "0.5": 0.0666, + "0.75": 0.0488, + "0.9": 0.0361, + "1.0": 0.0274 + }, + "heavy_tail_at_rel": { + "0.0": 1.167, + "0.1": 1.1616, + "0.25": 1.1535, + "0.5": 1.1381, + "0.75": 1.1114, + "0.9": 1.0831, + "1.0": 1.0642 + }, + "band": [ + 2, + 3 + ], + "dispersion_in_band": [ + 0.0571, + 0.1212 + ] + }, + { + "file": "lens_160m_n200_db128_pen_provenance.json", + "model": "EleutherAI/pythia-160m-deduped", + "n_layers": 12, + "d_model": 768, + "n_source": 10, + "target_penultimate": true, + "dispersion": [ + 0.641485, + 0.50993, + 0.438254, + 0.381029, + 0.333103, + 0.279857, + 0.235863, + 0.171063, + 0.116419, + 0.063732 + ], + "heavy_tail": [ + 1.3339, + 1.3458, + 1.3247, + 1.3153, + 1.2923, + 1.2053, + 1.1726, + 1.0926, + 1.0608, + 1.0283 + ], + "dispersion_at_rel": { + "0.0": 0.6415, + "0.1": 0.5231, + "0.25": 0.4239, + "0.5": 0.3065, + "0.75": 0.1873, + "0.9": 0.1112, + "1.0": 0.0637 + }, + "cancelled_fraction_at_rel": { + "0.0": 0.2195, + "0.1": 0.1897, + "0.25": 0.162, + "0.5": 0.1251, + "0.75": 0.0823, + "0.9": 0.0514, + "1.0": 0.0304 + }, + "heavy_tail_at_rel": { + "0.0": 1.3339, + "0.1": 1.3446, + "0.25": 1.3223, + "0.5": 1.2488, + "0.75": 1.1126, + "0.9": 1.0575, + "1.0": 1.0283 + }, + "band": [ + 5, + 9 + ], + "dispersion_in_band": [ + 0.0637, + 0.2799 + ] + }, + { + "file": "lens_1b_n200_db128_pen_provenance.json", + "model": "EleutherAI/pythia-1b-deduped", + "n_layers": 16, + "d_model": 2048, + "n_source": 14, + "target_penultimate": true, + "dispersion": [ + 1.73571, + 1.505767, + 1.166506, + 0.982771, + 0.875869, + 0.72398, + 0.594042, + 0.504802, + 0.41589, + 0.311256, + 0.239409, + 0.147058, + 0.085984, + 0.037524 + ], + "heavy_tail": [ + 1.344, + 1.355, + 1.3476, + 1.3291, + 1.2963, + 1.2578, + 1.2308, + 1.2071, + 1.1956, + 1.16, + 1.1446, + 1.1061, + 1.0783, + 1.0453 + ], + "dispersion_at_rel": { + "0.0": 1.7357, + "0.1": 1.404, + "0.25": 0.956, + "0.5": 0.5494, + "0.75": 0.2574, + "0.9": 0.1043, + "1.0": 0.0375 + }, + "cancelled_fraction_at_rel": { + "0.0": 0.3954, + "0.1": 0.355, + "0.25": 0.285, + "0.5": 0.1966, + "0.75": 0.1082, + "0.9": 0.0484, + "1.0": 0.0182 + }, + "heavy_tail_at_rel": { + "0.0": 1.344, + "0.1": 1.3528, + "0.25": 1.3209, + "0.5": 1.2189, + "0.75": 1.1484, + "0.9": 1.0866, + "1.0": 1.0453 + }, + "band": [ + 6, + 13 + ], + "dispersion_in_band": [ + 0.0375, + 0.594 + ] + }, + { + "file": "lens_1.4b_n200_db128_pen_provenance.json", + "model": "EleutherAI/pythia-1.4b-deduped", + "n_layers": 24, + "d_model": 2048, + "n_source": 22, + "target_penultimate": true, + "dispersion": [ + 2.897373, + 2.608921, + 1.988551, + 1.617573, + 1.354258, + 1.135337, + 0.959127, + 0.845753, + 0.732547, + 0.645362, + 0.584442, + 0.506679, + 0.414947, + 0.348885, + 0.277452, + 0.225428, + 0.176245, + 0.151089, + 0.107296, + 0.087375, + 0.06569, + 0.040788 + ], + "heavy_tail": [ + 1.5032, + 1.4749, + 1.3815, + 1.3593, + 1.3525, + 1.3354, + 1.3166, + 1.3033, + 1.2871, + 1.2795, + 1.2574, + 1.2078, + 1.1796, + 1.1549, + 1.1375, + 1.1258, + 1.0973, + 1.083, + 1.0618, + 1.0484, + 1.0593, + 1.0523 + ], + "dispersion_at_rel": { + "0.0": 2.8974, + "0.1": 1.9515, + "0.25": 1.0913, + "0.5": 0.5456, + "0.75": 0.1885, + "0.9": 0.0894, + "1.0": 0.0408 + }, + "cancelled_fraction_at_rel": { + "0.0": 0.4935, + "0.1": 0.4179, + "0.25": 0.3085, + "0.5": 0.1956, + "0.75": 0.0827, + "0.9": 0.0419, + "1.0": 0.0198 + }, + "heavy_tail_at_rel": { + "0.0": 1.5032, + "0.1": 1.3793, + "0.25": 1.3307, + "0.5": 1.2326, + "0.75": 1.1044, + "0.9": 1.0497, + "1.0": 1.0523 + }, + "band": [ + 9, + 21 + ], + "dispersion_in_band": [ + 0.0408, + 0.6454 + ] + }, + { + "file": "lens_410m_n200_db128_pen_provenance.json", + "model": "EleutherAI/pythia-410m-deduped", + "n_layers": 24, + "d_model": 1024, + "n_source": 22, + "target_penultimate": true, + "dispersion": [ + 2.364437, + 2.077954, + 1.651819, + 1.420183, + 1.16357, + 1.051403, + 0.924073, + 0.810342, + 0.730484, + 0.671794, + 0.612544, + 0.562712, + 0.466847, + 0.402353, + 0.345839, + 0.296264, + 0.222337, + 0.166069, + 0.12263, + 0.082939, + 0.047779, + 0.022615 + ], + "heavy_tail": [ + 1.4235, + 1.4275, + 1.3778, + 1.3538, + 1.358, + 1.337, + 1.3175, + 1.286, + 1.2701, + 1.2459, + 1.2255, + 1.2031, + 1.1605, + 1.1503, + 1.1425, + 1.1336, + 1.1096, + 1.0978, + 1.0766, + 1.0571, + 1.042, + 1.0266 + ], + "dispersion_at_rel": { + "0.0": 2.3644, + "0.1": 1.6287, + "0.25": 1.0196, + "0.5": 0.5876, + "0.75": 0.2408, + "0.9": 0.0869, + "1.0": 0.0226 + }, + "cancelled_fraction_at_rel": { + "0.0": 0.4548, + "0.1": 0.3832, + "0.25": 0.2963, + "0.5": 0.2063, + "0.75": 0.1023, + "0.9": 0.0408, + "1.0": 0.0111 + }, + "heavy_tail_at_rel": { + "0.0": 1.4235, + "0.1": 1.3754, + "0.25": 1.3321, + "0.5": 1.2143, + "0.75": 1.1156, + "0.9": 1.059, + "1.0": 1.0266 + }, + "band": [ + 9, + 21 + ], + "dispersion_in_band": [ + 0.0226, + 0.6718 + ] + }, + { + "file": "lens_2.8b_n200_db128_pen_provenance.json", + "model": "EleutherAI/pythia-2.8b-deduped", + "n_layers": 32, + "d_model": 2560, + "n_source": 30, + "target_penultimate": true, + "dispersion": [ + 4.418416, + 4.123117, + 3.288672, + 2.626305, + 2.1716, + 1.724826, + 1.427161, + 1.165801, + 1.006128, + 0.874959, + 0.767417, + 0.688895, + 0.588333, + 0.528614, + 0.44279, + 0.390658, + 0.333914, + 0.281205, + 0.237206, + 0.201721, + 0.175299, + 0.146459, + 0.124612, + 0.104266, + 0.085761, + 0.067878, + 0.054365, + 0.040424, + 0.025223, + 0.012407 + ], + "heavy_tail": [ + 1.7019, + 1.721, + 1.5934, + 1.5902, + 1.6588, + 1.4691, + 1.2971, + 1.2752, + 1.2349, + 1.2249, + 1.2117, + 1.2098, + 1.1987, + 1.1759, + 1.166, + 1.166, + 1.1594, + 1.1318, + 1.1194, + 1.1056, + 1.0908, + 1.0794, + 1.065, + 1.0608, + 1.0504, + 1.0422, + 1.0398, + 1.0315, + 1.0188, + 1.0097 + ], + "dispersion_at_rel": { + "0.0": 4.4184, + "0.1": 2.6925, + "0.25": 1.1259, + "0.5": 0.4167, + "0.75": 0.1301, + "0.9": 0.053, + "1.0": 0.0124 + }, + "cancelled_fraction_at_rel": { + "0.0": 0.5704, + "0.1": 0.4796, + "0.25": 0.3142, + "0.5": 0.1598, + "0.75": 0.0593, + "0.9": 0.0255, + "1.0": 0.0061 + }, + "heavy_tail_at_rel": { + "0.0": 1.7019, + "0.1": 1.5905, + "0.25": 1.2651, + "0.5": 1.166, + "0.75": 1.0686, + "0.9": 1.039, + "1.0": 1.0097 + }, + "band": [ + 12, + 29 + ], + "dispersion_in_band": [ + 0.0124, + 0.5883 + ] + } + ], + "collapse_across_models": { + "0.0": { + "min": 0.229, + "max": 4.4184, + "max_over_min": 19.294 + }, + "0.1": { + "min": 0.2127, + "max": 2.6925, + "max_over_min": 12.659 + }, + "0.25": { + "min": 0.1882, + "max": 1.1259, + "max_over_min": 5.982 + }, + "0.5": { + "min": 0.1479, + "max": 0.5876, + "max_over_min": 3.973 + }, + "0.75": { + "min": 0.1052, + "max": 0.2574, + "max_over_min": 2.447 + }, + "0.9": { + "min": 0.053, + "max": 0.1112, + "max_over_min": 2.098 + }, + "1.0": { + "min": 0.0124, + "max": 0.0637, + "max_over_min": 5.137 + } + }, + "note": "If the profile collapses under relative depth, dispersion is a property of FRACTIONAL depth rather than of absolute layer index or model size, and the raw growth of disp(layer 0) with model size is a distance-to-target artifact." +} \ No newline at end of file diff --git a/results/t14b_dispersion_scaling.json b/results/t14b_dispersion_scaling.json new file mode 100644 index 0000000000000000000000000000000000000000..3516d3c321c44e3a54ef0f23a5e7fc7d0b6db211 --- /dev/null +++ b/results/t14b_dispersion_scaling.json @@ -0,0 +1,62 @@ +{ + "experiment": "T14b \u2014 what does layer-0 dispersion scale with?", + "status": "EXPLORATORY", + "rows": [ + { + "model": "70m", + "n_layers": 6, + "d_model": 512, + "params": 70000000.0, + "disp_layer0": 0.229049, + "n_source": 4 + }, + { + "model": "160m", + "n_layers": 12, + "d_model": 768, + "params": 160000000.0, + "disp_layer0": 0.641485, + "n_source": 10 + }, + { + "model": "1b", + "n_layers": 16, + "d_model": 2048, + "params": 1000000000.0, + "disp_layer0": 1.73571, + "n_source": 14 + }, + { + "model": "1.4b", + "n_layers": 24, + "d_model": 2048, + "params": 1400000000.0, + "disp_layer0": 2.897373, + "n_source": 22 + }, + { + "model": "410m", + "n_layers": 24, + "d_model": 1024, + "params": 410000000.0, + "disp_layer0": 2.364437, + "n_source": 22 + }, + { + "model": "2.8b", + "n_layers": 32, + "d_model": 2560, + "params": 2800000000.0, + "disp_layer0": 4.418416, + "n_source": 30 + } + ], + "log_log_pearson_r": { + "n_layers": 0.9882, + "d_model": 0.8976, + "params": 0.9418 + }, + "power_law_exponent_vs_depth": 1.791, + "discriminating_case": "410m (24 layers, d=1024, 0.41B) has HIGHER layer-0 dispersion (2.364) than 1b (16 layers, d=2048, 1.0B; 1.736), so depth dominates parameter count and width", + "correction": "supersedes the 3-model reading in the first T14 commit, which reported mid-network scale-invariance; with 6 models there is NO collapse at any relative depth" +} \ No newline at end of file diff --git a/results/t15_shuffle_diagnostic_410m.json b/results/t15_shuffle_diagnostic_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..07c06a0476e79f885439f54ae5d05427c00c38f6 --- /dev/null +++ b/results/t15_shuffle_diagnostic_410m.json @@ -0,0 +1,285 @@ +{ + "experiment": "T15 \u2014 why does a layer-shuffled J still beat the logit lens?", + "status": "EXPLORATORY \u2014 diagnostic on burned cells", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "seed": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "derangement_layer_map": { + "0": 17, + "1": 8, + "2": 21, + "3": 18, + "4": 7, + "5": 12, + "6": 19, + "7": 1, + "8": 16, + "9": 14, + "10": 2, + "11": 6, + "12": 15, + "13": 11, + "14": 4, + "15": 22, + "16": 13, + "17": 10, + "18": 0, + "19": 3, + "20": 9, + "21": 20, + "22": 5 + }, + "hypotheses": { + "H_SIM": "Jacobians are near-identical across layers, so shuffling is a no-op", + "H_MIN": "min-over-layers launders the difference; per-layer, jlens should beat shuffled almost everywhere" + }, + "jacobian_similarity": { + "pairwise_cos_mean": 0.6291, + "pairwise_cos_min": 0.1734, + "pairwise_cos_max": 0.98, + "pairwise_cos_median": 0.6589, + "cos_with_identity_per_layer": [ + 0.168, + 0.1817, + 0.1893, + 0.2188, + 0.2357, + 0.2784, + 0.3006, + 0.3247, + 0.3481, + 0.39, + 0.428, + 0.4787, + 0.547, + 0.6007, + 0.6416, + 0.6747, + 0.7563, + 0.8179, + 0.8473, + 0.885, + 0.904, + 0.9235, + 0.9476 + ], + "cos_with_identity_mean": 0.5255, + "note": "pairwise_cos near 1 would mean the Jacobians are interchangeable and shuffling is a no-op (H-SIM). cos_with_identity near 1 would mean J ~ I, i.e. the J-lens is nearly the logit lens." + }, + "results": { + "multihop": { + "logit": { + "per_layer_auc": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0, + "11": 0.0015, + "12": 0.0, + "13": 0.0015, + "14": 0.0114, + "15": 0.0459, + "16": 0.0551, + "17": 0.0838, + "18": 0.111, + "19": 0.1504, + "20": 0.1314, + "21": 0.1336, + "22": 0.1421 + }, + "best_single_layer_auc": 0.1504, + "mean_over_layers_auc": 0.0377, + "min_over_layers_auc": 0.1994 + }, + "jlens": { + "per_layer_auc": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0029, + "5": 0.0, + "6": 0.0, + "7": 0.0015, + "8": 0.0, + "9": 0.0, + "10": 0.0029, + "11": 0.0029, + "12": 0.036, + "13": 0.0348, + "14": 0.0224, + "15": 0.0284, + "16": 0.0595, + "17": 0.1227, + "18": 0.1232, + "19": 0.1397, + "20": 0.1519, + "21": 0.1674, + "22": 0.1555 + }, + "best_single_layer_auc": 0.1674, + "mean_over_layers_auc": 0.0457, + "min_over_layers_auc": 0.2602 + }, + "shuffled": { + "per_layer_auc": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0097, + "6": 0.0015, + "7": 0.0015, + "8": 0.0029, + "9": 0.0068, + "10": 0.0015, + "11": 0.0015, + "12": 0.0092, + "13": 0.0015, + "14": 0.0015, + "15": 0.0629, + "16": 0.0431, + "17": 0.0516, + "18": 0.0, + "19": 0.0015, + "20": 0.0151, + "21": 0.1373, + "22": 0.0209 + }, + "best_single_layer_auc": 0.1373, + "mean_over_layers_auc": 0.0161, + "min_over_layers_auc": 0.1946 + }, + "n_layers_jlens_beats_shuffled": 13, + "n_layers": 23 + }, + "typo": { + "logit": { + "per_layer_auc": { + "0": 0.0, + "1": 0.0016, + "2": 0.0057, + "3": 0.0057, + "4": 0.0031, + "5": 0.0104, + "6": 0.0146, + "7": 0.0344, + "8": 0.0258, + "9": 0.0242, + "10": 0.0224, + "11": 0.0185, + "12": 0.0352, + "13": 0.0347, + "14": 0.0469, + "15": 0.0524, + "16": 0.0478, + "17": 0.0293, + "18": 0.0712, + "19": 0.0928, + "20": 0.0783, + "21": 0.1046, + "22": 0.1283 + }, + "best_single_layer_auc": 0.1283, + "mean_over_layers_auc": 0.0386, + "min_over_layers_auc": 0.2034 + }, + "jlens": { + "per_layer_auc": { + "0": 0.0138, + "1": 0.0169, + "2": 0.0149, + "3": 0.0104, + "4": 0.0394, + "5": 0.0235, + "6": 0.024, + "7": 0.0462, + "8": 0.0256, + "9": 0.0988, + "10": 0.0699, + "11": 0.0603, + "12": 0.1137, + "13": 0.1246, + "14": 0.1321, + "15": 0.1389, + "16": 0.1207, + "17": 0.1266, + "18": 0.1678, + "19": 0.1332, + "20": 0.1519, + "21": 0.1499, + "22": 0.1713 + }, + "best_single_layer_auc": 0.1713, + "mean_over_layers_auc": 0.0858, + "min_over_layers_auc": 0.3241 + }, + "shuffled": { + "per_layer_auc": { + "0": 0.0, + "1": 0.0115, + "2": 0.0016, + "3": 0.0063, + "4": 0.0529, + "5": 0.0858, + "6": 0.0104, + "7": 0.0235, + "8": 0.0894, + "9": 0.1285, + "10": 0.0303, + "11": 0.0258, + "12": 0.1485, + "13": 0.0709, + "14": 0.0579, + "15": 0.0571, + "16": 0.1155, + "17": 0.0512, + "18": 0.0183, + "19": 0.0454, + "20": 0.049, + "21": 0.1447, + "22": 0.0756 + }, + "best_single_layer_auc": 0.1485, + "mean_over_layers_auc": 0.0565, + "min_over_layers_auc": 0.3278 + }, + "n_layers_jlens_beats_shuffled": 18, + "n_layers": 23 + } + }, + "runtime_s": 68.7 +} \ No newline at end of file diff --git a/results/t15b_partition_aggregation_dependence.json b/results/t15b_partition_aggregation_dependence.json new file mode 100644 index 0000000000000000000000000000000000000000..c35da6159948d336e6720c506810c979edb3fb8e --- /dev/null +++ b/results/t15b_partition_aggregation_dependence.json @@ -0,0 +1,38 @@ +{ + "experiment": "T15b \u2014 is the generic/layer-specific partition aggregation-dependent?", + "status": "EXPLORATORY \u2014 reanalysis of t15, no new compute", + "finding": "YES. The clean bimodal partition reported in t13b exists ONLY under min-over-layers. Under best-single-layer and mean-over-layers, typo's layer-specific component is POSITIVE (+0.0228, +0.0293) rather than negative (-0.0037), and BOTH sets read as layer-specific. Since t15 independently showed min-over-layers launders the J-vs-shuffled difference, the partition computed under that same metric cannot be trusted.", + "consequence": "The t13b decomposition must be recomputed under a non-existential aggregation before any claim rests on it. The three 'generic-only' sets (poetry, typo, association) are NOT established as generic-only.", + "per_cell": { + "multihop|best single layer": { + "generic": -0.0131, + "specific": 0.0301, + "verdict": "LAYER-SPECIFIC" + }, + "multihop|mean over layers": { + "generic": -0.0216, + "specific": 0.0296, + "verdict": "LAYER-SPECIFIC" + }, + "multihop|MIN over layers": { + "generic": -0.0048, + "specific": 0.0656, + "verdict": "LAYER-SPECIFIC" + }, + "typo|best single layer": { + "generic": 0.0202, + "specific": 0.0228, + "verdict": "LAYER-SPECIFIC" + }, + "typo|mean over layers": { + "generic": 0.0179, + "specific": 0.0293, + "verdict": "LAYER-SPECIFIC" + }, + "typo|MIN over layers": { + "generic": 0.1244, + "specific": -0.0037, + "verdict": "generic only" + } + } +} \ No newline at end of file diff --git a/results/t16_robust_aggregation_1b.json b/results/t16_robust_aggregation_1b.json new file mode 100644 index 0000000000000000000000000000000000000000..ef47005d91d1a83e2c777f869ca1e9895f636a8d --- /dev/null +++ b/results/t16_robust_aggregation_1b.json @@ -0,0 +1,253 @@ +{ + "experiment": "T16 / E2 \u2014 robust aggregation control (mean vs median vs trimmed)", + "status": "EXPLORATORY \u2014 control; distribution held fixed, only the estimator varies", + "registered_prediction": "near-null: per-prompt Frobenius CV is 1.8-12%, so there are no magnitude outliers to trim", + "model": "EleutherAI/pythia-1b-deduped", + "corpus": "wikitext", + "n_prompts_used": 200, + "dim_batch": 128, + "max_seq_len": 128, + "target_layer_arg": -2, + "target_layer_effective": 14, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "trim_frac": 0.1, + "n_trimmed": 20, + "device": "cuda", + "dtype": "float32 fit, float16 retention", + "median_note": "torch.median on an even N returns the LOWER middle value, not the midpoint of the two central order statistics", + "summary": { + "min_cos_median_vs_mean": 0.997013, + "mean_cos_median_vs_mean": 0.998541, + "max_rel_dist_median_vs_mean": 0.077385, + "VERDICT": "NULL \u2014 median ~ mean; dispersion is directional heterogeneity, not estimator instability; the distribution programme survives" + }, + "per_layer": { + "0": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.077385, + "cos_median_vs_mean": 0.997013, + "rel_dist_trimmed_vs_mean": 0.054843, + "cos_trimmed_vs_mean": 0.998529, + "rel_dist_winsor_vs_mean": 0.006258, + "cos_winsor_vs_mean": 0.999996, + "fro_mean": 22.48, + "fro_median": 22.3043, + "fro_ratio_median_over_mean": 0.9922, + "per_prompt_fro_cv": 0.114, + "per_prompt_fro_max_over_rms": 1.344 + }, + "1": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.072969, + "cos_median_vs_mean": 0.997349, + "rel_dist_trimmed_vs_mean": 0.055547, + "cos_trimmed_vs_mean": 0.998504, + "rel_dist_winsor_vs_mean": 0.006483, + "cos_winsor_vs_mean": 0.999995, + "fro_mean": 20.9753, + "fro_median": 20.8074, + "fro_ratio_median_over_mean": 0.992, + "per_prompt_fro_cv": 0.1113, + "per_prompt_fro_max_over_rms": 1.355 + }, + "2": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.067399, + "cos_median_vs_mean": 0.997744, + "rel_dist_trimmed_vs_mean": 0.05506, + "cos_trimmed_vs_mean": 0.998558, + "rel_dist_winsor_vs_mean": 0.006881, + "cos_winsor_vs_mean": 0.999995, + "fro_mean": 23.8737, + "fro_median": 23.6777, + "fro_ratio_median_over_mean": 0.9918, + "per_prompt_fro_cv": 0.1085, + "per_prompt_fro_max_over_rms": 1.3476 + }, + "3": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.063664, + "cos_median_vs_mean": 0.997985, + "rel_dist_trimmed_vs_mean": 0.046395, + "cos_trimmed_vs_mean": 0.999002, + "rel_dist_winsor_vs_mean": 0.007086, + "cos_winsor_vs_mean": 0.999996, + "fro_mean": 25.1907, + "fro_median": 25.0101, + "fro_ratio_median_over_mean": 0.9928, + "per_prompt_fro_cv": 0.1014, + "per_prompt_fro_max_over_rms": 1.3291 + }, + "4": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.061383, + "cos_median_vs_mean": 0.998123, + "rel_dist_trimmed_vs_mean": 0.04382, + "cos_trimmed_vs_mean": 0.999108, + "rel_dist_winsor_vs_mean": 0.005786, + "cos_winsor_vs_mean": 0.999997, + "fro_mean": 26.8221, + "fro_median": 26.6594, + "fro_ratio_median_over_mean": 0.9939, + "per_prompt_fro_cv": 0.094, + "per_prompt_fro_max_over_rms": 1.2962 + }, + "5": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.056715, + "cos_median_vs_mean": 0.998394, + "rel_dist_trimmed_vs_mean": 0.039898, + "cos_trimmed_vs_mean": 0.999261, + "rel_dist_winsor_vs_mean": 0.004657, + "cos_winsor_vs_mean": 0.999998, + "fro_mean": 30.0501, + "fro_median": 29.9228, + "fro_ratio_median_over_mean": 0.9958, + "per_prompt_fro_cv": 0.0836, + "per_prompt_fro_max_over_rms": 1.2578 + }, + "6": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.053099, + "cos_median_vs_mean": 0.99859, + "rel_dist_trimmed_vs_mean": 0.036101, + "cos_trimmed_vs_mean": 0.999399, + "rel_dist_winsor_vs_mean": 0.004202, + "cos_winsor_vs_mean": 0.999999, + "fro_mean": 34.8626, + "fro_median": 34.7605, + "fro_ratio_median_over_mean": 0.9971, + "per_prompt_fro_cv": 0.0751, + "per_prompt_fro_max_over_rms": 1.2308 + }, + "7": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.050803, + "cos_median_vs_mean": 0.998709, + "rel_dist_trimmed_vs_mean": 0.030816, + "cos_trimmed_vs_mean": 0.999571, + "rel_dist_winsor_vs_mean": 0.003446, + "cos_winsor_vs_mean": 0.999999, + "fro_mean": 38.2243, + "fro_median": 38.1667, + "fro_ratio_median_over_mean": 0.9985, + "per_prompt_fro_cv": 0.0685, + "per_prompt_fro_max_over_rms": 1.2071 + }, + "8": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.049542, + "cos_median_vs_mean": 0.998772, + "rel_dist_trimmed_vs_mean": 0.029131, + "cos_trimmed_vs_mean": 0.999628, + "rel_dist_winsor_vs_mean": 0.003349, + "cos_winsor_vs_mean": 1.0, + "fro_mean": 44.3661, + "fro_median": 44.3326, + "fro_ratio_median_over_mean": 0.9992, + "per_prompt_fro_cv": 0.0645, + "per_prompt_fro_max_over_rms": 1.1956 + }, + "9": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.044578, + "cos_median_vs_mean": 0.999008, + "rel_dist_trimmed_vs_mean": 0.029435, + "cos_trimmed_vs_mean": 0.999594, + "rel_dist_winsor_vs_mean": 0.002597, + "cos_winsor_vs_mean": 1.0, + "fro_mean": 45.0726, + "fro_median": 45.1267, + "fro_ratio_median_over_mean": 1.0012, + "per_prompt_fro_cv": 0.0489, + "per_prompt_fro_max_over_rms": 1.1599 + }, + "10": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.042195, + "cos_median_vs_mean": 0.999112, + "rel_dist_trimmed_vs_mean": 0.027265, + "cos_trimmed_vs_mean": 0.999665, + "rel_dist_winsor_vs_mean": 0.002426, + "cos_winsor_vs_mean": 1.0, + "fro_mean": 51.1828, + "fro_median": 51.249, + "fro_ratio_median_over_mean": 1.0013, + "per_prompt_fro_cv": 0.0452, + "per_prompt_fro_max_over_rms": 1.1446 + }, + "11": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.035363, + "cos_median_vs_mean": 0.999376, + "rel_dist_trimmed_vs_mean": 0.018588, + "cos_trimmed_vs_mean": 0.999842, + "rel_dist_winsor_vs_mean": 0.001536, + "cos_winsor_vs_mean": 1.0, + "fro_mean": 51.3388, + "fro_median": 51.3933, + "fro_ratio_median_over_mean": 1.0011, + "per_prompt_fro_cv": 0.0328, + "per_prompt_fro_max_over_rms": 1.1061 + }, + "12": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.028692, + "cos_median_vs_mean": 0.99959, + "rel_dist_trimmed_vs_mean": 0.013631, + "cos_trimmed_vs_mean": 0.999916, + "rel_dist_winsor_vs_mean": 0.001156, + "cos_winsor_vs_mean": 1.0, + "fro_mean": 53.4741, + "fro_median": 53.5386, + "fro_ratio_median_over_mean": 1.0012, + "per_prompt_fro_cv": 0.0245, + "per_prompt_fro_max_over_rms": 1.0783 + }, + "13": { + "n_prompts": 200, + "rel_dist_median_vs_mean": 0.019285, + "cos_median_vs_mean": 0.999815, + "rel_dist_trimmed_vs_mean": 0.0082, + "cos_trimmed_vs_mean": 0.99997, + "rel_dist_winsor_vs_mean": 0.00066, + "cos_winsor_vs_mean": 1.0, + "fro_mean": 51.3959, + "fro_median": 51.4417, + "fro_ratio_median_over_mean": 1.0009, + "per_prompt_fro_cv": 0.0161, + "per_prompt_fro_max_over_rms": 1.0453 + } + }, + "saved_lenses": { + "median": { + "path": "pythia/results/t16_robust_aggregation_1b_median.pt", + "sha256": "d6b60b393d31338719af4f33370285dec800ad1ec8656014e9513b3e06e44117" + }, + "trimmed": { + "path": "pythia/results/t16_robust_aggregation_1b_trimmed.pt", + "sha256": "366538a394f1ee21392b755e9df0a2e3a54a59cc806b724f29135f141bbfc234" + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 1705.0 +} \ No newline at end of file diff --git a/results/t16b_reads_mean_1b.json b/results/t16b_reads_mean_1b.json new file mode 100644 index 0000000000000000000000000000000000000000..d42082793e829ed76da240819fb4efd740069d90 --- /dev/null +++ b/results/t16b_reads_mean_1b.json @@ -0,0 +1,360 @@ +{ + "experiment": "T7 lens comparison \u2014 anchor \u00a7A.6 pass@k, one metric, three transports", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prereg": null, + "prereg_sha256": null, + "primary_metric": "normalized_auc_over_logk_paper (\u00a7A.6 per-item binary)", + "n_win_paper": null, + "n_win_note": "n_win is only defined over all six sets; None means a partial run and is NOT evaluable against the pre-registered threshold", + "target_layer_effective": 14, + "model": "EleutherAI/pythia-1b-deduped", + "lens": "pythia/results/lens_1b_n200_db128_pen.pt", + "lens_n_prompts": 200, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers_used": 14, + "fold_norm_gain": false, + "eval_data_provenance": "anthropics/jacobian-lens @581d398, Apache-2.0, data/evaluations/*.json \u2014 vendored, not downloaded", + "metric": "pass@k = mean over ITEMS of the fraction of that item's intermediates whose min-over-layers rank <= k; AUC normalized over log k (anchor README)", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "results": { + "multihop": { + "logit_lens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.1068, + "2": 0.1553, + "5": 0.1748, + "10": 0.2039, + "25": 0.2816, + "100": 0.4563 + }, + "normalized_auc_over_logk_paper": 0.2404, + "pass_at_k": { + "1": 0.1183, + "2": 0.172, + "5": 0.1935, + "10": 0.2204, + "25": 0.3011, + "100": 0.4892 + }, + "normalized_auc_over_logk": 0.2602, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 144.0 + }, + "jlens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.0485, + "2": 0.1456, + "5": 0.2136, + "10": 0.2621, + "25": 0.3689, + "100": 0.5146 + }, + "normalized_auc_over_logk_paper": 0.2819, + "pass_at_k": { + "1": 0.0538, + "2": 0.1613, + "5": 0.2366, + "10": 0.2903, + "25": 0.4032, + "100": 0.5538 + }, + "normalized_auc_over_logk": 0.3085, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 97.0 + }, + "jlens_minus_logit_auc_paper": 0.0415, + "jlens_minus_logit_auc": 0.0483 + }, + "multilingual": { + "logit_lens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.0381, + "2": 0.0584, + "5": 0.099, + "10": 0.1269, + "25": 0.1853, + "100": 0.2919 + }, + "normalized_auc_over_logk_paper": 0.1428, + "pass_at_k": { + "1": 0.0382, + "2": 0.0584, + "5": 0.0997, + "10": 0.1262, + "25": 0.1846, + "100": 0.2889 + }, + "normalized_auc_over_logk": 0.1422, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 580.0 + }, + "jlens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.0355, + "2": 0.0457, + "5": 0.1015, + "10": 0.1269, + "25": 0.1878, + "100": 0.3122 + }, + "normalized_auc_over_logk_paper": 0.1445, + "pass_at_k": { + "1": 0.0343, + "2": 0.0444, + "5": 0.1005, + "10": 0.1246, + "25": 0.1861, + "100": 0.3076 + }, + "normalized_auc_over_logk": 0.1425, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 328.0 + }, + "jlens_minus_logit_auc_paper": 0.0017, + "jlens_minus_logit_auc": 0.0003 + }, + "typo": { + "logit_lens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.0312, + "2": 0.0312, + "5": 0.0833, + "10": 0.1458, + "25": 0.2396, + "100": 0.4375 + }, + "normalized_auc_over_logk_paper": 0.1736, + "pass_at_k": { + "1": 0.0312, + "2": 0.0312, + "5": 0.0833, + "10": 0.1458, + "25": 0.2396, + "100": 0.4375 + }, + "normalized_auc_over_logk": 0.1736, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 133.0 + }, + "jlens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.1562, + "2": 0.2188, + "5": 0.3333, + "10": 0.4062, + "25": 0.5417, + "100": 0.6875 + }, + "normalized_auc_over_logk_paper": 0.4181, + "pass_at_k": { + "1": 0.1562, + "2": 0.2188, + "5": 0.3333, + "10": 0.4062, + "25": 0.5417, + "100": 0.6875 + }, + "normalized_auc_over_logk": 0.4181, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 20.0 + }, + "jlens_minus_logit_auc_paper": 0.2445, + "jlens_minus_logit_auc": 0.2445 + } + }, + "runtime_s": 89.7 +} \ No newline at end of file diff --git a/results/t16b_reads_median_1b.json b/results/t16b_reads_median_1b.json new file mode 100644 index 0000000000000000000000000000000000000000..b8a8c7498f0949dbb6e711fed003509c79beaa76 --- /dev/null +++ b/results/t16b_reads_median_1b.json @@ -0,0 +1,360 @@ +{ + "experiment": "T7 lens comparison \u2014 anchor \u00a7A.6 pass@k, one metric, three transports", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prereg": null, + "prereg_sha256": null, + "primary_metric": "normalized_auc_over_logk_paper (\u00a7A.6 per-item binary)", + "n_win_paper": null, + "n_win_note": "n_win is only defined over all six sets; None means a partial run and is NOT evaluable against the pre-registered threshold", + "target_layer_effective": 14, + "model": "EleutherAI/pythia-1b-deduped", + "lens": "pythia/results/t16_robust_aggregation_1b_median.pt", + "lens_n_prompts": 200, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "n_layers_used": 14, + "fold_norm_gain": false, + "eval_data_provenance": "anthropics/jacobian-lens @581d398, Apache-2.0, data/evaluations/*.json \u2014 vendored, not downloaded", + "metric": "pass@k = mean over ITEMS of the fraction of that item's intermediates whose min-over-layers rank <= k; AUC normalized over log k (anchor README)", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "results": { + "multihop": { + "logit_lens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.1068, + "2": 0.1553, + "5": 0.1748, + "10": 0.2039, + "25": 0.2816, + "100": 0.4563 + }, + "normalized_auc_over_logk_paper": 0.2404, + "pass_at_k": { + "1": 0.1183, + "2": 0.172, + "5": 0.1935, + "10": 0.2204, + "25": 0.3011, + "100": 0.4892 + }, + "normalized_auc_over_logk": 0.2602, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 144.0 + }, + "jlens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.0583, + "2": 0.1553, + "5": 0.2039, + "10": 0.2718, + "25": 0.3689, + "100": 0.5243 + }, + "normalized_auc_over_logk_paper": 0.2858, + "pass_at_k": { + "1": 0.0645, + "2": 0.172, + "5": 0.2258, + "10": 0.3011, + "25": 0.4032, + "100": 0.5645 + }, + "normalized_auc_over_logk": 0.3128, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 92.0 + }, + "jlens_minus_logit_auc_paper": 0.0454, + "jlens_minus_logit_auc": 0.0526 + }, + "multilingual": { + "logit_lens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.0381, + "2": 0.0584, + "5": 0.099, + "10": 0.1269, + "25": 0.1853, + "100": 0.2919 + }, + "normalized_auc_over_logk_paper": 0.1428, + "pass_at_k": { + "1": 0.0382, + "2": 0.0584, + "5": 0.0997, + "10": 0.1262, + "25": 0.1846, + "100": 0.2889 + }, + "normalized_auc_over_logk": 0.1422, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 580.0 + }, + "jlens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.033, + "2": 0.0457, + "5": 0.1041, + "10": 0.1294, + "25": 0.1954, + "100": 0.3071 + }, + "normalized_auc_over_logk_paper": 0.1464, + "pass_at_k": { + "1": 0.0319, + "2": 0.0444, + "5": 0.1028, + "10": 0.1269, + "25": 0.1931, + "100": 0.3014 + }, + "normalized_auc_over_logk": 0.144, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 326.0 + }, + "jlens_minus_logit_auc_paper": 0.0036, + "jlens_minus_logit_auc": 0.0018 + }, + "typo": { + "logit_lens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.0312, + "2": 0.0312, + "5": 0.0833, + "10": 0.1458, + "25": 0.2396, + "100": 0.4375 + }, + "normalized_auc_over_logk_paper": 0.1736, + "pass_at_k": { + "1": 0.0312, + "2": 0.0312, + "5": 0.0833, + "10": 0.1458, + "25": 0.2396, + "100": 0.4375 + }, + "normalized_auc_over_logk": 0.1736, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 133.0 + }, + "jlens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k_paper": { + "1": 0.1562, + "2": 0.2188, + "5": 0.3333, + "10": 0.4271, + "25": 0.5521, + "100": 0.6875 + }, + "normalized_auc_over_logk_paper": 0.4244, + "pass_at_k": { + "1": 0.1562, + "2": 0.2188, + "5": 0.3333, + "10": 0.4271, + "25": 0.5521, + "100": 0.6875 + }, + "normalized_auc_over_logk": 0.4244, + "metric_note": "paper = per-(item,intermediate) binary on min-over-layers rank; readme = mean over items of the fraction of that item's intermediates recovered. Identical when items have 1 intermediate.", + "median_best_rank": 20.0 + }, + "jlens_minus_logit_auc_paper": 0.2508, + "jlens_minus_logit_auc": 0.2508 + } + }, + "runtime_s": 77.4 +} \ No newline at end of file diff --git a/results/t17_reaggregate_160m.json b/results/t17_reaggregate_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..a30fcaa7da189bf30fd2fe7007f96af90aeec8f1 --- /dev/null +++ b/results/t17_reaggregate_160m.json @@ -0,0 +1,473 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-160m-deduped", + "lens": "pythia/results/lens_160m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "band": [ + 5, + 6, + 7, + 8, + 9, + 10 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 2, + "best1L": 2, + "mean": 3, + "persist": 0 + }, + "n_sets_stable": 4, + "n_sets_aggregation_dependent": 2, + "derangement_layer_map": { + "0": 9, + "1": 7, + "2": 8, + "3": 10, + "4": 6, + "5": 1, + "6": 5, + "7": 4, + "8": 3, + "9": 2, + "10": 0 + }, + "results": { + "multihop": { + "logit": { + "min": 0.1244, + "best1L": 0.1054, + "best1L_layer": 10, + "mean": 0.0229, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0306 + }, + "jlens": { + "min": 0.1093, + "best1L": 0.0789, + "best1L_layer": 9, + "mean": 0.0215, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0112 + }, + "shuffled": { + "min": 0.0676, + "best1L": 0.0447, + "best1L_layer": 7, + "mean": 0.0085, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0058 + }, + "per_layer_jlens": { + "0": 0.0073, + "1": 0.0073, + "2": 0.0044, + "3": 0.0131, + "4": 0.0287, + "5": 0.0265, + "6": 0.0, + "7": 0.0, + "8": 0.0015, + "9": 0.0789, + "10": 0.069 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0073, + "6": 0.0112, + "7": 0.0447, + "8": 0.0088, + "9": 0.016, + "10": 0.0053 + }, + "layer_specific": { + "min": 0.0417, + "best1L": 0.0342, + "mean": 0.013, + "persist": 0.0053 + }, + "generic": { + "min": -0.0568, + "best1L": -0.0607, + "mean": -0.0144, + "persist": -0.0248 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.0391, + "best1L": 0.0287, + "best1L_layer": 9, + "mean": 0.0072, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0083 + }, + "jlens": { + "min": 0.03, + "best1L": 0.0158, + "best1L_layer": 5, + "mean": 0.0058, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0011 + }, + "shuffled": { + "min": 0.035, + "best1L": 0.0216, + "best1L_layer": 7, + "mean": 0.0061, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0092 + }, + "per_layer_jlens": { + "0": 0.003, + "1": 0.0046, + "2": 0.0084, + "3": 0.0085, + "4": 0.0149, + "5": 0.0158, + "6": 0.0018, + "7": 0.0, + "8": 0.0008, + "9": 0.0015, + "10": 0.0046 + }, + "per_layer_shuffled": { + "0": 0.0004, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0011, + "5": 0.0104, + "6": 0.0089, + "7": 0.0216, + "8": 0.0107, + "9": 0.0052, + "10": 0.0093 + }, + "layer_specific": { + "min": -0.005, + "best1L": -0.0058, + "mean": -0.0003, + "persist": -0.0081 + }, + "generic": { + "min": -0.0041, + "best1L": -0.0071, + "mean": -0.001, + "persist": 0.0009 + }, + "verdict": "STABLE: not layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.4493, + "best1L": 0.4098, + "best1L_layer": 10, + "mean": 0.1131, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.262 + }, + "jlens": { + "min": 0.4402, + "best1L": 0.2385, + "best1L_layer": 2, + "mean": 0.1205, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0421 + }, + "shuffled": { + "min": 0.2169, + "best1L": 0.1039, + "best1L_layer": 7, + "mean": 0.0414, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0456 + }, + "per_layer_jlens": { + "0": 0.1999, + "1": 0.237, + "2": 0.2385, + "3": 0.1197, + "4": 0.104, + "5": 0.0606, + "6": 0.028, + "7": 0.0214, + "8": 0.0182, + "9": 0.149, + "10": 0.1487 + }, + "per_layer_shuffled": { + "0": 0.0091, + "1": 0.0405, + "2": 0.018, + "3": 0.0314, + "4": 0.0109, + "5": 0.0715, + "6": 0.0579, + "7": 0.1039, + "8": 0.0742, + "9": 0.0169, + "10": 0.0214 + }, + "layer_specific": { + "min": 0.2233, + "best1L": 0.1347, + "mean": 0.079, + "persist": -0.0035 + }, + "generic": { + "min": -0.2324, + "best1L": -0.3059, + "mean": -0.0716, + "persist": -0.2164 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "poetry": { + "logit": { + "min": 0.0383, + "best1L": 0.0207, + "best1L_layer": 5, + "mean": 0.0078, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0102 + }, + "jlens": { + "min": 0.0031, + "best1L": 0.0015, + "best1L_layer": 3, + "mean": 0.0003, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0378, + "best1L": 0.021, + "best1L_layer": 7, + "mean": 0.0047, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0031 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0015, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0015, + "10": 0.0 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0015, + "6": 0.0, + "7": 0.021, + "8": 0.0118, + "9": 0.0056, + "10": 0.0118 + }, + "layer_specific": { + "min": -0.0348, + "best1L": -0.0194, + "mean": -0.0044, + "persist": -0.0031 + }, + "generic": { + "min": -0.0005, + "best1L": 0.0003, + "mean": -0.0031, + "persist": -0.0072 + }, + "verdict": "STABLE: not layer-specific" + }, + "typo": { + "logit": { + "min": 0.2423, + "best1L": 0.165, + "best1L_layer": 10, + "mean": 0.0606, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0905 + }, + "jlens": { + "min": 0.2808, + "best1L": 0.1797, + "best1L_layer": 5, + "mean": 0.096, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0654 + }, + "shuffled": { + "min": 0.2569, + "best1L": 0.1691, + "best1L_layer": 7, + "mean": 0.0779, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.1214 + }, + "per_layer_jlens": { + "0": 0.0448, + "1": 0.0576, + "2": 0.1154, + "3": 0.1459, + "4": 0.1735, + "5": 0.1797, + "6": 0.1381, + "7": 0.025, + "8": 0.0279, + "9": 0.074, + "10": 0.0735 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0089, + "2": 0.0112, + "3": 0.0136, + "4": 0.1217, + "5": 0.0779, + "6": 0.1553, + "7": 0.1691, + "8": 0.1587, + "9": 0.0909, + "10": 0.0498 + }, + "layer_specific": { + "min": 0.0239, + "best1L": 0.0106, + "mean": 0.018, + "persist": -0.056 + }, + "generic": { + "min": 0.0145, + "best1L": 0.0041, + "mean": 0.0173, + "persist": 0.031 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "association": { + "logit": { + "min": 0.0033, + "best1L": 0.0033, + "best1L_layer": 10, + "mean": 0.0006, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0 + }, + "jlens": { + "min": 0.0109, + "best1L": 0.006, + "best1L_layer": 1, + "mean": 0.0013, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0112, + "best1L": 0.0079, + "best1L_layer": 7, + "mean": 0.0022, + "persist_K": 3, + "persist_band_size": 6, + "persist": 0.0033 + }, + "per_layer_jlens": { + "0": 0.0016, + "1": 0.006, + "2": 0.0016, + "3": 0.0, + "4": 0.0033, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0016 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0016, + "4": 0.0, + "5": 0.0016, + "6": 0.0, + "7": 0.0079, + "8": 0.006, + "9": 0.0033, + "10": 0.0033 + }, + "layer_specific": { + "min": -0.0003, + "best1L": -0.0019, + "mean": -0.0009, + "persist": -0.0033 + }, + "generic": { + "min": 0.0079, + "best1L": 0.0046, + "mean": 0.0016, + "persist": 0.0033 + }, + "verdict": "STABLE: not layer-specific" + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 58.7 +} \ No newline at end of file diff --git a/results/t17_reaggregate_410m.json b/results/t17_reaggregate_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..47d0dde8684a80f46ff0bcf7216bb319cb510837 --- /dev/null +++ b/results/t17_reaggregate_410m.json @@ -0,0 +1,649 @@ +{ + "experiment": "T17 / E18 \u2014 re-score reads under non-existential aggregations", + "status": "EXPLORATORY \u2014 re-analysis of burned cells; no refit", + "decision_rule": "sign of (jlens - shuffled) stable across all four aggregations => the set's classification stands; otherwise AGGREGATION-DEPENDENT and excluded from any claim", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "persistence_kfrac": 0.5, + "seed": 0, + "aggregations": [ + "min", + "best1L", + "mean", + "persist" + ], + "n_win_vs_logit_by_aggregation": { + "min": 6, + "best1L": 4, + "mean": 6, + "persist": 2 + }, + "n_sets_stable": 3, + "n_sets_aggregation_dependent": 3, + "derangement_layer_map": { + "0": 17, + "1": 8, + "2": 21, + "3": 18, + "4": 7, + "5": 12, + "6": 19, + "7": 1, + "8": 16, + "9": 14, + "10": 2, + "11": 6, + "12": 15, + "13": 11, + "14": 4, + "15": 22, + "16": 13, + "17": 10, + "18": 0, + "19": 3, + "20": 9, + "21": 20, + "22": 5 + }, + "results": { + "multihop": { + "logit": { + "min": 0.1994, + "best1L": 0.1504, + "best1L_layer": 19, + "mean": 0.0377, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0476 + }, + "jlens": { + "min": 0.2602, + "best1L": 0.1674, + "best1L_layer": 21, + "mean": 0.0457, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.043 + }, + "shuffled": { + "min": 0.1946, + "best1L": 0.1373, + "best1L_layer": 21, + "mean": 0.0161, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0015 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0029, + "5": 0.0, + "6": 0.0, + "7": 0.0015, + "8": 0.0, + "9": 0.0, + "10": 0.0029, + "11": 0.0029, + "12": 0.036, + "13": 0.0348, + "14": 0.0224, + "15": 0.0284, + "16": 0.0595, + "17": 0.1227, + "18": 0.1232, + "19": 0.1397, + "20": 0.1519, + "21": 0.1674, + "22": 0.1555 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0097, + "6": 0.0015, + "7": 0.0015, + "8": 0.0029, + "9": 0.0068, + "10": 0.0015, + "11": 0.0015, + "12": 0.0092, + "13": 0.0015, + "14": 0.0015, + "15": 0.0629, + "16": 0.0431, + "17": 0.0516, + "18": 0.0, + "19": 0.0015, + "20": 0.0151, + "21": 0.1373, + "22": 0.0209 + }, + "layer_specific": { + "min": 0.0655, + "best1L": 0.0301, + "mean": 0.0296, + "persist": 0.0416 + }, + "generic": { + "min": -0.0048, + "best1L": -0.0131, + "mean": -0.0216, + "persist": -0.0462 + }, + "verdict": "STABLE: layer-specific" + }, + "multilingual": { + "logit": { + "min": 0.1153, + "best1L": 0.0846, + "best1L_layer": 17, + "mean": 0.0174, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0186 + }, + "jlens": { + "min": 0.2169, + "best1L": 0.1822, + "best1L_layer": 17, + "mean": 0.0302, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0332 + }, + "shuffled": { + "min": 0.1195, + "best1L": 0.0524, + "best1L_layer": 21, + "mean": 0.0103, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0015 + }, + "per_layer_jlens": { + "0": 0.0039, + "1": 0.0029, + "2": 0.0029, + "3": 0.0038, + "4": 0.0034, + "5": 0.0011, + "6": 0.0019, + "7": 0.0015, + "8": 0.0004, + "9": 0.0008, + "10": 0.0004, + "11": 0.0, + "12": 0.0004, + "13": 0.0056, + "14": 0.0161, + "15": 0.0378, + "16": 0.0998, + "17": 0.1822, + "18": 0.0737, + "19": 0.0681, + "20": 0.0772, + "21": 0.0631, + "22": 0.0476 + }, + "per_layer_shuffled": { + "0": 0.0065, + "1": 0.0024, + "2": 0.0015, + "3": 0.0015, + "4": 0.0015, + "5": 0.003, + "6": 0.0109, + "7": 0.0027, + "8": 0.0094, + "9": 0.0004, + "10": 0.0008, + "11": 0.0, + "12": 0.0052, + "13": 0.0011, + "14": 0.0057, + "15": 0.0302, + "16": 0.0193, + "17": 0.0373, + "18": 0.0014, + "19": 0.0073, + "20": 0.0106, + "21": 0.0524, + "22": 0.0255 + }, + "layer_specific": { + "min": 0.0974, + "best1L": 0.1298, + "mean": 0.0199, + "persist": 0.0317 + }, + "generic": { + "min": 0.0042, + "best1L": -0.0322, + "mean": -0.0071, + "persist": -0.017 + }, + "verdict": "STABLE: layer-specific" + }, + "order-ops": { + "logit": { + "min": 0.5234, + "best1L": 0.3911, + "best1L_layer": 19, + "mean": 0.1555, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.2899 + }, + "jlens": { + "min": 0.5679, + "best1L": 0.3138, + "best1L_layer": 21, + "mean": 0.1911, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.2368 + }, + "shuffled": { + "min": 0.5241, + "best1L": 0.3106, + "best1L_layer": 15, + "mean": 0.1041, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0891 + }, + "per_layer_jlens": { + "0": 0.0082, + "1": 0.057, + "2": 0.0652, + "3": 0.175, + "4": 0.182, + "5": 0.178, + "6": 0.1873, + "7": 0.1894, + "8": 0.1459, + "9": 0.0963, + "10": 0.1007, + "11": 0.0879, + "12": 0.2131, + "13": 0.2956, + "14": 0.269, + "15": 0.2151, + "16": 0.2072, + "17": 0.2631, + "18": 0.2545, + "19": 0.3111, + "20": 0.295, + "21": 0.3138, + "22": 0.2856 + }, + "per_layer_shuffled": { + "0": 0.0198, + "1": 0.0014, + "2": 0.0453, + "3": 0.0273, + "4": 0.0688, + "5": 0.0419, + "6": 0.0187, + "7": 0.1311, + "8": 0.0014, + "9": 0.0041, + "10": 0.0672, + "11": 0.1277, + "12": 0.0906, + "13": 0.2615, + "14": 0.1427, + "15": 0.3106, + "16": 0.2961, + "17": 0.2241, + "18": 0.0205, + "19": 0.0722, + "20": 0.1002, + "21": 0.2963, + "22": 0.0239 + }, + "layer_specific": { + "min": 0.0438, + "best1L": 0.0032, + "mean": 0.0871, + "persist": 0.1477 + }, + "generic": { + "min": 0.0007, + "best1L": -0.0806, + "mean": -0.0515, + "persist": -0.2008 + }, + "verdict": "STABLE: layer-specific" + }, + "poetry": { + "logit": { + "min": 0.0287, + "best1L": 0.0077, + "best1L_layer": 18, + "mean": 0.0022, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0435, + "best1L": 0.0256, + "best1L_layer": 22, + "mean": 0.0045, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0015 + }, + "shuffled": { + "min": 0.0718, + "best1L": 0.0148, + "best1L_layer": 4, + "mean": 0.004, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0031, + "1": 0.0015, + "2": 0.0, + "3": 0.0077, + "4": 0.0087, + "5": 0.0061, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0015, + "10": 0.0031, + "11": 0.0, + "12": 0.0015, + "13": 0.0, + "14": 0.0015, + "15": 0.0015, + "16": 0.0061, + "17": 0.0061, + "18": 0.0061, + "19": 0.0061, + "20": 0.0061, + "21": 0.0108, + "22": 0.0256 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0015, + "2": 0.0056, + "3": 0.0, + "4": 0.0148, + "5": 0.012, + "6": 0.0072, + "7": 0.0041, + "8": 0.0046, + "9": 0.0072, + "10": 0.0031, + "11": 0.0015, + "12": 0.0015, + "13": 0.0, + "14": 0.0072, + "15": 0.0031, + "16": 0.0, + "17": 0.0015, + "18": 0.0, + "19": 0.0, + "20": 0.0031, + "21": 0.0046, + "22": 0.0092 + }, + "layer_specific": { + "min": -0.0283, + "best1L": 0.0108, + "mean": 0.0005, + "persist": 0.0015 + }, + "generic": { + "min": 0.0432, + "best1L": 0.0071, + "mean": 0.0018, + "persist": -0.0015 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "typo": { + "logit": { + "min": 0.2034, + "best1L": 0.1283, + "best1L_layer": 22, + "mean": 0.0386, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0475 + }, + "jlens": { + "min": 0.3241, + "best1L": 0.1713, + "best1L_layer": 22, + "mean": 0.0858, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.1194 + }, + "shuffled": { + "min": 0.3278, + "best1L": 0.1485, + "best1L_layer": 12, + "mean": 0.0565, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0407 + }, + "per_layer_jlens": { + "0": 0.0138, + "1": 0.0169, + "2": 0.0149, + "3": 0.0104, + "4": 0.0394, + "5": 0.0235, + "6": 0.024, + "7": 0.0462, + "8": 0.0256, + "9": 0.0988, + "10": 0.0699, + "11": 0.0603, + "12": 0.1137, + "13": 0.1246, + "14": 0.1321, + "15": 0.1389, + "16": 0.1207, + "17": 0.1266, + "18": 0.1678, + "19": 0.1332, + "20": 0.1519, + "21": 0.1499, + "22": 0.1713 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0115, + "2": 0.0016, + "3": 0.0063, + "4": 0.0529, + "5": 0.0858, + "6": 0.0104, + "7": 0.0235, + "8": 0.0894, + "9": 0.1285, + "10": 0.0303, + "11": 0.0258, + "12": 0.1485, + "13": 0.0709, + "14": 0.0579, + "15": 0.0571, + "16": 0.1155, + "17": 0.0512, + "18": 0.0183, + "19": 0.0454, + "20": 0.049, + "21": 0.1447, + "22": 0.0756 + }, + "layer_specific": { + "min": -0.0037, + "best1L": 0.0227, + "mean": 0.0293, + "persist": 0.0787 + }, + "generic": { + "min": 0.1244, + "best1L": 0.0202, + "mean": 0.0179, + "persist": -0.0068 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "best1L_layer": 13, + "mean": 0.0005, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0 + }, + "jlens": { + "min": 0.0115, + "best1L": 0.0033, + "best1L_layer": 5, + "mean": 0.0006, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0 + }, + "shuffled": { + "min": 0.0313, + "best1L": 0.0076, + "best1L_layer": 18, + "mean": 0.0017, + "persist_K": 7, + "persist_band_size": 14, + "persist": 0.0 + }, + "per_layer_jlens": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0033, + "6": 0.0016, + "7": 0.0016, + "8": 0.0, + "9": 0.0, + "10": 0.0, + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0016, + "15": 0.0, + "16": 0.0016, + "17": 0.0033, + "18": 0.0, + "19": 0.0, + "20": 0.0, + "21": 0.0, + "22": 0.0 + }, + "per_layer_shuffled": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0016, + "8": 0.0, + "9": 0.0, + "10": 0.0, + "11": 0.0, + "12": 0.0033, + "13": 0.0, + "14": 0.0016, + "15": 0.0065, + "16": 0.0016, + "17": 0.0016, + "18": 0.0076, + "19": 0.0016, + "20": 0.0063, + "21": 0.0, + "22": 0.0076 + }, + "layer_specific": { + "min": -0.0199, + "best1L": -0.0044, + "mean": -0.0011, + "persist": 0.0 + }, + "generic": { + "min": 0.0237, + "best1L": 0.0033, + "mean": 0.0012, + "persist": 0.0 + }, + "verdict": "AGGREGATION-DEPENDENT \u2014 excluded from any claim" + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 154.3 +} \ No newline at end of file diff --git a/results/t18_n_sweep_410m.json b/results/t18_n_sweep_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..89b43967166b8f7d4591a16033cecf6a7ace6fe0 --- /dev/null +++ b/results/t18_n_sweep_410m.json @@ -0,0 +1,562 @@ +{ + "experiment": "T18 / E19 \u2014 does eps = sqrt(disp/n) hold for the averaged Jacobian?", + "status": "EXPLORATORY", + "prereg": null, + "decision_rule": "ACCEPT iff the log-log slope lies in [-0.6,-0.4] at EVERY layer", + "declared_bias": "J_ref at n=400 is the reference, so n=400 is degenerate and excluded from the fit; reference-free split-half / odd-even are reported alongside", + "model": "EleutherAI/pythia-410m-deduped", + "ns": [ + 10, + 25, + 50, + 100, + 200 + ], + "n_ref": 400, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "dim_batch": 128, + "target_layer_effective": 22, + "slope_min": -0.5649, + "slope_max": -0.4767, + "slope_mean": -0.5112, + "VERDICT": "ACCEPT \u2014 the sqrt(disp/n) law describes this estimator", + "n_layers_in_band": 22, + "per_layer": { + "0": { + "measured": { + "10": 0.780454, + "25": 0.56072, + "50": 0.361355, + "100": 0.237356, + "200": 0.147103 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.485594, + "25": 0.307117, + "50": 0.217164, + "100": 0.153558, + "200": 0.108582 + }, + "loglog_slope": -0.5649, + "in_decision_band": true, + "dispersion": 2.35802 + }, + "1": { + "measured": { + "10": 0.727365, + "25": 0.529647, + "50": 0.340408, + "100": 0.226371, + "200": 0.141751 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.455327, + "25": 0.287974, + "50": 0.203628, + "100": 0.143987, + "200": 0.101814 + }, + "loglog_slope": -0.5545, + "in_decision_band": true, + "dispersion": 2.073227 + }, + "2": { + "measured": { + "10": 0.680469, + "25": 0.493172, + "50": 0.316533, + "100": 0.213332, + "200": 0.134781 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.404229, + "25": 0.255657, + "50": 0.180776, + "100": 0.127828, + "200": 0.090388 + }, + "loglog_slope": -0.5488, + "in_decision_band": true, + "dispersion": 1.634007 + }, + "3": { + "measured": { + "10": 0.626601, + "25": 0.462076, + "50": 0.296606, + "100": 0.202898, + "200": 0.127869 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.374653, + "25": 0.236951, + "50": 0.16755, + "100": 0.118476, + "200": 0.083775 + }, + "loglog_slope": -0.5385, + "in_decision_band": true, + "dispersion": 1.403649 + }, + "4": { + "measured": { + "10": 0.565785, + "25": 0.420166, + "50": 0.275656, + "100": 0.188572, + "200": 0.118798 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.338004, + "25": 0.213772, + "50": 0.15116, + "100": 0.106886, + "200": 0.07558 + }, + "loglog_slope": -0.5277, + "in_decision_band": true, + "dispersion": 1.142466 + }, + "5": { + "measured": { + "10": 0.530948, + "25": 0.392351, + "50": 0.261229, + "100": 0.179631, + "200": 0.112693 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.320959, + "25": 0.202992, + "50": 0.143537, + "100": 0.101496, + "200": 0.071769 + }, + "loglog_slope": -0.5222, + "in_decision_band": true, + "dispersion": 1.030144 + }, + "6": { + "measured": { + "10": 0.501873, + "25": 0.374424, + "50": 0.24889, + "100": 0.172008, + "200": 0.107895 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.301149, + "25": 0.190463, + "50": 0.134678, + "100": 0.095232, + "200": 0.067339 + }, + "loglog_slope": -0.5182, + "in_decision_band": true, + "dispersion": 0.906906 + }, + "7": { + "measured": { + "10": 0.476191, + "25": 0.355859, + "50": 0.236778, + "100": 0.164728, + "200": 0.103163 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.281699, + "25": 0.178162, + "50": 0.125979, + "100": 0.089081, + "200": 0.06299 + }, + "loglog_slope": -0.5151, + "in_decision_band": true, + "dispersion": 0.793541 + }, + "8": { + "measured": { + "10": 0.465626, + "25": 0.348189, + "50": 0.23272, + "100": 0.160601, + "200": 0.100144 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.267188, + "25": 0.168984, + "50": 0.11949, + "100": 0.084492, + "200": 0.059745 + }, + "loglog_slope": -0.5174, + "in_decision_band": true, + "dispersion": 0.713894 + }, + "9": { + "measured": { + "10": 0.447141, + "25": 0.336681, + "50": 0.225115, + "100": 0.156963, + "200": 0.097115 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.256476, + "25": 0.16221, + "50": 0.1147, + "100": 0.081105, + "200": 0.05735 + }, + "loglog_slope": -0.5133, + "in_decision_band": true, + "dispersion": 0.657799 + }, + "10": { + "measured": { + "10": 0.431974, + "25": 0.328053, + "50": 0.218946, + "100": 0.153299, + "200": 0.094639 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.245146, + "25": 0.155044, + "50": 0.109633, + "100": 0.077522, + "200": 0.054816 + }, + "loglog_slope": -0.5105, + "in_decision_band": true, + "dispersion": 0.600968 + }, + "11": { + "measured": { + "10": 0.420771, + "25": 0.320197, + "50": 0.214118, + "100": 0.15059, + "200": 0.092169 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.234765, + "25": 0.148478, + "50": 0.10499, + "100": 0.074239, + "200": 0.052495 + }, + "loglog_slope": -0.5096, + "in_decision_band": true, + "dispersion": 0.551146 + }, + "12": { + "measured": { + "10": 0.375584, + "25": 0.292974, + "50": 0.197841, + "100": 0.140366, + "200": 0.086073 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.213726, + "25": 0.135172, + "50": 0.095581, + "100": 0.067586, + "200": 0.047791 + }, + "loglog_slope": -0.4946, + "in_decision_band": true, + "dispersion": 0.456789 + }, + "13": { + "measured": { + "10": 0.358388, + "25": 0.282087, + "50": 0.188687, + "100": 0.134662, + "200": 0.082907 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.198689, + "25": 0.125662, + "50": 0.088857, + "100": 0.062831, + "200": 0.044428 + }, + "loglog_slope": -0.4924, + "in_decision_band": true, + "dispersion": 0.394775 + }, + "14": { + "measured": { + "10": 0.347609, + "25": 0.275396, + "50": 0.181124, + "100": 0.128547, + "200": 0.080076 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.18392, + "25": 0.116321, + "50": 0.082252, + "100": 0.058161, + "200": 0.041126 + }, + "loglog_slope": -0.4966, + "in_decision_band": true, + "dispersion": 0.338266 + }, + "15": { + "measured": { + "10": 0.328565, + "25": 0.262637, + "50": 0.171903, + "100": 0.122094, + "200": 0.076331 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.170355, + "25": 0.107742, + "50": 0.076185, + "100": 0.053871, + "200": 0.038093 + }, + "loglog_slope": -0.4947, + "in_decision_band": true, + "dispersion": 0.290208 + }, + "16": { + "measured": { + "10": 0.286049, + "25": 0.23253, + "50": 0.151394, + "100": 0.109058, + "200": 0.068285 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.147129, + "25": 0.093053, + "50": 0.065798, + "100": 0.046526, + "200": 0.032899 + }, + "loglog_slope": -0.486, + "in_decision_band": true, + "dispersion": 0.21647 + }, + "17": { + "measured": { + "10": 0.258309, + "25": 0.210537, + "50": 0.133937, + "100": 0.096489, + "200": 0.060207 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.127152, + "25": 0.080418, + "50": 0.056864, + "100": 0.040209, + "200": 0.028432 + }, + "loglog_slope": -0.4955, + "in_decision_band": true, + "dispersion": 0.161676 + }, + "18": { + "measured": { + "10": 0.213925, + "25": 0.175987, + "50": 0.116356, + "100": 0.08324, + "200": 0.052609 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.10937, + "25": 0.069171, + "50": 0.048912, + "100": 0.034586, + "200": 0.024456 + }, + "loglog_slope": -0.4767, + "in_decision_band": true, + "dispersion": 0.119617 + }, + "19": { + "measured": { + "10": 0.179453, + "25": 0.147766, + "50": 0.097412, + "100": 0.069373, + "200": 0.04416 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.089916, + "25": 0.056868, + "50": 0.040212, + "100": 0.028434, + "200": 0.020106 + }, + "loglog_slope": -0.4775, + "in_decision_band": true, + "dispersion": 0.080849 + }, + "20": { + "measured": { + "10": 0.137639, + "25": 0.114324, + "50": 0.075673, + "100": 0.051811, + "200": 0.032129 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.068621, + "25": 0.0434, + "50": 0.030688, + "100": 0.0217, + "200": 0.015344 + }, + "loglog_slope": -0.4959, + "in_decision_band": true, + "dispersion": 0.047088 + }, + "21": { + "measured": { + "10": 0.094681, + "25": 0.078684, + "50": 0.051978, + "100": 0.036358, + "200": 0.021906 + }, + "predicted_sqrt_disp_over_n": { + "10": 0.047368, + "25": 0.029958, + "50": 0.021183, + "100": 0.014979, + "200": 0.010592 + }, + "loglog_slope": -0.4956, + "in_decision_band": true, + "dispersion": 0.022437 + } + }, + "reference_free": { + "0": { + "split_half_disagreement": 0.370292, + "odd_even_disagreement": 0.188242 + }, + "1": { + "split_half_disagreement": 0.353766, + "odd_even_disagreement": 0.174374 + }, + "2": { + "split_half_disagreement": 0.335846, + "odd_even_disagreement": 0.150169 + }, + "3": { + "split_half_disagreement": 0.319567, + "odd_even_disagreement": 0.136315 + }, + "4": { + "split_half_disagreement": 0.299298, + "odd_even_disagreement": 0.12073 + }, + "5": { + "split_half_disagreement": 0.286666, + "odd_even_disagreement": 0.11335 + }, + "6": { + "split_half_disagreement": 0.275121, + "odd_even_disagreement": 0.10423 + }, + "7": { + "split_half_disagreement": 0.262626, + "odd_even_disagreement": 0.096865 + }, + "8": { + "split_half_disagreement": 0.256636, + "odd_even_disagreement": 0.089396 + }, + "9": { + "split_half_disagreement": 0.250823, + "odd_even_disagreement": 0.084824 + }, + "10": { + "split_half_disagreement": 0.245001, + "odd_even_disagreement": 0.079553 + }, + "11": { + "split_half_disagreement": 0.240729, + "odd_even_disagreement": 0.074588 + }, + "12": { + "split_half_disagreement": 0.220828, + "odd_even_disagreement": 0.066754 + }, + "13": { + "split_half_disagreement": 0.211142, + "odd_even_disagreement": 0.060688 + }, + "14": { + "split_half_disagreement": 0.198759, + "odd_even_disagreement": 0.054532 + }, + "15": { + "split_half_disagreement": 0.187477, + "odd_even_disagreement": 0.049411 + }, + "16": { + "split_half_disagreement": 0.166556, + "odd_even_disagreement": 0.040934 + }, + "17": { + "split_half_disagreement": 0.14638, + "odd_even_disagreement": 0.034591 + }, + "18": { + "split_half_disagreement": 0.126297, + "odd_even_disagreement": 0.029467 + }, + "19": { + "split_half_disagreement": 0.104763, + "odd_even_disagreement": 0.023788 + }, + "20": { + "split_half_disagreement": 0.082088, + "odd_even_disagreement": 0.0178 + }, + "21": { + "split_half_disagreement": 0.057554, + "odd_even_disagreement": 0.012196 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "runtime_s": 2355.7 +} \ No newline at end of file diff --git a/results/t19_skipfirst_410m.json b/results/t19_skipfirst_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..d2bd146dd08e9e12b5d7dc939a6fb0b6631d744e --- /dev/null +++ b/results/t19_skipfirst_410m.json @@ -0,0 +1,426 @@ +{ + "experiment": "T19 / E17 \u2014 skip_first sweep 0 vs 4 vs 16", + "status": "EXPLORATORY \u2014 410m is fully observed, nothing burned", + "decision_rule": "n_win (best-single-layer) spread <=1 => immaterial; >=2 => live parameter", + "primary_outcome": "n_win vs logit under BEST-SINGLE-LAYER AUC", + "model": "EleutherAI/pythia-410m-deduped", + "skips": [ + 0, + 4, + 16 + ], + "n_prompts": 200, + "n_win_best1L_by_skip": { + "0": 5, + "4": 4, + "16": 4 + }, + "spread": 1, + "VERDICT": "IMMATERIAL \u2014 n_win spread <=1 across skip_first; keep 16 for continuity", + "results": { + "0": { + "n_prompts_used": 200, + "n_source_layers": 22, + "disp_L0": 1.899726, + "disp_last": 0.019707, + "per_set": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "mean": 0.033, + "persist": 0.0476 + }, + "jlens": { + "min": 0.2861, + "best1L": 0.1586, + "mean": 0.044, + "persist": 0.0379 + }, + "shuffled": { + "min": 0.2002, + "best1L": 0.1101, + "mean": 0.0158, + "persist": 0.0015 + } + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "mean": 0.0165, + "persist": 0.0212 + }, + "jlens": { + "min": 0.1582, + "best1L": 0.1266, + "mean": 0.0229, + "persist": 0.0323 + }, + "shuffled": { + "min": 0.091, + "best1L": 0.0479, + "mean": 0.0085, + "persist": 0.0023 + } + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "mean": 0.1449, + "persist": 0.2915 + }, + "jlens": { + "min": 0.5845, + "best1L": 0.3789, + "mean": 0.2127, + "persist": 0.2966 + }, + "shuffled": { + "min": 0.5859, + "best1L": 0.3966, + "mean": 0.1287, + "persist": 0.2087 + } + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "mean": 0.0021, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0327, + "best1L": 0.0102, + "mean": 0.0033, + "persist": 0.0015 + }, + "shuffled": { + "min": 0.0381, + "best1L": 0.0115, + "mean": 0.0024, + "persist": 0.0 + } + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "mean": 0.0345, + "persist": 0.0475 + }, + "jlens": { + "min": 0.4082, + "best1L": 0.224, + "mean": 0.1229, + "persist": 0.1884 + }, + "shuffled": { + "min": 0.3822, + "best1L": 0.2108, + "mean": 0.0787, + "persist": 0.0811 + } + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "mean": 0.0005, + "persist": 0.0 + }, + "jlens": { + "min": 0.0188, + "best1L": 0.0079, + "mean": 0.0016, + "persist": 0.0016 + }, + "shuffled": { + "min": 0.018, + "best1L": 0.0065, + "mean": 0.001, + "persist": 0.0 + } + } + }, + "n_win": { + "best1L": 5, + "min": 6, + "persist": 4 + } + }, + "4": { + "n_prompts_used": 200, + "n_source_layers": 22, + "disp_L0": 2.264748, + "disp_last": 0.021016, + "per_set": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "mean": 0.033, + "persist": 0.0476 + }, + "jlens": { + "min": 0.2968, + "best1L": 0.1594, + "mean": 0.0453, + "persist": 0.0411 + }, + "shuffled": { + "min": 0.2075, + "best1L": 0.1164, + "mean": 0.0167, + "persist": 0.0015 + } + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "mean": 0.0165, + "persist": 0.0212 + }, + "jlens": { + "min": 0.1599, + "best1L": 0.1294, + "mean": 0.023, + "persist": 0.0335 + }, + "shuffled": { + "min": 0.0932, + "best1L": 0.0485, + "mean": 0.0088, + "persist": 0.0031 + } + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "mean": 0.1449, + "persist": 0.2915 + }, + "jlens": { + "min": 0.5902, + "best1L": 0.3789, + "mean": 0.2196, + "persist": 0.2993 + }, + "shuffled": { + "min": 0.5822, + "best1L": 0.3952, + "mean": 0.1341, + "persist": 0.2125 + } + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "mean": 0.0021, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0333, + "best1L": 0.0108, + "mean": 0.0036, + "persist": 0.0015 + }, + "shuffled": { + "min": 0.0427, + "best1L": 0.0115, + "mean": 0.0026, + "persist": 0.0 + } + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "mean": 0.0345, + "persist": 0.0475 + }, + "jlens": { + "min": 0.372, + "best1L": 0.2074, + "mean": 0.1072, + "persist": 0.1741 + }, + "shuffled": { + "min": 0.3473, + "best1L": 0.1923, + "mean": 0.0645, + "persist": 0.0613 + } + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "mean": 0.0005, + "persist": 0.0 + }, + "jlens": { + "min": 0.0115, + "best1L": 0.0033, + "mean": 0.001, + "persist": 0.0016 + }, + "shuffled": { + "min": 0.0191, + "best1L": 0.0093, + "mean": 0.0011, + "persist": 0.0 + } + } + }, + "n_win": { + "best1L": 4, + "min": 6, + "persist": 4 + } + }, + "16": { + "n_prompts_used": 200, + "n_source_layers": 22, + "disp_L0": 2.364437, + "disp_last": 0.022615, + "per_set": { + "multihop": { + "logit": { + "min": 0.1868, + "best1L": 0.1504, + "mean": 0.033, + "persist": 0.0476 + }, + "jlens": { + "min": 0.2869, + "best1L": 0.156, + "mean": 0.0446, + "persist": 0.0435 + }, + "shuffled": { + "min": 0.2121, + "best1L": 0.1147, + "mean": 0.0176, + "persist": 0.0029 + } + }, + "multilingual": { + "logit": { + "min": 0.1115, + "best1L": 0.0846, + "mean": 0.0165, + "persist": 0.0212 + }, + "jlens": { + "min": 0.1555, + "best1L": 0.1292, + "mean": 0.0229, + "persist": 0.0347 + }, + "shuffled": { + "min": 0.0932, + "best1L": 0.049, + "mean": 0.0091, + "persist": 0.0038 + } + }, + "order-ops": { + "logit": { + "min": 0.515, + "best1L": 0.3911, + "mean": 0.1449, + "persist": 0.2915 + }, + "jlens": { + "min": 0.6124, + "best1L": 0.3789, + "mean": 0.2278, + "persist": 0.3011 + }, + "shuffled": { + "min": 0.5811, + "best1L": 0.3905, + "mean": 0.144, + "persist": 0.2276 + } + }, + "poetry": { + "logit": { + "min": 0.0256, + "best1L": 0.0077, + "mean": 0.0021, + "persist": 0.0015 + }, + "jlens": { + "min": 0.0343, + "best1L": 0.0118, + "mean": 0.0038, + "persist": 0.0015 + }, + "shuffled": { + "min": 0.0458, + "best1L": 0.013, + "mean": 0.0031, + "persist": 0.0 + } + }, + "typo": { + "logit": { + "min": 0.1757, + "best1L": 0.1046, + "mean": 0.0345, + "persist": 0.0475 + }, + "jlens": { + "min": 0.3491, + "best1L": 0.2003, + "mean": 0.0949, + "persist": 0.1564 + }, + "shuffled": { + "min": 0.3202, + "best1L": 0.1662, + "mean": 0.0552, + "persist": 0.0545 + } + }, + "association": { + "logit": { + "min": 0.0076, + "best1L": 0.0044, + "mean": 0.0005, + "persist": 0.0 + }, + "jlens": { + "min": 0.0082, + "best1L": 0.0033, + "mean": 0.001, + "persist": 0.0016 + }, + "shuffled": { + "min": 0.0253, + "best1L": 0.0112, + "mean": 0.0014, + "persist": 0.0 + } + } + }, + "n_win": { + "best1L": 4, + "min": 6, + "persist": 4 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124" + }, + "runtime_s": 1375.5 +} \ No newline at end of file diff --git a/results/t20_ablation_kl_160m.json b/results/t20_ablation_kl_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..778ef4dd183647f99f234fe4984efab6c6cd452b --- /dev/null +++ b/results/t20_ablation_kl_160m.json @@ -0,0 +1,3402 @@ +{ + "experiment": "T20 / E20 \u2014 ablation KL, the anchor's Figure 53 DV", + "status": "EXPLORATORY", + "prereg": null, + "prereg_sha256": null, + "decision_rule": "ratio KL(J^P)/KL(I), band mean, on multihop: >1.5 reproduces the anchor; <=1.0 falsifies at this scale; (1.0,1.5] inconclusive", + "primary_outcome": "band-mean ratio KL(jlens)/KL(logit) per set", + "layer_protocol": "one layer at a time, swept over the band; NO band compounding. Headline is the band MEAN of the per-layer ratio; band-max is stored as a secondary and is explicitly not primary (existential statistic).", + "cross_lens_caveat": "the logit lens has no lens-native coordinate write (external review B5/G6); applied here it is a SHARED hidden-state intervention parameterised by whichever lens supplies v_c", + "VERDICT": "REPRODUCES \u2014 multihop ratio 8.6095 > 1.5, the anchor's ~2x claim holds here", + "model": "EleutherAI/pythia-160m-deduped", + "lens": "results/lens_160m_n200_db128_pen.pt", + "lens_sha256": "65373e75993d2342ba95142055fcaea1c71206b3a4c8faa7de9a8e984cbd5b78", + "n_layers": 12, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "band": [ + 5, + 6, + 7, + 8, + 9 + ], + "layers_arg": "band", + "skip_positions": 1, + "max_items_effective": null, + "min_pairs_floor": 30, + "evals": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo", + "association" + ], + "arms": [ + "logit", + "jlens", + "shuffled", + "rand_match" + ], + "seed": 0, + "derangement_layer_map": { + "0": 2, + "1": 9, + "2": 1, + "3": 7, + "4": 6, + "5": 8, + "6": 4, + "7": 5, + "8": 0, + "9": 3 + }, + "device": "cpu", + "n_forward_passes": 18411, + "results": { + "multihop": { + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.005386, + "jlens": 0.046368, + "shuffled": 0.031355, + "rand_match": 0.003099 + }, + "PRIMARY_ratio_jlens_over_logit": 8.6095, + "ratio_jlens_over_shuffled": 1.4788, + "ratio_jlens_over_rand_match": 14.962, + "ratio_by_layer_jlens_over_logit": { + "5": 10.5134, + "6": 31.0472, + "7": 7.4575, + "8": 5.3913, + "9": 2.688 + }, + "band_max_ratio_SECONDARY": 31.0472, + "per_layer": { + "logit": { + "5": { + "kl_mean": 0.007251, + "dose_mean_per_position": 0.4997, + "dose_sum_over_positions": 12.3577 + }, + "6": { + "kl_mean": 0.002178, + "dose_mean_per_position": 0.399, + "dose_sum_over_positions": 10.1117 + }, + "7": { + "kl_mean": 0.004012, + "dose_mean_per_position": 0.5144, + "dose_sum_over_positions": 12.6813 + }, + "8": { + "kl_mean": 0.008066, + "dose_mean_per_position": 0.7469, + "dose_sum_over_positions": 16.6234 + }, + "9": { + "kl_mean": 0.005421, + "dose_mean_per_position": 0.935, + "dose_sum_over_positions": 20.6988 + } + }, + "jlens": { + "5": { + "kl_mean": 0.076228, + "dose_mean_per_position": 0.9747, + "dose_sum_over_positions": 21.1284 + }, + "6": { + "kl_mean": 0.067632, + "dose_mean_per_position": 1.1495, + "dose_sum_over_positions": 24.9237 + }, + "7": { + "kl_mean": 0.029921, + "dose_mean_per_position": 1.0011, + "dose_sum_over_positions": 21.6619 + }, + "8": { + "kl_mean": 0.043487, + "dose_mean_per_position": 1.2784, + "dose_sum_over_positions": 26.862 + }, + "9": { + "kl_mean": 0.014572, + "dose_mean_per_position": 1.1595, + "dose_sum_over_positions": 25.1232 + } + }, + "shuffled": { + "5": { + "kl_mean": 0.003851, + "dose_mean_per_position": 0.3737, + "dose_sum_over_positions": 8.3271 + }, + "6": { + "kl_mean": 0.055536, + "dose_mean_per_position": 1.326, + "dose_sum_over_positions": 28.8234 + }, + "7": { + "kl_mean": 0.058587, + "dose_mean_per_position": 1.7661, + "dose_sum_over_positions": 37.5593 + }, + "8": { + "kl_mean": 0.019283, + "dose_mean_per_position": 2.0122, + "dose_sum_over_positions": 44.3759 + }, + "9": { + "kl_mean": 0.019518, + "dose_mean_per_position": 2.0373, + "dose_sum_over_positions": 44.5738 + } + }, + "rand_match": { + "5": { + "kl_mean": 0.003458, + "dose_mean_per_position": 0.5127, + "dose_sum_over_positions": 11.6982 + }, + "6": { + "kl_mean": 0.003154, + "dose_mean_per_position": 0.5425, + "dose_sum_over_positions": 12.6487 + }, + "7": { + "kl_mean": 0.002803, + "dose_mean_per_position": 0.6194, + "dose_sum_over_positions": 14.0668 + }, + "8": { + "kl_mean": 0.003194, + "dose_mean_per_position": 0.7714, + "dose_sum_over_positions": 17.5099 + }, + "9": { + "kl_mean": 0.002887, + "dose_mean_per_position": 0.7854, + "dose_sum_over_positions": 17.5605 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "5": [ + 0.0, + 0.484, + 0.2203, + 0.1014, + 0.4928, + 0.051, + 0.1756, + 0.9009, + 0.2269, + 0.0717, + 0.4677, + 0.4307, + 1.034, + 0.3944, + 0.3632, + 0.3952, + 0.511, + 0.724, + 0.7498, + 0.5415 + ], + "6": [ + 0.0, + 0.3452, + 0.1453, + 0.2681, + 0.0746, + 0.3247, + 0.0871, + 0.2244, + 0.6035, + 0.0501, + 0.2198, + 0.4647, + 0.242, + 0.0931, + 0.2394, + 0.2132, + 0.1811, + 0.5353, + 0.3104, + 0.5013 + ], + "7": [ + 0.0, + 0.3456, + 0.6783, + 0.7134, + 0.2704, + 0.1006, + 0.0712, + 0.2637, + 0.5941, + 0.1705, + 0.0354, + 0.0496, + 0.0182, + 0.0839, + 0.422, + 0.0042, + 0.1516, + 0.0634, + 0.124, + 0.3142 + ], + "8": [ + 0.0, + 0.558, + 0.0721, + 0.5517, + 0.4184, + 1.0781, + 0.9684, + 1.2558, + 1.3874, + 0.9697, + 1.0764, + 1.0978, + 0.6557, + 0.715, + 0.5891, + 0.9777, + 0.6234, + 1.2897, + 0.5309, + 0.5091 + ], + "9": [ + 0.0, + 2.3993, + 2.2292, + 0.7908, + 0.1228, + 0.2092, + 0.6068, + 0.9231, + 0.7786, + 0.1928, + 1.4679, + 0.8969, + 0.2932, + 0.5063, + 0.8008, + 0.7586, + 0.1751, + 1.352, + 0.1157, + 0.4906 + ] + }, + "jlens": { + "5": [ + 0.0, + 0.1515, + 0.7395, + 1.0707, + 1.4468, + 1.3256, + 1.3507, + 1.2308, + 1.3309, + 1.079, + 1.9055, + 1.2407, + 0.3571, + 1.1002, + 0.872, + 0.424, + 1.3388, + 1.4358, + 1.1425, + 0.7725 + ], + "6": [ + 0.0, + 0.1595, + 0.5283, + 1.234, + 1.3808, + 1.8175, + 1.8556, + 1.4513, + 1.6105, + 1.2533, + 1.6004, + 1.4855, + 0.2785, + 1.3244, + 1.1826, + 1.2261, + 1.139, + 1.5497, + 1.1135, + 0.8464 + ], + "7": [ + 0.0, + 0.9395, + 0.3263, + 0.9134, + 0.5889, + 1.8647, + 1.807, + 1.2893, + 1.7076, + 1.512, + 1.6068, + 1.8042, + 0.4023, + 0.9144, + 1.3034, + 1.3641, + 1.0671, + 1.845, + 1.3132, + 0.986 + ], + "8": [ + 0.0, + 1.0313, + 0.6356, + 1.009, + 0.9728, + 1.9999, + 1.8702, + 1.7914, + 1.8623, + 1.5558, + 2.2668, + 2.2628, + 0.1528, + 1.4037, + 1.8447, + 1.4732, + 1.3459, + 2.6838, + 2.0469, + 1.4169 + ], + "9": [ + 0.0, + 2.4235, + 1.8932, + 0.2747, + 0.2451, + 1.133, + 1.0008, + 1.2956, + 1.9778, + 1.2629, + 2.1231, + 1.9085, + 0.5851, + 0.9671, + 1.8248, + 0.756, + 0.9443, + 2.8607, + 1.8262, + 1.0312 + ] + }, + "shuffled": { + "5": [ + 0.0, + 0.6747, + 0.1011, + 0.5618, + 0.0535, + 0.4801, + 0.4906, + 0.2088, + 0.4221, + 0.4093, + 0.4396, + 0.1873, + 0.8457, + 0.1779, + 0.3263, + 0.1029, + 0.43, + 0.5267, + 0.013, + 0.0513 + ], + "6": [ + 0.0, + 0.423, + 0.8105, + 1.3266, + 1.9946, + 1.8177, + 1.815, + 1.9942, + 1.671, + 1.6164, + 2.4433, + 1.4962, + 0.5141, + 1.5933, + 1.6077, + 1.2081, + 1.4332, + 1.9546, + 1.0453, + 1.1289 + ], + "7": [ + 0.0, + 0.3762, + 0.8187, + 1.8961, + 2.2869, + 2.7612, + 3.0005, + 2.5411, + 2.3167, + 2.3353, + 3.1179, + 2.2843, + 1.3273, + 1.8436, + 1.9296, + 1.926, + 1.9926, + 2.7939, + 1.8111, + 2.0674 + ], + "8": [ + 0.0, + 0.736, + 0.7636, + 2.7714, + 3.3671, + 2.1128, + 2.1783, + 2.8455, + 1.5054, + 1.9108, + 2.8959, + 3.0773, + 0.3556, + 1.8697, + 2.1156, + 1.7095, + 1.5383, + 2.1643, + 2.2267, + 2.1356 + ], + "9": [ + 0.0, + 0.1466, + 0.1893, + 2.2829, + 2.8854, + 2.3425, + 2.4256, + 3.0383, + 2.7966, + 2.4773, + 3.7938, + 3.0723, + 0.5818, + 1.9738, + 2.9675, + 2.1067, + 2.3619, + 2.458, + 3.0004, + 3.3282 + ] + }, + "rand_match": { + "5": [ + 0.0, + 0.1294, + 0.3706, + 0.5147, + 1.3844, + 0.7223, + 0.6078, + 1.2059, + 0.8858, + 0.6757, + 1.2753, + 0.9125, + 0.0765, + 0.4009, + 0.6312, + 0.9341, + 0.2765, + 0.1376, + 0.5855, + 0.8886 + ], + "6": [ + 0.0, + 1.1327, + 0.4718, + 0.3265, + 1.2965, + 0.417, + 0.481, + 0.9207, + 0.4932, + 0.3807, + 0.9563, + 1.1981, + 0.3648, + 1.3773, + 1.1817, + 0.9425, + 1.591, + 0.9607, + 0.4785, + 0.2449 + ], + "7": [ + 0.0, + 0.4846, + 0.0598, + 0.0801, + 0.9338, + 0.1376, + 0.1273, + 0.7403, + 0.2991, + 0.7895, + 0.8328, + 1.5979, + 0.3381, + 0.4014, + 0.8345, + 0.972, + 1.1278, + 0.7143, + 0.7287, + 0.8123 + ], + "8": [ + 0.0, + 0.7006, + 0.0704, + 0.2458, + 0.5739, + 0.2282, + 0.0717, + 0.4577, + 0.0659, + 0.138, + 0.5556, + 0.9582, + 0.7237, + 0.7215, + 0.3794, + 0.5243, + 0.0006, + 0.4664, + 0.2321, + 0.0505 + ], + "9": [ + 0.0, + 0.3432, + 1.123, + 0.2104, + 1.6571, + 0.4635, + 0.2298, + 0.0319, + 0.1618, + 0.7211, + 0.5122, + 0.7794, + 0.2328, + 1.3126, + 0.0671, + 0.7892, + 0.2735, + 0.4169, + 1.3707, + 0.4786 + ] + } + } + }, + "multilingual": { + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "band_mean_kl": { + "logit": 0.012599, + "jlens": 0.026345, + "shuffled": 0.014357, + "rand_match": 0.003734 + }, + "PRIMARY_ratio_jlens_over_logit": 2.091, + "ratio_jlens_over_shuffled": 1.8351, + "ratio_jlens_over_rand_match": 7.0562, + "ratio_by_layer_jlens_over_logit": { + "5": 1.0884, + "6": 3.0698, + "7": 4.0789, + "8": 5.0257, + "9": 1.6982 + }, + "band_max_ratio_SECONDARY": 5.0257, + "per_layer": { + "logit": { + "5": { + "kl_mean": 0.011294, + "dose_mean_per_position": 0.6243, + "dose_sum_over_positions": 7.2426 + }, + "6": { + "kl_mean": 0.00627, + "dose_mean_per_position": 0.5186, + "dose_sum_over_positions": 6.1014 + }, + "7": { + "kl_mean": 0.00435, + "dose_mean_per_position": 0.4111, + "dose_sum_over_positions": 4.8122 + }, + "8": { + "kl_mean": 0.003809, + "dose_mean_per_position": 0.5109, + "dose_sum_over_positions": 6.0942 + }, + "9": { + "kl_mean": 0.037274, + "dose_mean_per_position": 1.3688, + "dose_sum_over_positions": 16.6928 + } + }, + "jlens": { + "5": { + "kl_mean": 0.012293, + "dose_mean_per_position": 0.411, + "dose_sum_over_positions": 4.8444 + }, + "6": { + "kl_mean": 0.019248, + "dose_mean_per_position": 0.6361, + "dose_sum_over_positions": 7.7081 + }, + "7": { + "kl_mean": 0.017745, + "dose_mean_per_position": 0.6886, + "dose_sum_over_positions": 8.7575 + }, + "8": { + "kl_mean": 0.019141, + "dose_mean_per_position": 0.8361, + "dose_sum_over_positions": 10.5117 + }, + "9": { + "kl_mean": 0.063299, + "dose_mean_per_position": 1.4969, + "dose_sum_over_positions": 18.6596 + } + }, + "shuffled": { + "5": { + "kl_mean": 0.024047, + "dose_mean_per_position": 0.7256, + "dose_sum_over_positions": 8.6237 + }, + "6": { + "kl_mean": 0.014649, + "dose_mean_per_position": 0.6273, + "dose_sum_over_positions": 7.1068 + }, + "7": { + "kl_mean": 0.018059, + "dose_mean_per_position": 0.8555, + "dose_sum_over_positions": 10.1798 + }, + "8": { + "kl_mean": 0.010073, + "dose_mean_per_position": 0.8826, + "dose_sum_over_positions": 9.9773 + }, + "9": { + "kl_mean": 0.004955, + "dose_mean_per_position": 0.8841, + "dose_sum_over_positions": 10.3687 + } + }, + "rand_match": { + "5": { + "kl_mean": 0.003887, + "dose_mean_per_position": 0.4435, + "dose_sum_over_positions": 5.0992 + }, + "6": { + "kl_mean": 0.003698, + "dose_mean_per_position": 0.488, + "dose_sum_over_positions": 5.6545 + }, + "7": { + "kl_mean": 0.00382, + "dose_mean_per_position": 0.6046, + "dose_sum_over_positions": 6.9892 + }, + "8": { + "kl_mean": 0.004113, + "dose_mean_per_position": 0.6624, + "dose_sum_over_positions": 7.6499 + }, + "9": { + "kl_mean": 0.00315, + "dose_mean_per_position": 0.6499, + "dose_sum_over_positions": 7.4417 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "5": [ + 0.0, + 0.9614, + 1.281, + 1.569, + 0.6422, + 0.5721, + 1.2425, + 0.7737, + 0.312, + 0.8983, + 0.7608, + 0.9136 + ], + "6": [ + 0.0, + 0.7381, + 1.1209, + 1.3221, + 0.0481, + 0.7613, + 0.591, + 0.3429, + 0.5636, + 0.537, + 0.4559, + 0.7562 + ], + "7": [ + 0.0, + 0.462, + 0.7756, + 0.8389, + 0.1576, + 0.5385, + 0.3405, + 0.2277, + 0.3185, + 0.581, + 0.4008, + 0.3169 + ], + "8": [ + 0.0, + 0.4173, + 0.7801, + 0.484, + 0.8303, + 0.2627, + 0.2783, + 0.4846, + 0.4372, + 0.1969, + 0.4905, + 0.0213 + ], + "9": [ + 0.0, + 2.0285, + 1.716, + 0.71, + 0.0058, + 0.721, + 2.0697, + 1.0488, + 1.7858, + 0.8429, + 1.2092, + 1.5215 + ] + }, + "jlens": { + "5": [ + 0.0, + 0.2526, + 0.204, + 0.5844, + 1.0705, + 0.2699, + 0.1924, + 0.3653, + 0.3499, + 0.7551, + 0.8337, + 0.1648 + ], + "6": [ + 0.0, + 0.4125, + 0.0178, + 0.5895, + 2.0033, + 0.8311, + 0.0515, + 0.2581, + 1.1055, + 1.2368, + 1.0751, + 0.2061 + ], + "7": [ + 0.0, + 1.5231, + 0.5797, + 0.2355, + 1.6307, + 0.4087, + 0.436, + 0.4192, + 0.3373, + 0.6192, + 0.4143, + 0.0077 + ], + "8": [ + 0.0, + 1.2702, + 0.8985, + 0.5739, + 2.0382, + 0.6028, + 0.513, + 0.4911, + 0.0862, + 0.8611, + 0.4919, + 0.2996 + ], + "9": [ + 0.0, + 2.549, + 1.5527, + 0.9055, + 1.6105, + 0.0197, + 2.014, + 1.5051, + 0.6285, + 0.6508, + 0.2449, + 0.8776 + ] + }, + "shuffled": { + "5": [ + 0.0, + 1.4998, + 1.5571, + 1.3664, + 0.3034, + 0.7369, + 1.1616, + 1.0121, + 0.6101, + 0.5041, + 0.433, + 0.9515 + ], + "6": [ + 0.0, + 0.0838, + 0.4555, + 1.1079, + 1.4032, + 0.9081, + 0.3958, + 0.4307, + 1.2894, + 1.5153, + 1.0894, + 0.5203 + ], + "7": [ + 0.0, + 0.1085, + 0.8626, + 1.1362, + 2.3426, + 1.5534, + 0.533, + 0.8085, + 1.3682, + 1.8809, + 1.4936, + 0.8893 + ], + "8": [ + 0.0, + 0.4847, + 1.4239, + 1.6388, + 2.1302, + 1.2364, + 0.971, + 0.3442, + 2.035, + 1.8752, + 1.6974, + 1.7467 + ], + "9": [ + 0.0, + 0.0401, + 1.2068, + 1.5234, + 2.6932, + 1.6579, + 0.2696, + 0.107, + 1.4349, + 2.2091, + 1.8892, + 1.1726 + ] + }, + "rand_match": { + "5": [ + 0.0, + 0.6337, + 0.3197, + 0.2143, + 0.1206, + 0.4775, + 0.695, + 0.6174, + 0.3735, + 0.2596, + 0.943, + 0.6407 + ], + "6": [ + 0.0, + 0.1873, + 0.4107, + 0.0352, + 0.3293, + 0.0726, + 0.4179, + 0.2998, + 0.1685, + 0.0119, + 0.1074, + 0.7072 + ], + "7": [ + 0.0, + 0.0703, + 0.2655, + 0.1934, + 0.2343, + 0.2609, + 0.2246, + 0.4871, + 0.1559, + 0.26, + 0.1052, + 0.3784 + ], + "8": [ + 0.0, + 0.0702, + 1.0021, + 0.915, + 0.2244, + 0.1501, + 0.3239, + 0.0993, + 0.6596, + 0.4836, + 0.755, + 0.6782 + ], + "9": [ + 0.0, + 1.1863, + 1.4592, + 1.3398, + 1.5771, + 1.7205, + 0.113, + 0.9217, + 2.0018, + 1.7496, + 1.4003, + 1.1358 + ] + } + } + }, + "order-ops": { + "n_items_scored": 55, + "n_intermediate_pairs_scored": 110, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.00616, + "jlens": 0.012378, + "shuffled": 0.012955, + "rand_match": 0.001811 + }, + "PRIMARY_ratio_jlens_over_logit": 2.0095, + "ratio_jlens_over_shuffled": 0.9554, + "ratio_jlens_over_rand_match": 6.8337, + "ratio_by_layer_jlens_over_logit": { + "5": 3.8291, + "6": 3.1765, + "7": 1.3517, + "8": 5.2214, + "9": 0.5162 + }, + "band_max_ratio_SECONDARY": 5.2214, + "per_layer": { + "logit": { + "5": { + "kl_mean": 0.005966, + "dose_mean_per_position": 0.5698, + "dose_sum_over_positions": 5.4471 + }, + "6": { + "kl_mean": 0.00617, + "dose_mean_per_position": 0.6444, + "dose_sum_over_positions": 6.2498 + }, + "7": { + "kl_mean": 0.005071, + "dose_mean_per_position": 0.6856, + "dose_sum_over_positions": 6.663 + }, + "8": { + "kl_mean": 0.001185, + "dose_mean_per_position": 0.4468, + "dose_sum_over_positions": 4.2507 + }, + "9": { + "kl_mean": 0.012406, + "dose_mean_per_position": 1.7025, + "dose_sum_over_positions": 15.9246 + } + }, + "jlens": { + "5": { + "kl_mean": 0.022842, + "dose_mean_per_position": 0.4628, + "dose_sum_over_positions": 4.3936 + }, + "6": { + "kl_mean": 0.0196, + "dose_mean_per_position": 0.5364, + "dose_sum_over_positions": 5.0581 + }, + "7": { + "kl_mean": 0.006855, + "dose_mean_per_position": 0.4755, + "dose_sum_over_positions": 4.3706 + }, + "8": { + "kl_mean": 0.006186, + "dose_mean_per_position": 0.6056, + "dose_sum_over_positions": 5.5221 + }, + "9": { + "kl_mean": 0.006404, + "dose_mean_per_position": 1.3245, + "dose_sum_over_positions": 12.3236 + } + }, + "shuffled": { + "5": { + "kl_mean": 0.005269, + "dose_mean_per_position": 0.3769, + "dose_sum_over_positions": 3.4891 + }, + "6": { + "kl_mean": 0.01722, + "dose_mean_per_position": 0.8515, + "dose_sum_over_positions": 8.0503 + }, + "7": { + "kl_mean": 0.027609, + "dose_mean_per_position": 0.8876, + "dose_sum_over_positions": 8.6489 + }, + "8": { + "kl_mean": 0.011526, + "dose_mean_per_position": 1.0056, + "dose_sum_over_positions": 9.8843 + }, + "9": { + "kl_mean": 0.003153, + "dose_mean_per_position": 0.7237, + "dose_sum_over_positions": 7.0304 + } + }, + "rand_match": { + "5": { + "kl_mean": 0.00203, + "dose_mean_per_position": 0.4267, + "dose_sum_over_positions": 4.1565 + }, + "6": { + "kl_mean": 0.002379, + "dose_mean_per_position": 0.4796, + "dose_sum_over_positions": 4.6669 + }, + "7": { + "kl_mean": 0.001649, + "dose_mean_per_position": 0.5488, + "dose_sum_over_positions": 5.338 + }, + "8": { + "kl_mean": 0.001358, + "dose_mean_per_position": 0.5696, + "dose_sum_over_positions": 5.6089 + }, + "9": { + "kl_mean": 0.001641, + "dose_mean_per_position": 0.6662, + "dose_sum_over_positions": 6.196 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "5": [ + 0.0, + 0.9502, + 0.5008, + 0.7872, + 0.1137, + 0.2969, + 1.0741, + 0.3554, + 0.6955 + ], + "6": [ + 0.0, + 0.8537, + 0.7694, + 1.0837, + 0.6266, + 0.0923, + 1.3181, + 0.4187, + 0.6216 + ], + "7": [ + 0.0, + 0.9685, + 0.7146, + 1.1817, + 0.6879, + 0.348, + 1.5369, + 0.7977, + 0.8427 + ], + "8": [ + 0.0, + 0.7678, + 0.648, + 0.3549, + 0.0917, + 0.0866, + 0.8336, + 0.2816, + 0.2933 + ], + "9": [ + 0.0, + 3.4978, + 2.532, + 2.7501, + 1.7058, + 2.0517, + 3.1949, + 1.9662, + 1.5902 + ] + }, + "jlens": { + "5": [ + 0.0, + 0.3803, + 0.0037, + 0.618, + 0.4733, + 0.7406, + 0.4429, + 0.7874, + 0.5213 + ], + "6": [ + 0.0, + 0.2377, + 0.1985, + 0.9527, + 0.6325, + 0.2691, + 0.1427, + 0.6524, + 0.6817 + ], + "7": [ + 0.0, + 1.1012, + 0.8186, + 0.0911, + 0.0829, + 0.4211, + 0.8341, + 0.0618, + 0.2902 + ], + "8": [ + 0.0, + 1.5074, + 1.0207, + 0.1854, + 0.0563, + 0.1526, + 1.0077, + 0.1185, + 0.319 + ], + "9": [ + 0.0, + 3.293, + 2.5811, + 1.8971, + 1.8755, + 1.771, + 2.8368, + 1.6046, + 0.2804 + ] + }, + "shuffled": { + "5": [ + 0.0, + 0.6058, + 0.5886, + 0.2644, + 0.0381, + 0.252, + 0.7502, + 0.1211, + 0.652 + ], + "6": [ + 0.0, + 0.5458, + 0.7204, + 1.4795, + 1.0145, + 1.206, + 0.8307, + 1.0746, + 1.2862 + ], + "7": [ + 0.0, + 0.0769, + 0.5018, + 1.7485, + 1.6344, + 1.1269, + 0.9336, + 1.4478, + 1.5637 + ], + "8": [ + 0.0, + 0.2125, + 0.5386, + 2.1608, + 0.6578, + 0.9026, + 1.3177, + 1.5167, + 2.0281 + ], + "9": [ + 0.0, + 0.7613, + 0.4094, + 1.2835, + 0.9834, + 0.6, + 0.3985, + 1.047, + 1.196 + ] + }, + "rand_match": { + "5": [ + 0.0, + 0.3031, + 0.2762, + 0.1422, + 0.0318, + 0.0254, + 0.0264, + 0.26, + 0.1779 + ], + "6": [ + 0.0, + 0.0196, + 0.1241, + 0.0413, + 0.6273, + 0.1619, + 0.305, + 0.5324, + 0.5176 + ], + "7": [ + 0.0, + 0.2238, + 0.4258, + 0.5619, + 0.57, + 0.4867, + 0.6733, + 0.0037, + 0.6585 + ], + "8": [ + 0.0, + 0.502, + 0.9006, + 0.5467, + 1.2125, + 0.907, + 0.7944, + 0.7994, + 0.5963 + ], + "9": [ + 0.0, + 0.0236, + 0.7666, + 0.471, + 1.8172, + 1.2229, + 0.8041, + 1.5785, + 0.4568 + ] + } + } + }, + "poetry": { + "n_items_scored": 98, + "n_intermediate_pairs_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.003966, + "jlens": 0.056717, + "shuffled": 0.038688, + "rand_match": 0.004578 + }, + "PRIMARY_ratio_jlens_over_logit": 14.3011, + "ratio_jlens_over_shuffled": 1.466, + "ratio_jlens_over_rand_match": 12.39, + "ratio_by_layer_jlens_over_logit": { + "5": 63.7357, + "6": 17.3103, + "7": 24.4623, + "8": 5.1271, + "9": 2.6627 + }, + "band_max_ratio_SECONDARY": 63.7357, + "per_layer": { + "logit": { + "5": { + "kl_mean": 0.001849, + "dose_mean_per_position": 0.6662, + "dose_sum_over_positions": 18.1866 + }, + "6": { + "kl_mean": 0.004323, + "dose_mean_per_position": 0.7911, + "dose_sum_over_positions": 21.6285 + }, + "7": { + "kl_mean": 0.001511, + "dose_mean_per_position": 0.7082, + "dose_sum_over_positions": 19.334 + }, + "8": { + "kl_mean": 0.008775, + "dose_mean_per_position": 1.1866, + "dose_sum_over_positions": 32.4102 + }, + "9": { + "kl_mean": 0.003372, + "dose_mean_per_position": 1.6703, + "dose_sum_over_positions": 45.6513 + } + }, + "jlens": { + "5": { + "kl_mean": 0.117824, + "dose_mean_per_position": 1.0365, + "dose_sum_over_positions": 28.343 + }, + "6": { + "kl_mean": 0.074831, + "dose_mean_per_position": 1.3255, + "dose_sum_over_positions": 36.2588 + }, + "7": { + "kl_mean": 0.036961, + "dose_mean_per_position": 1.4189, + "dose_sum_over_positions": 38.8251 + }, + "8": { + "kl_mean": 0.044991, + "dose_mean_per_position": 1.7387, + "dose_sum_over_positions": 47.5668 + }, + "9": { + "kl_mean": 0.008979, + "dose_mean_per_position": 1.827, + "dose_sum_over_positions": 49.9707 + } + }, + "shuffled": { + "5": { + "kl_mean": 0.007787, + "dose_mean_per_position": 0.6596, + "dose_sum_over_positions": 18.0409 + }, + "6": { + "kl_mean": 0.059052, + "dose_mean_per_position": 1.411, + "dose_sum_over_positions": 38.5734 + }, + "7": { + "kl_mean": 0.065944, + "dose_mean_per_position": 1.8449, + "dose_sum_over_positions": 50.4866 + }, + "8": { + "kl_mean": 0.041274, + "dose_mean_per_position": 2.4209, + "dose_sum_over_positions": 66.1501 + }, + "9": { + "kl_mean": 0.019385, + "dose_mean_per_position": 2.4188, + "dose_sum_over_positions": 66.2027 + } + }, + "rand_match": { + "5": { + "kl_mean": 0.008051, + "dose_mean_per_position": 0.8041, + "dose_sum_over_positions": 21.8956 + }, + "6": { + "kl_mean": 0.00425, + "dose_mean_per_position": 0.828, + "dose_sum_over_positions": 22.6913 + }, + "7": { + "kl_mean": 0.004836, + "dose_mean_per_position": 0.9258, + "dose_sum_over_positions": 25.2663 + }, + "8": { + "kl_mean": 0.003637, + "dose_mean_per_position": 0.9575, + "dose_sum_over_positions": 26.1793 + }, + "9": { + "kl_mean": 0.002114, + "dose_mean_per_position": 0.9336, + "dose_sum_over_positions": 25.7278 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "5": [ + 0.0, + 1.0869, + 0.4101, + 1.5993, + 0.6444, + 0.0778, + 0.3093, + 0.0248, + 5.845, + 0.5261, + 0.1738, + 0.3153, + 0.1639, + 0.2008, + 0.6916, + 1.4199, + 0.1075, + 0.5076, + 0.1368, + 0.1673, + 0.2511, + 0.3513, + 0.0456, + 0.4016, + 0.5795, + 0.5055, + 0.4886, + 0.0874, + 0.0336 + ], + "6": [ + 0.0, + 1.4254, + 0.6388, + 0.7421, + 0.5236, + 0.1335, + 0.3946, + 0.0568, + 6.2872, + 0.3037, + 0.7056, + 0.2198, + 0.2684, + 0.526, + 0.1485, + 0.2617, + 0.1386, + 0.3346, + 0.6382, + 0.0588, + 0.6814, + 0.1662, + 0.1739, + 0.0282, + 0.846, + 1.2107, + 0.7929, + 0.4661, + 0.6748 + ], + "7": [ + 0.0, + 1.3713, + 0.5415, + 0.5221, + 0.3764, + 0.0862, + 0.4544, + 0.6599, + 6.3108, + 0.0058, + 0.6129, + 0.3076, + 0.048, + 0.15, + 0.0359, + 0.1135, + 0.011, + 0.3782, + 0.7329, + 0.1051, + 0.5482, + 0.1122, + 0.2063, + 0.0073, + 0.9184, + 1.239, + 0.7891, + 0.5152, + 0.6303 + ], + "8": [ + 0.0, + 1.424, + 0.1626, + 0.7186, + 0.2569, + 0.8728, + 0.9312, + 0.0349, + 6.2364, + 1.0268, + 1.4662, + 0.1869, + 0.5886, + 1.0264, + 0.415, + 0.0399, + 0.6699, + 0.9828, + 1.3807, + 0.4235, + 1.5672, + 0.8317, + 0.9176, + 0.6794, + 1.989, + 1.9791, + 1.8399, + 1.1133, + 2.1572 + ], + "9": [ + 0.0, + 2.6886, + 2.0825, + 2.0969, + 1.1725, + 0.5549, + 0.2535, + 2.1123, + 10.2984, + 0.3752, + 1.2517, + 0.8514, + 0.8868, + 0.8002, + 0.3652, + 0.3614, + 0.5044, + 1.1801, + 1.2062, + 0.076, + 1.8568, + 1.3114, + 1.3546, + 1.4799, + 2.2712, + 1.9841, + 1.2429, + 1.4064, + 1.5323 + ] + }, + "jlens": { + "5": [ + 0.0, + 0.2988, + 0.4896, + 0.2224, + 0.6302, + 1.7834, + 1.4094, + 0.9795, + 1.1857, + 1.2453, + 1.7625, + 0.8414, + 1.1703, + 1.3192, + 1.4336, + 0.1255, + 0.9408, + 0.9636, + 1.2363, + 0.5121, + 1.2599, + 1.6191, + 1.2819, + 1.1235, + 1.0537, + 1.3412, + 1.1866, + 1.054, + 1.122 + ], + "6": [ + 0.0, + 0.7338, + 0.3386, + 0.1069, + 0.6357, + 1.907, + 1.5917, + 0.5805, + 1.8851, + 1.4392, + 1.807, + 1.2469, + 1.412, + 1.767, + 1.3148, + 0.087, + 1.3029, + 1.69, + 1.4137, + 0.6216, + 1.2969, + 2.1772, + 1.6988, + 1.9845, + 1.7845, + 1.5201, + 1.2281, + 1.3816, + 1.3468 + ], + "7": [ + 0.0, + 1.2981, + 1.5045, + 0.7547, + 0.3595, + 1.5964, + 1.6235, + 0.0695, + 2.8168, + 1.3495, + 1.5283, + 1.3103, + 1.6574, + 1.3497, + 1.4025, + 0.5904, + 1.5024, + 1.4037, + 2.0324, + 0.5825, + 1.5568, + 1.9061, + 2.2828, + 1.5815, + 2.0913, + 2.1021, + 1.6229, + 1.3629, + 2.0132 + ], + "8": [ + 0.0, + 1.6009, + 1.3516, + 0.9046, + 0.6348, + 2.41, + 2.1395, + 0.2128, + 1.7372, + 1.8194, + 2.1481, + 1.5476, + 1.5845, + 1.4062, + 1.4783, + 0.5048, + 1.8489, + 1.9429, + 2.5764, + 0.718, + 2.323, + 2.2142, + 2.2353, + 2.0143, + 2.6281, + 2.4345, + 1.936, + 2.047, + 2.496 + ], + "9": [ + 0.0, + 2.4232, + 2.8077, + 2.5616, + 0.2508, + 1.9849, + 1.4992, + 1.2514, + 3.4155, + 1.3199, + 1.7654, + 1.4473, + 1.097, + 1.2411, + 1.1586, + 0.7929, + 1.4453, + 2.3143, + 2.2914, + 0.3682, + 2.4224, + 1.9145, + 1.8031, + 2.1219, + 2.2937, + 2.5392, + 1.718, + 2.004, + 2.0025 + ] + }, + "shuffled": { + "5": [ + 0.0, + 1.029, + 0.8708, + 1.3793, + 0.3307, + 0.5905, + 0.7795, + 0.0776, + 2.6121, + 0.7822, + 0.8442, + 0.2685, + 0.4479, + 0.4687, + 0.1016, + 1.2239, + 0.364, + 0.0602, + 0.4895, + 0.0886, + 0.4066, + 0.6861, + 0.7253, + 0.4775, + 0.693, + 0.5956, + 0.6809, + 0.0754, + 0.3793 + ], + "6": [ + 0.0, + 0.4249, + 0.8465, + 0.329, + 1.0101, + 2.3777, + 1.8262, + 1.1248, + 5.0668, + 1.3799, + 2.0906, + 1.2404, + 1.6903, + 1.6298, + 1.8398, + 0.5392, + 1.3431, + 1.8751, + 1.4584, + 0.8721, + 1.2315, + 1.7727, + 1.8607, + 1.8466, + 1.0629, + 1.0458, + 1.2545, + 2.0708, + 1.0311 + ], + "7": [ + 0.0, + 0.2726, + 0.8349, + 0.5095, + 1.0158, + 2.8195, + 2.4343, + 1.0407, + 1.0241, + 2.0476, + 2.6054, + 2.3533, + 2.7057, + 2.2481, + 2.7381, + 0.815, + 2.4403, + 1.8748, + 2.3468, + 1.3692, + 2.0726, + 3.0253, + 2.6645, + 2.2214, + 2.4292, + 2.2949, + 1.8061, + 2.2851, + 1.8401 + ], + "8": [ + 0.0, + 0.426, + 0.8426, + 0.3565, + 1.2273, + 2.7922, + 2.2636, + 1.0539, + 9.8636, + 3.4359, + 3.0714, + 2.6367, + 1.896, + 2.7925, + 2.6692, + 1.1503, + 2.7135, + 2.6073, + 2.9115, + 1.8952, + 2.9028, + 3.1462, + 1.9446, + 2.9301, + 2.5847, + 2.3417, + 2.9532, + 2.4601, + 2.7794 + ], + "9": [ + 0.0, + 0.4787, + 0.4099, + 0.917, + 1.5142, + 3.0159, + 2.2839, + 0.6547, + 3.3317, + 3.4647, + 4.0303, + 2.9616, + 2.6506, + 3.28, + 3.2139, + 1.2479, + 3.0139, + 3.3901, + 3.2498, + 1.8519, + 2.2228, + 4.4521, + 2.8975, + 2.9909, + 2.748, + 2.8541, + 3.0453, + 3.8539, + 2.6987 + ] + }, + "rand_match": { + "5": [ + 0.0, + 0.455, + 0.0689, + 0.3752, + 0.3695, + 0.396, + 0.4546, + 0.3173, + 12.4088, + 0.244, + 0.3208, + 0.3639, + 0.4936, + 0.377, + 0.5947, + 1.1794, + 0.2495, + 0.21, + 0.0156, + 0.176, + 0.0085, + 0.1411, + 0.0575, + 0.1893, + 0.0274, + 0.3007, + 0.082, + 0.1469, + 0.2001 + ], + "6": [ + 0.0, + 0.1578, + 0.64, + 0.1327, + 0.3649, + 0.3032, + 0.5301, + 0.0324, + 9.7522, + 0.2442, + 0.4471, + 0.5226, + 0.456, + 1.4909, + 0.3214, + 0.2108, + 0.5973, + 0.4561, + 0.3009, + 0.0059, + 0.7393, + 0.9848, + 0.9602, + 0.4633, + 0.7733, + 0.9404, + 0.595, + 0.3906, + 0.8599 + ], + "7": [ + 0.0, + 0.1079, + 1.1562, + 0.1198, + 0.5827, + 1.2704, + 1.4556, + 1.0345, + 5.3725, + 1.0508, + 0.169, + 0.7688, + 0.3769, + 0.9427, + 0.655, + 0.0202, + 0.505, + 1.2538, + 0.8806, + 0.9291, + 1.4001, + 0.9472, + 0.5468, + 1.0995, + 1.01, + 0.9185, + 0.7724, + 0.7474, + 0.6189 + ], + "8": [ + 0.0, + 0.3833, + 0.6418, + 1.4002, + 0.4467, + 0.04, + 0.5252, + 0.3189, + 3.3791, + 0.4902, + 0.2131, + 0.3274, + 0.0638, + 0.3761, + 0.3604, + 0.4512, + 0.6752, + 0.0275, + 0.3179, + 0.0414, + 0.0804, + 0.4313, + 0.1602, + 0.7879, + 0.0829, + 1.0312, + 0.4992, + 0.1189, + 1.0988 + ], + "9": [ + 0.0, + 0.3921, + 0.6233, + 0.7583, + 0.1702, + 0.5024, + 0.518, + 0.743, + 2.4271, + 0.0648, + 1.0297, + 1.0224, + 0.7141, + 0.3263, + 0.369, + 0.6074, + 0.4483, + 0.2592, + 0.241, + 0.2233, + 0.9685, + 0.207, + 1.0168, + 0.2585, + 1.2886, + 1.1404, + 1.1788, + 1.9377, + 1.7215 + ] + } + } + }, + "typo": { + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.006631, + "jlens": 0.110448, + "shuffled": 0.056442, + "rand_match": 0.004781 + }, + "PRIMARY_ratio_jlens_over_logit": 16.6564, + "ratio_jlens_over_shuffled": 1.9568, + "ratio_jlens_over_rand_match": 23.1034, + "ratio_by_layer_jlens_over_logit": { + "5": 26.668, + "6": 47.4244, + "7": 26.8122, + "8": 9.869, + "9": 5.1864 + }, + "band_max_ratio_SECONDARY": 47.4244, + "per_layer": { + "logit": { + "5": { + "kl_mean": 0.006121, + "dose_mean_per_position": 0.3855, + "dose_sum_over_positions": 3.7449 + }, + "6": { + "kl_mean": 0.003016, + "dose_mean_per_position": 0.4259, + "dose_sum_over_positions": 4.1634 + }, + "7": { + "kl_mean": 0.003082, + "dose_mean_per_position": 0.4331, + "dose_sum_over_positions": 4.2075 + }, + "8": { + "kl_mean": 0.011695, + "dose_mean_per_position": 0.9197, + "dose_sum_over_positions": 8.9614 + }, + "9": { + "kl_mean": 0.009241, + "dose_mean_per_position": 0.9545, + "dose_sum_over_positions": 9.3248 + } + }, + "jlens": { + "5": { + "kl_mean": 0.163239, + "dose_mean_per_position": 1.0017, + "dose_sum_over_positions": 9.7839 + }, + "6": { + "kl_mean": 0.143011, + "dose_mean_per_position": 1.1861, + "dose_sum_over_positions": 11.6049 + }, + "7": { + "kl_mean": 0.082646, + "dose_mean_per_position": 1.1034, + "dose_sum_over_positions": 10.8302 + }, + "8": { + "kl_mean": 0.115416, + "dose_mean_per_position": 1.4641, + "dose_sum_over_positions": 14.3596 + }, + "9": { + "kl_mean": 0.047926, + "dose_mean_per_position": 1.4115, + "dose_sum_over_positions": 13.8335 + } + }, + "shuffled": { + "5": { + "kl_mean": 0.012163, + "dose_mean_per_position": 0.4309, + "dose_sum_over_positions": 4.1584 + }, + "6": { + "kl_mean": 0.078322, + "dose_mean_per_position": 1.3549, + "dose_sum_over_positions": 13.2151 + }, + "7": { + "kl_mean": 0.117061, + "dose_mean_per_position": 1.7843, + "dose_sum_over_positions": 17.415 + }, + "8": { + "kl_mean": 0.030847, + "dose_mean_per_position": 1.8422, + "dose_sum_over_positions": 17.9418 + }, + "9": { + "kl_mean": 0.043819, + "dose_mean_per_position": 2.1352, + "dose_sum_over_positions": 20.8593 + } + }, + "rand_match": { + "5": { + "kl_mean": 0.005573, + "dose_mean_per_position": 0.4275, + "dose_sum_over_positions": 4.2197 + }, + "6": { + "kl_mean": 0.003133, + "dose_mean_per_position": 0.4448, + "dose_sum_over_positions": 4.3116 + }, + "7": { + "kl_mean": 0.00589, + "dose_mean_per_position": 0.6192, + "dose_sum_over_positions": 5.9948 + }, + "8": { + "kl_mean": 0.005839, + "dose_mean_per_position": 0.6235, + "dose_sum_over_positions": 5.9829 + }, + "9": { + "kl_mean": 0.003468, + "dose_mean_per_position": 0.6753, + "dose_sum_over_positions": 6.5685 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "5": [ + 0.0, + 0.5424, + 0.4061, + 0.5657, + 0.1351, + 0.12, + 0.3324, + 0.7149, + 0.4381, + 0.0424, + 1.4036, + 0.3051 + ], + "6": [ + 0.0, + 0.4265, + 0.0348, + 0.0527, + 0.1028, + 0.0699, + 0.0978, + 0.3221, + 0.2395, + 0.4244, + 0.8263, + 0.4459 + ], + "7": [ + 0.0, + 0.7704, + 0.0677, + 0.2597, + 0.0046, + 0.2155, + 0.4361, + 0.1509, + 0.2932, + 0.7169, + 0.5398, + 0.536 + ], + "8": [ + 0.0, + 0.1945, + 0.8627, + 1.216, + 0.7978, + 1.2729, + 1.1485, + 0.9868, + 0.9081, + 1.3255, + 0.095, + 1.3884 + ], + "9": [ + 0.0, + 1.6047, + 0.5335, + 1.1981, + 1.1466, + 0.4004, + 1.5312, + 0.3185, + 0.1518, + 0.6433, + 0.5834, + 1.5332 + ] + }, + "jlens": { + "5": [ + 0.0, + 0.7182, + 0.9934, + 1.1579, + 1.0326, + 1.0314, + 1.068, + 1.7022, + 0.7877, + 1.0147, + 1.332, + 1.6488 + ], + "6": [ + 0.0, + 0.4496, + 1.275, + 1.7741, + 1.3645, + 1.1117, + 1.7081, + 2.0322, + 1.1545, + 1.6101, + 1.1063, + 2.2243 + ], + "7": [ + 0.0, + 0.0771, + 0.6516, + 1.505, + 1.3955, + 1.061, + 1.4768, + 1.8299, + 1.1751, + 1.9005, + 0.1198, + 2.3003 + ], + "8": [ + 0.0, + 0.0594, + 1.4036, + 2.1713, + 1.9617, + 1.7325, + 2.2784, + 2.7163, + 1.945, + 2.5848, + 0.1097, + 3.0959 + ], + "9": [ + 0.0, + 0.7846, + 0.2426, + 1.8965, + 1.4335, + 1.0185, + 1.9286, + 1.8673, + 1.6884, + 1.8098, + 1.0709, + 3.2332 + ] + }, + "shuffled": { + "5": [ + 0.0, + 0.1179, + 0.1552, + 0.3902, + 0.4292, + 0.1174, + 0.1691, + 0.2556, + 0.2793, + 0.5621, + 0.8216, + 0.8284 + ], + "6": [ + 0.0, + 1.0383, + 1.7589, + 1.7437, + 1.6117, + 1.5203, + 2.6528, + 2.564, + 1.4976, + 1.6327, + 1.7003, + 2.4294 + ], + "7": [ + 0.0, + 0.8643, + 1.752, + 2.0054, + 1.5767, + 1.7507, + 2.5735, + 3.0327, + 2.0919, + 2.4741, + 1.9034, + 2.8596 + ], + "8": [ + 0.0, + 1.8048, + 2.9363, + 2.594, + 2.4567, + 2.614, + 2.9466, + 3.6348, + 2.4837, + 3.3805, + 2.1312, + 3.0932 + ], + "9": [ + 0.0, + 1.4629, + 2.309, + 2.8107, + 2.4675, + 2.4889, + 2.6038, + 3.189, + 2.8251, + 3.6379, + 1.8626, + 3.3927 + ] + }, + "rand_match": { + "5": [ + 0.0, + 0.4184, + 0.0167, + 0.3286, + 0.1981, + 0.2791, + 0.3378, + 0.729, + 0.5387, + 0.3492, + 1.3054, + 0.4845 + ], + "6": [ + 0.0, + 0.4971, + 0.0541, + 0.0727, + 0.1267, + 0.5842, + 0.0872, + 0.0953, + 0.2122, + 0.2518, + 0.2634, + 0.2195 + ], + "7": [ + 0.0, + 0.3118, + 0.2298, + 1.2749, + 0.7874, + 1.6442, + 0.6567, + 1.1853, + 1.0415, + 0.6276, + 0.1379, + 0.3262 + ], + "8": [ + 0.0, + 0.1603, + 0.7968, + 0.4208, + 0.969, + 1.3683, + 1.2087, + 0.7111, + 0.7028, + 0.8946, + 0.823, + 0.6833 + ], + "9": [ + 0.0, + 2.511, + 1.7152, + 1.2486, + 0.7577, + 0.9495, + 0.955, + 0.4373, + 0.3795, + 0.6798, + 0.0386, + 1.5695 + ] + } + } + }, + "association": { + "n_items_scored": 102, + "n_intermediate_pairs_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "band_mean_kl": { + "logit": 0.443593, + "jlens": 0.115276, + "shuffled": 0.42062, + "rand_match": 0.913425 + }, + "PRIMARY_ratio_jlens_over_logit": 0.2599, + "ratio_jlens_over_shuffled": 0.2741, + "ratio_jlens_over_rand_match": 0.1262, + "ratio_by_layer_jlens_over_logit": { + "5": 0.3012, + "6": 0.1213, + "7": 0.1158, + "8": 0.0767, + "9": 0.5346 + }, + "band_max_ratio_SECONDARY": 0.5346, + "per_layer": { + "logit": { + "5": { + "kl_mean": 0.384134, + "dose_mean_per_position": 0.5726, + "dose_sum_over_positions": 16.5469 + }, + "6": { + "kl_mean": 0.436895, + "dose_mean_per_position": 0.6695, + "dose_sum_over_positions": 19.4597 + }, + "7": { + "kl_mean": 0.408944, + "dose_mean_per_position": 0.6412, + "dose_sum_over_positions": 18.5271 + }, + "8": { + "kl_mean": 0.366521, + "dose_mean_per_position": 1.085, + "dose_sum_over_positions": 31.4743 + }, + "9": { + "kl_mean": 0.621472, + "dose_mean_per_position": 1.2015, + "dose_sum_over_positions": 34.9584 + } + }, + "jlens": { + "5": { + "kl_mean": 0.115717, + "dose_mean_per_position": 1.1525, + "dose_sum_over_positions": 33.5598 + }, + "6": { + "kl_mean": 0.05298, + "dose_mean_per_position": 1.3821, + "dose_sum_over_positions": 40.294 + }, + "7": { + "kl_mean": 0.04734, + "dose_mean_per_position": 1.4404, + "dose_sum_over_positions": 41.9415 + }, + "8": { + "kl_mean": 0.028124, + "dose_mean_per_position": 1.8073, + "dose_sum_over_positions": 52.6446 + }, + "9": { + "kl_mean": 0.332221, + "dose_mean_per_position": 1.714, + "dose_sum_over_positions": 49.9208 + } + }, + "shuffled": { + "5": { + "kl_mean": 0.055714, + "dose_mean_per_position": 0.4747, + "dose_sum_over_positions": 13.9112 + }, + "6": { + "kl_mean": 0.4952, + "dose_mean_per_position": 1.6261, + "dose_sum_over_positions": 47.2741 + }, + "7": { + "kl_mean": 0.094836, + "dose_mean_per_position": 2.0817, + "dose_sum_over_positions": 60.63 + }, + "8": { + "kl_mean": 1.26751, + "dose_mean_per_position": 2.432, + "dose_sum_over_positions": 70.6072 + }, + "9": { + "kl_mean": 0.189841, + "dose_mean_per_position": 2.6344, + "dose_sum_over_positions": 76.6433 + } + }, + "rand_match": { + "5": { + "kl_mean": 0.910524, + "dose_mean_per_position": 0.7586, + "dose_sum_over_positions": 21.8911 + }, + "6": { + "kl_mean": 1.125371, + "dose_mean_per_position": 0.7892, + "dose_sum_over_positions": 22.7675 + }, + "7": { + "kl_mean": 0.875114, + "dose_mean_per_position": 0.8724, + "dose_sum_over_positions": 25.1665 + }, + "8": { + "kl_mean": 0.986674, + "dose_mean_per_position": 1.0414, + "dose_sum_over_positions": 29.986 + }, + "9": { + "kl_mean": 0.669444, + "dose_mean_per_position": 1.0086, + "dose_sum_over_positions": 28.8899 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "5": [ + 0.0, + 0.5377, + 0.0434, + 0.1311, + 0.9143, + 1.1297, + 0.7715, + 0.4376, + 1.1929, + 0.267, + 0.8024, + 0.5111, + 1.0843, + 0.5718, + 0.1123, + 0.0608, + 0.2738, + 0.5264, + 0.071, + 0.1123, + 0.2762, + 0.9417, + 0.1496, + 0.2462, + 0.1694, + 0.1825, + 0.0958, + 0.1576, + 8.076 + ], + "6": [ + 0.0, + 0.2855, + 0.7667, + 0.5277, + 0.2007, + 0.1006, + 0.4026, + 0.2082, + 0.0798, + 0.8658, + 0.6715, + 0.0174, + 0.3013, + 0.4381, + 0.5369, + 1.049, + 0.7365, + 1.3091, + 0.6792, + 0.5727, + 0.2086, + 1.412, + 0.5695, + 0.566, + 1.5624, + 1.6445, + 0.5809, + 1.1132, + 8.3536 + ], + "7": [ + 0.0, + 0.4148, + 0.6525, + 0.9111, + 0.312, + 0.0379, + 0.1826, + 0.1455, + 0.0789, + 1.027, + 0.086, + 0.4506, + 0.2493, + 0.6972, + 1.0868, + 0.3613, + 0.2636, + 1.0395, + 1.0317, + 0.6682, + 0.5738, + 1.5531, + 0.6365, + 0.2238, + 1.1213, + 1.0327, + 0.6693, + 1.1967, + 8.1654 + ], + "8": [ + 0.0, + 0.1722, + 1.5558, + 1.943, + 0.1774, + 0.17, + 0.9372, + 0.3361, + 0.8691, + 1.8001, + 0.1661, + 0.3388, + 0.3387, + 1.3696, + 1.5483, + 0.4947, + 0.2517, + 1.3211, + 1.6014, + 1.807, + 0.7609, + 1.582, + 0.5043, + 0.8378, + 2.1756, + 1.8151, + 1.0316, + 1.8986, + 8.0603 + ], + "9": [ + 0.0, + 0.5295, + 1.5694, + 1.8435, + 1.013, + 0.3109, + 0.0576, + 0.31, + 0.8912, + 1.3162, + 0.2598, + 0.2187, + 0.3411, + 1.581, + 1.6394, + 0.2575, + 0.0768, + 0.8842, + 1.587, + 2.2746, + 0.3388, + 1.8217, + 1.1541, + 0.3639, + 1.5843, + 0.4498, + 1.303, + 1.4285, + 9.9823 + ] + }, + "jlens": { + "5": [ + 0.0, + 0.6916, + 1.4904, + 0.9889, + 0.9457, + 1.1442, + 0.731, + 1.2859, + 1.1179, + 1.6669, + 1.3974, + 1.458, + 1.1545, + 1.4685, + 1.4915, + 1.6189, + 1.6309, + 1.5582, + 1.8763, + 1.2906, + 0.7803, + 1.7348, + 1.0663, + 1.5265, + 1.4125, + 0.9029, + 0.932, + 1.2191, + 2.8398 + ], + "6": [ + 0.0, + 0.3291, + 1.8266, + 1.477, + 1.2891, + 1.5933, + 1.5133, + 2.0907, + 1.8904, + 1.6742, + 1.5938, + 1.7541, + 1.0758, + 1.9574, + 1.2432, + 1.5806, + 1.9458, + 2.2525, + 1.7284, + 1.714, + 1.231, + 1.7099, + 1.5249, + 1.9532, + 2.0273, + 1.5151, + 0.9217, + 1.8524, + 0.0737 + ], + "7": [ + 0.0, + 0.2777, + 2.355, + 2.1383, + 0.8974, + 1.5921, + 1.7683, + 2.1857, + 1.6447, + 2.1851, + 1.2316, + 1.2864, + 0.8908, + 1.5041, + 1.3395, + 2.1696, + 1.7206, + 2.2554, + 2.4143, + 1.8706, + 1.3714, + 1.9206, + 1.4501, + 1.9602, + 1.929, + 1.8834, + 1.0771, + 1.5529, + 1.3476 + ], + "8": [ + 0.0, + 0.0166, + 2.7991, + 2.6, + 1.2327, + 1.7403, + 2.1967, + 2.1167, + 2.1534, + 2.941, + 1.4964, + 1.3377, + 0.9391, + 2.045, + 1.8393, + 2.057, + 1.6684, + 2.1768, + 2.9674, + 2.7159, + 1.7726, + 2.3836, + 1.8326, + 1.9328, + 2.5862, + 2.0452, + 1.2752, + 2.1531, + 0.7167 + ], + "9": [ + 0.0, + 0.7228, + 2.3518, + 2.1978, + 0.4282, + 1.1972, + 1.2557, + 1.0366, + 1.8377, + 2.239, + 0.8837, + 1.3444, + 0.6568, + 2.5406, + 1.3196, + 1.4118, + 1.4912, + 1.61, + 2.499, + 2.9777, + 1.3956, + 2.0028, + 1.566, + 1.2118, + 2.5286, + 1.3339, + 1.4415, + 1.9825, + 4.8826 + ] + }, + "shuffled": { + "5": [ + 0.0, + 0.4536, + 0.6371, + 0.3625, + 0.0307, + 0.0749, + 0.0542, + 0.6333, + 0.2508, + 0.7727, + 0.706, + 0.6775, + 0.1725, + 0.4661, + 0.005, + 0.6442, + 0.5742, + 0.69, + 0.4709, + 0.2863, + 0.008, + 1.275, + 0.136, + 0.6607, + 1.1277, + 0.606, + 0.2065, + 0.7454, + 2.1219 + ], + "6": [ + 0.0, + 1.0808, + 1.6288, + 1.6416, + 1.9524, + 1.8325, + 1.4649, + 1.7656, + 1.7911, + 1.8128, + 1.4507, + 2.3738, + 1.4047, + 2.0518, + 1.5543, + 1.4738, + 2.488, + 2.1283, + 1.9005, + 1.4594, + 1.1204, + 1.9582, + 1.552, + 2.0333, + 2.3324, + 1.0603, + 1.5385, + 1.8919, + 6.2319 + ], + "7": [ + 0.0, + 1.2018, + 2.8895, + 2.6564, + 1.7403, + 2.0881, + 1.7706, + 2.519, + 2.2916, + 2.8413, + 1.8991, + 2.1071, + 2.05, + 2.4429, + 1.9652, + 2.7997, + 2.5488, + 2.7748, + 3.2142, + 2.2683, + 2.0142, + 2.6468, + 2.2451, + 2.8004, + 2.4945, + 2.1543, + 2.045, + 1.6874, + 2.9089 + ], + "8": [ + 0.0, + 1.4853, + 2.577, + 2.2365, + 2.2266, + 2.5164, + 1.5517, + 1.7626, + 2.292, + 2.946, + 2.0578, + 2.3414, + 1.3683, + 2.2748, + 2.1683, + 2.1211, + 2.3329, + 2.3144, + 2.5921, + 2.9718, + 2.1403, + 2.0716, + 1.4812, + 2.71, + 2.5559, + 1.6466, + 1.9069, + 2.1065, + 12.4588 + ], + "9": [ + 0.0, + 1.255, + 3.7578, + 3.2504, + 3.0634, + 3.2914, + 2.6096, + 1.9253, + 2.916, + 3.2956, + 2.0297, + 1.8748, + 2.0241, + 3.0154, + 1.9332, + 3.1905, + 3.0021, + 3.1613, + 3.3942, + 3.9731, + 2.5312, + 2.4356, + 3.0781, + 3.6016, + 3.4523, + 2.5169, + 2.8898, + 3.0035, + 4.2207 + ] + }, + "rand_match": { + "5": [ + 0.0, + 0.1321, + 0.0151, + 0.333, + 0.0412, + 0.1609, + 0.4831, + 0.5438, + 0.2969, + 0.457, + 0.5434, + 0.3725, + 0.2812, + 0.0336, + 0.5332, + 0.3453, + 0.0857, + 0.368, + 0.4504, + 0.0336, + 0.1406, + 0.139, + 0.1909, + 0.1739, + 0.0626, + 0.3383, + 0.9532, + 0.6601, + 9.8835 + ], + "6": [ + 0.0, + 1.2204, + 1.101, + 1.1397, + 0.9258, + 1.2187, + 0.9025, + 0.9706, + 1.2333, + 0.7876, + 1.0394, + 0.9884, + 0.1221, + 0.5195, + 0.5852, + 0.6718, + 0.7556, + 0.6066, + 0.7108, + 0.3473, + 0.5392, + 0.5839, + 0.8624, + 0.2845, + 0.4904, + 0.0789, + 0.8981, + 0.1085, + 2.8927 + ], + "7": [ + 0.0, + 0.7098, + 0.2778, + 0.127, + 0.0707, + 0.1766, + 0.5008, + 0.0546, + 0.2268, + 0.6824, + 0.1801, + 0.4654, + 0.1378, + 0.141, + 0.0252, + 0.2983, + 0.3885, + 0.9662, + 0.1385, + 0.1076, + 0.3272, + 0.4767, + 0.0719, + 0.3819, + 0.8431, + 0.0629, + 0.7621, + 0.1468, + 1.2413 + ], + "8": [ + 0.0, + 0.4177, + 1.6813, + 1.6777, + 1.1379, + 0.7606, + 1.2394, + 0.8342, + 0.8362, + 0.3398, + 0.271, + 0.0973, + 0.6726, + 0.4159, + 0.1457, + 0.0178, + 0.1738, + 0.0991, + 0.3471, + 1.1322, + 0.8079, + 0.0617, + 0.1061, + 0.3636, + 0.1281, + 0.3106, + 0.9777, + 0.3321, + 13.0073 + ], + "9": [ + 0.0, + 0.4809, + 0.1601, + 0.0576, + 0.2326, + 0.107, + 0.8798, + 1.5209, + 0.2015, + 0.1177, + 0.5471, + 0.4184, + 0.5388, + 0.0296, + 0.0427, + 0.4891, + 0.5042, + 0.2077, + 0.082, + 0.2627, + 0.534, + 0.6538, + 0.5758, + 0.2679, + 0.1077, + 0.6574, + 0.9019, + 0.0821, + 0.8763 + ] + } + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "argv": [ + "t20_ablation_kl.py", + "--model", + "EleutherAI/pythia-160m-deduped", + "--lens", + "results/lens_160m_n200_db128_pen.pt", + "--evals", + "all", + "--out", + "results/t20_ablation_kl_160m.json" + ], + "runtime_s": 737.5, + "provenance": { + "payload_sha256_before_redaction": "c767c5cf4863b6d46e9b3a22d872db8365bdd51e975e4b5baebc6c3088a76081", + "payload_sha256": "2be674957f58dcbbc0795fc8385b4ab4225b861ee906b20ee07f0d5c58d2239c", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t20_ablation_kl_410m.json b/results/t20_ablation_kl_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..d571e8d7ca0a9b7a9d54bf9ff72b64ec05043c4d --- /dev/null +++ b/results/t20_ablation_kl_410m.json @@ -0,0 +1,8378 @@ +{ + "experiment": "T20 / E20 \u2014 ablation KL, the anchor's Figure 53 DV", + "status": "EXPLORATORY", + "prereg": null, + "prereg_sha256": null, + "decision_rule": "ratio KL(J^P)/KL(I), band mean, on multihop: >1.5 reproduces the anchor; <=1.0 falsifies at this scale; (1.0,1.5] inconclusive", + "primary_outcome": "band-mean ratio KL(jlens)/KL(logit) per set", + "layer_protocol": "one layer at a time, swept over the band; NO band compounding. Headline is the band MEAN of the per-layer ratio; band-max is stored as a secondary and is explicitly not primary (existential statistic).", + "cross_lens_caveat": "the logit lens has no lens-native coordinate write (external review B5/G6); applied here it is a SHARED hidden-state intervention parameterised by whichever lens supplies v_c", + "VERDICT": "REPRODUCES \u2014 multihop ratio 2.21 > 1.5, the anchor's ~2x claim holds here", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/lens_410m_n200_db128_pen.pt", + "lens_sha256": "4bad084ffea3a88708029f5b916bc9bc56a46600532ede1bd0604947ea45872a", + "n_layers": 24, + "source_layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "layers_arg": "band", + "skip_positions": 1, + "max_items_effective": null, + "min_pairs_floor": 30, + "evals": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo", + "association" + ], + "arms": [ + "logit", + "jlens", + "shuffled", + "rand_match" + ], + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "device": "cpu", + "n_forward_passes": 46987, + "results": { + "multihop": { + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.017073, + "jlens": 0.037733, + "shuffled": 0.021937, + "rand_match": 0.001772 + }, + "PRIMARY_ratio_jlens_over_logit": 2.21, + "ratio_jlens_over_shuffled": 1.72, + "ratio_jlens_over_rand_match": 21.2976, + "ratio_by_layer_jlens_over_logit": { + "9": 36.0438, + "10": 23.9178, + "11": 26.871, + "12": 20.404, + "13": 12.962, + "14": 8.4969, + "15": 5.4321, + "16": 2.7099, + "17": 1.3044, + "18": 1.4656, + "19": 1.1501, + "20": 1.3711, + "21": 1.2186 + }, + "band_max_ratio_SECONDARY": 36.0438, + "per_layer": { + "logit": { + "9": { + "kl_mean": 0.000933, + "dose_mean_per_position": 0.4776, + "dose_sum_over_positions": 10.0099 + }, + "10": { + "kl_mean": 0.001182, + "dose_mean_per_position": 0.4864, + "dose_sum_over_positions": 10.2226 + }, + "11": { + "kl_mean": 0.001068, + "dose_mean_per_position": 0.4306, + "dose_sum_over_positions": 9.2082 + }, + "12": { + "kl_mean": 0.001378, + "dose_mean_per_position": 0.469, + "dose_sum_over_positions": 9.9927 + }, + "13": { + "kl_mean": 0.00193, + "dose_mean_per_position": 0.517, + "dose_sum_over_positions": 11.0497 + }, + "14": { + "kl_mean": 0.003774, + "dose_mean_per_position": 0.6112, + "dose_sum_over_positions": 12.9445 + }, + "15": { + "kl_mean": 0.006322, + "dose_mean_per_position": 0.6733, + "dose_sum_over_positions": 14.6124 + }, + "16": { + "kl_mean": 0.011387, + "dose_mean_per_position": 0.8946, + "dose_sum_over_positions": 19.6286 + }, + "17": { + "kl_mean": 0.029213, + "dose_mean_per_position": 1.3282, + "dose_sum_over_positions": 28.267 + }, + "18": { + "kl_mean": 0.029716, + "dose_mean_per_position": 1.3093, + "dose_sum_over_positions": 27.1731 + }, + "19": { + "kl_mean": 0.04242, + "dose_mean_per_position": 1.5368, + "dose_sum_over_positions": 31.3871 + }, + "20": { + "kl_mean": 0.040647, + "dose_mean_per_position": 1.6732, + "dose_sum_over_positions": 33.5284 + }, + "21": { + "kl_mean": 0.051984, + "dose_mean_per_position": 1.7027, + "dose_sum_over_positions": 35.1288 + } + }, + "jlens": { + "9": { + "kl_mean": 0.033636, + "dose_mean_per_position": 1.1448, + "dose_sum_over_positions": 24.4161 + }, + "10": { + "kl_mean": 0.028274, + "dose_mean_per_position": 1.1163, + "dose_sum_over_positions": 23.8402 + }, + "11": { + "kl_mean": 0.028685, + "dose_mean_per_position": 1.139, + "dose_sum_over_positions": 24.8031 + }, + "12": { + "kl_mean": 0.028126, + "dose_mean_per_position": 1.2076, + "dose_sum_over_positions": 26.5718 + }, + "13": { + "kl_mean": 0.025021, + "dose_mean_per_position": 1.2517, + "dose_sum_over_positions": 27.8563 + }, + "14": { + "kl_mean": 0.032064, + "dose_mean_per_position": 1.2636, + "dose_sum_over_positions": 27.4534 + }, + "15": { + "kl_mean": 0.034342, + "dose_mean_per_position": 1.2754, + "dose_sum_over_positions": 27.3733 + }, + "16": { + "kl_mean": 0.030857, + "dose_mean_per_position": 1.143, + "dose_sum_over_positions": 24.1749 + }, + "17": { + "kl_mean": 0.038106, + "dose_mean_per_position": 1.2349, + "dose_sum_over_positions": 25.5478 + }, + "18": { + "kl_mean": 0.04355, + "dose_mean_per_position": 1.2049, + "dose_sum_over_positions": 24.1017 + }, + "19": { + "kl_mean": 0.048787, + "dose_mean_per_position": 1.3037, + "dose_sum_over_positions": 25.9716 + }, + "20": { + "kl_mean": 0.05573, + "dose_mean_per_position": 1.4469, + "dose_sum_over_positions": 29.0745 + }, + "21": { + "kl_mean": 0.063347, + "dose_mean_per_position": 1.5679, + "dose_sum_over_positions": 32.3822 + } + }, + "shuffled": { + "9": { + "kl_mean": 0.028867, + "dose_mean_per_position": 1.0889, + "dose_sum_over_positions": 23.6574 + }, + "10": { + "kl_mean": 0.019289, + "dose_mean_per_position": 0.9983, + "dose_sum_over_positions": 21.7636 + }, + "11": { + "kl_mean": 0.001005, + "dose_mean_per_position": 0.411, + "dose_sum_over_positions": 8.8286 + }, + "12": { + "kl_mean": 0.021607, + "dose_mean_per_position": 1.1745, + "dose_sum_over_positions": 26.0845 + }, + "13": { + "kl_mean": 0.019531, + "dose_mean_per_position": 1.0935, + "dose_sum_over_positions": 23.6092 + }, + "14": { + "kl_mean": 0.025035, + "dose_mean_per_position": 1.3379, + "dose_sum_over_positions": 29.1566 + }, + "15": { + "kl_mean": 0.010668, + "dose_mean_per_position": 0.7057, + "dose_sum_over_positions": 14.9647 + }, + "16": { + "kl_mean": 0.013007, + "dose_mean_per_position": 0.763, + "dose_sum_over_positions": 16.3309 + }, + "17": { + "kl_mean": 0.01749, + "dose_mean_per_position": 1.6242, + "dose_sum_over_positions": 33.6422 + }, + "18": { + "kl_mean": 0.005319, + "dose_mean_per_position": 1.3391, + "dose_sum_over_positions": 28.8869 + }, + "19": { + "kl_mean": 0.018693, + "dose_mean_per_position": 1.93, + "dose_sum_over_positions": 39.7942 + }, + "20": { + "kl_mean": 0.045586, + "dose_mean_per_position": 2.3043, + "dose_sum_over_positions": 48.9559 + }, + "21": { + "kl_mean": 0.059089, + "dose_mean_per_position": 1.457, + "dose_sum_over_positions": 30.0715 + } + }, + "rand_match": { + "9": { + "kl_mean": 0.001044, + "dose_mean_per_position": 0.4619, + "dose_sum_over_positions": 9.7499 + }, + "10": { + "kl_mean": 0.000973, + "dose_mean_per_position": 0.4726, + "dose_sum_over_positions": 9.9745 + }, + "11": { + "kl_mean": 0.001125, + "dose_mean_per_position": 0.4629, + "dose_sum_over_positions": 9.8506 + }, + "12": { + "kl_mean": 0.000943, + "dose_mean_per_position": 0.4874, + "dose_sum_over_positions": 10.5258 + }, + "13": { + "kl_mean": 0.001071, + "dose_mean_per_position": 0.5237, + "dose_sum_over_positions": 11.5831 + }, + "14": { + "kl_mean": 0.001048, + "dose_mean_per_position": 0.555, + "dose_sum_over_positions": 11.9466 + }, + "15": { + "kl_mean": 0.001454, + "dose_mean_per_position": 0.653, + "dose_sum_over_positions": 14.3897 + }, + "16": { + "kl_mean": 0.002146, + "dose_mean_per_position": 0.7587, + "dose_sum_over_positions": 16.187 + }, + "17": { + "kl_mean": 0.001764, + "dose_mean_per_position": 0.8514, + "dose_sum_over_positions": 19.078 + }, + "18": { + "kl_mean": 0.002499, + "dose_mean_per_position": 1.1121, + "dose_sum_over_positions": 25.2187 + }, + "19": { + "kl_mean": 0.002393, + "dose_mean_per_position": 1.1027, + "dose_sum_over_positions": 23.6826 + }, + "20": { + "kl_mean": 0.003265, + "dose_mean_per_position": 1.146, + "dose_sum_over_positions": 25.3469 + }, + "21": { + "kl_mean": 0.003305, + "dose_mean_per_position": 1.2056, + "dose_sum_over_positions": 24.7324 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "9": [ + 0.0, + 0.7127, + 0.2895, + 0.2021, + 0.6804, + 0.4557, + 0.7323, + 1.3041, + 0.7567, + 0.8888, + 1.1986, + 0.5514, + 0.072, + 0.1894, + 1.0631, + 1.4128, + 1.5534, + 1.3805, + 0.5787, + 0.977 + ], + "10": [ + 0.0, + 0.3011, + 0.0339, + 0.3693, + 1.0242, + 0.5959, + 0.8567, + 1.2923, + 0.5874, + 0.8644, + 1.4129, + 0.8013, + 0.1481, + 0.4009, + 0.95, + 1.0777, + 1.5315, + 1.2466, + 0.8232, + 0.6973 + ], + "11": [ + 0.0, + 0.2226, + 0.0262, + 0.1562, + 0.7737, + 0.2583, + 0.4576, + 0.9817, + 0.2955, + 0.7364, + 1.1723, + 0.8563, + 0.3032, + 0.4267, + 1.1113, + 1.1521, + 1.2663, + 1.7982, + 0.8483, + 0.4774 + ], + "12": [ + 0.0, + 0.3691, + 0.0646, + 0.1264, + 0.7642, + 0.0044, + 0.025, + 0.8051, + 0.2697, + 0.0444, + 1.0017, + 0.2459, + 0.1259, + 0.4548, + 1.0542, + 0.9454, + 1.1867, + 1.2953, + 0.9003, + 0.3545 + ], + "13": [ + 0.0, + 0.0632, + 0.1537, + 0.386, + 1.0964, + 0.1872, + 0.5163, + 1.3974, + 0.2312, + 0.4838, + 1.6091, + 0.9512, + 0.2827, + 0.8112, + 1.4614, + 1.0243, + 1.6069, + 2.0086, + 1.0678, + 0.7769 + ], + "14": [ + 0.0, + 0.2371, + 0.1713, + 0.3396, + 0.8166, + 0.3778, + 0.4905, + 1.4559, + 0.0987, + 0.446, + 1.8293, + 1.3569, + 0.0514, + 0.6403, + 1.4434, + 1.0236, + 1.6448, + 2.2244, + 1.2975, + 1.1241 + ], + "15": [ + 0.0, + 0.0751, + 0.2941, + 0.6535, + 1.1319, + 0.6409, + 0.6418, + 1.5586, + 0.0643, + 0.4912, + 1.6976, + 1.2447, + 0.8621, + 1.2706, + 1.1589, + 0.8746, + 1.4772, + 2.1577, + 0.9922, + 1.1951 + ], + "16": [ + 0.0, + 0.3085, + 0.8853, + 1.4699, + 1.4812, + 1.0692, + 0.9588, + 1.7393, + 0.6502, + 0.8313, + 1.8946, + 1.2368, + 0.7537, + 0.8144, + 1.4007, + 1.2434, + 1.8943, + 2.4459, + 1.3181, + 1.4176 + ], + "17": [ + 0.0, + 0.751, + 1.5137, + 2.0881, + 2.0939, + 0.9248, + 1.3385, + 2.5602, + 0.4297, + 1.3709, + 2.3102, + 0.8865, + 0.8749, + 0.2697, + 1.6177, + 1.801, + 2.5433, + 2.6368, + 1.08, + 1.0556 + ], + "18": [ + 0.0, + 0.7003, + 1.5359, + 1.8499, + 1.9713, + 0.8405, + 1.2819, + 2.5024, + 0.5814, + 1.585, + 2.3884, + 0.5989, + 0.741, + 0.3046, + 1.5591, + 1.4178, + 2.3344, + 2.3212, + 0.2543, + 0.4462 + ], + "19": [ + 0.0, + 0.7605, + 2.2199, + 2.6987, + 2.1876, + 0.849, + 0.9346, + 2.059, + 1.4478, + 1.1872, + 1.6847, + 0.9454, + 0.8894, + 0.1349, + 1.0244, + 1.4753, + 2.0618, + 1.9189, + 0.5691, + 0.3753 + ], + "20": [ + 0.0, + 1.3715, + 2.8427, + 3.4087, + 2.4858, + 1.6096, + 1.7445, + 2.3386, + 1.2052, + 1.6583, + 2.0099, + 1.7462, + 1.5975, + 0.8992, + 1.3962, + 1.8873, + 2.5152, + 2.3513, + 0.1184, + 0.6076 + ], + "21": [ + 0.0, + 1.4501, + 1.2725, + 2.3571, + 1.824, + 0.8427, + 1.7722, + 0.111, + 1.2569, + 1.7643, + 0.7287, + 0.3754, + 1.5281, + 0.152, + 0.6155, + 1.153, + 1.9751, + 1.1304, + 0.7562, + 0.5689 + ] + }, + "jlens": { + "9": [ + 0.0, + 0.7264, + 0.5119, + 0.9403, + 0.4496, + 0.1838, + 0.3882, + 0.342, + 0.1128, + 0.4853, + 0.815, + 0.6581, + 0.2119, + 1.4188, + 0.8769, + 1.0549, + 0.0654, + 0.4949, + 0.3721, + 0.6428 + ], + "10": [ + 0.0, + 0.3474, + 0.32, + 0.7456, + 0.3258, + 0.0392, + 0.1583, + 0.2623, + 0.0592, + 0.4034, + 0.5445, + 0.4496, + 0.2336, + 1.4336, + 0.7943, + 0.7689, + 0.0069, + 0.1782, + 0.2647, + 0.3737 + ], + "11": [ + 0.0, + 0.3044, + 0.3518, + 0.6502, + 0.3572, + 0.1353, + 0.0079, + 0.336, + 0.025, + 0.2537, + 0.5875, + 0.4301, + 0.1394, + 1.1718, + 0.5542, + 0.8276, + 0.0243, + 0.1957, + 0.6605, + 0.4333 + ], + "12": [ + 0.0, + 0.1038, + 0.3977, + 0.5376, + 0.1272, + 0.1951, + 0.0578, + 0.3623, + 0.2329, + 0.1785, + 0.6808, + 0.3701, + 0.5041, + 1.039, + 0.5217, + 0.5682, + 0.226, + 0.3571, + 0.7472, + 0.522 + ], + "13": [ + 0.0, + 0.0778, + 0.6658, + 0.6924, + 0.3311, + 0.254, + 0.0568, + 0.3873, + 0.3723, + 0.029, + 0.5208, + 0.2949, + 0.5204, + 0.8381, + 0.822, + 0.9266, + 0.7777, + 0.4728, + 0.6625, + 0.6249 + ], + "14": [ + 0.0, + 0.3226, + 0.5505, + 0.5241, + 0.519, + 0.0357, + 0.209, + 0.2943, + 0.6258, + 0.1473, + 0.0689, + 0.2954, + 0.5568, + 1.0864, + 1.1112, + 1.1158, + 1.0457, + 0.5849, + 0.6418, + 0.5337 + ], + "15": [ + 0.0, + 0.432, + 0.4712, + 0.442, + 0.0948, + 0.1306, + 0.2888, + 0.5154, + 0.6219, + 0.2679, + 0.4474, + 0.1026, + 0.3287, + 0.4623, + 0.9669, + 0.9138, + 0.9955, + 0.9051, + 0.403, + 0.6706 + ], + "16": [ + 0.0, + 0.7712, + 0.617, + 0.7881, + 0.419, + 0.1726, + 0.5531, + 0.6973, + 1.33, + 0.2798, + 0.3876, + 0.4338, + 0.5054, + 0.3656, + 1.0025, + 1.0743, + 1.1861, + 0.6965, + 0.1386, + 0.7085 + ], + "17": [ + 0.0, + 0.9969, + 0.9905, + 1.0263, + 0.7132, + 0.2186, + 0.6563, + 1.0359, + 1.2968, + 0.3881, + 0.1123, + 0.0954, + 0.9802, + 0.2253, + 0.439, + 0.9483, + 1.1705, + 0.8918, + 0.2912, + 0.4408 + ], + "18": [ + 0.0, + 1.3176, + 1.0698, + 1.3249, + 1.0637, + 0.4819, + 1.151, + 1.3193, + 1.6728, + 0.4934, + 0.0286, + 0.2247, + 1.1665, + 0.3247, + 0.7401, + 1.1252, + 1.4864, + 1.1689, + 0.4785, + 0.455 + ], + "19": [ + 0.0, + 1.2564, + 1.0062, + 1.4579, + 1.4932, + 0.4143, + 1.0123, + 1.3697, + 2.4666, + 0.2272, + 0.0926, + 0.1502, + 1.7596, + 0.2345, + 0.8673, + 1.6228, + 1.8716, + 1.0463, + 0.9396, + 0.0837 + ], + "20": [ + 0.0, + 1.2962, + 1.0585, + 1.7146, + 1.4049, + 0.2159, + 1.0362, + 0.9074, + 2.747, + 0.3159, + 0.211, + 0.1731, + 1.6301, + 0.0389, + 0.4074, + 0.87, + 1.4571, + 1.0307, + 1.1954, + 0.3289 + ], + "21": [ + 0.0, + 1.1944, + 0.5841, + 1.5455, + 1.073, + 0.0391, + 0.9369, + 0.2656, + 2.6428, + 0.675, + 0.0604, + 0.6864, + 1.3204, + 0.34, + 0.2243, + 0.045, + 0.9881, + 0.2448, + 1.4033, + 0.095 + ] + }, + "shuffled": { + "9": [ + 0.0, + 0.1392, + 0.4413, + 0.6615, + 0.4201, + 0.0568, + 0.1378, + 0.3953, + 0.2071, + 0.3952, + 0.4891, + 0.6856, + 0.2898, + 1.5056, + 0.8528, + 0.981, + 0.1805, + 0.3595, + 0.2912, + 0.5385 + ], + "10": [ + 0.0, + 0.1224, + 0.1114, + 0.5952, + 0.9827, + 0.324, + 0.0297, + 0.357, + 0.0679, + 0.0216, + 0.9128, + 0.5247, + 0.7105, + 1.7676, + 0.2569, + 0.3825, + 0.6889, + 0.3521, + 0.0251, + 0.0111 + ], + "11": [ + 0.0, + 0.1947, + 0.1769, + 0.3036, + 0.8502, + 0.2567, + 0.6009, + 1.0973, + 0.4331, + 0.7023, + 0.9229, + 0.9966, + 0.8678, + 1.0136, + 1.4176, + 1.7468, + 1.6066, + 1.7459, + 1.0318, + 0.7223 + ], + "12": [ + 0.0, + 0.7278, + 0.9714, + 0.9576, + 0.2289, + 0.3904, + 0.5217, + 0.4835, + 0.3871, + 0.5249, + 0.8042, + 0.336, + 0.0682, + 1.2557, + 0.8274, + 0.6027, + 0.5661, + 0.5794, + 0.8512, + 0.597 + ], + "13": [ + 0.0, + 0.0659, + 0.4844, + 0.5084, + 0.6299, + 0.3757, + 0.0079, + 0.191, + 0.5896, + 0.0886, + 0.6027, + 0.336, + 0.4605, + 1.3317, + 0.9583, + 0.9262, + 0.7262, + 0.4719, + 0.598, + 0.5174 + ], + "14": [ + 0.0, + 0.6663, + 0.7023, + 0.6001, + 0.5653, + 0.008, + 0.0437, + 0.4313, + 0.4534, + 0.1116, + 0.3326, + 0.2836, + 0.2036, + 0.7915, + 0.3573, + 0.3381, + 0.5131, + 0.0788, + 0.3111, + 0.2654 + ], + "15": [ + 0.0, + 0.8965, + 0.4461, + 0.6653, + 0.4651, + 0.4945, + 0.6779, + 0.5233, + 0.5228, + 0.2743, + 0.2085, + 0.72, + 0.7775, + 1.1288, + 1.0851, + 1.4732, + 1.4531, + 1.3295, + 0.4915, + 1.026 + ], + "16": [ + 0.0, + 0.097, + 0.0017, + 0.7689, + 0.5648, + 0.4966, + 0.465, + 1.0166, + 1.3297, + 0.2521, + 0.8068, + 0.5844, + 0.3759, + 0.4589, + 0.6863, + 0.5435, + 1.2991, + 1.5871, + 0.246, + 0.6835 + ], + "17": [ + 0.0, + 0.4779, + 0.1889, + 0.2993, + 1.3311, + 0.9894, + 0.9765, + 0.5134, + 1.7154, + 1.0961, + 1.4746, + 2.1737, + 0.4861, + 2.5496, + 1.3925, + 1.5818, + 1.3098, + 0.4912, + 1.6399, + 2.0847 + ], + "18": [ + 0.0, + 0.7683, + 0.3623, + 0.0532, + 1.1941, + 0.4567, + 0.9509, + 0.6164, + 0.2368, + 0.7954, + 0.883, + 1.0314, + 1.8878, + 2.1817, + 1.0853, + 1.1609, + 1.4479, + 0.7349, + 1.2997, + 1.62 + ], + "19": [ + 0.0, + 0.3533, + 0.0456, + 0.1271, + 0.1078, + 1.6131, + 1.1082, + 0.6711, + 1.2691, + 0.9004, + 1.3664, + 2.8186, + 1.0141, + 1.7807, + 1.3659, + 1.6601, + 1.8101, + 1.2389, + 2.2115, + 2.306 + ], + "20": [ + 0.0, + 0.5409, + 0.41, + 1.1879, + 0.337, + 2.3913, + 2.4421, + 1.247, + 3.1346, + 2.4693, + 2.2305, + 3.0862, + 0.4098, + 2.142, + 1.9862, + 1.2496, + 1.378, + 1.5065, + 3.6642, + 3.6228 + ], + "21": [ + 0.0, + 1.2887, + 0.2644, + 0.8879, + 0.9588, + 0.5223, + 0.592, + 0.1151, + 3.0835, + 0.0447, + 0.5595, + 1.2166, + 1.3925, + 0.4477, + 0.3437, + 0.043, + 0.8675, + 0.0743, + 1.6327, + 0.4897 + ] + }, + "rand_match": { + "9": [ + 0.0, + 0.507, + 0.9008, + 0.2944, + 0.936, + 0.1774, + 0.1031, + 0.3016, + 0.1821, + 0.1348, + 0.1157, + 0.9449, + 0.4288, + 0.0176, + 0.846, + 1.1964, + 0.2802, + 0.7611, + 0.4812, + 0.0492 + ], + "10": [ + 0.0, + 0.5388, + 0.5068, + 0.1457, + 0.6368, + 0.4331, + 0.0876, + 0.2017, + 0.4247, + 0.638, + 0.3389, + 0.261, + 0.0148, + 0.0637, + 0.0101, + 0.4791, + 0.1194, + 0.3614, + 0.2844, + 0.3478 + ], + "11": [ + 0.0, + 0.0312, + 0.2119, + 0.3003, + 0.0043, + 0.3044, + 0.3469, + 0.2626, + 0.4, + 0.1281, + 0.5105, + 0.1579, + 0.6394, + 0.1672, + 0.0649, + 0.163, + 0.2761, + 0.5278, + 0.2209, + 0.0028 + ], + "12": [ + 0.0, + 0.106, + 0.4952, + 0.7749, + 1.1153, + 0.4717, + 0.5585, + 0.9739, + 0.395, + 0.571, + 0.1661, + 0.3585, + 0.642, + 1.4656, + 0.8967, + 0.1882, + 0.6493, + 0.3779, + 0.0313, + 0.4937 + ], + "13": [ + 0.0, + 0.3699, + 0.0781, + 0.3388, + 0.6015, + 0.9849, + 1.1199, + 0.0714, + 0.7299, + 0.8861, + 0.6851, + 1.1211, + 0.5571, + 0.475, + 0.8269, + 1.2105, + 0.7727, + 0.7087, + 0.1628, + 0.6803 + ], + "14": [ + 0.0, + 0.8508, + 0.8806, + 0.8506, + 0.4768, + 0.5131, + 0.8375, + 0.1119, + 1.058, + 0.6541, + 0.2543, + 0.5733, + 0.2124, + 1.246, + 0.8574, + 0.5609, + 1.1698, + 0.0275, + 0.8716, + 0.5496 + ], + "15": [ + 0.0, + 0.111, + 0.1698, + 1.227, + 1.4087, + 0.7362, + 1.1087, + 1.2089, + 0.9148, + 1.2309, + 1.2934, + 0.897, + 1.474, + 2.4392, + 0.7733, + 0.2943, + 0.0384, + 1.5003, + 1.293, + 1.5778 + ], + "16": [ + 0.0, + 0.0552, + 0.5032, + 0.6545, + 0.551, + 0.5858, + 0.7107, + 0.5938, + 1.3319, + 1.0541, + 1.1715, + 1.4185, + 0.2971, + 0.7401, + 1.1277, + 0.2145, + 0.1969, + 0.4322, + 0.5429, + 0.5918 + ], + "17": [ + 0.0, + 1.8374, + 2.3725, + 2.412, + 1.6027, + 2.4101, + 2.4056, + 1.9833, + 2.2249, + 2.1413, + 2.3912, + 2.835, + 1.0275, + 1.1418, + 0.9927, + 1.5122, + 1.0544, + 0.7533, + 1.9316, + 2.3426 + ], + "18": [ + 0.0, + 1.4802, + 1.6641, + 1.4456, + 1.5166, + 0.2804, + 0.7313, + 1.795, + 1.282, + 1.385, + 1.959, + 0.1568, + 1.011, + 0.4988, + 1.0057, + 1.8734, + 1.4373, + 1.0463, + 0.107, + 0.1687 + ], + "19": [ + 0.0, + 1.8219, + 1.2584, + 1.589, + 0.7973, + 1.6807, + 1.11, + 0.6611, + 1.6414, + 1.4014, + 1.1569, + 0.3593, + 1.0811, + 0.345, + 0.389, + 2.4354, + 0.6575, + 0.4815, + 0.2486, + 1.2103 + ], + "20": [ + 0.0, + 1.1837, + 1.1424, + 2.1649, + 0.9698, + 0.684, + 0.1827, + 0.2192, + 1.7366, + 1.0689, + 1.0363, + 1.0932, + 0.1001, + 0.9125, + 0.8405, + 0.3196, + 0.6245, + 0.0016, + 0.1878, + 1.0534 + ], + "21": [ + 0.0, + 1.68, + 3.3456, + 3.0812, + 3.1153, + 2.5072, + 3.3116, + 2.8196, + 2.8505, + 2.9625, + 2.73, + 2.1665, + 1.3413, + 2.0085, + 1.6588, + 1.253, + 1.5638, + 3.3214, + 2.2687, + 3.3048 + ] + } + } + }, + "multilingual": { + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "band_mean_kl": { + "logit": 0.004615, + "jlens": 0.013061, + "shuffled": 0.007859, + "rand_match": 0.001346 + }, + "PRIMARY_ratio_jlens_over_logit": 2.8299, + "ratio_jlens_over_shuffled": 1.6618, + "ratio_jlens_over_rand_match": 9.7045, + "ratio_by_layer_jlens_over_logit": { + "9": 18.6663, + "10": 14.8722, + "11": 16.5987, + "12": 9.9632, + "13": 9.5749, + "14": 8.0809, + "15": 6.8742, + "16": 3.9891, + "17": 1.6928, + "18": 1.6789, + "19": 1.2451, + "20": 1.1021, + "21": 0.9307 + }, + "band_max_ratio_SECONDARY": 18.6663, + "per_layer": { + "logit": { + "9": { + "kl_mean": 0.000629, + "dose_mean_per_position": 0.4724, + "dose_sum_over_positions": 5.4661 + }, + "10": { + "kl_mean": 0.00083, + "dose_mean_per_position": 0.4792, + "dose_sum_over_positions": 5.5819 + }, + "11": { + "kl_mean": 0.000772, + "dose_mean_per_position": 0.4429, + "dose_sum_over_positions": 5.143 + }, + "12": { + "kl_mean": 0.000945, + "dose_mean_per_position": 0.4687, + "dose_sum_over_positions": 5.4328 + }, + "13": { + "kl_mean": 0.001227, + "dose_mean_per_position": 0.4982, + "dose_sum_over_positions": 5.8208 + }, + "14": { + "kl_mean": 0.001863, + "dose_mean_per_position": 0.5057, + "dose_sum_over_positions": 5.8607 + }, + "15": { + "kl_mean": 0.002474, + "dose_mean_per_position": 0.5063, + "dose_sum_over_positions": 5.882 + }, + "16": { + "kl_mean": 0.004159, + "dose_mean_per_position": 0.6218, + "dose_sum_over_positions": 7.172 + }, + "17": { + "kl_mean": 0.010842, + "dose_mean_per_position": 0.9831, + "dose_sum_over_positions": 11.0183 + }, + "18": { + "kl_mean": 0.008502, + "dose_mean_per_position": 0.9577, + "dose_sum_over_positions": 10.8 + }, + "19": { + "kl_mean": 0.009404, + "dose_mean_per_position": 1.0777, + "dose_sum_over_positions": 12.1402 + }, + "20": { + "kl_mean": 0.009676, + "dose_mean_per_position": 1.1964, + "dose_sum_over_positions": 13.6806 + }, + "21": { + "kl_mean": 0.008675, + "dose_mean_per_position": 1.1874, + "dose_sum_over_positions": 13.5545 + } + }, + "jlens": { + "9": { + "kl_mean": 0.011735, + "dose_mean_per_position": 1.1376, + "dose_sum_over_positions": 12.6591 + }, + "10": { + "kl_mean": 0.012343, + "dose_mean_per_position": 1.0883, + "dose_sum_over_positions": 12.0912 + }, + "11": { + "kl_mean": 0.012814, + "dose_mean_per_position": 1.0942, + "dose_sum_over_positions": 12.1579 + }, + "12": { + "kl_mean": 0.009417, + "dose_mean_per_position": 1.1162, + "dose_sum_over_positions": 12.3118 + }, + "13": { + "kl_mean": 0.011748, + "dose_mean_per_position": 1.1395, + "dose_sum_over_positions": 12.6466 + }, + "14": { + "kl_mean": 0.015055, + "dose_mean_per_position": 1.222, + "dose_sum_over_positions": 13.5868 + }, + "15": { + "kl_mean": 0.01701, + "dose_mean_per_position": 1.3342, + "dose_sum_over_positions": 14.8399 + }, + "16": { + "kl_mean": 0.01659, + "dose_mean_per_position": 1.1232, + "dose_sum_over_positions": 12.5497 + }, + "17": { + "kl_mean": 0.018353, + "dose_mean_per_position": 1.1572, + "dose_sum_over_positions": 12.9213 + }, + "18": { + "kl_mean": 0.014274, + "dose_mean_per_position": 1.0387, + "dose_sum_over_positions": 11.7218 + }, + "19": { + "kl_mean": 0.01171, + "dose_mean_per_position": 1.0366, + "dose_sum_over_positions": 11.7821 + }, + "20": { + "kl_mean": 0.010665, + "dose_mean_per_position": 1.0484, + "dose_sum_over_positions": 11.9684 + }, + "21": { + "kl_mean": 0.008074, + "dose_mean_per_position": 0.9974, + "dose_sum_over_positions": 11.3909 + } + }, + "shuffled": { + "9": { + "kl_mean": 0.009647, + "dose_mean_per_position": 1.0613, + "dose_sum_over_positions": 11.8204 + }, + "10": { + "kl_mean": 0.005722, + "dose_mean_per_position": 0.8519, + "dose_sum_over_positions": 9.4827 + }, + "11": { + "kl_mean": 0.000928, + "dose_mean_per_position": 0.4409, + "dose_sum_over_positions": 5.152 + }, + "12": { + "kl_mean": 0.008678, + "dose_mean_per_position": 1.0643, + "dose_sum_over_positions": 11.8554 + }, + "13": { + "kl_mean": 0.008823, + "dose_mean_per_position": 1.0265, + "dose_sum_over_positions": 11.4711 + }, + "14": { + "kl_mean": 0.011364, + "dose_mean_per_position": 1.2764, + "dose_sum_over_positions": 14.2007 + }, + "15": { + "kl_mean": 0.004817, + "dose_mean_per_position": 0.6204, + "dose_sum_over_positions": 7.1514 + }, + "16": { + "kl_mean": 0.004696, + "dose_mean_per_position": 0.5722, + "dose_sum_over_positions": 6.5965 + }, + "17": { + "kl_mean": 0.00726, + "dose_mean_per_position": 1.315, + "dose_sum_over_positions": 14.3704 + }, + "18": { + "kl_mean": 0.003302, + "dose_mean_per_position": 1.0746, + "dose_sum_over_positions": 12.0623 + }, + "19": { + "kl_mean": 0.008354, + "dose_mean_per_position": 1.5733, + "dose_sum_over_positions": 17.4465 + }, + "20": { + "kl_mean": 0.020452, + "dose_mean_per_position": 1.9747, + "dose_sum_over_positions": 21.9248 + }, + "21": { + "kl_mean": 0.008126, + "dose_mean_per_position": 0.9893, + "dose_sum_over_positions": 11.3244 + } + }, + "rand_match": { + "9": { + "kl_mean": 0.000498, + "dose_mean_per_position": 0.4526, + "dose_sum_over_positions": 5.2458 + }, + "10": { + "kl_mean": 0.000589, + "dose_mean_per_position": 0.4517, + "dose_sum_over_positions": 5.2938 + }, + "11": { + "kl_mean": 0.00065, + "dose_mean_per_position": 0.4585, + "dose_sum_over_positions": 5.3284 + }, + "12": { + "kl_mean": 0.00058, + "dose_mean_per_position": 0.4557, + "dose_sum_over_positions": 5.3434 + }, + "13": { + "kl_mean": 0.000711, + "dose_mean_per_position": 0.4812, + "dose_sum_over_positions": 5.5696 + }, + "14": { + "kl_mean": 0.00077, + "dose_mean_per_position": 0.5158, + "dose_sum_over_positions": 5.9271 + }, + "15": { + "kl_mean": 0.000782, + "dose_mean_per_position": 0.5329, + "dose_sum_over_positions": 6.1451 + }, + "16": { + "kl_mean": 0.001234, + "dose_mean_per_position": 0.6386, + "dose_sum_over_positions": 7.3102 + }, + "17": { + "kl_mean": 0.001847, + "dose_mean_per_position": 0.7687, + "dose_sum_over_positions": 8.871 + }, + "18": { + "kl_mean": 0.00221, + "dose_mean_per_position": 0.8518, + "dose_sum_over_positions": 9.7328 + }, + "19": { + "kl_mean": 0.002191, + "dose_mean_per_position": 0.8592, + "dose_sum_over_positions": 9.8169 + }, + "20": { + "kl_mean": 0.002859, + "dose_mean_per_position": 1.0099, + "dose_sum_over_positions": 11.6333 + }, + "21": { + "kl_mean": 0.002574, + "dose_mean_per_position": 0.9824, + "dose_sum_over_positions": 11.4031 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "9": [ + 0.0, + 0.2659, + 0.3311, + 0.1533, + 0.845, + 0.1661, + 0.6567, + 0.9172, + 0.4948, + 1.1081, + 0.0789, + 0.5652 + ], + "10": [ + 0.0, + 0.5085, + 0.1117, + 0.6448, + 0.2057, + 0.01, + 0.5295, + 0.6053, + 0.5783, + 0.7113, + 0.2925, + 0.76 + ], + "11": [ + 0.0, + 0.6848, + 0.1853, + 0.7726, + 0.0434, + 0.2285, + 0.3408, + 0.5518, + 0.51, + 0.3246, + 0.1199, + 0.6345 + ], + "12": [ + 0.0, + 0.4391, + 0.0242, + 0.5596, + 0.6393, + 0.0683, + 0.4032, + 0.956, + 0.5616, + 0.5699, + 0.1318, + 0.631 + ], + "13": [ + 0.0, + 1.2779, + 0.08, + 0.9868, + 0.0217, + 0.3889, + 0.0033, + 0.7226, + 0.1644, + 0.0884, + 0.1578, + 0.4638 + ], + "14": [ + 0.0, + 1.1066, + 0.3825, + 0.6246, + 0.0107, + 0.3801, + 0.0578, + 1.0707, + 0.4691, + 0.1397, + 0.1704, + 0.6388 + ], + "15": [ + 0.0, + 1.2379, + 0.0392, + 0.6659, + 0.5385, + 0.8593, + 0.2631, + 0.7607, + 0.7803, + 0.0883, + 0.0923, + 0.2098 + ], + "16": [ + 0.0, + 1.8441, + 0.0157, + 1.0, + 1.1243, + 1.2429, + 0.8512, + 0.6459, + 1.1455, + 0.5805, + 0.1426, + 0.2549 + ], + "17": [ + 0.0, + 1.4788, + 0.0723, + 0.607, + 0.0759, + 0.2442, + 0.5171, + 1.9474, + 0.684, + 0.1784, + 0.353, + 0.8306 + ], + "18": [ + 0.0, + 1.9842, + 0.1589, + 1.1661, + 0.7658, + 1.5479, + 0.2691, + 1.0808, + 1.5308, + 0.6486, + 0.1218, + 0.0297 + ], + "19": [ + 0.0, + 1.9426, + 0.2094, + 1.4996, + 1.6942, + 2.4939, + 1.1646, + 0.9554, + 2.0448, + 1.3507, + 0.0049, + 0.6797 + ], + "20": [ + 0.0, + 2.2025, + 0.148, + 1.4033, + 2.1956, + 2.8427, + 1.786, + 0.349, + 2.455, + 1.5372, + 0.3847, + 1.3568 + ], + "21": [ + 0.0, + 2.0489, + 0.3905, + 1.8659, + 1.7159, + 2.3522, + 1.5304, + 0.6919, + 2.0432, + 1.2168, + 0.1437, + 1.5995 + ] + }, + "jlens": { + "9": [ + 0.0, + 0.5222, + 0.4063, + 0.6716, + 2.0154, + 0.8505, + 0.1786, + 0.0294, + 1.9007, + 0.4533, + 1.7387, + 0.0519 + ], + "10": [ + 0.0, + 0.7952, + 0.3264, + 0.3906, + 1.7578, + 0.6005, + 0.1316, + 0.0771, + 1.707, + 0.4932, + 1.445, + 0.2467 + ], + "11": [ + 0.0, + 0.6211, + 0.3821, + 0.8983, + 1.3899, + 0.3832, + 0.1149, + 0.0976, + 1.3365, + 0.3656, + 1.6297, + 0.2429 + ], + "12": [ + 0.0, + 0.1219, + 0.6968, + 1.4482, + 1.3587, + 0.3022, + 0.0821, + 0.6078, + 1.1225, + 0.3729, + 1.2236, + 0.1066 + ], + "13": [ + 0.0, + 0.6196, + 0.006, + 0.9635, + 2.2023, + 1.0035, + 0.0749, + 0.3131, + 1.6416, + 0.6961, + 1.9572, + 0.4603 + ], + "14": [ + 0.0, + 0.263, + 0.319, + 0.843, + 2.1261, + 1.189, + 0.1076, + 0.6799, + 1.6462, + 1.1032, + 2.0448, + 0.4514 + ], + "15": [ + 0.0, + 0.4193, + 0.0056, + 0.9011, + 1.7853, + 0.8803, + 0.3138, + 1.1836, + 1.139, + 0.92, + 2.1753, + 0.5619 + ], + "16": [ + 0.0, + 0.213, + 0.0622, + 1.1005, + 1.6035, + 0.1934, + 0.1045, + 0.201, + 0.0036, + 0.4509, + 1.6889, + 0.2636 + ], + "17": [ + 0.0, + 1.0612, + 0.2495, + 0.892, + 0.8241, + 0.0141, + 0.5845, + 1.5689, + 0.6969, + 0.6872, + 1.0523, + 0.4355 + ], + "18": [ + 0.0, + 1.6337, + 0.7055, + 1.1704, + 0.8, + 0.0724, + 0.7716, + 1.3643, + 0.7022, + 0.7418, + 0.8261, + 0.613 + ], + "19": [ + 0.0, + 2.1931, + 0.554, + 1.3612, + 0.0172, + 0.681, + 0.0564, + 0.8996, + 1.2566, + 0.0406, + 0.0508, + 0.1726 + ], + "20": [ + 0.0, + 2.0731, + 0.4851, + 1.3444, + 0.3474, + 0.7472, + 0.1169, + 0.5074, + 1.1804, + 0.0482, + 0.0257, + 0.3513 + ], + "21": [ + 0.0, + 2.2588, + 0.5921, + 1.6974, + 0.8679, + 1.2264, + 0.8106, + 0.4307, + 1.6481, + 0.4923, + 0.0167, + 1.2522 + ] + }, + "shuffled": { + "9": [ + 0.0, + 1.1111, + 0.1056, + 0.2565, + 2.4035, + 1.2101, + 0.5171, + 0.2092, + 2.0622, + 0.7099, + 1.7281, + 0.3442 + ], + "10": [ + 0.0, + 0.6708, + 0.4216, + 0.46, + 1.9526, + 0.7111, + 0.1893, + 0.6655, + 1.6996, + 0.2405, + 1.5144, + 0.2875 + ], + "11": [ + 0.0, + 0.864, + 0.5177, + 1.0258, + 0.12, + 0.1731, + 0.1365, + 0.365, + 0.4699, + 0.1941, + 0.3694, + 0.6384 + ], + "12": [ + 0.0, + 0.8096, + 0.1201, + 1.0351, + 1.9046, + 0.872, + 0.4322, + 0.0035, + 2.0293, + 0.9402, + 1.9882, + 0.0859 + ], + "13": [ + 0.0, + 0.3161, + 0.0206, + 1.0921, + 2.215, + 1.1707, + 0.2083, + 0.5339, + 1.9418, + 0.9187, + 1.9228, + 0.4187 + ], + "14": [ + 0.0, + 0.7339, + 0.2032, + 0.7716, + 2.0884, + 0.9222, + 0.0669, + 0.0717, + 1.2926, + 0.6587, + 1.873, + 0.0855 + ], + "15": [ + 0.0, + 1.3687, + 0.5429, + 1.0392, + 0.0357, + 0.2936, + 0.1635, + 0.5361, + 0.6667, + 0.3522, + 0.5416, + 0.4076 + ], + "16": [ + 0.0, + 1.4919, + 0.1821, + 0.6726, + 0.3417, + 0.4361, + 0.1984, + 0.7869, + 0.8294, + 0.0136, + 0.0486, + 0.3144 + ], + "17": [ + 0.0, + 1.265, + 0.2104, + 0.7846, + 2.8401, + 1.8193, + 1.0312, + 0.6004, + 2.0982, + 1.5182, + 2.1964, + 0.9759 + ], + "18": [ + 0.0, + 1.2808, + 0.4474, + 0.4461, + 1.9377, + 1.6838, + 0.6058, + 1.0311, + 1.0973, + 1.1307, + 2.1735, + 0.6261 + ], + "19": [ + 0.0, + 0.1674, + 1.7127, + 1.6079, + 2.2112, + 2.0777, + 0.3147, + 0.6698, + 1.2597, + 1.3043, + 1.8223, + 0.2311 + ], + "20": [ + 0.0, + 0.3695, + 0.7124, + 0.7915, + 2.9175, + 2.3435, + 0.9918, + 0.8957, + 0.6838, + 1.9693, + 2.2821, + 0.4429 + ], + "21": [ + 0.0, + 2.1343, + 0.7374, + 1.8335, + 0.348, + 0.6175, + 0.4245, + 0.6539, + 1.117, + 0.0689, + 0.3512, + 1.0994 + ] + }, + "rand_match": { + "9": [ + 0.0, + 0.2442, + 0.9079, + 0.729, + 0.0373, + 0.1266, + 0.3463, + 0.6048, + 0.7536, + 0.123, + 0.7789, + 0.4887 + ], + "10": [ + 0.0, + 0.5818, + 0.1896, + 0.5861, + 0.5592, + 0.5087, + 0.0217, + 0.6696, + 0.3828, + 0.0285, + 0.5152, + 0.4988 + ], + "11": [ + 0.0, + 0.2527, + 0.0512, + 0.2761, + 0.9185, + 0.107, + 0.2711, + 0.0038, + 0.8886, + 0.0376, + 0.5202, + 0.2743 + ], + "12": [ + 0.0, + 1.1617, + 0.4601, + 0.846, + 0.1077, + 0.3242, + 0.1596, + 0.3786, + 0.0252, + 0.3555, + 0.5555, + 0.3952 + ], + "13": [ + 0.0, + 0.2923, + 0.2444, + 0.3269, + 0.6174, + 0.0922, + 0.1427, + 0.9142, + 0.2842, + 0.244, + 0.2669, + 0.2584 + ], + "14": [ + 0.0, + 0.5947, + 0.0906, + 0.0924, + 0.7713, + 0.7472, + 1.1492, + 0.6136, + 0.6253, + 1.542, + 0.286, + 1.5658 + ], + "15": [ + 0.0, + 0.0968, + 0.243, + 0.2341, + 0.5861, + 0.0398, + 0.0478, + 0.6226, + 0.0833, + 0.0956, + 0.0367, + 0.4409 + ], + "16": [ + 0.0, + 0.168, + 0.3583, + 0.3785, + 0.1584, + 0.0932, + 0.1078, + 0.3689, + 0.0504, + 0.0134, + 0.7039, + 0.1369 + ], + "17": [ + 0.0, + 0.5714, + 0.1704, + 0.3237, + 0.1186, + 1.1349, + 0.1749, + 0.5955, + 0.1618, + 0.3645, + 0.8228, + 0.619 + ], + "18": [ + 0.0, + 0.4313, + 0.8467, + 0.5129, + 0.6021, + 0.0908, + 0.0369, + 0.2734, + 0.1827, + 0.9381, + 0.0918, + 0.1701 + ], + "19": [ + 0.0, + 1.5105, + 0.4644, + 0.3772, + 2.4192, + 1.7093, + 1.5688, + 0.5122, + 1.7767, + 1.0838, + 0.0868, + 0.971 + ], + "20": [ + 0.0, + 1.5305, + 2.1353, + 2.4952, + 4.2246, + 3.0463, + 3.707, + 2.3172, + 3.2902, + 3.9559, + 1.927, + 3.1293 + ], + "21": [ + 0.0, + 2.1981, + 1.0808, + 2.1988, + 2.1904, + 2.1951, + 1.2965, + 0.8656, + 1.1515, + 1.6936, + 0.1236, + 1.0674 + ] + } + } + }, + "order-ops": { + "n_items_scored": 55, + "n_intermediate_pairs_scored": 110, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.027357, + "jlens": 0.054238, + "shuffled": 0.027594, + "rand_match": 0.000907 + }, + "PRIMARY_ratio_jlens_over_logit": 1.9826, + "ratio_jlens_over_shuffled": 1.9655, + "ratio_jlens_over_rand_match": 59.7739, + "ratio_by_layer_jlens_over_logit": { + "9": 13.953, + "10": 7.3194, + "11": 10.9988, + "12": 7.9771, + "13": 3.1849, + "14": 4.4794, + "15": 5.2474, + "16": 4.5346, + "17": 2.2474, + "18": 1.5896, + "19": 1.2853, + "20": 1.1926, + "21": 1.2149 + }, + "band_max_ratio_SECONDARY": 13.953, + "per_layer": { + "logit": { + "9": { + "kl_mean": 0.001465, + "dose_mean_per_position": 0.4652, + "dose_sum_over_positions": 4.3448 + }, + "10": { + "kl_mean": 0.002406, + "dose_mean_per_position": 0.4942, + "dose_sum_over_positions": 4.5798 + }, + "11": { + "kl_mean": 0.00196, + "dose_mean_per_position": 0.4641, + "dose_sum_over_positions": 4.3486 + }, + "12": { + "kl_mean": 0.002647, + "dose_mean_per_position": 0.4793, + "dose_sum_over_positions": 4.4542 + }, + "13": { + "kl_mean": 0.012278, + "dose_mean_per_position": 0.5919, + "dose_sum_over_positions": 5.4967 + }, + "14": { + "kl_mean": 0.010337, + "dose_mean_per_position": 0.6172, + "dose_sum_over_positions": 5.7758 + }, + "15": { + "kl_mean": 0.012398, + "dose_mean_per_position": 0.6641, + "dose_sum_over_positions": 6.1847 + }, + "16": { + "kl_mean": 0.012633, + "dose_mean_per_position": 0.7243, + "dose_sum_over_positions": 6.7343 + }, + "17": { + "kl_mean": 0.033007, + "dose_mean_per_position": 1.0458, + "dose_sum_over_positions": 9.6729 + }, + "18": { + "kl_mean": 0.042095, + "dose_mean_per_position": 1.1049, + "dose_sum_over_positions": 10.1657 + }, + "19": { + "kl_mean": 0.064626, + "dose_mean_per_position": 1.76, + "dose_sum_over_positions": 16.1981 + }, + "20": { + "kl_mean": 0.075321, + "dose_mean_per_position": 2.1007, + "dose_sum_over_positions": 19.2724 + }, + "21": { + "kl_mean": 0.084472, + "dose_mean_per_position": 2.0733, + "dose_sum_over_positions": 19.1579 + } + }, + "jlens": { + "9": { + "kl_mean": 0.020439, + "dose_mean_per_position": 1.4357, + "dose_sum_over_positions": 13.2242 + }, + "10": { + "kl_mean": 0.017613, + "dose_mean_per_position": 1.4313, + "dose_sum_over_positions": 13.2414 + }, + "11": { + "kl_mean": 0.021562, + "dose_mean_per_position": 1.7348, + "dose_sum_over_positions": 16.0124 + }, + "12": { + "kl_mean": 0.021112, + "dose_mean_per_position": 1.7458, + "dose_sum_over_positions": 16.0484 + }, + "13": { + "kl_mean": 0.039103, + "dose_mean_per_position": 1.9429, + "dose_sum_over_positions": 17.9091 + }, + "14": { + "kl_mean": 0.046303, + "dose_mean_per_position": 1.8687, + "dose_sum_over_positions": 17.1548 + }, + "15": { + "kl_mean": 0.065058, + "dose_mean_per_position": 1.9657, + "dose_sum_over_positions": 18.0831 + }, + "16": { + "kl_mean": 0.057284, + "dose_mean_per_position": 1.7912, + "dose_sum_over_positions": 16.4128 + }, + "17": { + "kl_mean": 0.07418, + "dose_mean_per_position": 1.7626, + "dose_sum_over_positions": 16.1193 + }, + "18": { + "kl_mean": 0.066916, + "dose_mean_per_position": 1.6894, + "dose_sum_over_positions": 15.323 + }, + "19": { + "kl_mean": 0.083063, + "dose_mean_per_position": 1.8647, + "dose_sum_over_positions": 16.9851 + }, + "20": { + "kl_mean": 0.089829, + "dose_mean_per_position": 2.0559, + "dose_sum_over_positions": 18.7665 + }, + "21": { + "kl_mean": 0.102628, + "dose_mean_per_position": 2.1032, + "dose_sum_over_positions": 19.4185 + } + }, + "shuffled": { + "9": { + "kl_mean": 0.019133, + "dose_mean_per_position": 1.411, + "dose_sum_over_positions": 13.0335 + }, + "10": { + "kl_mean": 0.004778, + "dose_mean_per_position": 0.9534, + "dose_sum_over_positions": 8.8413 + }, + "11": { + "kl_mean": 0.002421, + "dose_mean_per_position": 0.5344, + "dose_sum_over_positions": 4.9472 + }, + "12": { + "kl_mean": 0.01259, + "dose_mean_per_position": 1.6248, + "dose_sum_over_positions": 15.0894 + }, + "13": { + "kl_mean": 0.036899, + "dose_mean_per_position": 1.6965, + "dose_sum_over_positions": 15.5548 + }, + "14": { + "kl_mean": 0.023836, + "dose_mean_per_position": 2.0308, + "dose_sum_over_positions": 18.6915 + }, + "15": { + "kl_mean": 0.023917, + "dose_mean_per_position": 0.8392, + "dose_sum_over_positions": 7.6708 + }, + "16": { + "kl_mean": 0.017448, + "dose_mean_per_position": 0.7885, + "dose_sum_over_positions": 7.3192 + }, + "17": { + "kl_mean": 0.009327, + "dose_mean_per_position": 1.4627, + "dose_sum_over_positions": 13.5188 + }, + "18": { + "kl_mean": 0.005373, + "dose_mean_per_position": 1.3714, + "dose_sum_over_positions": 12.6042 + }, + "19": { + "kl_mean": 0.017387, + "dose_mean_per_position": 2.4369, + "dose_sum_over_positions": 22.4622 + }, + "20": { + "kl_mean": 0.096188, + "dose_mean_per_position": 3.5847, + "dose_sum_over_positions": 32.9703 + }, + "21": { + "kl_mean": 0.089427, + "dose_mean_per_position": 1.8414, + "dose_sum_over_positions": 16.928 + } + }, + "rand_match": { + "9": { + "kl_mean": 0.000956, + "dose_mean_per_position": 0.4384, + "dose_sum_over_positions": 4.0897 + }, + "10": { + "kl_mean": 0.000921, + "dose_mean_per_position": 0.4574, + "dose_sum_over_positions": 4.2801 + }, + "11": { + "kl_mean": 0.000874, + "dose_mean_per_position": 0.4406, + "dose_sum_over_positions": 4.046 + }, + "12": { + "kl_mean": 0.000829, + "dose_mean_per_position": 0.4552, + "dose_sum_over_positions": 4.214 + }, + "13": { + "kl_mean": 0.000497, + "dose_mean_per_position": 0.4309, + "dose_sum_over_positions": 4.0519 + }, + "14": { + "kl_mean": 0.000625, + "dose_mean_per_position": 0.5331, + "dose_sum_over_positions": 4.9927 + }, + "15": { + "kl_mean": 0.000565, + "dose_mean_per_position": 0.5444, + "dose_sum_over_positions": 5.1105 + }, + "16": { + "kl_mean": 0.000774, + "dose_mean_per_position": 0.6699, + "dose_sum_over_positions": 6.3496 + }, + "17": { + "kl_mean": 0.000856, + "dose_mean_per_position": 0.691, + "dose_sum_over_positions": 6.3926 + }, + "18": { + "kl_mean": 0.001098, + "dose_mean_per_position": 0.7965, + "dose_sum_over_positions": 7.4516 + }, + "19": { + "kl_mean": 0.001217, + "dose_mean_per_position": 0.8523, + "dose_sum_over_positions": 7.7458 + }, + "20": { + "kl_mean": 0.001185, + "dose_mean_per_position": 0.9064, + "dose_sum_over_positions": 8.6222 + }, + "21": { + "kl_mean": 0.0014, + "dose_mean_per_position": 1.0336, + "dose_sum_over_positions": 9.7748 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "9": [ + 0.0, + 0.0084, + 0.2013, + 1.1142, + 0.572, + 0.1142, + 0.1966, + 0.6876, + 0.6651 + ], + "10": [ + 0.0, + 0.0337, + 0.1992, + 1.162, + 0.4619, + 0.4908, + 0.413, + 0.5399, + 1.0668 + ], + "11": [ + 0.0, + 0.2415, + 0.1093, + 0.7812, + 0.8114, + 0.1104, + 0.2764, + 0.972, + 0.7188 + ], + "12": [ + 0.0, + 0.2602, + 0.0911, + 0.4102, + 0.4202, + 0.0285, + 0.3274, + 0.783, + 0.9973 + ], + "13": [ + 0.0, + 0.2949, + 0.1637, + 0.352, + 0.4376, + 0.1069, + 0.3456, + 0.8452, + 2.3738 + ], + "14": [ + 0.0, + 0.512, + 0.4524, + 0.054, + 0.7258, + 0.0966, + 0.582, + 0.967, + 2.0892 + ], + "15": [ + 0.0, + 0.1372, + 0.3934, + 0.3283, + 0.6333, + 0.1539, + 0.3232, + 0.7342, + 2.4332 + ], + "16": [ + 0.0, + 0.1108, + 0.1961, + 0.228, + 0.3221, + 0.4343, + 0.1615, + 0.6249, + 2.427 + ], + "17": [ + 0.0, + 0.6841, + 0.3236, + 1.0496, + 0.3233, + 1.3061, + 0.7471, + 0.0602, + 3.0756 + ], + "18": [ + 0.0, + 1.0454, + 0.1424, + 1.138, + 0.4242, + 1.6702, + 0.8419, + 0.0421, + 2.8813 + ], + "19": [ + 0.0, + 1.4481, + 0.1728, + 1.4581, + 0.072, + 2.0469, + 1.1505, + 0.0846, + 3.2047 + ], + "20": [ + 0.0, + 1.927, + 0.2826, + 1.8349, + 0.568, + 2.5978, + 1.4116, + 0.4877, + 3.5208 + ], + "21": [ + 0.0, + 1.6556, + 0.4028, + 1.4387, + 0.1299, + 1.9224, + 1.1775, + 0.221, + 3.4786 + ] + }, + "jlens": { + "9": [ + 0.0, + 2.0126, + 1.7247, + 2.2462, + 1.1208, + 1.4971, + 1.3352, + 1.5187, + 1.3809 + ], + "10": [ + 0.0, + 1.7467, + 1.8508, + 2.2146, + 1.0979, + 1.8435, + 1.4185, + 1.4371, + 1.4275 + ], + "11": [ + 0.0, + 2.0413, + 2.5109, + 3.0202, + 1.9408, + 2.1469, + 1.7813, + 1.7861, + 1.5856 + ], + "12": [ + 0.0, + 1.9322, + 2.3607, + 2.7339, + 1.7992, + 2.5158, + 1.8729, + 1.7209, + 2.0592 + ], + "13": [ + 0.0, + 1.9682, + 1.8243, + 3.1579, + 1.7181, + 2.5562, + 2.0288, + 1.4275, + 3.4315 + ], + "14": [ + 0.0, + 1.7509, + 1.8171, + 2.9122, + 1.4956, + 2.3599, + 1.9089, + 1.4379, + 3.5077 + ], + "15": [ + 0.0, + 1.9312, + 2.0377, + 2.7869, + 1.6957, + 2.4691, + 1.837, + 1.7209, + 3.8369 + ], + "16": [ + 0.0, + 1.7725, + 1.6906, + 2.2291, + 1.3966, + 2.0411, + 1.6603, + 1.2063, + 3.4222 + ], + "17": [ + 0.0, + 1.4607, + 1.0494, + 2.0235, + 0.8688, + 2.0307, + 1.5595, + 0.516, + 3.3306 + ], + "18": [ + 0.0, + 1.3988, + 0.369, + 1.5502, + 0.4663, + 1.957, + 1.0525, + 0.1676, + 2.8562 + ], + "19": [ + 0.0, + 1.6632, + 0.3154, + 1.7296, + 0.1235, + 1.8842, + 1.3372, + 0.186, + 3.2532 + ], + "20": [ + 0.0, + 1.9478, + 0.1844, + 1.6386, + 0.1908, + 2.2088, + 1.1906, + 0.2379, + 3.3383 + ], + "21": [ + 0.0, + 2.0156, + 0.4445, + 1.9319, + 0.2407, + 2.113, + 1.5562, + 0.242, + 3.6908 + ] + }, + "shuffled": { + "9": [ + 0.0, + 1.8929, + 1.725, + 2.1689, + 0.9769, + 1.6705, + 1.3269, + 1.4505, + 1.4523 + ], + "10": [ + 0.0, + 1.3557, + 2.1773, + 1.5503, + 1.3784, + 1.1408, + 0.0011, + 1.6174, + 0.9553 + ], + "11": [ + 0.0, + 0.2736, + 0.0726, + 1.2138, + 0.7378, + 0.2272, + 0.2688, + 0.8146, + 0.6319 + ], + "12": [ + 0.0, + 2.2832, + 2.6366, + 2.1704, + 1.8716, + 2.2926, + 1.0797, + 1.8561, + 1.3859 + ], + "13": [ + 0.0, + 1.4702, + 1.5061, + 2.6597, + 1.1298, + 1.9645, + 1.4111, + 0.9627, + 3.4065 + ], + "14": [ + 0.0, + 2.6455, + 2.6783, + 3.3995, + 2.3451, + 3.2037, + 2.5706, + 2.0901, + 2.7778 + ], + "15": [ + 0.0, + 0.2474, + 0.0853, + 1.0052, + 0.4255, + 0.5708, + 0.0333, + 0.4428, + 2.6628 + ], + "16": [ + 0.0, + 0.2276, + 0.1029, + 0.4317, + 0.1728, + 0.5648, + 0.0214, + 0.4379, + 2.4088 + ], + "17": [ + 0.0, + 2.359, + 1.9641, + 2.133, + 2.0183, + 2.0681, + 1.1851, + 1.4208, + 1.4143 + ], + "18": [ + 0.0, + 2.2108, + 0.7432, + 1.7892, + 0.9368, + 1.6467, + 0.8742, + 1.0887, + 1.5488 + ], + "19": [ + 0.0, + 4.3582, + 3.0249, + 3.1826, + 2.5281, + 3.7771, + 2.5513, + 2.0252, + 2.5211 + ], + "20": [ + 0.0, + 4.2765, + 2.2438, + 3.8016, + 2.5782, + 4.4419, + 3.4707, + 2.325, + 4.2088 + ], + "21": [ + 0.0, + 1.7383, + 0.9492, + 1.4782, + 0.215, + 1.6591, + 1.0507, + 0.1242, + 3.382 + ] + }, + "rand_match": { + "9": [ + 0.0, + 1.0924, + 0.3845, + 0.1077, + 0.1443, + 0.3532, + 1.0707, + 0.3663, + 0.9837 + ], + "10": [ + 0.0, + 0.1509, + 0.2528, + 0.2898, + 0.3412, + 0.7398, + 0.0932, + 0.0792, + 0.6166 + ], + "11": [ + 0.0, + 0.8619, + 0.3915, + 0.3399, + 0.0543, + 1.0336, + 0.2305, + 0.4676, + 1.678 + ], + "12": [ + 0.0, + 0.3951, + 0.1106, + 0.6761, + 0.3841, + 0.0418, + 1.0932, + 0.3784, + 0.8432 + ], + "13": [ + 0.0, + 0.3684, + 0.2945, + 0.5451, + 0.8257, + 0.0154, + 0.0667, + 0.9479, + 0.12 + ], + "14": [ + 0.0, + 0.3632, + 0.8231, + 0.7431, + 0.1871, + 1.0528, + 1.0499, + 0.0899, + 0.354 + ], + "15": [ + 0.0, + 0.8299, + 1.0552, + 0.9043, + 1.7087, + 1.2874, + 0.9055, + 1.5456, + 0.6162 + ], + "16": [ + 0.0, + 0.0891, + 0.3252, + 0.7067, + 0.2147, + 0.3212, + 0.733, + 0.392, + 0.2002 + ], + "17": [ + 0.0, + 0.906, + 0.8837, + 0.6367, + 1.3763, + 1.2141, + 0.7532, + 0.8033, + 0.2598 + ], + "18": [ + 0.0, + 1.3101, + 1.7832, + 1.6648, + 1.1951, + 1.3281, + 1.0024, + 1.5943, + 1.3287 + ], + "19": [ + 0.0, + 1.0077, + 1.0469, + 0.8478, + 0.2519, + 1.188, + 1.5827, + 0.0075, + 1.6388 + ], + "20": [ + 0.0, + 0.9831, + 0.4952, + 0.1506, + 0.0529, + 0.341, + 0.9713, + 0.946, + 0.611 + ], + "21": [ + 0.0, + 2.913, + 1.5125, + 3.7057, + 1.9303, + 1.4909, + 2.8096, + 0.2216, + 2.2341 + ] + } + } + }, + "poetry": { + "n_items_scored": 98, + "n_intermediate_pairs_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.003754, + "jlens": 0.008368, + "shuffled": 0.00623, + "rand_match": 0.001747 + }, + "PRIMARY_ratio_jlens_over_logit": 2.2292, + "ratio_jlens_over_shuffled": 1.3431, + "ratio_jlens_over_rand_match": 4.7894, + "ratio_by_layer_jlens_over_logit": { + "9": 6.7999, + "10": 7.0777, + "11": 8.0094, + "12": 7.5349, + "13": 6.9419, + "14": 6.0526, + "15": 5.5974, + "16": 2.8061, + "17": 1.0791, + "18": 1.0941, + "19": 0.98, + "20": 0.8892, + "21": 0.8505 + }, + "band_max_ratio_SECONDARY": 8.0094, + "per_layer": { + "logit": { + "9": { + "kl_mean": 0.001393, + "dose_mean_per_position": 1.179, + "dose_sum_over_positions": 32.1876 + }, + "10": { + "kl_mean": 0.001225, + "dose_mean_per_position": 1.1867, + "dose_sum_over_positions": 32.4292 + }, + "11": { + "kl_mean": 0.001106, + "dose_mean_per_position": 1.1684, + "dose_sum_over_positions": 31.9041 + }, + "12": { + "kl_mean": 0.001039, + "dose_mean_per_position": 1.2293, + "dose_sum_over_positions": 33.5851 + }, + "13": { + "kl_mean": 0.001161, + "dose_mean_per_position": 1.2758, + "dose_sum_over_positions": 34.8778 + }, + "14": { + "kl_mean": 0.001535, + "dose_mean_per_position": 1.3705, + "dose_sum_over_positions": 37.4767 + }, + "15": { + "kl_mean": 0.001969, + "dose_mean_per_position": 1.4539, + "dose_sum_over_positions": 39.7348 + }, + "16": { + "kl_mean": 0.00376, + "dose_mean_per_position": 1.6774, + "dose_sum_over_positions": 45.8325 + }, + "17": { + "kl_mean": 0.008338, + "dose_mean_per_position": 2.2711, + "dose_sum_over_positions": 61.9797 + }, + "18": { + "kl_mean": 0.00701, + "dose_mean_per_position": 2.2929, + "dose_sum_over_positions": 62.5885 + }, + "19": { + "kl_mean": 0.006737, + "dose_mean_per_position": 2.4625, + "dose_sum_over_positions": 67.2178 + }, + "20": { + "kl_mean": 0.006751, + "dose_mean_per_position": 2.5297, + "dose_sum_over_positions": 69.0665 + }, + "21": { + "kl_mean": 0.006777, + "dose_mean_per_position": 2.5875, + "dose_sum_over_positions": 70.6201 + } + }, + "jlens": { + "9": { + "kl_mean": 0.009471, + "dose_mean_per_position": 1.7508, + "dose_sum_over_positions": 47.8421 + }, + "10": { + "kl_mean": 0.00867, + "dose_mean_per_position": 1.7153, + "dose_sum_over_positions": 46.8942 + }, + "11": { + "kl_mean": 0.008856, + "dose_mean_per_position": 1.6039, + "dose_sum_over_positions": 43.8244 + }, + "12": { + "kl_mean": 0.007828, + "dose_mean_per_position": 1.6841, + "dose_sum_over_positions": 45.9135 + }, + "13": { + "kl_mean": 0.008058, + "dose_mean_per_position": 1.6319, + "dose_sum_over_positions": 44.5122 + }, + "14": { + "kl_mean": 0.009292, + "dose_mean_per_position": 1.7546, + "dose_sum_over_positions": 47.8331 + }, + "15": { + "kl_mean": 0.011023, + "dose_mean_per_position": 1.854, + "dose_sum_over_positions": 50.5269 + }, + "16": { + "kl_mean": 0.010552, + "dose_mean_per_position": 1.7256, + "dose_sum_over_positions": 47.0356 + }, + "17": { + "kl_mean": 0.008997, + "dose_mean_per_position": 1.8011, + "dose_sum_over_positions": 49.1778 + }, + "18": { + "kl_mean": 0.007669, + "dose_mean_per_position": 1.8004, + "dose_sum_over_positions": 49.1807 + }, + "19": { + "kl_mean": 0.006603, + "dose_mean_per_position": 1.9736, + "dose_sum_over_positions": 53.9352 + }, + "20": { + "kl_mean": 0.006003, + "dose_mean_per_position": 2.0823, + "dose_sum_over_positions": 56.9034 + }, + "21": { + "kl_mean": 0.005764, + "dose_mean_per_position": 2.2668, + "dose_sum_over_positions": 61.9288 + } + }, + "shuffled": { + "9": { + "kl_mean": 0.007556, + "dose_mean_per_position": 1.7174, + "dose_sum_over_positions": 46.9375 + }, + "10": { + "kl_mean": 0.005658, + "dose_mean_per_position": 1.6443, + "dose_sum_over_positions": 44.9522 + }, + "11": { + "kl_mean": 0.000646, + "dose_mean_per_position": 0.7237, + "dose_sum_over_positions": 19.8341 + }, + "12": { + "kl_mean": 0.008282, + "dose_mean_per_position": 1.9144, + "dose_sum_over_positions": 52.3609 + }, + "13": { + "kl_mean": 0.005483, + "dose_mean_per_position": 1.4666, + "dose_sum_over_positions": 40.0054 + }, + "14": { + "kl_mean": 0.006631, + "dose_mean_per_position": 1.8378, + "dose_sum_over_positions": 50.1811 + }, + "15": { + "kl_mean": 0.002833, + "dose_mean_per_position": 1.1178, + "dose_sum_over_positions": 30.5803 + }, + "16": { + "kl_mean": 0.003267, + "dose_mean_per_position": 1.342, + "dose_sum_over_positions": 36.6787 + }, + "17": { + "kl_mean": 0.007996, + "dose_mean_per_position": 2.8665, + "dose_sum_over_positions": 78.2716 + }, + "18": { + "kl_mean": 0.003887, + "dose_mean_per_position": 2.556, + "dose_sum_over_positions": 70.0729 + }, + "19": { + "kl_mean": 0.008352, + "dose_mean_per_position": 3.3435, + "dose_sum_over_positions": 91.3301 + }, + "20": { + "kl_mean": 0.015817, + "dose_mean_per_position": 3.3164, + "dose_sum_over_positions": 90.5416 + }, + "21": { + "kl_mean": 0.004588, + "dose_mean_per_position": 2.0886, + "dose_sum_over_positions": 57.1044 + } + }, + "rand_match": { + "9": { + "kl_mean": 0.001854, + "dose_mean_per_position": 1.4707, + "dose_sum_over_positions": 40.2252 + }, + "10": { + "kl_mean": 0.001615, + "dose_mean_per_position": 1.4039, + "dose_sum_over_positions": 38.1953 + }, + "11": { + "kl_mean": 0.001578, + "dose_mean_per_position": 1.4393, + "dose_sum_over_positions": 39.4648 + }, + "12": { + "kl_mean": 0.001297, + "dose_mean_per_position": 1.3207, + "dose_sum_over_positions": 36.2704 + }, + "13": { + "kl_mean": 0.001271, + "dose_mean_per_position": 1.5157, + "dose_sum_over_positions": 41.4845 + }, + "14": { + "kl_mean": 0.001255, + "dose_mean_per_position": 1.4752, + "dose_sum_over_positions": 40.3665 + }, + "15": { + "kl_mean": 0.00176, + "dose_mean_per_position": 1.6247, + "dose_sum_over_positions": 44.3663 + }, + "16": { + "kl_mean": 0.001882, + "dose_mean_per_position": 1.6484, + "dose_sum_over_positions": 45.0415 + }, + "17": { + "kl_mean": 0.001709, + "dose_mean_per_position": 1.7408, + "dose_sum_over_positions": 47.4403 + }, + "18": { + "kl_mean": 0.002155, + "dose_mean_per_position": 1.8845, + "dose_sum_over_positions": 51.4974 + }, + "19": { + "kl_mean": 0.002066, + "dose_mean_per_position": 1.9135, + "dose_sum_over_positions": 52.0256 + }, + "20": { + "kl_mean": 0.002561, + "dose_mean_per_position": 2.0213, + "dose_sum_over_positions": 55.1507 + }, + "21": { + "kl_mean": 0.001711, + "dose_mean_per_position": 1.8249, + "dose_sum_over_positions": 49.9572 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "9": [ + 0.0, + 0.6561, + 0.2612, + 1.1837, + 1.7282, + 0.6189, + 0.9184, + 0.7311, + 14.2727, + 0.5104, + 1.0853, + 0.9454, + 1.6119, + 1.744, + 0.7362, + 0.0701, + 0.8141, + 1.2673, + 0.6948, + 0.3011, + 1.4035, + 0.0117, + 0.6271, + 0.3044, + 0.8792, + 1.8336, + 1.5793, + 0.7413, + 1.5184 + ], + "10": [ + 0.0, + 0.4731, + 0.2868, + 0.9611, + 1.5795, + 0.3016, + 0.8376, + 0.3511, + 14.3265, + 0.2159, + 1.1464, + 1.0814, + 1.5702, + 1.2475, + 0.6433, + 0.0138, + 0.7269, + 1.2012, + 0.718, + 0.2681, + 1.6335, + 0.0932, + 0.8688, + 0.0822, + 1.5494, + 1.9765, + 1.9287, + 0.5917, + 1.5917 + ], + "11": [ + 0.0, + 0.7742, + 0.0813, + 1.0302, + 1.8662, + 0.3614, + 1.2563, + 0.7024, + 14.4064, + 0.828, + 1.5176, + 1.0871, + 1.2915, + 1.6718, + 0.5584, + 0.2707, + 1.3698, + 1.0878, + 0.6825, + 0.4025, + 1.6206, + 0.0958, + 0.6003, + 0.3931, + 1.5108, + 1.6377, + 1.5561, + 0.4911, + 1.542 + ], + "12": [ + 0.0, + 0.5461, + 0.1155, + 1.2251, + 1.7284, + 0.4447, + 1.3128, + 0.7411, + 14.3755, + 0.7471, + 1.6432, + 0.8027, + 1.5621, + 1.5313, + 1.5344, + 0.0195, + 1.6059, + 1.4943, + 1.075, + 0.6252, + 2.4879, + 0.5717, + 0.6603, + 0.1106, + 1.7357, + 1.8154, + 1.78, + 0.7696, + 1.603 + ], + "13": [ + 0.0, + 0.7195, + 0.1249, + 1.3321, + 1.9737, + 0.864, + 1.6386, + 1.046, + 14.1518, + 0.8291, + 1.7522, + 0.9621, + 1.3683, + 1.4137, + 1.4366, + 0.0528, + 1.4646, + 1.2325, + 0.8676, + 0.3804, + 2.1522, + 0.6102, + 0.6679, + 0.1058, + 1.6983, + 2.3101, + 2.1539, + 1.0232, + 1.9283 + ], + "14": [ + 0.0, + 0.5404, + 0.0914, + 1.3767, + 1.9494, + 0.7855, + 1.4735, + 1.0246, + 14.0359, + 1.3067, + 2.0976, + 1.3108, + 1.6381, + 1.6076, + 2.1486, + 0.1092, + 1.6587, + 1.574, + 1.3646, + 0.7769, + 2.4788, + 0.7823, + 1.4536, + 0.9075, + 2.4754, + 2.7586, + 2.5601, + 1.1835, + 2.3969 + ], + "15": [ + 0.0, + 0.7244, + 0.2906, + 1.394, + 2.2163, + 0.9732, + 1.6163, + 1.0388, + 14.122, + 0.9738, + 1.9357, + 1.2615, + 1.5079, + 1.7271, + 1.9368, + 0.2581, + 1.6989, + 1.5887, + 1.3229, + 0.6398, + 2.2536, + 0.648, + 1.4072, + 0.9425, + 2.4664, + 2.7728, + 2.8735, + 1.387, + 2.6845 + ], + "16": [ + 0.0, + 0.6798, + 0.2462, + 0.9555, + 1.7388, + 0.7507, + 0.9258, + 0.4119, + 14.1957, + 1.075, + 1.771, + 1.155, + 1.956, + 2.0234, + 1.7339, + 0.086, + 1.9764, + 1.4332, + 1.0409, + 0.7485, + 2.2919, + 0.5066, + 1.2759, + 0.8058, + 2.5671, + 2.8826, + 2.9008, + 1.3495, + 2.8683 + ], + "17": [ + 0.0, + 0.9723, + 0.1599, + 0.8748, + 1.8499, + 0.6927, + 1.0588, + 0.6081, + 14.0385, + 1.2596, + 1.9339, + 1.2275, + 1.789, + 2.1456, + 2.0784, + 0.1138, + 1.7724, + 1.5453, + 1.8167, + 1.0577, + 3.104, + 0.011, + 1.5376, + 0.971, + 3.6105, + 3.6942, + 3.4409, + 1.0552, + 3.5994 + ], + "18": [ + 0.0, + 0.9951, + 0.3222, + 0.5641, + 1.6039, + 0.8664, + 1.0877, + 0.8, + 13.9431, + 1.6224, + 1.8613, + 1.7385, + 2.1616, + 2.6787, + 2.7227, + 0.0628, + 2.3105, + 1.7512, + 2.1278, + 1.1196, + 3.4594, + 0.3755, + 1.6758, + 1.445, + 4.1863, + 4.2295, + 3.9178, + 1.2765, + 4.3257 + ], + "19": [ + 0.0, + 0.7689, + 0.217, + 0.1403, + 1.7028, + 0.7375, + 0.9512, + 0.7097, + 13.1192, + 1.3019, + 1.5362, + 1.3395, + 1.4747, + 2.3562, + 2.5872, + 0.3021, + 2.2484, + 1.292, + 1.6578, + 0.8795, + 3.2711, + 0.2338, + 1.7274, + 1.134, + 3.8847, + 4.3803, + 3.7917, + 1.4195, + 4.7359 + ], + "20": [ + 0.0, + 0.6692, + 0.2559, + 0.1548, + 1.5383, + 0.5751, + 0.2631, + 0.2086, + 12.3104, + 0.8256, + 1.0993, + 0.8955, + 1.8361, + 2.3786, + 2.4031, + 0.5733, + 2.3965, + 1.2498, + 1.5088, + 0.7987, + 2.8924, + 0.81, + 2.5603, + 1.5118, + 3.9934, + 4.2584, + 3.5507, + 0.8918, + 5.2589 + ], + "21": [ + 0.0, + 0.9981, + 0.2323, + 0.0934, + 2.1627, + 1.1869, + 1.0888, + 1.0899, + 8.9324, + 1.6415, + 1.985, + 1.9049, + 2.3562, + 2.9161, + 3.3085, + 0.2876, + 2.9913, + 2.5356, + 2.9679, + 1.1205, + 4.2954, + 2.4587, + 2.953, + 3.1181, + 4.8874, + 5.3412, + 4.3521, + 2.5662, + 6.3181 + ] + }, + "jlens": { + "9": [ + 0.0, + 1.3817, + 0.9791, + 1.1383, + 2.7112, + 1.0432, + 1.8896, + 1.0637, + 5.2238, + 0.4766, + 3.0821, + 1.5415, + 1.4281, + 4.312, + 1.1772, + 1.0287, + 1.5288, + 2.9287, + 0.77, + 0.5971, + 1.3731, + 1.5975, + 1.6305, + 1.0644, + 1.5509, + 1.5894, + 1.6107, + 1.633, + 1.2244 + ], + "10": [ + 0.0, + 1.2741, + 0.8382, + 0.9157, + 2.767, + 1.0676, + 1.991, + 1.0961, + 11.5563, + 0.4333, + 2.87, + 1.296, + 1.3559, + 3.3837, + 1.514, + 1.0628, + 1.6525, + 2.8458, + 1.1589, + 0.8042, + 1.973, + 1.4963, + 1.5746, + 0.9964, + 1.809, + 1.8046, + 1.8824, + 1.4072, + 1.497 + ], + "11": [ + 0.0, + 1.4219, + 0.5291, + 0.459, + 2.6675, + 1.1011, + 2.1701, + 1.4706, + 6.7161, + 0.8, + 2.6168, + 1.384, + 1.1344, + 3.6629, + 1.8864, + 1.0918, + 1.8215, + 2.3686, + 1.1948, + 1.0319, + 1.831, + 1.0411, + 1.1582, + 0.7444, + 1.5785, + 1.1965, + 1.6429, + 1.2062, + 1.4912 + ], + "12": [ + 0.0, + 1.4577, + 0.3106, + 0.9734, + 2.684, + 1.2906, + 2.2477, + 1.7186, + 8.845, + 0.7006, + 2.703, + 1.3779, + 1.7012, + 3.3837, + 2.6482, + 1.3327, + 2.5371, + 2.978, + 1.7301, + 0.8095, + 2.3275, + 1.5936, + 1.7458, + 1.1358, + 2.0598, + 1.8576, + 2.1677, + 1.5079, + 1.7363 + ], + "13": [ + 0.0, + 1.2573, + 0.8153, + 1.2655, + 2.7182, + 1.2082, + 2.3582, + 1.7532, + 2.1813, + 0.5889, + 2.5781, + 1.4476, + 2.1215, + 3.3602, + 2.5015, + 1.2271, + 2.2581, + 3.0212, + 1.8981, + 0.8757, + 2.5555, + 1.8727, + 2.2293, + 1.6135, + 2.6717, + 2.448, + 2.4174, + 1.8454, + 2.2896 + ], + "14": [ + 0.0, + 0.9579, + 0.1925, + 0.9819, + 2.3763, + 1.4366, + 2.1387, + 1.3146, + 2.6859, + 0.9197, + 2.3652, + 1.2366, + 1.7625, + 2.9767, + 3.1439, + 0.9493, + 2.0962, + 2.8535, + 1.9266, + 1.2768, + 2.6034, + 1.5669, + 2.3998, + 1.9669, + 2.99, + 2.7998, + 2.5397, + 1.9682, + 2.624 + ], + "15": [ + 0.0, + 0.5208, + 0.2395, + 0.3742, + 2.2254, + 0.9382, + 1.8767, + 1.3063, + 11.325, + 0.9577, + 2.3101, + 1.1233, + 1.3102, + 2.8575, + 2.5955, + 0.3124, + 1.8175, + 2.4739, + 2.0539, + 1.0776, + 2.7014, + 1.326, + 2.1501, + 1.5977, + 2.7656, + 2.714, + 2.6512, + 1.4642, + 2.3076 + ], + "16": [ + 0.0, + 0.2307, + 0.6433, + 0.2266, + 1.9187, + 0.3675, + 0.9907, + 0.6819, + 15.3827, + 1.0825, + 1.7094, + 0.7723, + 1.6807, + 2.8321, + 2.3709, + 0.0646, + 2.2807, + 1.8647, + 1.3198, + 0.7013, + 2.45, + 0.7042, + 1.8269, + 1.1906, + 2.6953, + 2.9594, + 2.9348, + 0.8546, + 2.7731 + ], + "17": [ + 0.0, + 0.547, + 0.6375, + 0.1392, + 2.2383, + 0.5559, + 1.2207, + 0.5092, + 14.2506, + 1.4182, + 1.9972, + 0.9207, + 2.0869, + 3.1104, + 2.798, + 0.148, + 2.262, + 1.9499, + 1.765, + 0.8849, + 3.0323, + 0.5966, + 2.1641, + 1.2332, + 3.5984, + 3.3536, + 3.6297, + 0.8592, + 3.2472 + ], + "18": [ + 0.0, + 0.5634, + 0.6135, + 0.0808, + 1.9727, + 0.6631, + 1.0979, + 0.5452, + 13.9165, + 1.7637, + 1.7078, + 1.3342, + 2.4788, + 3.4982, + 3.3631, + 0.148, + 3.1555, + 1.8507, + 1.9126, + 0.8411, + 3.284, + 0.86, + 2.442, + 1.4954, + 3.9297, + 3.9833, + 4.3352, + 0.9316, + 4.1122 + ], + "19": [ + 0.0, + 0.3608, + 0.909, + 0.1981, + 2.1085, + 0.5609, + 1.0694, + 0.8694, + 11.5374, + 2.0898, + 1.7182, + 1.4706, + 2.3104, + 3.0736, + 3.3333, + 0.3271, + 3.1728, + 1.5649, + 2.087, + 0.9327, + 3.8313, + 1.3816, + 2.8745, + 1.5372, + 4.1363, + 4.5186, + 4.8971, + 1.4655, + 4.9281 + ], + "20": [ + 0.0, + 0.6273, + 0.5657, + 0.0189, + 2.1313, + 0.6999, + 0.8246, + 0.9692, + 8.3234, + 2.1297, + 1.8644, + 1.564, + 2.3907, + 3.2279, + 3.5206, + 0.3357, + 3.4059, + 1.7278, + 2.2779, + 0.9642, + 3.8331, + 2.0173, + 3.3926, + 2.2049, + 4.3751, + 5.0852, + 5.1419, + 1.629, + 5.7484 + ], + "21": [ + 0.0, + 0.7251, + 0.7407, + 0.2355, + 2.0504, + 0.6959, + 0.7159, + 1.0601, + 3.5945, + 1.8832, + 1.962, + 1.8978, + 2.4752, + 3.1094, + 3.8854, + 0.231, + 3.2883, + 2.17, + 2.8798, + 1.0282, + 4.1865, + 2.3414, + 3.6221, + 2.7075, + 5.0003, + 5.4322, + 4.9256, + 2.1337, + 6.5375 + ] + }, + "shuffled": { + "9": [ + 0.0, + 1.4239, + 0.9909, + 1.1583, + 2.8001, + 1.1688, + 2.0333, + 1.1787, + 11.2097, + 0.5887, + 2.9075, + 1.4472, + 1.6109, + 3.8782, + 1.1196, + 0.8567, + 1.4575, + 2.5353, + 0.8228, + 0.5856, + 1.5919, + 1.3475, + 1.4919, + 0.7592, + 1.5102, + 1.5953, + 1.5957, + 1.5165, + 1.4059 + ], + "10": [ + 0.0, + 0.7082, + 0.0993, + 0.1095, + 1.395, + 0.4194, + 0.5246, + 0.4498, + 11.7617, + 0.1891, + 3.1689, + 1.021, + 0.6222, + 3.8908, + 0.9435, + 0.87, + 1.2457, + 3.0401, + 0.5557, + 0.1294, + 1.1862, + 0.9796, + 1.172, + 0.8885, + 0.8911, + 1.1855, + 1.1959, + 0.831, + 0.8321 + ], + "11": [ + 0.0, + 0.2653, + 0.499, + 0.7055, + 1.5924, + 0.0065, + 0.6357, + 0.4418, + 11.1609, + 0.4722, + 0.9551, + 0.6563, + 1.3083, + 1.2891, + 0.3039, + 0.0836, + 1.2549, + 0.5088, + 0.3549, + 0.3585, + 1.4692, + 0.1541, + 0.8138, + 0.845, + 1.1176, + 1.3221, + 1.3282, + 0.0189, + 1.2297 + ], + "12": [ + 0.0, + 1.2174, + 1.2106, + 1.363, + 2.902, + 1.323, + 2.0136, + 1.0981, + 22.0257, + 0.3678, + 3.2028, + 1.4406, + 1.1492, + 4.0642, + 2.2218, + 1.6541, + 2.2594, + 2.7223, + 1.1709, + 0.0022, + 1.5071, + 1.6017, + 1.3116, + 1.2003, + 1.4825, + 1.8866, + 2.1188, + 1.8357, + 1.7966 + ], + "13": [ + 0.0, + 0.9743, + 0.0615, + 0.8867, + 2.2955, + 1.0726, + 2.2223, + 1.4921, + 2.8471, + 0.5363, + 1.9156, + 0.9986, + 1.5481, + 2.6971, + 2.2708, + 0.6153, + 1.8274, + 2.4201, + 1.458, + 0.7002, + 2.1975, + 1.4369, + 1.6973, + 1.1849, + 2.2716, + 2.3886, + 2.3893, + 1.5788, + 2.1329 + ], + "14": [ + 0.0, + 1.2935, + 1.0522, + 0.705, + 2.6984, + 1.5415, + 2.4911, + 1.6334, + 7.5241, + 0.6732, + 2.8239, + 1.1546, + 1.1686, + 3.0006, + 2.9099, + 1.1173, + 2.3763, + 2.9164, + 1.5709, + 0.4727, + 2.2844, + 1.6677, + 1.9035, + 2.2768, + 2.7638, + 2.0498, + 1.9047, + 1.6604, + 2.1338 + ], + "15": [ + 0.0, + 0.1075, + 0.5661, + 0.6078, + 2.0691, + 0.4303, + 1.1194, + 0.6355, + 14.0908, + 0.8425, + 1.4478, + 0.6191, + 1.4517, + 2.0044, + 2.0738, + 0.0429, + 1.7144, + 1.1508, + 1.1247, + 0.4063, + 2.0619, + 0.3754, + 1.5502, + 0.5506, + 2.5239, + 2.6346, + 2.9046, + 0.6711, + 2.3861 + ], + "16": [ + 0.0, + 0.4069, + 0.235, + 0.7426, + 1.5303, + 0.6502, + 0.8353, + 0.4443, + 7.4133, + 1.0981, + 1.6677, + 1.0938, + 2.0163, + 2.0896, + 1.9546, + 0.1895, + 1.9818, + 1.4737, + 1.1962, + 0.7998, + 2.3356, + 0.5996, + 1.6144, + 1.0601, + 2.7619, + 3.0151, + 3.0714, + 1.3312, + 2.9383 + ], + "17": [ + 0.0, + 0.5054, + 1.0823, + 0.7306, + 2.5698, + 1.7047, + 2.763, + 1.6616, + 12.9669, + 2.316, + 3.9883, + 1.5759, + 1.8431, + 4.3084, + 3.1017, + 1.1479, + 3.2001, + 4.3765, + 3.1788, + 1.3697, + 3.5768, + 1.429, + 2.22, + 3.2214, + 3.4495, + 3.463, + 3.8906, + 2.6772, + 3.2942 + ], + "18": [ + 0.0, + 2.3428, + 2.0523, + 1.3627, + 4.1372, + 2.084, + 3.1513, + 3.0959, + 58.4142, + 3.4657, + 3.8812, + 2.1134, + 2.0697, + 4.2685, + 3.087, + 1.0192, + 3.9658, + 4.1447, + 3.4258, + 2.1106, + 3.9218, + 2.1563, + 1.6877, + 3.2774, + 3.8001, + 3.2414, + 4.1205, + 3.1951, + 3.066 + ], + "19": [ + 0.0, + 1.048, + 0.4058, + 0.877, + 3.289, + 2.0841, + 3.0394, + 1.9717, + 14.7775, + 3.3602, + 3.7064, + 2.0842, + 1.6229, + 4.1912, + 3.6306, + 0.8836, + 3.7207, + 4.1381, + 2.8602, + 0.7632, + 3.8165, + 2.4159, + 1.6725, + 3.5374, + 4.6219, + 4.4688, + 5.1583, + 3.7467, + 4.7513 + ], + "20": [ + 0.0, + 2.1486, + 1.2389, + 1.7758, + 4.6429, + 3.3369, + 4.0565, + 3.0067, + 4.375, + 3.7189, + 4.2193, + 3.1819, + 3.7775, + 5.421, + 5.4849, + 1.592, + 5.0218, + 4.6044, + 3.9695, + 2.5767, + 5.1869, + 3.5893, + 4.0227, + 4.5052, + 5.7729, + 5.7388, + 6.3964, + 3.3523, + 6.219 + ], + "21": [ + 0.0, + 0.7141, + 0.534, + 0.1803, + 2.4249, + 0.6553, + 0.6415, + 0.8574, + 9.2056, + 2.2342, + 1.9477, + 1.711, + 2.8025, + 3.4795, + 4.0499, + 0.109, + 3.7299, + 1.7793, + 2.7029, + 0.8058, + 4.356, + 2.428, + 4.4204, + 2.3486, + 5.0971, + 5.2728, + 5.5277, + 1.9333, + 6.4885 + ] + }, + "rand_match": { + "9": [ + 0.0, + 0.5898, + 0.6975, + 0.0225, + 0.6956, + 0.1095, + 0.2913, + 0.3286, + 18.449, + 0.2795, + 0.5206, + 0.5763, + 0.288, + 0.6323, + 0.2901, + 0.1052, + 0.022, + 0.5663, + 0.141, + 0.3954, + 0.8266, + 0.475, + 0.0965, + 0.8281, + 0.5207, + 0.3272, + 0.2839, + 1.933, + 0.3593 + ], + "10": [ + 0.0, + 1.408, + 0.2621, + 0.2612, + 0.9743, + 0.7765, + 1.3778, + 0.8146, + 13.806, + 0.8846, + 0.9787, + 0.1845, + 0.2516, + 0.225, + 0.1137, + 0.9467, + 0.792, + 0.6052, + 0.648, + 0.1336, + 0.5737, + 0.2517, + 0.2759, + 0.67, + 0.1921, + 0.036, + 0.4115, + 0.2824, + 0.7473 + ], + "11": [ + 0.0, + 1.1244, + 1.5178, + 0.0569, + 1.3177, + 0.7153, + 0.8364, + 0.7027, + 65.4885, + 0.9299, + 1.2936, + 0.9149, + 0.4674, + 1.4689, + 0.6301, + 0.7806, + 0.7666, + 0.4746, + 0.7302, + 1.2073, + 1.1474, + 0.3723, + 0.3243, + 0.4928, + 0.5613, + 0.504, + 0.6959, + 1.6169, + 0.9353 + ], + "12": [ + 0.0, + 0.8115, + 0.71, + 0.7364, + 0.6696, + 0.7298, + 0.5398, + 0.8612, + 3.3438, + 0.213, + 0.3041, + 0.1179, + 0.0117, + 0.6554, + 0.1894, + 0.3748, + 0.0999, + 0.5938, + 0.1774, + 0.5362, + 0.712, + 0.3888, + 0.8013, + 0.2586, + 1.0943, + 1.071, + 0.786, + 0.9339, + 0.4226 + ], + "13": [ + 0.0, + 0.0172, + 0.8742, + 0.8708, + 0.324, + 0.1896, + 0.3199, + 0.9906, + 13.9717, + 0.2939, + 0.1182, + 0.1438, + 0.7088, + 0.0513, + 0.0717, + 0.8939, + 0.2634, + 0.76, + 0.3633, + 0.3604, + 0.2263, + 0.273, + 0.4798, + 0.2218, + 0.0765, + 0.0205, + 0.1903, + 0.2214, + 0.2773 + ], + "14": [ + 0.0, + 0.4113, + 0.9578, + 1.1889, + 1.198, + 0.7638, + 1.0684, + 0.5223, + 39.9258, + 0.4853, + 1.2728, + 0.1477, + 0.1347, + 0.1065, + 1.1357, + 1.1701, + 0.968, + 0.2815, + 0.2799, + 0.0796, + 0.8677, + 0.2725, + 0.3301, + 0.252, + 0.6945, + 0.9679, + 0.5315, + 1.1777, + 0.8057 + ], + "15": [ + 0.0, + 1.2938, + 0.0042, + 0.1893, + 0.0212, + 0.1204, + 0.5847, + 0.0167, + 33.7046, + 0.1474, + 0.0153, + 0.3079, + 0.627, + 0.9273, + 0.8746, + 0.5105, + 0.3272, + 0.3568, + 0.4603, + 0.669, + 0.5643, + 1.171, + 0.9638, + 0.9752, + 0.8217, + 0.2221, + 0.1075, + 0.6486, + 0.1881 + ], + "16": [ + 0.0, + 0.0025, + 0.7228, + 0.1081, + 1.0865, + 0.6174, + 1.3066, + 0.803, + 7.4613, + 0.9971, + 0.3279, + 0.4949, + 0.1848, + 0.1691, + 0.2121, + 1.2608, + 0.7321, + 0.6905, + 1.1914, + 1.2818, + 1.1188, + 0.3178, + 0.2321, + 1.2134, + 0.4412, + 0.0275, + 0.2317, + 0.574, + 0.2653 + ], + "17": [ + 0.0, + 0.262, + 0.2524, + 0.0686, + 0.4946, + 1.2733, + 0.6687, + 0.9266, + 25.9554, + 0.482, + 1.2367, + 0.3274, + 1.2827, + 0.0761, + 0.2922, + 0.22, + 1.4685, + 0.4538, + 0.8342, + 0.625, + 1.0953, + 0.9365, + 1.0069, + 0.0696, + 0.1514, + 0.8691, + 0.7102, + 0.8274, + 0.7723 + ], + "18": [ + 0.0, + 1.7639, + 1.2657, + 1.1493, + 1.9997, + 1.9384, + 2.1341, + 2.0132, + 14.3193, + 0.9078, + 2.1458, + 1.054, + 1.1746, + 0.8027, + 1.7828, + 0.2794, + 1.7967, + 1.0681, + 1.6524, + 0.4839, + 2.3087, + 0.242, + 0.0579, + 0.877, + 0.918, + 2.0499, + 1.5413, + 0.6021, + 2.3303 + ], + "19": [ + 0.0, + 3.1805, + 4.215, + 2.2968, + 4.1054, + 3.1206, + 3.4811, + 3.9383, + 24.0266, + 2.7721, + 3.7259, + 3.9033, + 3.1828, + 1.8723, + 1.9023, + 0.817, + 2.5278, + 2.3342, + 2.2404, + 3.1989, + 2.7704, + 0.1651, + 1.5038, + 1.2087, + 1.4355, + 2.6064, + 2.2176, + 1.2971, + 1.5324 + ], + "20": [ + 0.0, + 4.668, + 2.7469, + 1.9086, + 4.098, + 4.0648, + 2.7321, + 3.3982, + 66.5952, + 3.5569, + 3.6699, + 1.7077, + 2.5031, + 3.4523, + 3.4482, + 1.2191, + 4.0431, + 1.5527, + 1.9414, + 3.9508, + 2.6176, + 1.0604, + 2.6731, + 1.3379, + 1.2487, + 0.7308, + 3.5167, + 0.1057, + 1.1904 + ], + "21": [ + 0.0, + 0.1351, + 0.4698, + 0.0914, + 0.1958, + 1.0186, + 1.7845, + 1.9603, + 16.9094, + 2.4662, + 1.4342, + 1.9199, + 0.0126, + 0.306, + 0.1821, + 2.0116, + 0.3697, + 1.5828, + 1.6723, + 0.0225, + 1.8998, + 2.5123, + 0.0766, + 2.75, + 1.7539, + 0.9716, + 0.4815, + 0.4525, + 0.095 + ] + } + } + }, + "typo": { + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.031943, + "jlens": 0.067202, + "shuffled": 0.040041, + "rand_match": 0.002011 + }, + "PRIMARY_ratio_jlens_over_logit": 2.1038, + "ratio_jlens_over_shuffled": 1.6783, + "ratio_jlens_over_rand_match": 33.411, + "ratio_by_layer_jlens_over_logit": { + "9": 17.4801, + "10": 13.2939, + "11": 12.0608, + "12": 6.5672, + "13": 6.2353, + "14": 5.4704, + "15": 4.9929, + "16": 2.686, + "17": 1.3342, + "18": 1.3976, + "19": 1.0951, + "20": 1.1111, + "21": 1.0013 + }, + "band_max_ratio_SECONDARY": 17.4801, + "per_layer": { + "logit": { + "9": { + "kl_mean": 0.003011, + "dose_mean_per_position": 0.6083, + "dose_sum_over_positions": 5.947 + }, + "10": { + "kl_mean": 0.004027, + "dose_mean_per_position": 0.643, + "dose_sum_over_positions": 6.2741 + }, + "11": { + "kl_mean": 0.004759, + "dose_mean_per_position": 0.59, + "dose_sum_over_positions": 5.7424 + }, + "12": { + "kl_mean": 0.007749, + "dose_mean_per_position": 0.6951, + "dose_sum_over_positions": 6.79 + }, + "13": { + "kl_mean": 0.008771, + "dose_mean_per_position": 0.7758, + "dose_sum_over_positions": 7.5701 + }, + "14": { + "kl_mean": 0.012616, + "dose_mean_per_position": 0.8957, + "dose_sum_over_positions": 8.739 + }, + "15": { + "kl_mean": 0.01615, + "dose_mean_per_position": 1.0632, + "dose_sum_over_positions": 10.3932 + }, + "16": { + "kl_mean": 0.026228, + "dose_mean_per_position": 1.397, + "dose_sum_over_positions": 13.6344 + }, + "17": { + "kl_mean": 0.049754, + "dose_mean_per_position": 2.1361, + "dose_sum_over_positions": 20.6531 + }, + "18": { + "kl_mean": 0.053322, + "dose_mean_per_position": 2.2644, + "dose_sum_over_positions": 21.8802 + }, + "19": { + "kl_mean": 0.068774, + "dose_mean_per_position": 2.6766, + "dose_sum_over_positions": 25.8524 + }, + "20": { + "kl_mean": 0.071678, + "dose_mean_per_position": 2.7231, + "dose_sum_over_positions": 26.2575 + }, + "21": { + "kl_mean": 0.088414, + "dose_mean_per_position": 2.9455, + "dose_sum_over_positions": 28.5094 + } + }, + "jlens": { + "9": { + "kl_mean": 0.052631, + "dose_mean_per_position": 1.7023, + "dose_sum_over_positions": 16.4532 + }, + "10": { + "kl_mean": 0.05353, + "dose_mean_per_position": 1.5829, + "dose_sum_over_positions": 15.3319 + }, + "11": { + "kl_mean": 0.057401, + "dose_mean_per_position": 1.5353, + "dose_sum_over_positions": 14.8667 + }, + "12": { + "kl_mean": 0.05089, + "dose_mean_per_position": 1.5503, + "dose_sum_over_positions": 15.0064 + }, + "13": { + "kl_mean": 0.054687, + "dose_mean_per_position": 1.6497, + "dose_sum_over_positions": 15.9892 + }, + "14": { + "kl_mean": 0.069014, + "dose_mean_per_position": 1.7564, + "dose_sum_over_positions": 16.9981 + }, + "15": { + "kl_mean": 0.080635, + "dose_mean_per_position": 1.9717, + "dose_sum_over_positions": 19.0806 + }, + "16": { + "kl_mean": 0.070446, + "dose_mean_per_position": 1.8988, + "dose_sum_over_positions": 18.3736 + }, + "17": { + "kl_mean": 0.066383, + "dose_mean_per_position": 2.0897, + "dose_sum_over_positions": 20.1743 + }, + "18": { + "kl_mean": 0.074521, + "dose_mean_per_position": 2.2225, + "dose_sum_over_positions": 21.4705 + }, + "19": { + "kl_mean": 0.075313, + "dose_mean_per_position": 2.5081, + "dose_sum_over_positions": 24.2731 + }, + "20": { + "kl_mean": 0.079641, + "dose_mean_per_position": 2.6725, + "dose_sum_over_positions": 25.8747 + }, + "21": { + "kl_mean": 0.088532, + "dose_mean_per_position": 2.7797, + "dose_sum_over_positions": 26.9325 + } + }, + "shuffled": { + "9": { + "kl_mean": 0.043251, + "dose_mean_per_position": 1.5956, + "dose_sum_over_positions": 15.4409 + }, + "10": { + "kl_mean": 0.027135, + "dose_mean_per_position": 1.2771, + "dose_sum_over_positions": 12.3325 + }, + "11": { + "kl_mean": 0.004824, + "dose_mean_per_position": 0.6254, + "dose_sum_over_positions": 6.1014 + }, + "12": { + "kl_mean": 0.0459, + "dose_mean_per_position": 1.6067, + "dose_sum_over_positions": 15.4688 + }, + "13": { + "kl_mean": 0.048159, + "dose_mean_per_position": 1.492, + "dose_sum_over_positions": 14.4909 + }, + "14": { + "kl_mean": 0.048079, + "dose_mean_per_position": 1.7201, + "dose_sum_over_positions": 16.6034 + }, + "15": { + "kl_mean": 0.026975, + "dose_mean_per_position": 1.2056, + "dose_sum_over_positions": 11.7094 + }, + "16": { + "kl_mean": 0.027428, + "dose_mean_per_position": 1.3915, + "dose_sum_over_positions": 13.5874 + }, + "17": { + "kl_mean": 0.032227, + "dose_mean_per_position": 2.3099, + "dose_sum_over_positions": 22.2809 + }, + "18": { + "kl_mean": 0.011487, + "dose_mean_per_position": 1.6193, + "dose_sum_over_positions": 15.7416 + }, + "19": { + "kl_mean": 0.039816, + "dose_mean_per_position": 2.739, + "dose_sum_over_positions": 26.3152 + }, + "20": { + "kl_mean": 0.082244, + "dose_mean_per_position": 3.4508, + "dose_sum_over_positions": 33.294 + }, + "21": { + "kl_mean": 0.083009, + "dose_mean_per_position": 2.6662, + "dose_sum_over_positions": 25.9228 + } + }, + "rand_match": { + "9": { + "kl_mean": 0.001135, + "dose_mean_per_position": 0.4846, + "dose_sum_over_positions": 4.6731 + }, + "10": { + "kl_mean": 0.000701, + "dose_mean_per_position": 0.4268, + "dose_sum_over_positions": 4.2086 + }, + "11": { + "kl_mean": 0.001139, + "dose_mean_per_position": 0.4683, + "dose_sum_over_positions": 4.5391 + }, + "12": { + "kl_mean": 0.000745, + "dose_mean_per_position": 0.437, + "dose_sum_over_positions": 4.2696 + }, + "13": { + "kl_mean": 0.0012, + "dose_mean_per_position": 0.4876, + "dose_sum_over_positions": 4.7436 + }, + "14": { + "kl_mean": 0.001167, + "dose_mean_per_position": 0.4902, + "dose_sum_over_positions": 4.7858 + }, + "15": { + "kl_mean": 0.001519, + "dose_mean_per_position": 0.5637, + "dose_sum_over_positions": 5.5024 + }, + "16": { + "kl_mean": 0.002643, + "dose_mean_per_position": 0.679, + "dose_sum_over_positions": 6.5737 + }, + "17": { + "kl_mean": 0.00208, + "dose_mean_per_position": 0.7207, + "dose_sum_over_positions": 7.039 + }, + "18": { + "kl_mean": 0.003913, + "dose_mean_per_position": 0.8787, + "dose_sum_over_positions": 8.6031 + }, + "19": { + "kl_mean": 0.002521, + "dose_mean_per_position": 0.8777, + "dose_sum_over_positions": 8.552 + }, + "20": { + "kl_mean": 0.003964, + "dose_mean_per_position": 1.1881, + "dose_sum_over_positions": 11.6789 + }, + "21": { + "kl_mean": 0.003421, + "dose_mean_per_position": 1.1342, + "dose_sum_over_positions": 11.0043 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "9": [ + 0.0, + 0.4523, + 0.9596, + 0.4291, + 0.6165, + 0.2689, + 0.5598, + 0.4153, + 0.2302, + 0.3378, + 0.0645, + 0.5288 + ], + "10": [ + 0.0, + 0.2584, + 0.7473, + 0.3322, + 0.7151, + 0.0137, + 0.3937, + 0.0446, + 0.3, + 0.2354, + 0.09, + 0.6552 + ], + "11": [ + 0.0, + 0.4931, + 0.5385, + 0.3101, + 0.5957, + 0.1411, + 0.472, + 0.183, + 0.219, + 0.5125, + 0.1136, + 0.1817 + ], + "12": [ + 0.0, + 0.6832, + 0.6176, + 0.21, + 0.3709, + 0.2822, + 0.0993, + 0.1669, + 0.7552, + 0.8923, + 0.0309, + 0.7515 + ], + "13": [ + 0.0, + 0.6335, + 1.1715, + 0.4653, + 0.4552, + 0.4605, + 0.2822, + 0.5073, + 0.6257, + 1.0862, + 0.7872, + 1.2123 + ], + "14": [ + 0.0, + 0.9272, + 0.8004, + 0.4639, + 0.4557, + 0.3252, + 0.3853, + 0.952, + 1.1361, + 0.9392, + 0.0498, + 1.0597 + ], + "15": [ + 0.0, + 0.4018, + 0.7553, + 0.5563, + 0.4086, + 0.5848, + 0.2139, + 1.3298, + 1.5264, + 1.3795, + 0.387, + 1.1615 + ], + "16": [ + 0.0, + 0.5182, + 0.9744, + 0.3831, + 0.2155, + 0.7823, + 0.1119, + 0.994, + 1.5819, + 1.6958, + 0.1562, + 1.3573 + ], + "17": [ + 0.0, + 0.3968, + 1.126, + 0.3733, + 0.23, + 0.8205, + 0.1158, + 1.2726, + 2.0321, + 2.5946, + 0.7687, + 1.9793 + ], + "18": [ + 0.0, + 0.9511, + 0.6698, + 0.584, + 0.8453, + 0.0065, + 0.6811, + 0.9633, + 1.5134, + 2.3061, + 0.7173, + 1.5677 + ], + "19": [ + 0.0, + 0.1344, + 1.4776, + 0.0197, + 0.2167, + 1.0267, + 0.0457, + 1.5812, + 2.7698, + 3.4758, + 1.4806, + 2.8637 + ], + "20": [ + 0.0, + 0.5852, + 0.9779, + 0.4389, + 0.7777, + 0.5515, + 0.7182, + 0.5397, + 2.0675, + 3.3898, + 1.075, + 2.2113 + ], + "21": [ + 0.0, + 0.9264, + 2.0725, + 1.2294, + 0.7306, + 1.8516, + 0.0807, + 2.1944, + 3.63, + 4.4579, + 1.8817, + 3.1478 + ] + }, + "jlens": { + "9": [ + 0.0, + 0.0804, + 1.9936, + 1.9554, + 1.4918, + 0.8323, + 1.8662, + 1.787, + 1.4048, + 0.7277, + 4.4844, + 4.5963 + ], + "10": [ + 0.0, + 0.6293, + 1.8607, + 2.1075, + 1.707, + 0.7888, + 1.7328, + 1.7617, + 1.3029, + 0.7637, + 4.1327, + 4.3385 + ], + "11": [ + 0.0, + 0.8508, + 2.318, + 2.1236, + 1.5706, + 0.8964, + 1.5479, + 1.9717, + 1.371, + 1.0836, + 4.3418, + 3.7068 + ], + "12": [ + 0.0, + 0.6467, + 2.0943, + 1.9741, + 1.527, + 1.0229, + 1.6863, + 1.9335, + 1.5694, + 1.2307, + 4.5126, + 3.7805 + ], + "13": [ + 0.0, + 0.8044, + 2.1665, + 1.7013, + 1.5738, + 0.9305, + 1.4867, + 1.7529, + 1.2771, + 1.3748, + 5.1257, + 4.3551 + ], + "14": [ + 0.0, + 0.9316, + 2.7325, + 1.7373, + 1.4897, + 1.1415, + 1.8637, + 2.5148, + 2.1121, + 1.854, + 5.568, + 4.9752 + ], + "15": [ + 0.0, + 1.6226, + 3.334, + 2.6092, + 1.8333, + 1.7022, + 2.2446, + 3.1379, + 2.845, + 2.567, + 6.0115, + 5.1382 + ], + "16": [ + 0.0, + 1.7278, + 2.6955, + 2.2519, + 1.5732, + 2.1289, + 1.8554, + 2.8042, + 3.1516, + 2.9615, + 4.2952, + 4.4877 + ], + "17": [ + 0.0, + 1.9697, + 2.9201, + 1.8607, + 1.4413, + 2.3884, + 1.392, + 2.928, + 3.8753, + 3.7574, + 3.8795, + 4.5337 + ], + "18": [ + 0.0, + 1.8922, + 2.6236, + 1.7755, + 1.3011, + 2.4847, + 1.2067, + 3.286, + 4.3604, + 4.0677, + 3.5543, + 4.5503 + ], + "19": [ + 0.0, + 1.9519, + 2.6958, + 1.9638, + 1.5134, + 2.8384, + 1.0915, + 3.3548, + 5.0292, + 4.8413, + 1.5303, + 3.9957 + ], + "20": [ + 0.0, + 1.8406, + 2.5741, + 2.1402, + 1.8146, + 2.8954, + 1.0583, + 3.0529, + 4.8718, + 4.8825, + 1.4963, + 3.8738 + ], + "21": [ + 0.0, + 1.6421, + 2.365, + 2.2074, + 1.62, + 2.7442, + 0.6093, + 3.2565, + 5.1241, + 4.9401, + 1.3366, + 3.5328 + ] + }, + "shuffled": { + "9": [ + 0.0, + 0.7975, + 2.2925, + 2.4489, + 1.9406, + 1.0859, + 1.7848, + 1.9709, + 1.6391, + 1.0693, + 4.3333, + 4.4893 + ], + "10": [ + 0.0, + 0.1746, + 1.7674, + 1.95, + 1.2325, + 0.8941, + 1.7572, + 1.6112, + 1.1199, + 0.6183, + 3.7774, + 3.2026 + ], + "11": [ + 0.0, + 0.1248, + 0.8746, + 0.7643, + 1.1007, + 0.4954, + 0.1393, + 0.2189, + 0.8425, + 0.7288, + 0.1438, + 0.1862 + ], + "12": [ + 0.0, + 0.0747, + 1.7224, + 1.3268, + 1.04, + 0.6055, + 1.559, + 1.5575, + 1.294, + 1.4108, + 4.3038, + 3.685 + ], + "13": [ + 0.0, + 0.8827, + 2.3191, + 1.6898, + 1.5334, + 0.9355, + 1.4279, + 1.6373, + 1.5592, + 1.5965, + 5.414, + 4.3269 + ], + "14": [ + 0.0, + 1.0089, + 2.5324, + 1.4326, + 1.5723, + 0.8679, + 1.7236, + 1.9154, + 1.3767, + 1.2062, + 4.8538, + 4.0936 + ], + "15": [ + 0.0, + 0.67, + 1.4922, + 1.5035, + 0.9161, + 1.3969, + 0.8093, + 1.9931, + 2.3914, + 1.7101, + 2.3331, + 2.6188 + ], + "16": [ + 0.0, + 0.4363, + 1.853, + 1.4068, + 1.0672, + 1.673, + 1.0299, + 1.9392, + 2.4664, + 2.1289, + 0.7242, + 2.3495 + ], + "17": [ + 0.0, + 0.9115, + 1.9376, + 1.973, + 1.5542, + 1.3822, + 1.7845, + 1.91, + 1.4005, + 1.47, + 4.2775, + 3.9496 + ], + "18": [ + 0.0, + 1.2451, + 2.5508, + 2.6402, + 1.9947, + 2.2082, + 3.2013, + 2.5897, + 1.5144, + 1.8259, + 4.9727, + 3.8039 + ], + "19": [ + 0.0, + 0.5036, + 1.4224, + 0.9081, + 0.559, + 0.5546, + 0.7611, + 0.7228, + 0.4838, + 1.079, + 3.4089, + 3.5135 + ], + "20": [ + 0.0, + 1.376, + 1.8674, + 1.2554, + 0.3356, + 0.8071, + 0.4823, + 1.9915, + 2.1367, + 2.8004, + 3.5024, + 5.0367 + ], + "21": [ + 0.0, + 2.5604, + 3.0371, + 2.8986, + 2.4207, + 3.6256, + 1.2178, + 3.9814, + 6.0885, + 5.6368, + 1.6972, + 3.9797 + ] + }, + "rand_match": { + "9": [ + 0.0, + 0.6843, + 1.3166, + 0.7209, + 0.5903, + 0.9458, + 0.5795, + 0.132, + 0.193, + 0.0652, + 0.4602, + 0.9116 + ], + "10": [ + 0.0, + 0.1006, + 1.2003, + 0.7309, + 0.7557, + 0.0778, + 0.3925, + 0.654, + 0.5259, + 0.2262, + 0.7728, + 0.6489 + ], + "11": [ + 0.0, + 0.0834, + 0.0786, + 0.5401, + 0.4932, + 0.4346, + 1.1755, + 0.4496, + 0.6914, + 0.0892, + 0.0652, + 0.3609 + ], + "12": [ + 0.0, + 0.2739, + 0.3431, + 0.3333, + 0.2261, + 0.3822, + 0.2692, + 0.1982, + 0.0284, + 0.3439, + 0.0355, + 0.3806 + ], + "13": [ + 0.0, + 0.3151, + 0.2077, + 0.1747, + 0.2335, + 0.0727, + 0.5917, + 0.3163, + 0.2378, + 0.1836, + 0.5435, + 0.1839 + ], + "14": [ + 0.0, + 1.4628, + 2.1086, + 1.195, + 0.9214, + 1.3882, + 1.4766, + 0.6667, + 0.707, + 1.2296, + 0.3402, + 0.0844 + ], + "15": [ + 0.0, + 0.2126, + 0.6148, + 0.5867, + 0.4112, + 0.7529, + 0.3697, + 0.3521, + 0.2378, + 0.094, + 0.3567, + 0.8437 + ], + "16": [ + 0.0, + 0.038, + 1.5497, + 0.9519, + 0.7533, + 0.2766, + 0.7358, + 0.4967, + 0.4595, + 0.2615, + 1.6343, + 0.4901 + ], + "17": [ + 0.0, + 0.688, + 0.7941, + 0.7675, + 0.8296, + 0.4477, + 1.2804, + 0.4087, + 0.1715, + 0.3926, + 0.2271, + 0.7254 + ], + "18": [ + 0.0, + 3.5907, + 3.4397, + 3.147, + 2.8973, + 3.1662, + 3.0285, + 3.9317, + 3.7705, + 2.4114, + 2.219, + 2.2853 + ], + "19": [ + 0.0, + 0.9755, + 0.1437, + 0.2076, + 0.5556, + 0.7421, + 0.5216, + 0.2948, + 0.1795, + 0.3135, + 1.2422, + 0.1651 + ], + "20": [ + 0.0, + 1.194, + 1.9048, + 0.2769, + 0.3358, + 1.3456, + 0.8788, + 0.474, + 1.9991, + 1.4214, + 0.1903, + 0.2918 + ], + "21": [ + 0.0, + 0.8443, + 1.1472, + 1.035, + 1.1471, + 1.4081, + 0.5976, + 1.1141, + 1.5467, + 1.0445, + 0.1077, + 1.4376 + ] + } + } + }, + "association": { + "n_items_scored": 102, + "n_intermediate_pairs_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "band_mean_kl": { + "logit": 0.001279, + "jlens": 0.001497, + "shuffled": 0.001215, + "rand_match": 0.000533 + }, + "PRIMARY_ratio_jlens_over_logit": 1.1701, + "ratio_jlens_over_shuffled": 1.2317, + "ratio_jlens_over_rand_match": 2.8061, + "ratio_by_layer_jlens_over_logit": { + "9": 12.8028, + "10": 10.0564, + "11": 12.076, + "12": 9.0982, + "13": 5.7813, + "14": 5.2462, + "15": 3.9739, + "16": 1.0452, + "17": 0.6426, + "18": 0.4575, + "19": 0.3362, + "20": 0.1928, + "21": 0.3944 + }, + "band_max_ratio_SECONDARY": 12.8028, + "per_layer": { + "logit": { + "9": { + "kl_mean": 0.000177, + "dose_mean_per_position": 0.4816, + "dose_sum_over_positions": 14.1578 + }, + "10": { + "kl_mean": 0.000214, + "dose_mean_per_position": 0.51, + "dose_sum_over_positions": 15.0779 + }, + "11": { + "kl_mean": 0.00018, + "dose_mean_per_position": 0.4547, + "dose_sum_over_positions": 13.4506 + }, + "12": { + "kl_mean": 0.000171, + "dose_mean_per_position": 0.4663, + "dose_sum_over_positions": 13.8211 + }, + "13": { + "kl_mean": 0.000255, + "dose_mean_per_position": 0.5678, + "dose_sum_over_positions": 16.7575 + }, + "14": { + "kl_mean": 0.00031, + "dose_mean_per_position": 0.5884, + "dose_sum_over_positions": 17.3774 + }, + "15": { + "kl_mean": 0.000494, + "dose_mean_per_position": 0.6936, + "dose_sum_over_positions": 20.3625 + }, + "16": { + "kl_mean": 0.001349, + "dose_mean_per_position": 1.052, + "dose_sum_over_positions": 30.5976 + }, + "17": { + "kl_mean": 0.001778, + "dose_mean_per_position": 1.5039, + "dose_sum_over_positions": 43.8406 + }, + "18": { + "kl_mean": 0.002359, + "dose_mean_per_position": 1.6895, + "dose_sum_over_positions": 49.129 + }, + "19": { + "kl_mean": 0.003068, + "dose_mean_per_position": 1.8825, + "dose_sum_over_positions": 54.8938 + }, + "20": { + "kl_mean": 0.004387, + "dose_mean_per_position": 2.1549, + "dose_sum_over_positions": 62.7498 + }, + "21": { + "kl_mean": 0.001887, + "dose_mean_per_position": 1.8391, + "dose_sum_over_positions": 53.84 + } + }, + "jlens": { + "9": { + "kl_mean": 0.002265, + "dose_mean_per_position": 0.9104, + "dose_sum_over_positions": 26.8801 + }, + "10": { + "kl_mean": 0.002155, + "dose_mean_per_position": 0.8362, + "dose_sum_over_positions": 24.7376 + }, + "11": { + "kl_mean": 0.00217, + "dose_mean_per_position": 0.7703, + "dose_sum_over_positions": 22.7609 + }, + "12": { + "kl_mean": 0.001552, + "dose_mean_per_position": 0.7864, + "dose_sum_over_positions": 23.1869 + }, + "13": { + "kl_mean": 0.001473, + "dose_mean_per_position": 0.8781, + "dose_sum_over_positions": 25.8333 + }, + "14": { + "kl_mean": 0.001625, + "dose_mean_per_position": 0.8708, + "dose_sum_over_positions": 25.5397 + }, + "15": { + "kl_mean": 0.001962, + "dose_mean_per_position": 0.9584, + "dose_sum_over_positions": 28.0919 + }, + "16": { + "kl_mean": 0.00141, + "dose_mean_per_position": 1.001, + "dose_sum_over_positions": 29.1689 + }, + "17": { + "kl_mean": 0.001143, + "dose_mean_per_position": 1.0984, + "dose_sum_over_positions": 32.2183 + }, + "18": { + "kl_mean": 0.001079, + "dose_mean_per_position": 1.1764, + "dose_sum_over_positions": 34.4011 + }, + "19": { + "kl_mean": 0.001031, + "dose_mean_per_position": 1.335, + "dose_sum_over_positions": 39.1063 + }, + "20": { + "kl_mean": 0.000846, + "dose_mean_per_position": 1.4148, + "dose_sum_over_positions": 41.5107 + }, + "21": { + "kl_mean": 0.000744, + "dose_mean_per_position": 1.4186, + "dose_sum_over_positions": 41.5563 + } + }, + "shuffled": { + "9": { + "kl_mean": 0.001585, + "dose_mean_per_position": 0.8526, + "dose_sum_over_positions": 25.2077 + }, + "10": { + "kl_mean": 0.000941, + "dose_mean_per_position": 0.7351, + "dose_sum_over_positions": 21.6288 + }, + "11": { + "kl_mean": 0.000189, + "dose_mean_per_position": 0.4339, + "dose_sum_over_positions": 12.7483 + }, + "12": { + "kl_mean": 0.001472, + "dose_mean_per_position": 0.8987, + "dose_sum_over_positions": 26.5502 + }, + "13": { + "kl_mean": 0.000954, + "dose_mean_per_position": 0.7584, + "dose_sum_over_positions": 22.2789 + }, + "14": { + "kl_mean": 0.001595, + "dose_mean_per_position": 0.8686, + "dose_sum_over_positions": 25.7127 + }, + "15": { + "kl_mean": 0.000442, + "dose_mean_per_position": 0.6845, + "dose_sum_over_positions": 20.0568 + }, + "16": { + "kl_mean": 0.000491, + "dose_mean_per_position": 0.7606, + "dose_sum_over_positions": 22.1532 + }, + "17": { + "kl_mean": 0.001574, + "dose_mean_per_position": 1.3841, + "dose_sum_over_positions": 40.5965 + }, + "18": { + "kl_mean": 0.001211, + "dose_mean_per_position": 1.2775, + "dose_sum_over_positions": 37.6137 + }, + "19": { + "kl_mean": 0.002482, + "dose_mean_per_position": 1.7516, + "dose_sum_over_positions": 51.7069 + }, + "20": { + "kl_mean": 0.002208, + "dose_mean_per_position": 1.8346, + "dose_sum_over_positions": 54.0029 + }, + "21": { + "kl_mean": 0.000653, + "dose_mean_per_position": 1.351, + "dose_sum_over_positions": 39.5687 + } + }, + "rand_match": { + "9": { + "kl_mean": 0.000207, + "dose_mean_per_position": 0.4705, + "dose_sum_over_positions": 13.7812 + }, + "10": { + "kl_mean": 0.000236, + "dose_mean_per_position": 0.4611, + "dose_sum_over_positions": 13.441 + }, + "11": { + "kl_mean": 0.000218, + "dose_mean_per_position": 0.461, + "dose_sum_over_positions": 13.395 + }, + "12": { + "kl_mean": 0.000298, + "dose_mean_per_position": 0.4641, + "dose_sum_over_positions": 13.6546 + }, + "13": { + "kl_mean": 0.000238, + "dose_mean_per_position": 0.4968, + "dose_sum_over_positions": 14.5888 + }, + "14": { + "kl_mean": 0.000306, + "dose_mean_per_position": 0.5384, + "dose_sum_over_positions": 15.8323 + }, + "15": { + "kl_mean": 0.000294, + "dose_mean_per_position": 0.5607, + "dose_sum_over_positions": 16.4353 + }, + "16": { + "kl_mean": 0.000574, + "dose_mean_per_position": 0.7288, + "dose_sum_over_positions": 21.3419 + }, + "17": { + "kl_mean": 0.000648, + "dose_mean_per_position": 0.7916, + "dose_sum_over_positions": 23.3436 + }, + "18": { + "kl_mean": 0.001045, + "dose_mean_per_position": 1.018, + "dose_sum_over_positions": 29.7176 + }, + "19": { + "kl_mean": 0.000924, + "dose_mean_per_position": 1.0521, + "dose_sum_over_positions": 31.0704 + }, + "20": { + "kl_mean": 0.001057, + "dose_mean_per_position": 1.1941, + "dose_sum_over_positions": 34.7283 + }, + "21": { + "kl_mean": 0.000891, + "dose_mean_per_position": 1.1922, + "dose_sum_over_positions": 35.0853 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "9": [ + 0.0, + 1.624, + 1.1303, + 0.7403, + 0.7708, + 0.5137, + 0.963, + 0.4064, + 0.1647, + 0.7673, + 1.4448, + 1.2167, + 0.1049, + 0.7632, + 0.0183, + 0.3599, + 0.2491, + 0.5669, + 0.3937, + 1.3735, + 0.1744, + 0.4115, + 0.6039, + 0.9988, + 1.1265, + 0.1221, + 0.8868, + 1.0477, + 0.6112 + ], + "10": [ + 0.0, + 1.6397, + 1.0336, + 0.7501, + 1.1044, + 1.0101, + 1.3478, + 0.7299, + 0.4634, + 0.7349, + 1.5438, + 1.6192, + 0.2635, + 0.735, + 0.248, + 0.0435, + 0.805, + 0.6453, + 0.0525, + 0.8669, + 0.003, + 0.7138, + 0.5284, + 0.7078, + 1.6906, + 0.7235, + 1.1162, + 1.15, + 0.4913 + ], + "11": [ + 0.0, + 1.5156, + 1.0739, + 0.3459, + 0.7917, + 0.7592, + 1.0311, + 0.5742, + 0.7475, + 0.4831, + 1.4592, + 1.6341, + 0.0744, + 0.8919, + 0.4683, + 0.1081, + 0.7835, + 0.365, + 0.191, + 0.8841, + 0.0075, + 0.2465, + 0.0297, + 0.4155, + 1.4815, + 0.5551, + 0.5888, + 0.9158, + 0.4992 + ], + "12": [ + 0.0, + 1.5477, + 1.2073, + 0.3907, + 1.0537, + 0.882, + 1.3332, + 0.9323, + 1.3276, + 0.8052, + 1.5206, + 1.9664, + 0.2337, + 1.1824, + 0.1387, + 0.4528, + 1.0697, + 0.6923, + 0.3364, + 0.3957, + 0.0478, + 0.5857, + 0.2897, + 0.7968, + 2.1287, + 0.6318, + 0.7513, + 1.084, + 0.5505 + ], + "13": [ + 0.0, + 1.9597, + 1.5807, + 0.7566, + 1.5905, + 1.6395, + 1.5949, + 1.0749, + 1.8127, + 1.4337, + 1.5996, + 2.2329, + 0.8916, + 1.5388, + 0.6203, + 0.8204, + 1.5676, + 0.9086, + 0.9009, + 0.1416, + 0.8775, + 1.4651, + 1.2601, + 1.4231, + 2.4906, + 1.0901, + 1.2785, + 1.6966, + 0.79 + ], + "14": [ + 0.0, + 1.8069, + 1.509, + 0.6255, + 1.6773, + 1.9923, + 1.9094, + 1.248, + 1.6989, + 1.38, + 1.7377, + 2.195, + 1.1666, + 1.2547, + 0.484, + 0.773, + 1.741, + 1.473, + 0.9374, + 0.1456, + 0.8779, + 1.2945, + 1.1937, + 1.3497, + 2.433, + 1.1023, + 1.3526, + 1.9355, + 0.75 + ], + "15": [ + 0.0, + 1.8337, + 1.9298, + 0.9994, + 2.2663, + 2.5574, + 2.1773, + 1.5797, + 2.0935, + 2.02, + 2.2062, + 2.5946, + 1.3703, + 1.6821, + 1.0335, + 1.5029, + 2.0324, + 1.8567, + 1.5766, + 0.8949, + 0.9742, + 1.716, + 1.627, + 2.0201, + 2.768, + 1.1528, + 1.595, + 2.1857, + 1.2011 + ], + "16": [ + 0.0, + 2.3815, + 2.4339, + 1.6239, + 3.0443, + 3.2588, + 2.9864, + 2.6355, + 2.9935, + 3.2621, + 2.9318, + 3.4528, + 2.4374, + 2.9026, + 2.1671, + 2.6394, + 2.8011, + 2.7478, + 3.1569, + 2.3504, + 2.3628, + 2.9028, + 2.7997, + 3.0384, + 4.0515, + 2.2895, + 1.9301, + 3.328, + 2.6328 + ], + "17": [ + 0.0, + 2.7749, + 3.2207, + 1.9689, + 3.7672, + 3.635, + 3.5636, + 3.4827, + 3.8329, + 3.5946, + 4.0042, + 3.3992, + 2.4177, + 3.4623, + 2.9245, + 3.4437, + 3.1464, + 3.5766, + 3.0784, + 2.82, + 3.3466, + 3.7017, + 3.5772, + 3.862, + 3.8409, + 2.2362, + 2.4402, + 4.2696, + 3.5114 + ], + "18": [ + 0.0, + 3.8732, + 4.3212, + 3.5384, + 4.5832, + 4.4002, + 4.7651, + 4.7019, + 4.8219, + 4.8753, + 4.5502, + 3.9486, + 3.6342, + 4.4674, + 3.55, + 3.7858, + 3.4332, + 3.8815, + 3.8991, + 3.9907, + 4.5281, + 4.0126, + 4.0366, + 5.0245, + 4.772, + 2.8693, + 3.1156, + 5.2055, + 4.618 + ], + "19": [ + 0.0, + 3.3923, + 3.7355, + 2.9543, + 4.5593, + 4.6837, + 4.5658, + 4.1294, + 4.8563, + 4.4783, + 4.134, + 3.9663, + 4.1818, + 4.217, + 3.5075, + 3.1426, + 3.1514, + 3.3798, + 3.8148, + 3.7339, + 4.2895, + 3.3624, + 3.5746, + 3.9655, + 4.253, + 2.4296, + 2.2967, + 4.3814, + 4.4238 + ], + "20": [ + 0.0, + 4.4807, + 4.3937, + 3.4708, + 4.9431, + 4.7056, + 5.1233, + 5.4391, + 5.1988, + 5.03, + 4.6229, + 4.7965, + 4.9175, + 4.7791, + 3.6919, + 3.6824, + 3.332, + 3.4295, + 4.4408, + 4.233, + 5.4004, + 4.3845, + 4.3015, + 4.355, + 4.2492, + 3.5878, + 3.0251, + 5.0329, + 5.199 + ], + "21": [ + 0.0, + 3.5995, + 2.5365, + 2.1067, + 4.3247, + 4.3148, + 4.179, + 4.9868, + 4.0204, + 3.1413, + 3.1016, + 4.175, + 4.5482, + 3.4642, + 3.0765, + 2.277, + 2.8498, + 2.3953, + 2.1216, + 2.0015, + 3.7813, + 3.6346, + 3.3449, + 2.3508, + 3.323, + 2.828, + 2.2939, + 2.9293, + 3.1301 + ] + }, + "jlens": { + "9": [ + 0.0, + 1.7076, + 0.48, + 0.1956, + 1.708, + 0.6555, + 0.553, + 0.5082, + 0.9563, + 0.9094, + 0.9334, + 0.1017, + 0.0746, + 0.0692, + 0.0067, + 0.7747, + 0.7007, + 1.2603, + 0.3711, + 0.4232, + 0.2586, + 0.6759, + 0.546, + 0.2442, + 0.2202, + 0.5315, + 0.6046, + 1.0545, + 0.4124 + ], + "10": [ + 0.0, + 1.4941, + 0.4887, + 0.0037, + 2.1218, + 1.2223, + 0.8068, + 0.6309, + 1.1161, + 0.6118, + 1.0798, + 0.2808, + 0.1136, + 0.3358, + 0.3758, + 0.3828, + 0.2732, + 0.8236, + 0.3689, + 0.4752, + 0.5416, + 0.3832, + 0.1047, + 0.0485, + 0.1518, + 0.2217, + 0.4192, + 0.7268, + 0.2951 + ], + "11": [ + 0.0, + 1.4939, + 0.2643, + 0.1156, + 1.9124, + 1.107, + 0.4494, + 0.4099, + 1.1229, + 0.4369, + 0.9559, + 0.5904, + 0.2402, + 0.5568, + 0.2866, + 0.3093, + 0.1525, + 0.5974, + 0.2291, + 0.1689, + 0.0588, + 0.3937, + 0.275, + 0.2967, + 0.3951, + 0.1142, + 0.2919, + 0.3699, + 0.1946 + ], + "12": [ + 0.0, + 1.2729, + 0.2412, + 0.0964, + 1.1896, + 0.5994, + 0.3007, + 0.1483, + 0.6369, + 0.2168, + 0.892, + 0.078, + 0.1656, + 0.3646, + 0.3955, + 0.0974, + 0.374, + 0.8909, + 0.3303, + 0.1878, + 0.3739, + 0.4498, + 0.4434, + 0.2275, + 0.0337, + 0.1338, + 0.3261, + 0.147, + 0.1562 + ], + "13": [ + 0.0, + 1.3459, + 0.3005, + 0.1035, + 1.4767, + 0.7256, + 0.4996, + 0.3647, + 0.8223, + 0.2439, + 1.3324, + 0.84, + 0.878, + 0.3531, + 0.0671, + 0.4176, + 0.202, + 0.364, + 0.4085, + 0.0003, + 0.1752, + 0.0258, + 0.3245, + 0.9362, + 0.357, + 0.7213, + 0.2376, + 0.0868, + 0.2773 + ], + "14": [ + 0.0, + 1.7483, + 0.6509, + 0.0378, + 1.6564, + 1.0802, + 1.0956, + 0.7317, + 0.7377, + 0.3702, + 1.3264, + 0.9968, + 1.3358, + 0.5184, + 0.0192, + 0.634, + 0.5056, + 0.277, + 0.2734, + 0.06, + 0.5171, + 0.0416, + 0.289, + 0.9638, + 0.492, + 1.0371, + 0.2825, + 0.4552, + 0.0857 + ], + "15": [ + 0.0, + 1.6966, + 0.7913, + 0.0476, + 1.724, + 1.4951, + 1.1582, + 1.0885, + 1.12, + 0.43, + 1.8584, + 1.5267, + 2.3004, + 1.1551, + 0.5675, + 1.2654, + 0.7575, + 0.6098, + 0.1526, + 0.4881, + 0.8253, + 0.5369, + 0.736, + 1.5902, + 1.4162, + 1.4994, + 1.4085, + 1.2953, + 0.2224 + ], + "16": [ + 0.0, + 1.9287, + 0.6202, + 0.3285, + 2.0142, + 1.6945, + 1.7004, + 2.0301, + 1.3212, + 0.8221, + 2.0502, + 1.7469, + 2.6193, + 1.1939, + 1.0689, + 1.4523, + 1.0904, + 0.7867, + 0.1677, + 0.4017, + 1.324, + 0.9764, + 1.2173, + 1.6634, + 1.6218, + 1.7214, + 1.2665, + 1.2603, + 0.5249 + ], + "17": [ + 0.0, + 2.2053, + 0.7897, + 0.2389, + 2.3334, + 2.2182, + 2.243, + 2.1945, + 1.59, + 0.8697, + 2.398, + 2.0349, + 2.2617, + 1.3524, + 1.1346, + 1.3469, + 1.1597, + 0.8487, + 0.0204, + 0.381, + 1.5899, + 1.5381, + 1.5849, + 1.6136, + 1.528, + 1.3028, + 1.3063, + 1.3954, + 0.539 + ], + "18": [ + 0.0, + 2.411, + 1.1075, + 0.4662, + 2.908, + 2.4423, + 2.9286, + 3.1139, + 2.2334, + 1.0545, + 2.7259, + 2.2778, + 2.852, + 1.6857, + 1.1583, + 1.5472, + 1.1896, + 1.2548, + 0.0127, + 0.5764, + 1.8903, + 1.556, + 1.6607, + 1.986, + 1.9663, + 0.8657, + 1.5702, + 1.8573, + 0.6065 + ], + "19": [ + 0.0, + 2.8819, + 1.221, + 0.3441, + 3.0154, + 2.8765, + 3.1218, + 3.6374, + 2.827, + 1.2985, + 2.6529, + 2.6547, + 3.743, + 1.6099, + 1.5226, + 1.4503, + 1.51, + 1.5772, + 0.2887, + 0.4929, + 2.3188, + 1.5867, + 1.7418, + 1.8096, + 1.851, + 0.6174, + 1.8007, + 1.8505, + 0.5483 + ], + "20": [ + 0.0, + 3.1611, + 1.0981, + 0.469, + 3.3132, + 2.5808, + 3.244, + 3.8657, + 2.9667, + 1.2542, + 2.747, + 3.4477, + 3.7108, + 1.9352, + 1.4341, + 1.4313, + 1.0563, + 1.4452, + 0.2026, + 0.4407, + 2.3147, + 1.8081, + 1.3615, + 1.8452, + 1.9888, + 0.8109, + 1.4697, + 2.0284, + 0.706 + ], + "21": [ + 0.0, + 3.3823, + 1.2338, + 0.5326, + 3.49, + 2.9571, + 3.2302, + 4.1047, + 3.1966, + 1.3773, + 2.6308, + 3.3885, + 3.917, + 2.1825, + 1.7147, + 1.2446, + 1.2023, + 1.5005, + 0.1209, + 0.4318, + 2.5654, + 2.3426, + 1.6998, + 1.5238, + 2.2184, + 1.1366, + 1.2648, + 2.2451, + 0.7933 + ] + }, + "shuffled": { + "9": [ + 0.0, + 1.5118, + 0.5101, + 0.1208, + 1.6699, + 0.5737, + 0.7035, + 0.7413, + 1.0661, + 0.8904, + 1.3368, + 0.0773, + 0.0317, + 0.0601, + 0.046, + 0.335, + 0.5857, + 1.14, + 0.3508, + 0.2401, + 0.2494, + 0.6871, + 0.4212, + 0.4382, + 0.3003, + 0.5488, + 0.4893, + 1.0407, + 0.4765 + ], + "10": [ + 0.0, + 1.2995, + 0.3651, + 0.2634, + 1.9743, + 0.6878, + 0.5627, + 0.63, + 1.3479, + 0.7871, + 1.305, + 0.9238, + 0.596, + 0.7662, + 0.684, + 0.2186, + 0.0169, + 0.084, + 0.224, + 0.2546, + 0.0987, + 0.0377, + 0.297, + 0.0496, + 0.6696, + 0.1279, + 0.5867, + 0.3927, + 0.5467 + ], + "11": [ + 0.0, + 1.1103, + 0.2615, + 0.4236, + 0.4617, + 0.22, + 0.5183, + 0.3505, + 0.3477, + 0.293, + 1.2733, + 0.9761, + 0.2374, + 0.0284, + 1.0376, + 0.3487, + 0.2213, + 0.0424, + 0.4884, + 1.3293, + 0.433, + 0.5845, + 0.4278, + 0.0676, + 0.3694, + 0.1094, + 0.1005, + 0.0609, + 0.3859 + ], + "12": [ + 0.0, + 1.9735, + 0.8691, + 0.9161, + 2.7891, + 1.7021, + 1.0991, + 1.4029, + 2.0109, + 1.0489, + 1.2128, + 1.635, + 1.1141, + 1.0343, + 0.5563, + 0.0382, + 0.6646, + 0.2437, + 0.3166, + 0.1954, + 0.0898, + 0.0884, + 0.0842, + 0.5523, + 1.2389, + 0.684, + 0.3078, + 0.2939, + 0.8556 + ], + "13": [ + 0.0, + 1.3602, + 0.3411, + 0.0178, + 1.2498, + 0.5154, + 0.4521, + 0.2566, + 0.6029, + 0.2152, + 1.125, + 0.6546, + 0.7556, + 0.2649, + 0.1846, + 0.3819, + 0.2086, + 0.3734, + 0.2739, + 0.2207, + 0.0947, + 0.1757, + 0.0251, + 0.6947, + 0.204, + 0.628, + 0.0141, + 0.2834, + 0.1015 + ], + "14": [ + 0.0, + 2.0655, + 0.9906, + 0.7231, + 2.6612, + 1.9058, + 1.1956, + 1.0691, + 1.7792, + 0.8679, + 1.4565, + 1.3596, + 1.0395, + 0.9553, + 0.218, + 0.1777, + 0.3786, + 0.1386, + 0.133, + 0.851, + 0.4408, + 0.4374, + 0.6449, + 0.8772, + 0.6887, + 0.4828, + 0.0677, + 1.1464, + 0.6459 + ], + "15": [ + 0.0, + 1.4493, + 0.6136, + 0.0719, + 1.5706, + 1.4147, + 1.679, + 1.6564, + 1.127, + 0.1882, + 1.7959, + 1.9069, + 1.8442, + 0.6896, + 0.0572, + 0.873, + 1.2431, + 1.5184, + 0.4645, + 0.2382, + 0.2339, + 0.2761, + 0.8263, + 1.5221, + 1.7173, + 0.9038, + 0.8888, + 1.1062, + 0.5149 + ], + "16": [ + 0.0, + 1.7166, + 0.807, + 0.225, + 1.954, + 1.777, + 2.1587, + 2.0177, + 1.5813, + 1.1043, + 2.3111, + 2.2926, + 1.9714, + 1.2947, + 0.4056, + 1.4802, + 1.4684, + 1.6412, + 0.6238, + 0.2994, + 1.3113, + 1.2567, + 1.4133, + 2.1148, + 2.4582, + 1.3137, + 0.6908, + 1.8975, + 0.2456 + ], + "17": [ + 0.0, + 2.2956, + 0.6979, + 1.6006, + 3.5573, + 2.4964, + 1.2837, + 2.1771, + 1.9468, + 1.154, + 2.1395, + 1.193, + 0.8588, + 0.651, + 1.0777, + 0.4429, + 0.2349, + 0.0976, + 0.1409, + 1.1297, + 0.0632, + 0.4435, + 0.2367, + 0.4322, + 0.9014, + 0.3969, + 0.3822, + 0.2638, + 2.1965 + ], + "18": [ + 0.0, + 0.4349, + 1.4793, + 1.6122, + 1.6337, + 1.2729, + 0.2086, + 1.0802, + 0.2323, + 1.1769, + 0.6095, + 0.6329, + 0.597, + 0.8508, + 0.5549, + 0.645, + 0.7338, + 1.1747, + 1.3566, + 1.0952, + 1.0965, + 0.8088, + 0.7261, + 1.128, + 1.2076, + 0.2452, + 0.1906, + 0.4449, + 0.9876 + ], + "19": [ + 0.0, + 3.9998, + 0.9699, + 2.2535, + 3.3977, + 2.1971, + 1.4853, + 2.299, + 2.345, + 1.1057, + 2.0619, + 2.479, + 2.4322, + 0.9497, + 0.5319, + 0.2061, + 0.1874, + 0.1422, + 0.1385, + 0.6413, + 1.0384, + 0.1783, + 0.7945, + 0.2225, + 1.1119, + 0.0214, + 0.4987, + 0.0759, + 2.1037 + ], + "20": [ + 0.0, + 1.9889, + 2.6796, + 2.0457, + 1.0113, + 0.4364, + 1.536, + 1.4145, + 1.2261, + 2.2185, + 0.497, + 0.1576, + 1.971, + 2.729, + 1.8739, + 1.636, + 0.5725, + 2.4359, + 3.6947, + 3.4695, + 2.0743, + 1.76, + 1.3951, + 2.0991, + 2.3498, + 1.381, + 3.7335, + 2.5795, + 2.6629 + ], + "21": [ + 0.0, + 2.9288, + 0.3705, + 0.4711, + 2.9273, + 2.5151, + 2.6296, + 3.3434, + 2.8028, + 0.6854, + 2.0015, + 3.0078, + 3.3652, + 1.736, + 1.158, + 0.602, + 0.5177, + 0.9816, + 0.5879, + 0.3261, + 1.6695, + 1.6204, + 1.0968, + 0.9648, + 1.9144, + 0.5233, + 1.124, + 1.7184, + 0.0411 + ] + }, + "rand_match": { + "9": [ + 0.0, + 0.3645, + 1.5531, + 2.0406, + 2.3242, + 2.9565, + 1.0161, + 0.8854, + 2.0599, + 1.3834, + 1.9966, + 1.8063, + 1.4514, + 1.5222, + 1.1235, + 1.9352, + 1.4219, + 1.1712, + 1.162, + 1.0548, + 1.1729, + 0.9815, + 0.9887, + 1.7818, + 1.6087, + 1.5691, + 0.6879, + 0.4475, + 0.8427 + ], + "10": [ + 0.0, + 0.8003, + 0.0308, + 0.1113, + 0.2136, + 0.93, + 0.7266, + 0.1807, + 0.7635, + 0.4386, + 0.0573, + 0.0775, + 0.362, + 0.2062, + 0.3487, + 0.1586, + 0.4627, + 0.0686, + 0.0317, + 0.0853, + 0.1874, + 0.7494, + 0.1257, + 0.6792, + 0.5273, + 0.1551, + 0.4173, + 0.704, + 0.3004 + ], + "11": [ + 0.0, + 0.0487, + 0.25, + 0.0655, + 0.1954, + 0.7143, + 0.6156, + 0.0713, + 0.1097, + 0.2332, + 0.4304, + 0.1223, + 0.1907, + 0.188, + 0.0112, + 0.1808, + 0.0932, + 1.0057, + 0.603, + 0.0188, + 0.6897, + 0.1865, + 0.454, + 0.0305, + 0.1257, + 0.4475, + 0.4428, + 0.2012, + 0.2078 + ], + "12": [ + 0.0, + 1.0022, + 1.1813, + 0.1402, + 1.098, + 0.1436, + 0.005, + 0.321, + 0.2866, + 0.0981, + 1.0628, + 1.0547, + 0.6884, + 0.926, + 0.309, + 0.0684, + 0.1742, + 0.4711, + 0.3194, + 0.6232, + 0.8665, + 0.5369, + 1.1431, + 1.1116, + 1.0942, + 0.5292, + 0.6765, + 0.0084, + 0.9091 + ], + "13": [ + 0.0, + 0.0304, + 0.7353, + 0.2472, + 0.136, + 0.1144, + 0.5657, + 0.8981, + 0.303, + 0.2306, + 0.9541, + 0.2876, + 0.7692, + 0.2914, + 0.4818, + 1.1314, + 0.9902, + 0.8406, + 0.4873, + 0.655, + 0.7597, + 0.3932, + 0.2321, + 0.6025, + 0.6882, + 0.3375, + 0.0054, + 0.3068, + 0.0616 + ], + "14": [ + 0.0, + 0.1304, + 0.0235, + 0.3432, + 0.7615, + 0.0844, + 0.6993, + 0.6385, + 0.454, + 0.4374, + 0.3664, + 0.462, + 1.3631, + 0.1144, + 0.6127, + 0.6854, + 0.0919, + 0.181, + 0.2337, + 0.8144, + 0.4748, + 0.2819, + 0.6317, + 1.1592, + 0.6881, + 1.2215, + 0.3143, + 0.2608, + 0.1359 + ], + "15": [ + 0.0, + 0.1013, + 0.974, + 0.2041, + 0.6159, + 0.5974, + 0.2496, + 0.1731, + 0.9218, + 0.5908, + 1.0242, + 0.1658, + 0.1474, + 0.3457, + 0.1663, + 0.1169, + 0.3344, + 1.2444, + 0.2131, + 0.294, + 0.5566, + 0.5252, + 0.0399, + 0.0081, + 0.3017, + 0.1472, + 0.3143, + 0.2936, + 0.4154 + ], + "16": [ + 0.0, + 0.5637, + 0.2837, + 0.6862, + 0.1449, + 0.6458, + 0.9709, + 1.3358, + 1.4487, + 1.3923, + 0.376, + 1.0131, + 0.9083, + 0.8408, + 2.142, + 0.5091, + 0.2609, + 0.6146, + 1.6961, + 1.5076, + 1.0731, + 1.1988, + 1.6611, + 1.1122, + 1.7385, + 0.6604, + 0.3118, + 1.3057, + 1.6277 + ], + "17": [ + 0.0, + 0.1045, + 0.6103, + 0.1153, + 0.4102, + 0.9163, + 0.1964, + 0.262, + 1.4368, + 0.0843, + 0.0077, + 1.0363, + 0.6515, + 0.8801, + 0.0718, + 0.3998, + 0.0841, + 0.7261, + 0.3353, + 0.4014, + 0.468, + 0.2632, + 0.8021, + 0.6309, + 0.4228, + 1.3052, + 0.5197, + 0.5212, + 0.7812 + ], + "18": [ + 0.0, + 0.744, + 0.5664, + 0.5135, + 0.3457, + 0.057, + 0.5666, + 0.8063, + 0.1408, + 0.0808, + 0.4268, + 0.1989, + 0.8807, + 0.3977, + 0.2888, + 0.4895, + 0.2233, + 0.0113, + 0.3741, + 0.8079, + 0.6854, + 0.8122, + 0.5018, + 1.0533, + 0.1358, + 0.7491, + 1.0082, + 0.2855, + 0.0262 + ], + "19": [ + 0.0, + 0.3666, + 0.7386, + 0.4972, + 0.7947, + 2.2296, + 0.3036, + 1.7663, + 0.4797, + 0.7304, + 0.1491, + 0.8701, + 0.9529, + 0.4752, + 0.2341, + 0.8447, + 1.3263, + 1.2423, + 0.3564, + 0.4259, + 0.7696, + 0.1665, + 0.6496, + 0.2275, + 0.1616, + 1.866, + 0.8944, + 0.4107, + 0.2799 + ], + "20": [ + 0.0, + 1.007, + 1.3834, + 0.7848, + 1.089, + 0.3502, + 0.721, + 2.0288, + 0.1113, + 0.506, + 1.0822, + 0.0298, + 0.8306, + 0.6869, + 1.105, + 0.5274, + 0.3002, + 0.3645, + 1.1195, + 0.7734, + 1.1394, + 0.1012, + 1.1254, + 0.1254, + 0.3736, + 0.4148, + 3.1737, + 0.9609, + 0.4138 + ], + "21": [ + 0.0, + 0.9962, + 0.9763, + 0.7098, + 1.1124, + 0.9644, + 0.5902, + 1.4268, + 0.7706, + 0.813, + 0.3696, + 1.4599, + 0.4575, + 1.5699, + 1.1373, + 0.4888, + 0.6836, + 0.659, + 0.7429, + 0.1149, + 0.859, + 1.8775, + 0.1327, + 0.3953, + 0.8612, + 0.0926, + 0.402, + 0.0696, + 0.3506 + ] + } + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "argv": [ + "t20_ablation_kl.py", + "--model", + "EleutherAI/pythia-410m-deduped", + "--lens", + "results/lens_410m_n200_db128_pen.pt", + "--evals", + "all", + "--out", + "results/t20_ablation_kl_410m.json" + ], + "runtime_s": 2965.3, + "provenance": { + "payload_sha256_before_redaction": "2634a6a7310cca78ca3b41c61751ace513224b31c3300c79e9b8a6cd4b02fbe5", + "payload_sha256": "704c014931b61576ccea82471444e252c73f4be7ad8f160eb85e809ca92fae30", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t20_ablation_kl_70m.json b/results/t20_ablation_kl_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..b571b089012e0917642f9fb10fe69265900ebc6c --- /dev/null +++ b/results/t20_ablation_kl_70m.json @@ -0,0 +1,1533 @@ +{ + "experiment": "T20 / E20 \u2014 ablation KL, the anchor's Figure 53 DV", + "status": "EXPLORATORY", + "prereg": null, + "prereg_sha256": null, + "decision_rule": "ratio KL(J^P)/KL(I), band mean, on multihop: >1.5 reproduces the anchor; <=1.0 falsifies at this scale; (1.0,1.5] inconclusive", + "primary_outcome": "band-mean ratio KL(jlens)/KL(logit) per set", + "layer_protocol": "one layer at a time, swept over the band; NO band compounding. Headline is the band MEAN of the per-layer ratio; band-max is stored as a secondary and is explicitly not primary (existential statistic).", + "cross_lens_caveat": "the logit lens has no lens-native coordinate write (external review B5/G6); applied here it is a SHARED hidden-state intervention parameterised by whichever lens supplies v_c", + "VERDICT": "FALSIFIED at this scale \u2014 multihop ratio 0.5832 <= 1.0", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/lens_70m_n200_db128_pen.pt", + "lens_sha256": "b53b45cd4f89a5174109f70fcffb7d6bce639093afc9b6dcecb34a88ab71a390", + "n_layers": 6, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "layers_arg": "band", + "skip_positions": 1, + "max_items_effective": null, + "min_pairs_floor": 30, + "evals": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo", + "association" + ], + "arms": [ + "logit", + "jlens", + "shuffled", + "rand_match" + ], + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "device": "cpu", + "n_forward_passes": 7695, + "results": { + "multihop": { + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.014876, + "jlens": 0.008675, + "shuffled": 0.004166, + "rand_match": 0.004407 + }, + "PRIMARY_ratio_jlens_over_logit": 0.5832, + "ratio_jlens_over_shuffled": 2.0826, + "ratio_jlens_over_rand_match": 1.9684, + "ratio_by_layer_jlens_over_logit": { + "2": 0.5131, + "3": 0.712 + }, + "band_max_ratio_SECONDARY": 0.712, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019273, + "dose_mean_per_position": 0.6468, + "dose_sum_over_positions": 16.2733 + }, + "3": { + "kl_mean": 0.010479, + "dose_mean_per_position": 0.6897, + "dose_sum_over_positions": 17.2888 + } + }, + "jlens": { + "2": { + "kl_mean": 0.009889, + "dose_mean_per_position": 0.5063, + "dose_sum_over_positions": 11.3354 + }, + "3": { + "kl_mean": 0.007461, + "dose_mean_per_position": 0.4958, + "dose_sum_over_positions": 10.8898 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.002604, + "dose_mean_per_position": 0.3997, + "dose_sum_over_positions": 9.1783 + }, + "3": { + "kl_mean": 0.005727, + "dose_mean_per_position": 0.736, + "dose_sum_over_positions": 16.7834 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.004383, + "dose_mean_per_position": 0.4441, + "dose_sum_over_positions": 9.8799 + }, + "3": { + "kl_mean": 0.004432, + "dose_mean_per_position": 0.5476, + "dose_sum_over_positions": 12.3821 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0143, + 0.4263, + 0.3879, + 0.1754, + 0.3134, + 0.3558, + 1.574, + 0.955, + 0.3888, + 1.4085, + 0.8029, + 1.2517, + 1.3781, + 1.7634, + 0.9378, + 0.959, + 1.1397, + 0.7515, + 0.1324 + ], + "3": [ + 0.0, + 0.1675, + 0.1087, + 0.4415, + 0.0303, + 0.1676, + 0.091, + 1.5784, + 0.6766, + 0.3601, + 1.4793, + 0.1837, + 1.0786, + 1.4508, + 1.5462, + 1.293, + 1.1929, + 1.6134, + 0.9143, + 0.1636 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.6623, + 0.0925, + 0.0031, + 0.3292, + 0.3877, + 0.6776, + 0.9563, + 1.0446, + 0.8933, + 1.3874, + 1.2347, + 0.257, + 0.7211, + 0.9779, + 0.9562, + 0.6455, + 0.7008, + 0.6493, + 0.5784 + ], + "3": [ + 0.0, + 0.8514, + 0.3954, + 0.43, + 0.302, + 0.1863, + 0.3956, + 0.5269, + 0.9101, + 0.9479, + 1.0015, + 0.5102, + 0.1176, + 0.9017, + 1.3558, + 1.2925, + 1.4045, + 1.486, + 0.8727, + 1.0185 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.194, + 0.489, + 0.4936, + 0.1278, + 0.5256, + 0.1374, + 0.488, + 0.5194, + 0.4274, + 1.1903, + 0.7843, + 0.0975, + 0.0743, + 0.27, + 0.3438, + 0.3889, + 0.2784, + 0.2385, + 0.0398 + ], + "3": [ + 0.0, + 0.2169, + 0.4983, + 0.5618, + 0.3599, + 0.9683, + 0.8581, + 1.1536, + 0.9928, + 1.1977, + 1.6801, + 1.531, + 1.0058, + 1.3477, + 1.1968, + 1.1042, + 1.1441, + 1.4616, + 0.8578, + 1.114 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.8595, + 0.0865, + 0.4166, + 0.224, + 0.1023, + 0.3016, + 0.6538, + 0.1718, + 0.4688, + 0.7481, + 0.0053, + 0.5808, + 0.3543, + 0.3642, + 0.1474, + 1.1441, + 0.4846, + 0.2117, + 0.3653 + ], + "3": [ + 0.0, + 0.082, + 0.0965, + 0.1922, + 0.4986, + 0.051, + 0.5813, + 0.6251, + 0.6909, + 0.6936, + 0.7641, + 0.3953, + 0.4381, + 0.0576, + 1.0085, + 0.4477, + 0.1701, + 0.2303, + 0.437, + 0.2175 + ] + } + } + }, + "multilingual": { + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "band_mean_kl": { + "logit": 0.003209, + "jlens": 0.005331, + "shuffled": 0.003418, + "rand_match": 0.003698 + }, + "PRIMARY_ratio_jlens_over_logit": 1.6615, + "ratio_jlens_over_shuffled": 1.5595, + "ratio_jlens_over_rand_match": 1.4417, + "ratio_by_layer_jlens_over_logit": { + "2": 0.9708, + "3": 2.3677 + }, + "band_max_ratio_SECONDARY": 2.3677, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003244, + "dose_mean_per_position": 0.4899, + "dose_sum_over_positions": 5.5071 + }, + "3": { + "kl_mean": 0.003173, + "dose_mean_per_position": 0.523, + "dose_sum_over_positions": 5.8039 + } + }, + "jlens": { + "2": { + "kl_mean": 0.00315, + "dose_mean_per_position": 0.3435, + "dose_sum_over_positions": 3.9641 + }, + "3": { + "kl_mean": 0.007512, + "dose_mean_per_position": 0.4969, + "dose_sum_over_positions": 6.0645 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.002469, + "dose_mean_per_position": 0.4206, + "dose_sum_over_positions": 4.8332 + }, + "3": { + "kl_mean": 0.004368, + "dose_mean_per_position": 0.5877, + "dose_sum_over_positions": 6.6602 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.00377, + "dose_mean_per_position": 0.4726, + "dose_sum_over_positions": 5.4467 + }, + "3": { + "kl_mean": 0.003626, + "dose_mean_per_position": 0.5427, + "dose_sum_over_positions": 6.1871 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 1.3924, + 3.5021, + 0.0182, + 0.3026, + 0.7169, + 0.272, + 0.1566, + 0.032, + 0.2212, + 0.5358, + 0.3988 + ], + "3": [ + 0.0, + 1.1118, + 3.2379, + 0.5082, + 1.2603, + 0.095, + 0.2086, + 0.1731, + 0.2063, + 0.5958, + 0.0088, + 0.2401 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.2714, + 1.5745, + 0.4594, + 0.4065, + 0.0372, + 0.0075, + 0.2193, + 0.2221, + 0.217, + 0.109, + 0.1541 + ], + "3": [ + 0.0, + 0.8444, + 0.3159, + 0.3394, + 0.7543, + 0.2179, + 0.1748, + 0.7529, + 0.1414, + 0.1516, + 0.2037, + 0.5538 + ] + }, + "shuffled": { + "2": [ + 0.0, + 1.0523, + 2.9048, + 0.1427, + 0.9285, + 0.3804, + 0.5343, + 0.0895, + 0.091, + 0.235, + 0.1922, + 0.1476 + ], + "3": [ + 0.0, + 1.1782, + 2.8782, + 0.3279, + 0.9923, + 0.7141, + 0.6811, + 0.41, + 0.5636, + 0.4673, + 0.4591, + 0.0251 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5647, + 1.365, + 0.2132, + 0.0545, + 0.2211, + 0.1914, + 0.1958, + 0.1866, + 0.4777, + 0.3089, + 0.0994 + ], + "3": [ + 0.0, + 3.2869, + 4.0832, + 0.5296, + 1.0719, + 0.795, + 0.853, + 0.9589, + 0.6516, + 1.0234, + 1.1701, + 1.124 + ] + } + } + }, + "order-ops": { + "n_items_scored": 55, + "n_intermediate_pairs_scored": 110, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.002976, + "jlens": 0.004422, + "shuffled": 0.00073, + "rand_match": 0.002122 + }, + "PRIMARY_ratio_jlens_over_logit": 1.486, + "ratio_jlens_over_shuffled": 6.0606, + "ratio_jlens_over_rand_match": 2.0838, + "ratio_by_layer_jlens_over_logit": { + "2": 1.5133, + "3": 1.4458 + }, + "band_max_ratio_SECONDARY": 1.5133, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003541, + "dose_mean_per_position": 0.3877, + "dose_sum_over_positions": 3.5915 + }, + "3": { + "kl_mean": 0.002411, + "dose_mean_per_position": 0.4374, + "dose_sum_over_positions": 4.0395 + } + }, + "jlens": { + "2": { + "kl_mean": 0.005358, + "dose_mean_per_position": 0.4693, + "dose_sum_over_positions": 4.4148 + }, + "3": { + "kl_mean": 0.003486, + "dose_mean_per_position": 0.5082, + "dose_sum_over_positions": 4.796 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.000663, + "dose_mean_per_position": 0.2736, + "dose_sum_over_positions": 2.5218 + }, + "3": { + "kl_mean": 0.000797, + "dose_mean_per_position": 0.3182, + "dose_sum_over_positions": 2.9571 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.002213, + "dose_mean_per_position": 0.4331, + "dose_sum_over_positions": 4.0239 + }, + "3": { + "kl_mean": 0.002031, + "dose_mean_per_position": 0.5105, + "dose_sum_over_positions": 4.6388 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.2968, + 0.0493, + 0.3018, + 0.3017, + 0.2819, + 0.188, + 0.0994, + 0.2883 + ], + "3": [ + 0.0, + 0.5021, + 0.0337, + 0.0088, + 0.1413, + 0.5405, + 0.094, + 0.3283, + 0.4194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.6402, + 0.437, + 0.4858, + 0.3469, + 0.4636, + 1.0861, + 0.4492, + 0.5671 + ], + "3": [ + 0.0, + 1.0186, + 0.7788, + 0.2082, + 0.3805, + 0.4281, + 0.9815, + 0.5318, + 0.707 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.0777, + 0.1117, + 0.024, + 0.1802, + 0.126, + 0.2975, + 0.1513, + 0.288 + ], + "3": [ + 0.0, + 0.0068, + 0.0642, + 0.2346, + 0.0403, + 0.1537, + 0.1252, + 0.4869, + 0.1431 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.0945, + 0.2806, + 0.0689, + 0.3952, + 0.1473, + 0.2637, + 0.2711, + 0.2763 + ], + "3": [ + 0.0, + 0.4252, + 0.0526, + 0.3179, + 0.3876, + 0.5907, + 0.3064, + 0.4023, + 0.0867 + ] + } + } + }, + "poetry": { + "n_items_scored": 98, + "n_intermediate_pairs_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.011893, + "jlens": 0.007317, + "shuffled": 0.003084, + "rand_match": 0.00537 + }, + "PRIMARY_ratio_jlens_over_logit": 0.6153, + "ratio_jlens_over_shuffled": 2.3725, + "ratio_jlens_over_rand_match": 1.3625, + "ratio_by_layer_jlens_over_logit": { + "2": 0.885, + "3": 0.3436 + }, + "band_max_ratio_SECONDARY": 0.885, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.011934, + "dose_mean_per_position": 0.941, + "dose_sum_over_positions": 25.7454 + }, + "3": { + "kl_mean": 0.011851, + "dose_mean_per_position": 1.1137, + "dose_sum_over_positions": 30.4835 + } + }, + "jlens": { + "2": { + "kl_mean": 0.010562, + "dose_mean_per_position": 0.6966, + "dose_sum_over_positions": 19.0541 + }, + "3": { + "kl_mean": 0.004072, + "dose_mean_per_position": 0.712, + "dose_sum_over_positions": 19.4936 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.003819, + "dose_mean_per_position": 0.5525, + "dose_sum_over_positions": 15.0866 + }, + "3": { + "kl_mean": 0.002349, + "dose_mean_per_position": 0.9474, + "dose_sum_over_positions": 25.9074 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.006512, + "dose_mean_per_position": 0.5916, + "dose_sum_over_positions": 16.1283 + }, + "3": { + "kl_mean": 0.004228, + "dose_mean_per_position": 0.6559, + "dose_sum_over_positions": 17.9902 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.1594, + 0.5944, + 0.2455, + 0.5992, + 0.2572, + 0.9429, + 0.2139, + 11.1563, + 0.0825, + 0.2446, + 0.1937, + 0.292, + 0.0767, + 0.7736, + 0.4319, + 0.6853, + 0.9142, + 0.684, + 0.1378, + 0.081, + 0.7477, + 0.352, + 1.0318, + 0.4303, + 0.3018, + 0.195, + 0.7437, + 0.2196 + ], + "3": [ + 0.0, + 0.3802, + 1.2164, + 1.0455, + 0.1044, + 0.4311, + 0.8451, + 0.2006, + 11.3244, + 0.1434, + 0.5504, + 0.3074, + 0.2068, + 0.2864, + 1.1519, + 0.7277, + 0.8769, + 1.4379, + 1.5382, + 0.4364, + 0.248, + 0.3947, + 0.1714, + 1.1808, + 0.2771, + 0.5054, + 0.6367, + 1.1076, + 0.0233 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.5363, + 1.0409, + 0.8239, + 0.3393, + 0.928, + 0.0076, + 0.2143, + 6.179, + 0.3734, + 0.2519, + 0.1417, + 0.5389, + 0.2871, + 0.4323, + 0.3388, + 0.3333, + 0.165, + 0.6128, + 0.0897, + 0.1882, + 0.4519, + 0.7009, + 0.2105, + 0.7565, + 0.5372, + 1.0561, + 0.7595, + 0.9813 + ], + "3": [ + 0.0, + 0.8695, + 1.8387, + 1.4261, + 0.072, + 0.4671, + 0.486, + 0.0849, + 4.3845, + 0.1819, + 0.2994, + 0.1543, + 0.5511, + 0.6005, + 0.6457, + 0.3153, + 0.786, + 0.6519, + 0.8409, + 0.2791, + 0.2566, + 0.5298, + 0.6217, + 0.7155, + 0.7212, + 0.8781, + 1.2163, + 0.8723, + 0.7372 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.024, + 0.6105, + 0.1992, + 0.4088, + 0.8369, + 0.3329, + 0.4876, + 6.6796, + 0.4834, + 0.0042, + 0.04, + 0.2165, + 0.2141, + 0.4093, + 0.1065, + 0.1611, + 0.3177, + 0.0653, + 0.1427, + 0.2272, + 0.1519, + 0.3524, + 0.2682, + 0.6104, + 0.3172, + 0.3774, + 0.3786, + 0.1673 + ], + "3": [ + 0.0, + 0.4201, + 0.6198, + 0.5297, + 0.6387, + 1.3139, + 0.4536, + 0.8194, + 7.5326, + 1.2214, + 0.4404, + 0.3185, + 0.6228, + 0.5222, + 0.7127, + 0.3821, + 0.3199, + 0.5916, + 0.6715, + 0.0348, + 0.5079, + 0.8701, + 0.9366, + 0.9909, + 1.2813, + 1.2567, + 1.3568, + 1.3506, + 1.1092 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5663, + 0.0349, + 0.3271, + 0.0591, + 0.224, + 0.5588, + 0.0436, + 4.7702, + 0.2856, + 0.5625, + 0.8871, + 0.5497, + 1.1638, + 0.4989, + 0.2201, + 0.4187, + 0.339, + 0.6105, + 0.1274, + 0.2521, + 0.149, + 0.1882, + 0.2987, + 0.2137, + 0.1715, + 0.3117, + 0.0553, + 0.1175 + ], + "3": [ + 0.0, + 0.1785, + 0.1694, + 0.2338, + 0.1877, + 0.4362, + 0.2648, + 0.0485, + 5.4696, + 0.0288, + 0.8456, + 0.4643, + 0.385, + 0.61, + 0.0647, + 0.5367, + 0.7664, + 0.5221, + 0.2946, + 0.5209, + 0.0216, + 0.6684, + 0.4557, + 0.4048, + 0.2026, + 0.1609, + 0.202, + 0.3115, + 0.2607 + ] + } + } + }, + "typo": { + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.019955, + "jlens": 0.018562, + "shuffled": 0.009763, + "rand_match": 0.009575 + }, + "PRIMARY_ratio_jlens_over_logit": 0.9302, + "ratio_jlens_over_shuffled": 1.9013, + "ratio_jlens_over_rand_match": 1.9385, + "ratio_by_layer_jlens_over_logit": { + "2": 0.9288, + "3": 0.9316 + }, + "band_max_ratio_SECONDARY": 0.9316, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019932, + "dose_mean_per_position": 0.486, + "dose_sum_over_positions": 4.7988 + }, + "3": { + "kl_mean": 0.019977, + "dose_mean_per_position": 0.566, + "dose_sum_over_positions": 5.5739 + } + }, + "jlens": { + "2": { + "kl_mean": 0.018513, + "dose_mean_per_position": 0.3994, + "dose_sum_over_positions": 3.9353 + }, + "3": { + "kl_mean": 0.018611, + "dose_mean_per_position": 0.466, + "dose_sum_over_positions": 4.6062 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.005711, + "dose_mean_per_position": 0.3309, + "dose_sum_over_positions": 3.2255 + }, + "3": { + "kl_mean": 0.013815, + "dose_mean_per_position": 0.7338, + "dose_sum_over_positions": 7.1977 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.011462, + "dose_mean_per_position": 0.3907, + "dose_sum_over_positions": 3.742 + }, + "3": { + "kl_mean": 0.007688, + "dose_mean_per_position": 0.4858, + "dose_sum_over_positions": 4.6135 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0693, + 0.62, + 0.4626, + 0.3928, + 0.1845, + 1.245, + 1.0045, + 0.4486, + 0.5024, + 1.0145, + 0.4979 + ], + "3": [ + 0.0, + 0.2168, + 0.838, + 0.2187, + 0.25, + 0.1403, + 0.75, + 0.7185, + 0.1199, + 0.3759, + 0.8352, + 0.6194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.3168, + 0.138, + 0.5001, + 0.0057, + 0.4807, + 0.4461, + 0.1679, + 0.2171, + 0.1087, + 0.1121, + 0.5984 + ], + "3": [ + 0.0, + 0.2437, + 0.2473, + 0.2785, + 0.2282, + 0.1662, + 0.3243, + 0.1133, + 0.1568, + 0.0246, + 0.3042, + 0.3147 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.3512, + 0.2997, + 1.0322, + 0.0369, + 0.3231, + 0.269, + 0.1599, + 0.0258, + 0.2044, + 0.1855, + 0.5673 + ], + "3": [ + 0.0, + 0.9526, + 0.6304, + 0.266, + 0.3302, + 1.0701, + 0.7317, + 0.2008, + 0.8737, + 0.5399, + 0.7279, + 0.3512 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.4129, + 0.2709, + 0.1557, + 0.1544, + 0.5417, + 0.0378, + 0.2531, + 0.5354, + 0.2084, + 0.4157, + 0.5005 + ], + "3": [ + 0.0, + 0.5479, + 1.0458, + 0.7085, + 1.2307, + 0.8921, + 1.3774, + 0.9829, + 0.4017, + 0.3321, + 0.5386, + 0.2979 + ] + } + } + }, + "association": { + "n_items_scored": 102, + "n_intermediate_pairs_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "band_mean_kl": { + "logit": 0.761053, + "jlens": 0.415847, + "shuffled": 0.388224, + "rand_match": 0.331446 + }, + "PRIMARY_ratio_jlens_over_logit": 0.5464, + "ratio_jlens_over_shuffled": 1.0712, + "ratio_jlens_over_rand_match": 1.2546, + "ratio_by_layer_jlens_over_logit": { + "2": 0.7784, + "3": 0.3177 + }, + "band_max_ratio_SECONDARY": 0.7784, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.755566, + "dose_mean_per_position": 1.0786, + "dose_sum_over_positions": 31.3456 + }, + "3": { + "kl_mean": 0.766541, + "dose_mean_per_position": 1.1292, + "dose_sum_over_positions": 32.8565 + } + }, + "jlens": { + "2": { + "kl_mean": 0.588167, + "dose_mean_per_position": 0.6948, + "dose_sum_over_positions": 20.2208 + }, + "3": { + "kl_mean": 0.243527, + "dose_mean_per_position": 0.6557, + "dose_sum_over_positions": 19.1827 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.302941, + "dose_mean_per_position": 0.5293, + "dose_sum_over_positions": 15.3161 + }, + "3": { + "kl_mean": 0.473506, + "dose_mean_per_position": 1.0033, + "dose_sum_over_positions": 29.0827 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.367691, + "dose_mean_per_position": 0.6371, + "dose_sum_over_positions": 18.6053 + }, + "3": { + "kl_mean": 0.2952, + "dose_mean_per_position": 0.7068, + "dose_sum_over_positions": 20.5207 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.261, + 0.7127, + 0.4662, + 0.3029, + 0.3495, + 0.053, + 0.077, + 0.9719, + 0.7852, + 0.6237, + 1.3751, + 0.6093, + 2.0121, + 0.6116, + 1.0531, + 1.7732, + 1.1089, + 0.6423, + 0.3501, + 0.7945, + 1.2471, + 1.3578, + 1.1713, + 1.1945, + 0.3451, + 0.3122, + 1.3023, + 12.4952 + ], + "3": [ + 0.0, + 0.4806, + 0.662, + 0.7729, + 0.4281, + 0.7686, + 0.5767, + 0.175, + 1.3067, + 1.3245, + 0.9131, + 1.5172, + 0.5693, + 1.5528, + 0.9205, + 0.2771, + 1.0986, + 0.5817, + 1.0954, + 0.5864, + 0.5879, + 0.9137, + 1.0181, + 0.8501, + 2.2945, + 0.7854, + 0.5198, + 1.2149, + 12.637 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.3777, + 0.9143, + 0.8838, + 0.2639, + 0.2482, + 0.4919, + 0.3879, + 0.4617, + 0.819, + 0.0005, + 0.5881, + 0.6534, + 0.4676, + 0.6865, + 0.1444, + 0.698, + 0.6088, + 1.1263, + 0.9245, + 0.4415, + 0.7409, + 0.9108, + 0.734, + 0.7395, + 0.8245, + 0.2266, + 0.9195, + 7.3296 + ], + "3": [ + 0.0, + 0.3898, + 0.6414, + 1.0928, + 0.2518, + 0.6737, + 0.894, + 0.4134, + 0.6821, + 0.9603, + 0.4735, + 0.7585, + 0.5314, + 0.6162, + 0.7196, + 0.3617, + 0.7151, + 0.4274, + 0.8596, + 1.0831, + 0.6172, + 0.6645, + 0.6348, + 0.3482, + 1.5696, + 1.2316, + 0.791, + 0.9497, + 4.1373 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.2662, + 0.4465, + 0.4857, + 0.151, + 0.0337, + 0.8919, + 0.3742, + 0.4535, + 0.4575, + 0.5418, + 0.4828, + 0.573, + 0.0046, + 0.1377, + 0.0706, + 0.4465, + 0.5784, + 0.7146, + 0.4004, + 0.3112, + 0.3225, + 0.057, + 0.3107, + 0.5244, + 0.0977, + 0.1752, + 0.3224, + 8.116 + ], + "3": [ + 0.0, + 0.9924, + 1.0788, + 1.112, + 0.9255, + 0.5769, + 1.0172, + 1.1296, + 1.2089, + 0.9752, + 0.2851, + 0.7733, + 1.1845, + 0.6452, + 0.4377, + 0.609, + 1.278, + 1.0092, + 1.3125, + 0.8599, + 1.0815, + 1.1788, + 0.5046, + 0.4941, + 0.8467, + 0.9104, + 0.6822, + 0.9745, + 8.1162 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5339, + 0.0311, + 0.0312, + 0.4897, + 0.5893, + 0.0718, + 0.1459, + 0.131, + 0.2004, + 0.8931, + 0.3031, + 0.3423, + 0.4818, + 0.5813, + 0.1457, + 0.071, + 0.0664, + 0.512, + 0.5542, + 0.1682, + 0.216, + 0.1395, + 0.2062, + 0.1179, + 0.0305, + 0.5685, + 0.2558, + 0.7061 + ], + "3": [ + 0.0, + 1.4369, + 0.589, + 0.4872, + 0.274, + 0.3277, + 0.2342, + 0.3975, + 0.6986, + 0.3046, + 0.1022, + 0.5765, + 0.6712, + 0.5935, + 0.3667, + 0.3788, + 0.7403, + 0.1303, + 0.1983, + 0.0263, + 0.2366, + 0.1407, + 0.0562, + 0.1564, + 0.1818, + 0.1215, + 0.3484, + 0.4221, + 7.1495 + ] + } + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "argv": [ + "t20_ablation_kl.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--lens", + "results/lens_70m_n200_db128_pen.pt", + "--evals", + "all", + "--out", + "results/t20_ablation_kl_70m.json" + ], + "runtime_s": 120.3, + "provenance": { + "payload_sha256_before_redaction": "5546a89ddf449d276317f0d24c96d8cdb72302ba9c54aedef5f66d3c6e8020dd", + "payload_sha256": "34e6ef1501bb9f53b72a4067855d334c91c18cdf8239b20e594f6ac66cb3e256", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t20_ablation_kl_70m_Pcode.json b/results/t20_ablation_kl_70m_Pcode.json new file mode 100644 index 0000000000000000000000000000000000000000..b8fbee9fb5e6269eabfffea254be47f0ff029ba8 --- /dev/null +++ b/results/t20_ablation_kl_70m_Pcode.json @@ -0,0 +1,1533 @@ +{ + "experiment": "T20 / E20 \u2014 ablation KL, the anchor's Figure 53 DV", + "status": "EXPLORATORY", + "prereg": null, + "prereg_sha256": null, + "decision_rule": "ratio KL(J^P)/KL(I), band mean, on multihop: >1.5 reproduces the anchor; <=1.0 falsifies at this scale; (1.0,1.5] inconclusive", + "primary_outcome": "band-mean ratio KL(jlens)/KL(logit) per set", + "layer_protocol": "one layer at a time, swept over the band; NO band compounding. Headline is the band MEAN of the per-layer ratio; band-max is stored as a secondary and is explicitly not primary (existential statistic).", + "cross_lens_caveat": "the logit lens has no lens-native coordinate write (external review B5/G6); applied here it is a SHARED hidden-state intervention parameterised by whichever lens supplies v_c", + "VERDICT": "INCONCLUSIVE \u2014 multihop ratio 1.139 in (1.0, 1.5]", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/plad_70m_code_A.pt", + "lens_sha256": "d182821d8e428041d8ef9497740330c231bd497c6d3d1253691f445aefad07cd", + "n_layers": 6, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "layers_arg": "band", + "skip_positions": 1, + "max_items_effective": null, + "min_pairs_floor": 30, + "evals": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo", + "association" + ], + "arms": [ + "logit", + "jlens", + "shuffled", + "rand_match" + ], + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "device": "cpu", + "n_forward_passes": 7695, + "results": { + "multihop": { + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.014876, + "jlens": 0.016944, + "shuffled": 0.004772, + "rand_match": 0.004407 + }, + "PRIMARY_ratio_jlens_over_logit": 1.139, + "ratio_jlens_over_shuffled": 3.5504, + "ratio_jlens_over_rand_match": 3.8449, + "ratio_by_layer_jlens_over_logit": { + "2": 0.8285, + "3": 1.7101 + }, + "band_max_ratio_SECONDARY": 1.7101, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019273, + "dose_mean_per_position": 0.6468, + "dose_sum_over_positions": 16.2733 + }, + "3": { + "kl_mean": 0.010479, + "dose_mean_per_position": 0.6897, + "dose_sum_over_positions": 17.2888 + } + }, + "jlens": { + "2": { + "kl_mean": 0.015967, + "dose_mean_per_position": 0.5624, + "dose_sum_over_positions": 12.3444 + }, + "3": { + "kl_mean": 0.01792, + "dose_mean_per_position": 0.706, + "dose_sum_over_positions": 16.4106 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.00319, + "dose_mean_per_position": 0.4706, + "dose_sum_over_positions": 11.0055 + }, + "3": { + "kl_mean": 0.006355, + "dose_mean_per_position": 0.8668, + "dose_sum_over_positions": 19.7551 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.004381, + "dose_mean_per_position": 0.4441, + "dose_sum_over_positions": 9.8799 + }, + "3": { + "kl_mean": 0.004432, + "dose_mean_per_position": 0.5476, + "dose_sum_over_positions": 12.3821 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0143, + 0.4263, + 0.3879, + 0.1754, + 0.3134, + 0.3558, + 1.574, + 0.955, + 0.3888, + 1.4085, + 0.8029, + 1.2517, + 1.3781, + 1.7634, + 0.9378, + 0.959, + 1.1397, + 0.7515, + 0.1324 + ], + "3": [ + 0.0, + 0.1675, + 0.1087, + 0.4415, + 0.0303, + 0.1676, + 0.091, + 1.5784, + 0.6766, + 0.3601, + 1.4793, + 0.1837, + 1.0786, + 1.4508, + 1.5462, + 1.293, + 1.1929, + 1.6134, + 0.9143, + 0.1636 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.4413, + 0.0358, + 0.1683, + 0.1265, + 0.5923, + 0.7818, + 0.7044, + 1.2214, + 0.9741, + 1.2669, + 1.2944, + 0.467, + 0.7673, + 1.2138, + 1.081, + 0.741, + 0.7057, + 0.8942, + 0.6961 + ], + "3": [ + 0.0, + 0.2203, + 0.0476, + 0.0431, + 0.2329, + 0.6867, + 1.0158, + 0.8473, + 1.4268, + 1.3805, + 1.1698, + 1.0023, + 0.5224, + 1.3416, + 1.7848, + 1.7083, + 1.8106, + 1.7094, + 1.232, + 1.549 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.1037, + 0.4787, + 0.1626, + 0.1053, + 0.8659, + 0.1727, + 0.364, + 0.6732, + 0.4351, + 0.9116, + 0.7864, + 0.3209, + 0.37, + 0.5165, + 0.4462, + 0.6161, + 0.4707, + 0.4842, + 0.2482 + ], + "3": [ + 0.0, + 0.372, + 0.38, + 0.469, + 0.4465, + 1.2019, + 0.9511, + 1.1299, + 1.0085, + 1.2315, + 1.5019, + 1.4621, + 1.2759, + 1.576, + 1.4921, + 1.3394, + 1.3404, + 1.5885, + 0.9739, + 1.1207 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.8595, + 0.0865, + 0.4166, + 0.224, + 0.1023, + 0.3016, + 0.6538, + 0.1718, + 0.4688, + 0.7481, + 0.0053, + 0.5808, + 0.3543, + 0.3642, + 0.1474, + 1.1441, + 0.4846, + 0.2117, + 0.3653 + ], + "3": [ + 0.0, + 0.082, + 0.0965, + 0.1922, + 0.4986, + 0.051, + 0.5813, + 0.6251, + 0.6909, + 0.6936, + 0.7641, + 0.3953, + 0.4381, + 0.0576, + 1.0085, + 0.4477, + 0.1701, + 0.2303, + 0.437, + 0.2175 + ] + } + } + }, + "multilingual": { + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "band_mean_kl": { + "logit": 0.003209, + "jlens": 0.003439, + "shuffled": 0.002516, + "rand_match": 0.003698 + }, + "PRIMARY_ratio_jlens_over_logit": 1.0718, + "ratio_jlens_over_shuffled": 1.3667, + "ratio_jlens_over_rand_match": 0.93, + "ratio_by_layer_jlens_over_logit": { + "2": 0.7898, + "3": 1.3602 + }, + "band_max_ratio_SECONDARY": 1.3602, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003244, + "dose_mean_per_position": 0.4899, + "dose_sum_over_positions": 5.5071 + }, + "3": { + "kl_mean": 0.003173, + "dose_mean_per_position": 0.523, + "dose_sum_over_positions": 5.8039 + } + }, + "jlens": { + "2": { + "kl_mean": 0.002562, + "dose_mean_per_position": 0.3029, + "dose_sum_over_positions": 3.4696 + }, + "3": { + "kl_mean": 0.004316, + "dose_mean_per_position": 0.4231, + "dose_sum_over_positions": 4.9227 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.00186, + "dose_mean_per_position": 0.3676, + "dose_sum_over_positions": 4.1987 + }, + "3": { + "kl_mean": 0.003173, + "dose_mean_per_position": 0.5887, + "dose_sum_over_positions": 6.7019 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.00377, + "dose_mean_per_position": 0.4726, + "dose_sum_over_positions": 5.4467 + }, + "3": { + "kl_mean": 0.003625, + "dose_mean_per_position": 0.5427, + "dose_sum_over_positions": 6.1871 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 1.3924, + 3.5021, + 0.0182, + 0.3026, + 0.7169, + 0.272, + 0.1566, + 0.032, + 0.2212, + 0.5358, + 0.3988 + ], + "3": [ + 0.0, + 1.1118, + 3.2379, + 0.5082, + 1.2603, + 0.095, + 0.2086, + 0.1731, + 0.2063, + 0.5958, + 0.0088, + 0.2401 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.1377, + 0.8255, + 0.3055, + 0.6183, + 0.3076, + 0.0136, + 0.0212, + 0.0222, + 0.3628, + 0.3822, + 0.1236 + ], + "3": [ + 0.0, + 0.2675, + 0.7223, + 0.0012, + 1.0935, + 0.4294, + 0.1277, + 0.3356, + 0.3723, + 0.5956, + 0.1815, + 0.2424 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.9913, + 2.3115, + 0.0251, + 0.9138, + 0.4519, + 0.2268, + 0.1826, + 0.0442, + 0.2353, + 0.1007, + 0.1925 + ], + "3": [ + 0.0, + 1.1692, + 2.4088, + 0.3253, + 1.0008, + 0.8895, + 0.5608, + 0.3077, + 0.6418, + 0.4143, + 0.5475, + 0.2497 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5647, + 1.365, + 0.2132, + 0.0545, + 0.2211, + 0.1914, + 0.1958, + 0.1866, + 0.4777, + 0.3089, + 0.0994 + ], + "3": [ + 0.0, + 3.2869, + 4.0832, + 0.5296, + 1.0719, + 0.795, + 0.853, + 0.9589, + 0.6516, + 1.0234, + 1.1701, + 1.124 + ] + } + } + }, + "order-ops": { + "n_items_scored": 55, + "n_intermediate_pairs_scored": 110, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.002976, + "jlens": 0.001829, + "shuffled": 0.000849, + "rand_match": 0.002122 + }, + "PRIMARY_ratio_jlens_over_logit": 0.6147, + "ratio_jlens_over_shuffled": 2.1556, + "ratio_jlens_over_rand_match": 0.8621, + "ratio_by_layer_jlens_over_logit": { + "2": 0.5443, + "3": 0.7181 + }, + "band_max_ratio_SECONDARY": 0.7181, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003541, + "dose_mean_per_position": 0.3877, + "dose_sum_over_positions": 3.5915 + }, + "3": { + "kl_mean": 0.002411, + "dose_mean_per_position": 0.4374, + "dose_sum_over_positions": 4.0395 + } + }, + "jlens": { + "2": { + "kl_mean": 0.001927, + "dose_mean_per_position": 0.247, + "dose_sum_over_positions": 2.3253 + }, + "3": { + "kl_mean": 0.001732, + "dose_mean_per_position": 0.302, + "dose_sum_over_positions": 2.7947 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.000822, + "dose_mean_per_position": 0.2732, + "dose_sum_over_positions": 2.559 + }, + "3": { + "kl_mean": 0.000876, + "dose_mean_per_position": 0.3423, + "dose_sum_over_positions": 3.2295 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.002213, + "dose_mean_per_position": 0.4331, + "dose_sum_over_positions": 4.0239 + }, + "3": { + "kl_mean": 0.002031, + "dose_mean_per_position": 0.5105, + "dose_sum_over_positions": 4.6388 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.2968, + 0.0493, + 0.3018, + 0.3017, + 0.2819, + 0.188, + 0.0994, + 0.2883 + ], + "3": [ + 0.0, + 0.5021, + 0.0337, + 0.0088, + 0.1413, + 0.5405, + 0.094, + 0.3283, + 0.4194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.1865, + 0.0604, + 0.0642, + 0.0042, + 0.1585, + 0.8585, + 0.1487, + 0.1963 + ], + "3": [ + 0.0, + 0.2937, + 0.0653, + 0.4173, + 0.2782, + 0.126, + 0.4932, + 0.0153, + 0.1508 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.2071, + 0.28, + 0.2073, + 0.185, + 0.0267, + 0.2737, + 0.0105, + 0.1903 + ], + "3": [ + 0.0, + 0.2096, + 0.2698, + 0.0102, + 0.165, + 0.0604, + 0.3215, + 0.5014, + 0.0624 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.0945, + 0.2806, + 0.0689, + 0.3952, + 0.1473, + 0.2637, + 0.2711, + 0.2763 + ], + "3": [ + 0.0, + 0.4252, + 0.0526, + 0.3179, + 0.3875, + 0.5907, + 0.3064, + 0.4023, + 0.0867 + ] + } + } + }, + "poetry": { + "n_items_scored": 98, + "n_intermediate_pairs_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.011893, + "jlens": 0.016125, + "shuffled": 0.004672, + "rand_match": 0.00537 + }, + "PRIMARY_ratio_jlens_over_logit": 1.3559, + "ratio_jlens_over_shuffled": 3.4513, + "ratio_jlens_over_rand_match": 3.0029, + "ratio_by_layer_jlens_over_logit": { + "2": 1.0824, + "3": 1.6313 + }, + "band_max_ratio_SECONDARY": 1.6313, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.011934, + "dose_mean_per_position": 0.941, + "dose_sum_over_positions": 25.7454 + }, + "3": { + "kl_mean": 0.011851, + "dose_mean_per_position": 1.1137, + "dose_sum_over_positions": 30.4835 + } + }, + "jlens": { + "2": { + "kl_mean": 0.012918, + "dose_mean_per_position": 0.6124, + "dose_sum_over_positions": 16.759 + }, + "3": { + "kl_mean": 0.019332, + "dose_mean_per_position": 0.9499, + "dose_sum_over_positions": 26.0198 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.007105, + "dose_mean_per_position": 0.6225, + "dose_sum_over_positions": 16.9821 + }, + "3": { + "kl_mean": 0.002239, + "dose_mean_per_position": 0.8903, + "dose_sum_over_positions": 24.3354 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.006512, + "dose_mean_per_position": 0.5916, + "dose_sum_over_positions": 16.1283 + }, + "3": { + "kl_mean": 0.004227, + "dose_mean_per_position": 0.6559, + "dose_sum_over_positions": 17.9902 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.1594, + 0.5944, + 0.2455, + 0.5992, + 0.2572, + 0.9429, + 0.2139, + 11.1563, + 0.0825, + 0.2446, + 0.1937, + 0.292, + 0.0767, + 0.7736, + 0.4319, + 0.6853, + 0.9142, + 0.684, + 0.1378, + 0.081, + 0.7477, + 0.352, + 1.0318, + 0.4303, + 0.3018, + 0.195, + 0.7437, + 0.2196 + ], + "3": [ + 0.0, + 0.3802, + 1.2164, + 1.0455, + 0.1044, + 0.4311, + 0.8451, + 0.2006, + 11.3244, + 0.1434, + 0.5504, + 0.3074, + 0.2068, + 0.2864, + 1.1519, + 0.7277, + 0.8769, + 1.4379, + 1.5382, + 0.4364, + 0.248, + 0.3947, + 0.1714, + 1.1808, + 0.2771, + 0.5054, + 0.6367, + 1.1076, + 0.0233 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.213, + 0.9306, + 0.5918, + 0.4013, + 0.8899, + 0.1917, + 0.4831, + 2.8724, + 0.6135, + 0.0143, + 0.1559, + 0.4587, + 0.2191, + 0.4398, + 0.5752, + 0.6734, + 0.419, + 0.6809, + 0.1712, + 0.2748, + 0.5712, + 0.5936, + 0.2623, + 0.9894, + 0.4983, + 1.0669, + 0.7419, + 0.9294 + ], + "3": [ + 0.0, + 0.2655, + 1.342, + 1.0173, + 0.0796, + 0.6819, + 0.7821, + 0.5298, + 4.0512, + 0.7125, + 0.3422, + 0.6076, + 0.9449, + 1.0348, + 1.123, + 0.9248, + 1.3085, + 1.2141, + 1.4503, + 0.7392, + 0.898, + 1.0419, + 0.9421, + 1.181, + 1.3165, + 1.2246, + 1.5514, + 1.0328, + 0.9903 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.1304, + 0.8056, + 0.4974, + 0.4802, + 0.7082, + 0.1034, + 0.3906, + 7.8381, + 0.6633, + 0.0422, + 0.186, + 0.1258, + 0.0018, + 0.4743, + 0.1732, + 0.3666, + 0.1196, + 0.2139, + 0.2363, + 0.0249, + 0.3553, + 0.3421, + 0.2374, + 0.7592, + 0.4178, + 0.6945, + 0.5653, + 0.5255 + ], + "3": [ + 0.0, + 0.6007, + 0.614, + 0.3259, + 0.6552, + 1.0463, + 0.5077, + 0.7627, + 7.1587, + 1.2437, + 0.2538, + 0.2266, + 0.305, + 0.4564, + 0.646, + 0.4866, + 0.4909, + 0.483, + 0.4656, + 0.1111, + 0.2861, + 0.8759, + 0.5602, + 0.8874, + 1.1647, + 1.3096, + 1.2461, + 1.1746, + 1.0795 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5663, + 0.0349, + 0.3271, + 0.0591, + 0.224, + 0.5588, + 0.0436, + 4.7702, + 0.2856, + 0.5625, + 0.8871, + 0.5497, + 1.1638, + 0.4989, + 0.2201, + 0.4187, + 0.339, + 0.6105, + 0.1274, + 0.2521, + 0.149, + 0.1882, + 0.2987, + 0.2137, + 0.1715, + 0.3117, + 0.0553, + 0.1175 + ], + "3": [ + 0.0, + 0.1785, + 0.1694, + 0.2338, + 0.1877, + 0.4362, + 0.2648, + 0.0485, + 5.4696, + 0.0288, + 0.8456, + 0.4643, + 0.385, + 0.61, + 0.0647, + 0.5367, + 0.7664, + 0.5221, + 0.2946, + 0.5209, + 0.0216, + 0.6684, + 0.4557, + 0.4048, + 0.2026, + 0.1609, + 0.202, + 0.3115, + 0.2607 + ] + } + } + }, + "typo": { + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.019955, + "jlens": 0.03108, + "shuffled": 0.01256, + "rand_match": 0.009576 + }, + "PRIMARY_ratio_jlens_over_logit": 1.5576, + "ratio_jlens_over_shuffled": 2.4746, + "ratio_jlens_over_rand_match": 3.2457, + "ratio_by_layer_jlens_over_logit": { + "2": 1.0951, + "3": 2.019 + }, + "band_max_ratio_SECONDARY": 2.019, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019932, + "dose_mean_per_position": 0.486, + "dose_sum_over_positions": 4.7988 + }, + "3": { + "kl_mean": 0.019977, + "dose_mean_per_position": 0.566, + "dose_sum_over_positions": 5.5739 + } + }, + "jlens": { + "2": { + "kl_mean": 0.021827, + "dose_mean_per_position": 0.4233, + "dose_sum_over_positions": 4.1739 + }, + "3": { + "kl_mean": 0.040334, + "dose_mean_per_position": 0.7185, + "dose_sum_over_positions": 7.092 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.005245, + "dose_mean_per_position": 0.3471, + "dose_sum_over_positions": 3.3901 + }, + "3": { + "kl_mean": 0.019874, + "dose_mean_per_position": 0.701, + "dose_sum_over_positions": 6.855 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.011462, + "dose_mean_per_position": 0.3907, + "dose_sum_over_positions": 3.742 + }, + "3": { + "kl_mean": 0.00769, + "dose_mean_per_position": 0.4858, + "dose_sum_over_positions": 4.6135 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0693, + 0.62, + 0.4626, + 0.3928, + 0.1845, + 1.245, + 1.0045, + 0.4486, + 0.5024, + 1.0145, + 0.4979 + ], + "3": [ + 0.0, + 0.2168, + 0.838, + 0.2187, + 0.25, + 0.1403, + 0.75, + 0.7185, + 0.1199, + 0.3759, + 0.8352, + 0.6194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.559, + 0.2879, + 0.0404, + 0.2123, + 0.7116, + 0.3727, + 0.3724, + 0.3405, + 0.1068, + 0.1192, + 0.2891 + ], + "3": [ + 0.0, + 0.7288, + 0.6559, + 0.8335, + 0.7473, + 0.558, + 0.3651, + 0.5253, + 0.8257, + 0.4974, + 0.6148, + 0.1478 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.5148, + 0.75, + 0.3829, + 0.2622, + 0.7209, + 0.4784, + 0.352, + 0.1922, + 0.0097, + 0.7032, + 0.0915 + ], + "3": [ + 0.0, + 0.9579, + 0.9196, + 0.0612, + 0.3469, + 1.3157, + 0.6778, + 0.3969, + 0.9489, + 0.587, + 1.0191, + 0.103 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.4129, + 0.2709, + 0.1557, + 0.1544, + 0.5417, + 0.0378, + 0.2531, + 0.5354, + 0.2084, + 0.4157, + 0.5005 + ], + "3": [ + 0.0, + 0.5479, + 1.0458, + 0.7085, + 1.2307, + 0.8921, + 1.3775, + 0.9829, + 0.4017, + 0.3321, + 0.5386, + 0.2979 + ] + } + } + }, + "association": { + "n_items_scored": 102, + "n_intermediate_pairs_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "band_mean_kl": { + "logit": 0.761053, + "jlens": 0.232467, + "shuffled": 0.33096, + "rand_match": 0.331445 + }, + "PRIMARY_ratio_jlens_over_logit": 0.3055, + "ratio_jlens_over_shuffled": 0.7024, + "ratio_jlens_over_rand_match": 0.7014, + "ratio_by_layer_jlens_over_logit": { + "2": 0.2533, + "3": 0.3568 + }, + "band_max_ratio_SECONDARY": 0.3568, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.755566, + "dose_mean_per_position": 1.0786, + "dose_sum_over_positions": 31.3456 + }, + "3": { + "kl_mean": 0.766541, + "dose_mean_per_position": 1.1292, + "dose_sum_over_positions": 32.8565 + } + }, + "jlens": { + "2": { + "kl_mean": 0.191416, + "dose_mean_per_position": 0.6128, + "dose_sum_over_positions": 17.895 + }, + "3": { + "kl_mean": 0.273518, + "dose_mean_per_position": 0.9843, + "dose_sum_over_positions": 28.7633 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.352742, + "dose_mean_per_position": 0.5855, + "dose_sum_over_positions": 16.9344 + }, + "3": { + "kl_mean": 0.309177, + "dose_mean_per_position": 0.8934, + "dose_sum_over_positions": 25.9721 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.367695, + "dose_mean_per_position": 0.6371, + "dose_sum_over_positions": 18.6053 + }, + "3": { + "kl_mean": 0.295195, + "dose_mean_per_position": 0.7068, + "dose_sum_over_positions": 20.5207 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.261, + 0.7127, + 0.4662, + 0.3029, + 0.3495, + 0.053, + 0.077, + 0.9719, + 0.7852, + 0.6237, + 1.3751, + 0.6093, + 2.0121, + 0.6116, + 1.0531, + 1.7732, + 1.1089, + 0.6423, + 0.3501, + 0.7945, + 1.2471, + 1.3578, + 1.1713, + 1.1945, + 0.3451, + 0.3122, + 1.3023, + 12.4952 + ], + "3": [ + 0.0, + 0.4806, + 0.662, + 0.7729, + 0.4281, + 0.7686, + 0.5767, + 0.175, + 1.3067, + 1.3245, + 0.9131, + 1.5172, + 0.5693, + 1.5528, + 0.9205, + 0.2771, + 1.0986, + 0.5817, + 1.0954, + 0.5864, + 0.5879, + 0.9137, + 1.0181, + 0.8501, + 2.2945, + 0.7854, + 0.5198, + 1.2149, + 12.637 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.4581, + 0.6715, + 0.5647, + 0.0928, + 0.2748, + 0.6003, + 0.4271, + 0.5025, + 0.8482, + 0.1449, + 0.859, + 0.7347, + 0.4655, + 0.7042, + 0.3922, + 0.8621, + 0.6225, + 1.0079, + 0.9787, + 0.592, + 0.6747, + 0.8711, + 0.6717, + 0.8223, + 0.8184, + 0.1886, + 0.9041, + 3.7391 + ], + "3": [ + 0.0, + 0.871, + 0.8641, + 1.2426, + 0.3936, + 0.9154, + 1.3368, + 0.9465, + 1.1001, + 1.5294, + 1.314, + 1.4067, + 1.1088, + 0.9699, + 1.3219, + 1.0725, + 1.3858, + 0.8019, + 1.5069, + 1.566, + 1.1432, + 1.0461, + 1.0786, + 0.8008, + 1.8483, + 1.5464, + 1.0009, + 1.2482, + 3.9756 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.2286, + 0.139, + 0.1008, + 0.116, + 0.2455, + 0.9528, + 0.1895, + 0.113, + 0.3953, + 0.513, + 0.3406, + 0.5625, + 0.0021, + 0.1106, + 0.0397, + 0.3957, + 0.3585, + 0.641, + 0.4431, + 0.1847, + 0.2105, + 0.0301, + 0.3075, + 0.4418, + 0.1159, + 0.0997, + 0.6177, + 8.8024 + ], + "3": [ + 0.0, + 0.8221, + 0.7438, + 0.6624, + 0.9362, + 0.6686, + 1.0323, + 1.1338, + 1.3397, + 0.7623, + 0.2658, + 0.9038, + 1.1294, + 0.7591, + 0.5445, + 0.557, + 1.0999, + 0.7122, + 1.0421, + 0.6156, + 0.8791, + 0.7914, + 0.0743, + 0.1021, + 0.6414, + 0.7106, + 0.522, + 0.8654, + 7.7949 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5339, + 0.0311, + 0.0312, + 0.4897, + 0.5893, + 0.0718, + 0.1459, + 0.131, + 0.2004, + 0.8931, + 0.3031, + 0.3423, + 0.4818, + 0.5813, + 0.1457, + 0.071, + 0.0664, + 0.512, + 0.5542, + 0.1682, + 0.216, + 0.1395, + 0.2062, + 0.1179, + 0.0305, + 0.5685, + 0.2558, + 0.7061 + ], + "3": [ + 0.0, + 1.4369, + 0.589, + 0.4872, + 0.274, + 0.3277, + 0.2342, + 0.3975, + 0.6986, + 0.3046, + 0.1022, + 0.5765, + 0.6712, + 0.5935, + 0.3667, + 0.3788, + 0.7403, + 0.1303, + 0.1983, + 0.0263, + 0.2366, + 0.1407, + 0.0562, + 0.1564, + 0.1818, + 0.1215, + 0.3484, + 0.4221, + 7.1495 + ] + } + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "argv": [ + "t20_ablation_kl.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--lens", + "results/plad_70m_code_A.pt", + "--evals", + "all", + "--out", + "results/t20_ablation_kl_70m_Pcode.json" + ], + "runtime_s": 186.1, + "provenance": { + "payload_sha256_before_redaction": "f7b65ede7917302bf7e79bac7daaf6d76709ed5db72e25835ca44ae47aa7c11c", + "payload_sha256": "5750b3805e47aba9aaf291aca3ff81341afe5a12f48d0082f82b418071069d84", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t20_ablation_kl_70m_Ppile.json b/results/t20_ablation_kl_70m_Ppile.json new file mode 100644 index 0000000000000000000000000000000000000000..36e8f827fcfa619428c0a2bbcef99f1a982cd562 --- /dev/null +++ b/results/t20_ablation_kl_70m_Ppile.json @@ -0,0 +1,1533 @@ +{ + "experiment": "T20 / E20 \u2014 ablation KL, the anchor's Figure 53 DV", + "status": "EXPLORATORY", + "prereg": null, + "prereg_sha256": null, + "decision_rule": "ratio KL(J^P)/KL(I), band mean, on multihop: >1.5 reproduces the anchor; <=1.0 falsifies at this scale; (1.0,1.5] inconclusive", + "primary_outcome": "band-mean ratio KL(jlens)/KL(logit) per set", + "layer_protocol": "one layer at a time, swept over the band; NO band compounding. Headline is the band MEAN of the per-layer ratio; band-max is stored as a secondary and is explicitly not primary (existential statistic).", + "cross_lens_caveat": "the logit lens has no lens-native coordinate write (external review B5/G6); applied here it is a SHARED hidden-state intervention parameterised by whichever lens supplies v_c", + "VERDICT": "FALSIFIED at this scale \u2014 multihop ratio 0.8686 <= 1.0", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/plad_70m_pile_A.pt", + "lens_sha256": "d49584ea7e7778d96e2073fa5149ee39294e7702d5c06d5e9f8b6d8d77247486", + "n_layers": 6, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "layers_arg": "band", + "skip_positions": 1, + "max_items_effective": null, + "min_pairs_floor": 30, + "evals": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo", + "association" + ], + "arms": [ + "logit", + "jlens", + "shuffled", + "rand_match" + ], + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "device": "cpu", + "n_forward_passes": 7695, + "results": { + "multihop": { + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.014876, + "jlens": 0.012921, + "shuffled": 0.006234, + "rand_match": 0.004407 + }, + "PRIMARY_ratio_jlens_over_logit": 0.8686, + "ratio_jlens_over_shuffled": 2.0727, + "ratio_jlens_over_rand_match": 2.9322, + "ratio_by_layer_jlens_over_logit": { + "2": 0.8145, + "3": 0.968 + }, + "band_max_ratio_SECONDARY": 0.968, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019273, + "dose_mean_per_position": 0.6468, + "dose_sum_over_positions": 16.2733 + }, + "3": { + "kl_mean": 0.010479, + "dose_mean_per_position": 0.6897, + "dose_sum_over_positions": 17.2888 + } + }, + "jlens": { + "2": { + "kl_mean": 0.015698, + "dose_mean_per_position": 0.5537, + "dose_sum_over_positions": 12.3445 + }, + "3": { + "kl_mean": 0.010144, + "dose_mean_per_position": 0.5508, + "dose_sum_over_positions": 12.9256 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.003446, + "dose_mean_per_position": 0.5058, + "dose_sum_over_positions": 11.9031 + }, + "3": { + "kl_mean": 0.009021, + "dose_mean_per_position": 1.0404, + "dose_sum_over_positions": 23.5772 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.004381, + "dose_mean_per_position": 0.4441, + "dose_sum_over_positions": 9.8799 + }, + "3": { + "kl_mean": 0.004432, + "dose_mean_per_position": 0.5476, + "dose_sum_over_positions": 12.3821 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0143, + 0.4263, + 0.3879, + 0.1754, + 0.3134, + 0.3558, + 1.574, + 0.955, + 0.3888, + 1.4085, + 0.8029, + 1.2517, + 1.3781, + 1.7634, + 0.9378, + 0.959, + 1.1397, + 0.7515, + 0.1324 + ], + "3": [ + 0.0, + 0.1675, + 0.1087, + 0.4415, + 0.0303, + 0.1676, + 0.091, + 1.5784, + 0.6766, + 0.3601, + 1.4793, + 0.1837, + 1.0786, + 1.4508, + 1.5462, + 1.293, + 1.1929, + 1.6134, + 0.9143, + 0.1636 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.3713, + 0.1061, + 0.1579, + 0.1869, + 0.5852, + 0.8527, + 0.8208, + 1.1298, + 1.0148, + 1.2736, + 1.2194, + 0.409, + 0.8127, + 1.2051, + 1.0751, + 0.701, + 0.608, + 0.8337, + 0.6933 + ], + "3": [ + 0.0, + 0.4829, + 0.0637, + 0.1856, + 0.3988, + 0.3925, + 0.7437, + 0.6694, + 1.104, + 1.1212, + 0.9729, + 0.6354, + 0.0612, + 0.9411, + 1.4561, + 1.4402, + 1.4968, + 1.4913, + 1.0003, + 1.2928 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.128, + 0.4786, + 0.238, + 0.1085, + 0.9536, + 0.3003, + 0.6106, + 0.792, + 0.4872, + 1.0711, + 0.8562, + 0.2641, + 0.2707, + 0.4723, + 0.5013, + 0.6901, + 0.5159, + 0.5346, + 0.2786 + ], + "3": [ + 0.0, + 0.5427, + 0.6617, + 0.6993, + 0.6306, + 1.3484, + 1.1465, + 1.4544, + 1.1238, + 1.4358, + 1.8291, + 1.7108, + 1.3959, + 1.754, + 1.6094, + 1.5181, + 1.5349, + 1.8063, + 1.1283, + 1.366 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.8595, + 0.0865, + 0.4166, + 0.224, + 0.1023, + 0.3016, + 0.6538, + 0.1718, + 0.4688, + 0.7481, + 0.0053, + 0.5808, + 0.3543, + 0.3642, + 0.1474, + 1.1441, + 0.4846, + 0.2117, + 0.3653 + ], + "3": [ + 0.0, + 0.082, + 0.0965, + 0.1922, + 0.4986, + 0.051, + 0.5813, + 0.6251, + 0.6909, + 0.6936, + 0.7641, + 0.3953, + 0.4381, + 0.0576, + 1.0085, + 0.4477, + 0.1701, + 0.2303, + 0.437, + 0.2175 + ] + } + } + }, + "multilingual": { + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "band_mean_kl": { + "logit": 0.003209, + "jlens": 0.003591, + "shuffled": 0.003445, + "rand_match": 0.003698 + }, + "PRIMARY_ratio_jlens_over_logit": 1.1192, + "ratio_jlens_over_shuffled": 1.0424, + "ratio_jlens_over_rand_match": 0.971, + "ratio_by_layer_jlens_over_logit": { + "2": 0.7094, + "3": 1.5382 + }, + "band_max_ratio_SECONDARY": 1.5382, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003244, + "dose_mean_per_position": 0.4899, + "dose_sum_over_positions": 5.5071 + }, + "3": { + "kl_mean": 0.003173, + "dose_mean_per_position": 0.523, + "dose_sum_over_positions": 5.8039 + } + }, + "jlens": { + "2": { + "kl_mean": 0.002301, + "dose_mean_per_position": 0.3146, + "dose_sum_over_positions": 3.586 + }, + "3": { + "kl_mean": 0.00488, + "dose_mean_per_position": 0.4408, + "dose_sum_over_positions": 5.2914 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.001967, + "dose_mean_per_position": 0.391, + "dose_sum_over_positions": 4.4677 + }, + "3": { + "kl_mean": 0.004923, + "dose_mean_per_position": 0.7343, + "dose_sum_over_positions": 8.3311 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.003771, + "dose_mean_per_position": 0.4726, + "dose_sum_over_positions": 5.4467 + }, + "3": { + "kl_mean": 0.003625, + "dose_mean_per_position": 0.5427, + "dose_sum_over_positions": 6.1871 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 1.3924, + 3.5021, + 0.0182, + 0.3026, + 0.7169, + 0.272, + 0.1566, + 0.032, + 0.2212, + 0.5358, + 0.3988 + ], + "3": [ + 0.0, + 1.1118, + 3.2379, + 0.5082, + 1.2603, + 0.095, + 0.2086, + 0.1731, + 0.2063, + 0.5958, + 0.0088, + 0.2401 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.3062, + 1.3334, + 0.2597, + 0.5993, + 0.2761, + 0.0619, + 0.0329, + 0.0378, + 0.3733, + 0.3578, + 0.0951 + ], + "3": [ + 0.0, + 0.5161, + 0.7367, + 0.1971, + 0.9677, + 0.0459, + 0.121, + 0.6047, + 0.0464, + 0.3482, + 0.0687, + 0.4285 + ] + }, + "shuffled": { + "2": [ + 0.0, + 1.0638, + 2.6824, + 0.0243, + 0.9414, + 0.3942, + 0.4264, + 0.0474, + 0.0423, + 0.4143, + 0.1362, + 0.0586 + ], + "3": [ + 0.0, + 1.6503, + 3.502, + 0.5399, + 1.1728, + 0.9328, + 0.7319, + 0.5571, + 0.8072, + 0.6457, + 0.675, + 0.0293 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5647, + 1.365, + 0.2132, + 0.0545, + 0.2211, + 0.1914, + 0.1958, + 0.1866, + 0.4777, + 0.3089, + 0.0994 + ], + "3": [ + 0.0, + 3.2869, + 4.0832, + 0.5296, + 1.0719, + 0.795, + 0.853, + 0.9589, + 0.6516, + 1.0234, + 1.1701, + 1.124 + ] + } + } + }, + "order-ops": { + "n_items_scored": 55, + "n_intermediate_pairs_scored": 110, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.002976, + "jlens": 0.001717, + "shuffled": 0.001329, + "rand_match": 0.002122 + }, + "PRIMARY_ratio_jlens_over_logit": 0.5768, + "ratio_jlens_over_shuffled": 1.2917, + "ratio_jlens_over_rand_match": 0.8091, + "ratio_by_layer_jlens_over_logit": { + "2": 0.5574, + "3": 0.6053 + }, + "band_max_ratio_SECONDARY": 0.6053, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003541, + "dose_mean_per_position": 0.3877, + "dose_sum_over_positions": 3.5915 + }, + "3": { + "kl_mean": 0.002411, + "dose_mean_per_position": 0.4374, + "dose_sum_over_positions": 4.0395 + } + }, + "jlens": { + "2": { + "kl_mean": 0.001974, + "dose_mean_per_position": 0.2606, + "dose_sum_over_positions": 2.4565 + }, + "3": { + "kl_mean": 0.00146, + "dose_mean_per_position": 0.3322, + "dose_sum_over_positions": 3.1123 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.000913, + "dose_mean_per_position": 0.2851, + "dose_sum_over_positions": 2.6682 + }, + "3": { + "kl_mean": 0.001745, + "dose_mean_per_position": 0.498, + "dose_sum_over_positions": 4.6491 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.002213, + "dose_mean_per_position": 0.4331, + "dose_sum_over_positions": 4.0239 + }, + "3": { + "kl_mean": 0.00203, + "dose_mean_per_position": 0.5105, + "dose_sum_over_positions": 4.6388 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.2968, + 0.0493, + 0.3018, + 0.3017, + 0.2819, + 0.188, + 0.0994, + 0.2883 + ], + "3": [ + 0.0, + 0.5021, + 0.0337, + 0.0088, + 0.1413, + 0.5405, + 0.094, + 0.3283, + 0.4194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.2463, + 0.1364, + 0.1764, + 0.0566, + 0.1154, + 0.8286, + 0.1516, + 0.2451 + ], + "3": [ + 0.0, + 0.5846, + 0.3775, + 0.1169, + 0.0807, + 0.0918, + 0.6329, + 0.2507, + 0.3703 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.2164, + 0.2322, + 0.279, + 0.15, + 0.1445, + 0.2286, + 0.0203, + 0.1499 + ], + "3": [ + 0.0, + 0.4017, + 0.4803, + 0.193, + 0.3638, + 0.3346, + 0.0574, + 0.7288, + 0.1202 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.0945, + 0.2806, + 0.0689, + 0.3952, + 0.1473, + 0.2637, + 0.2711, + 0.2763 + ], + "3": [ + 0.0, + 0.4252, + 0.0526, + 0.3179, + 0.3876, + 0.5907, + 0.3064, + 0.4023, + 0.0867 + ] + } + } + }, + "poetry": { + "n_items_scored": 98, + "n_intermediate_pairs_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.011893, + "jlens": 0.012681, + "shuffled": 0.004858, + "rand_match": 0.005369 + }, + "PRIMARY_ratio_jlens_over_logit": 1.0663, + "ratio_jlens_over_shuffled": 2.6103, + "ratio_jlens_over_rand_match": 2.3619, + "ratio_by_layer_jlens_over_logit": { + "2": 1.2291, + "3": 0.9024 + }, + "band_max_ratio_SECONDARY": 1.2291, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.011934, + "dose_mean_per_position": 0.941, + "dose_sum_over_positions": 25.7454 + }, + "3": { + "kl_mean": 0.011851, + "dose_mean_per_position": 1.1137, + "dose_sum_over_positions": 30.4835 + } + }, + "jlens": { + "2": { + "kl_mean": 0.014669, + "dose_mean_per_position": 0.6566, + "dose_sum_over_positions": 17.9682 + }, + "3": { + "kl_mean": 0.010694, + "dose_mean_per_position": 0.8415, + "dose_sum_over_positions": 23.0462 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.006431, + "dose_mean_per_position": 0.6517, + "dose_sum_over_positions": 17.7809 + }, + "3": { + "kl_mean": 0.003286, + "dose_mean_per_position": 1.1179, + "dose_sum_over_positions": 30.5649 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.006511, + "dose_mean_per_position": 0.5916, + "dose_sum_over_positions": 16.1283 + }, + "3": { + "kl_mean": 0.004227, + "dose_mean_per_position": 0.6559, + "dose_sum_over_positions": 17.9902 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.1594, + 0.5944, + 0.2455, + 0.5992, + 0.2572, + 0.9429, + 0.2139, + 11.1563, + 0.0825, + 0.2446, + 0.1937, + 0.292, + 0.0767, + 0.7736, + 0.4319, + 0.6853, + 0.9142, + 0.684, + 0.1378, + 0.081, + 0.7477, + 0.352, + 1.0318, + 0.4303, + 0.3018, + 0.195, + 0.7437, + 0.2196 + ], + "3": [ + 0.0, + 0.3802, + 1.2164, + 1.0455, + 0.1044, + 0.4311, + 0.8451, + 0.2006, + 11.3244, + 0.1434, + 0.5504, + 0.3074, + 0.2068, + 0.2864, + 1.1519, + 0.7277, + 0.8769, + 1.4379, + 1.5382, + 0.4364, + 0.248, + 0.3947, + 0.1714, + 1.1808, + 0.2771, + 0.5054, + 0.6367, + 1.1076, + 0.0233 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.2397, + 0.9163, + 0.5735, + 0.4124, + 0.9036, + 0.2136, + 0.4645, + 4.326, + 0.5366, + 0.0332, + 0.109, + 0.4185, + 0.2108, + 0.4679, + 0.5424, + 0.5687, + 0.3693, + 0.7072, + 0.1914, + 0.2833, + 0.5192, + 0.5998, + 0.2508, + 0.9666, + 0.4591, + 1.1093, + 0.8237, + 0.9268 + ], + "3": [ + 0.0, + 0.5497, + 1.6877, + 1.2766, + 0.0797, + 0.5042, + 0.6338, + 0.3416, + 4.8703, + 0.4455, + 0.1415, + 0.2985, + 0.6553, + 0.6673, + 0.833, + 0.6259, + 1.0291, + 1.0395, + 1.2228, + 0.5381, + 0.5935, + 0.7404, + 0.7299, + 0.9767, + 1.0374, + 0.9712, + 1.3341, + 0.9281, + 0.728 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.1082, + 0.8263, + 0.4296, + 0.3595, + 0.7484, + 0.1373, + 0.4543, + 8.3653, + 0.5998, + 0.0583, + 0.1688, + 0.1157, + 0.077, + 0.444, + 0.2021, + 0.3403, + 0.2641, + 0.24, + 0.1604, + 0.0172, + 0.2885, + 0.3232, + 0.2742, + 0.7988, + 0.5008, + 0.666, + 0.6573, + 0.5637 + ], + "3": [ + 0.0, + 0.7016, + 0.4889, + 0.1206, + 0.7884, + 1.3049, + 0.6921, + 0.9116, + 9.8194, + 1.3676, + 0.4528, + 0.3748, + 0.4804, + 0.6135, + 0.7457, + 0.614, + 0.5051, + 0.7177, + 0.6703, + 0.0318, + 0.4339, + 0.9379, + 0.7586, + 1.1001, + 1.3537, + 1.4245, + 1.4151, + 1.4503, + 1.185 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5663, + 0.0349, + 0.3271, + 0.0591, + 0.224, + 0.5588, + 0.0436, + 4.7702, + 0.2856, + 0.5625, + 0.8871, + 0.5497, + 1.1638, + 0.4989, + 0.2201, + 0.4187, + 0.339, + 0.6105, + 0.1274, + 0.2521, + 0.149, + 0.1882, + 0.2987, + 0.2137, + 0.1715, + 0.3117, + 0.0553, + 0.1175 + ], + "3": [ + 0.0, + 0.1785, + 0.1694, + 0.2338, + 0.1877, + 0.4362, + 0.2648, + 0.0485, + 5.4696, + 0.0288, + 0.8456, + 0.4643, + 0.385, + 0.61, + 0.0647, + 0.5367, + 0.7664, + 0.5221, + 0.2946, + 0.5209, + 0.0216, + 0.6684, + 0.4557, + 0.4048, + 0.2026, + 0.1609, + 0.202, + 0.3115, + 0.2607 + ] + } + } + }, + "typo": { + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.019955, + "jlens": 0.024041, + "shuffled": 0.01688, + "rand_match": 0.009578 + }, + "PRIMARY_ratio_jlens_over_logit": 1.2048, + "ratio_jlens_over_shuffled": 1.4242, + "ratio_jlens_over_rand_match": 2.5099, + "ratio_by_layer_jlens_over_logit": { + "2": 1.1729, + "3": 1.2365 + }, + "band_max_ratio_SECONDARY": 1.2365, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019932, + "dose_mean_per_position": 0.486, + "dose_sum_over_positions": 4.7988 + }, + "3": { + "kl_mean": 0.019977, + "dose_mean_per_position": 0.566, + "dose_sum_over_positions": 5.5739 + } + }, + "jlens": { + "2": { + "kl_mean": 0.023379, + "dose_mean_per_position": 0.4391, + "dose_sum_over_positions": 4.3229 + }, + "3": { + "kl_mean": 0.024702, + "dose_mean_per_position": 0.5637, + "dose_sum_over_positions": 5.567 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.005551, + "dose_mean_per_position": 0.361, + "dose_sum_over_positions": 3.5383 + }, + "3": { + "kl_mean": 0.028209, + "dose_mean_per_position": 0.9121, + "dose_sum_over_positions": 8.9066 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.011465, + "dose_mean_per_position": 0.3907, + "dose_sum_over_positions": 3.742 + }, + "3": { + "kl_mean": 0.007691, + "dose_mean_per_position": 0.4858, + "dose_sum_over_positions": 4.6135 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0693, + 0.62, + 0.4626, + 0.3928, + 0.1845, + 1.245, + 1.0045, + 0.4486, + 0.5024, + 1.0145, + 0.4979 + ], + "3": [ + 0.0, + 0.2168, + 0.838, + 0.2187, + 0.25, + 0.1403, + 0.75, + 0.7185, + 0.1199, + 0.3759, + 0.8352, + 0.6194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.5421, + 0.274, + 0.179, + 0.0675, + 0.5924, + 0.3675, + 0.3195, + 0.3187, + 0.1476, + 0.2399, + 0.3066 + ], + "3": [ + 0.0, + 0.5077, + 0.4687, + 0.5938, + 0.4243, + 0.178, + 0.1562, + 0.2917, + 0.4583, + 0.1821, + 0.4411, + 0.0741 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.482, + 0.6278, + 0.5468, + 0.2364, + 0.6268, + 0.4601, + 0.1795, + 0.0451, + 0.0521, + 0.6029, + 0.2043 + ], + "3": [ + 0.0, + 1.1907, + 1.0907, + 0.1535, + 0.576, + 1.3787, + 0.9572, + 0.488, + 0.9461, + 0.7098, + 1.1508, + 0.139 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.4129, + 0.2709, + 0.1557, + 0.1544, + 0.5417, + 0.0378, + 0.2531, + 0.5354, + 0.2084, + 0.4157, + 0.5005 + ], + "3": [ + 0.0, + 0.5479, + 1.0458, + 0.7085, + 1.2307, + 0.8921, + 1.3775, + 0.9829, + 0.4017, + 0.3321, + 0.5386, + 0.2979 + ] + } + } + }, + "association": { + "n_items_scored": 102, + "n_intermediate_pairs_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "band_mean_kl": { + "logit": 0.761053, + "jlens": 0.276686, + "shuffled": 0.484876, + "rand_match": 0.331444 + }, + "PRIMARY_ratio_jlens_over_logit": 0.3636, + "ratio_jlens_over_shuffled": 0.5706, + "ratio_jlens_over_rand_match": 0.8348, + "ratio_by_layer_jlens_over_logit": { + "2": 0.3688, + "3": 0.3584 + }, + "band_max_ratio_SECONDARY": 0.3688, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.755566, + "dose_mean_per_position": 1.0786, + "dose_sum_over_positions": 31.3456 + }, + "3": { + "kl_mean": 0.766541, + "dose_mean_per_position": 1.1292, + "dose_sum_over_positions": 32.8565 + } + }, + "jlens": { + "2": { + "kl_mean": 0.27862, + "dose_mean_per_position": 0.6709, + "dose_sum_over_positions": 19.5512 + }, + "3": { + "kl_mean": 0.274752, + "dose_mean_per_position": 0.826, + "dose_sum_over_positions": 24.1157 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.411564, + "dose_mean_per_position": 0.6398, + "dose_sum_over_positions": 18.4947 + }, + "3": { + "kl_mean": 0.558189, + "dose_mean_per_position": 1.1938, + "dose_sum_over_positions": 34.6382 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.367691, + "dose_mean_per_position": 0.6371, + "dose_sum_over_positions": 18.6053 + }, + "3": { + "kl_mean": 0.295196, + "dose_mean_per_position": 0.7068, + "dose_sum_over_positions": 20.5207 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.261, + 0.7127, + 0.4662, + 0.3029, + 0.3495, + 0.053, + 0.077, + 0.9719, + 0.7852, + 0.6237, + 1.3751, + 0.6093, + 2.0121, + 0.6116, + 1.0531, + 1.7732, + 1.1089, + 0.6423, + 0.3501, + 0.7945, + 1.2471, + 1.3578, + 1.1713, + 1.1945, + 0.3451, + 0.3122, + 1.3023, + 12.4952 + ], + "3": [ + 0.0, + 0.4806, + 0.662, + 0.7729, + 0.4281, + 0.7686, + 0.5767, + 0.175, + 1.3067, + 1.3245, + 0.9131, + 1.5172, + 0.5693, + 1.5528, + 0.9205, + 0.2771, + 1.0986, + 0.5817, + 1.0954, + 0.5864, + 0.5879, + 0.9137, + 1.0181, + 0.8501, + 2.2945, + 0.7854, + 0.5198, + 1.2149, + 12.637 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.4864, + 0.7627, + 0.6705, + 0.1495, + 0.3529, + 0.6287, + 0.5086, + 0.5293, + 0.8793, + 0.0535, + 0.8382, + 0.7476, + 0.4156, + 0.7608, + 0.2897, + 0.8475, + 0.6868, + 1.086, + 1.0109, + 0.5391, + 0.6061, + 0.8097, + 0.5686, + 0.8408, + 0.8169, + 0.1009, + 0.8316, + 5.1029 + ], + "3": [ + 0.0, + 0.6838, + 0.736, + 1.1192, + 0.2329, + 0.7401, + 1.0527, + 0.7176, + 0.8717, + 1.2771, + 0.9393, + 1.1668, + 0.8246, + 0.7452, + 1.0441, + 0.6859, + 1.0633, + 0.5459, + 1.2567, + 1.3271, + 0.8562, + 0.8286, + 0.8776, + 0.4814, + 1.6601, + 1.2777, + 0.6813, + 0.9674, + 4.6319 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.2421, + 0.1308, + 0.1194, + 0.0944, + 0.2286, + 1.0628, + 0.3158, + 0.3416, + 0.4525, + 0.5664, + 0.3626, + 0.5368, + 0.0261, + 0.1152, + 0.0543, + 0.4351, + 0.5156, + 0.7181, + 0.5425, + 0.1963, + 0.3377, + 0.1379, + 0.2169, + 0.4943, + 0.2113, + 0.078, + 0.5686, + 9.4161 + ], + "3": [ + 0.0, + 1.0947, + 1.0203, + 0.9397, + 1.2207, + 0.9753, + 1.2356, + 1.3465, + 1.688, + 0.9857, + 0.4583, + 1.1508, + 1.3571, + 1.034, + 0.692, + 0.767, + 1.4141, + 1.0605, + 1.288, + 0.8821, + 1.0555, + 1.1446, + 0.42, + 0.3623, + 0.911, + 0.9344, + 0.7271, + 1.1, + 10.3736 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5339, + 0.0311, + 0.0312, + 0.4897, + 0.5893, + 0.0718, + 0.1459, + 0.131, + 0.2004, + 0.8931, + 0.3031, + 0.3423, + 0.4818, + 0.5813, + 0.1457, + 0.071, + 0.0664, + 0.512, + 0.5542, + 0.1682, + 0.216, + 0.1395, + 0.2062, + 0.1179, + 0.0305, + 0.5685, + 0.2558, + 0.7061 + ], + "3": [ + 0.0, + 1.4369, + 0.589, + 0.4872, + 0.274, + 0.3277, + 0.2342, + 0.3975, + 0.6986, + 0.3046, + 0.1022, + 0.5765, + 0.6712, + 0.5935, + 0.3667, + 0.3788, + 0.7403, + 0.1303, + 0.1983, + 0.0263, + 0.2366, + 0.1407, + 0.0562, + 0.1564, + 0.1818, + 0.1215, + 0.3484, + 0.4221, + 7.1495 + ] + } + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "argv": [ + "t20_ablation_kl.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--lens", + "results/plad_70m_pile_A.pt", + "--evals", + "all", + "--out", + "results/t20_ablation_kl_70m_Ppile.json" + ], + "runtime_s": 185.8, + "provenance": { + "payload_sha256_before_redaction": "1bc19f1b6b7a160123d4332719ce840edb0bbad0b8dfb3313a7050552023e24e", + "payload_sha256": "b36c1e0ea9e3e92f7e755eeac09d0dec54fa28ee968748ef160e19403f23915d", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t20_ablation_kl_70m_Pwikitext.json b/results/t20_ablation_kl_70m_Pwikitext.json new file mode 100644 index 0000000000000000000000000000000000000000..bd1fa84e4fb35068fe06300a65fd3757bb0ad2d5 --- /dev/null +++ b/results/t20_ablation_kl_70m_Pwikitext.json @@ -0,0 +1,1533 @@ +{ + "experiment": "T20 / E20 \u2014 ablation KL, the anchor's Figure 53 DV", + "status": "EXPLORATORY", + "prereg": null, + "prereg_sha256": null, + "decision_rule": "ratio KL(J^P)/KL(I), band mean, on multihop: >1.5 reproduces the anchor; <=1.0 falsifies at this scale; (1.0,1.5] inconclusive", + "primary_outcome": "band-mean ratio KL(jlens)/KL(logit) per set", + "layer_protocol": "one layer at a time, swept over the band; NO band compounding. Headline is the band MEAN of the per-layer ratio; band-max is stored as a secondary and is explicitly not primary (existential statistic).", + "cross_lens_caveat": "the logit lens has no lens-native coordinate write (external review B5/G6); applied here it is a SHARED hidden-state intervention parameterised by whichever lens supplies v_c", + "VERDICT": "FALSIFIED at this scale \u2014 multihop ratio 0.5839 <= 1.0", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/plad_70m_wikitext_A.pt", + "lens_sha256": "ac0cb01cd47db95dbf1ea68d3d2e0120f26ed58c251b4b1ad2f84b61f76ec6f4", + "n_layers": 6, + "source_layers": [ + 0, + 1, + 2, + 3 + ], + "band": [ + 2, + 3 + ], + "layers_arg": "band", + "skip_positions": 1, + "max_items_effective": null, + "min_pairs_floor": 30, + "evals": [ + "multihop", + "multilingual", + "order-ops", + "poetry", + "typo", + "association" + ], + "arms": [ + "logit", + "jlens", + "shuffled", + "rand_match" + ], + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "device": "cpu", + "n_forward_passes": 7695, + "results": { + "multihop": { + "n_items_scored": 93, + "n_intermediate_pairs_scored": 103, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.014876, + "jlens": 0.008686, + "shuffled": 0.004229, + "rand_match": 0.004407 + }, + "PRIMARY_ratio_jlens_over_logit": 0.5839, + "ratio_jlens_over_shuffled": 2.054, + "ratio_jlens_over_rand_match": 1.971, + "ratio_by_layer_jlens_over_logit": { + "2": 0.5183, + "3": 0.7044 + }, + "band_max_ratio_SECONDARY": 0.7044, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019273, + "dose_mean_per_position": 0.6468, + "dose_sum_over_positions": 16.2733 + }, + "3": { + "kl_mean": 0.010479, + "dose_mean_per_position": 0.6897, + "dose_sum_over_positions": 17.2888 + } + }, + "jlens": { + "2": { + "kl_mean": 0.009989, + "dose_mean_per_position": 0.5031, + "dose_sum_over_positions": 11.3007 + }, + "3": { + "kl_mean": 0.007382, + "dose_mean_per_position": 0.4797, + "dose_sum_over_positions": 10.4315 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.002671, + "dose_mean_per_position": 0.4261, + "dose_sum_over_positions": 9.8264 + }, + "3": { + "kl_mean": 0.005787, + "dose_mean_per_position": 0.741, + "dose_sum_over_positions": 16.9316 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.004382, + "dose_mean_per_position": 0.4441, + "dose_sum_over_positions": 9.8799 + }, + "3": { + "kl_mean": 0.004431, + "dose_mean_per_position": 0.5476, + "dose_sum_over_positions": 12.3821 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0143, + 0.4263, + 0.3879, + 0.1754, + 0.3134, + 0.3558, + 1.574, + 0.955, + 0.3888, + 1.4085, + 0.8029, + 1.2517, + 1.3781, + 1.7634, + 0.9378, + 0.959, + 1.1397, + 0.7515, + 0.1324 + ], + "3": [ + 0.0, + 0.1675, + 0.1087, + 0.4415, + 0.0303, + 0.1676, + 0.091, + 1.5784, + 0.6766, + 0.3601, + 1.4793, + 0.1837, + 1.0786, + 1.4508, + 1.5462, + 1.293, + 1.1929, + 1.6134, + 0.9143, + 0.1636 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.6582, + 0.0746, + 0.0077, + 0.2852, + 0.3706, + 0.6841, + 0.9593, + 1.0702, + 0.9137, + 1.3892, + 1.2489, + 0.2299, + 0.7641, + 1.0204, + 0.9719, + 0.655, + 0.7111, + 0.6593, + 0.5828 + ], + "3": [ + 0.0, + 0.8561, + 0.3905, + 0.4474, + 0.3875, + 0.1536, + 0.3955, + 0.4935, + 0.9084, + 0.9588, + 0.9519, + 0.4786, + 0.148, + 0.8679, + 1.3601, + 1.289, + 1.4184, + 1.4532, + 0.8618, + 1.0435 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.2253, + 0.4868, + 0.4528, + 0.1709, + 0.6235, + 0.155, + 0.5719, + 0.5789, + 0.4355, + 1.2506, + 0.8464, + 0.0036, + 0.2331, + 0.3242, + 0.3839, + 0.4826, + 0.3357, + 0.269, + 0.0191 + ], + "3": [ + 0.0, + 0.2219, + 0.4975, + 0.5461, + 0.3222, + 0.9782, + 0.8187, + 1.1391, + 1.0117, + 1.1803, + 1.6638, + 1.5222, + 1.0483, + 1.3899, + 1.2017, + 1.0969, + 1.1314, + 1.4697, + 0.8453, + 1.0504 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.8595, + 0.0865, + 0.4166, + 0.224, + 0.1023, + 0.3016, + 0.6538, + 0.1718, + 0.4688, + 0.7481, + 0.0053, + 0.5808, + 0.3543, + 0.3642, + 0.1474, + 1.1441, + 0.4846, + 0.2117, + 0.3653 + ], + "3": [ + 0.0, + 0.082, + 0.0965, + 0.1922, + 0.4986, + 0.051, + 0.5813, + 0.6251, + 0.6909, + 0.6936, + 0.7641, + 0.3953, + 0.4381, + 0.0576, + 1.0085, + 0.4477, + 0.1701, + 0.2303, + 0.437, + 0.2175 + ] + } + } + }, + "multilingual": { + "n_items_scored": 107, + "n_intermediate_pairs_scored": 394, + "n_intermediates_unscorable_multitoken": 34, + "band_mean_kl": { + "logit": 0.003209, + "jlens": 0.005583, + "shuffled": 0.003309, + "rand_match": 0.003698 + }, + "PRIMARY_ratio_jlens_over_logit": 1.7399, + "ratio_jlens_over_shuffled": 1.6869, + "ratio_jlens_over_rand_match": 1.5096, + "ratio_by_layer_jlens_over_logit": { + "2": 1.0146, + "3": 2.4817 + }, + "band_max_ratio_SECONDARY": 2.4817, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003244, + "dose_mean_per_position": 0.4899, + "dose_sum_over_positions": 5.5071 + }, + "3": { + "kl_mean": 0.003173, + "dose_mean_per_position": 0.523, + "dose_sum_over_positions": 5.8039 + } + }, + "jlens": { + "2": { + "kl_mean": 0.003292, + "dose_mean_per_position": 0.3477, + "dose_sum_over_positions": 4.0056 + }, + "3": { + "kl_mean": 0.007874, + "dose_mean_per_position": 0.4963, + "dose_sum_over_positions": 6.0704 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.002589, + "dose_mean_per_position": 0.4384, + "dose_sum_over_positions": 5.0195 + }, + "3": { + "kl_mean": 0.00403, + "dose_mean_per_position": 0.5853, + "dose_sum_over_positions": 6.6365 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.003771, + "dose_mean_per_position": 0.4726, + "dose_sum_over_positions": 5.4467 + }, + "3": { + "kl_mean": 0.003626, + "dose_mean_per_position": 0.5427, + "dose_sum_over_positions": 6.1871 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 1.3924, + 3.5021, + 0.0182, + 0.3026, + 0.7169, + 0.272, + 0.1566, + 0.032, + 0.2212, + 0.5358, + 0.3988 + ], + "3": [ + 0.0, + 1.1118, + 3.2379, + 0.5082, + 1.2603, + 0.095, + 0.2086, + 0.1731, + 0.2063, + 0.5958, + 0.0088, + 0.2401 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.3226, + 1.7273, + 0.4251, + 0.4436, + 0.0923, + 0.0003, + 0.1942, + 0.1912, + 0.1917, + 0.137, + 0.1575 + ], + "3": [ + 0.0, + 0.9097, + 0.2038, + 0.3397, + 0.7697, + 0.1493, + 0.1888, + 0.78, + 0.1505, + 0.1204, + 0.1687, + 0.5462 + ] + }, + "shuffled": { + "2": [ + 0.0, + 1.4098, + 3.4299, + 0.2508, + 0.969, + 0.3988, + 0.5157, + 0.0471, + 0.1392, + 0.2924, + 0.1388, + 0.1448 + ], + "3": [ + 0.0, + 1.2841, + 3.0637, + 0.4233, + 1.0404, + 0.7501, + 0.6794, + 0.4404, + 0.5862, + 0.4999, + 0.461, + 0.0152 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5647, + 1.365, + 0.2132, + 0.0545, + 0.2211, + 0.1914, + 0.1958, + 0.1866, + 0.4777, + 0.3089, + 0.0994 + ], + "3": [ + 0.0, + 3.2869, + 4.0832, + 0.5296, + 1.0719, + 0.795, + 0.853, + 0.9589, + 0.6516, + 1.0234, + 1.1701, + 1.124 + ] + } + } + }, + "order-ops": { + "n_items_scored": 55, + "n_intermediate_pairs_scored": 110, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.002976, + "jlens": 0.004701, + "shuffled": 0.000745, + "rand_match": 0.002122 + }, + "PRIMARY_ratio_jlens_over_logit": 1.5798, + "ratio_jlens_over_shuffled": 6.3066, + "ratio_jlens_over_rand_match": 2.2157, + "ratio_by_layer_jlens_over_logit": { + "2": 1.5728, + "3": 1.59 + }, + "band_max_ratio_SECONDARY": 1.59, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.003541, + "dose_mean_per_position": 0.3877, + "dose_sum_over_positions": 3.5915 + }, + "3": { + "kl_mean": 0.002411, + "dose_mean_per_position": 0.4374, + "dose_sum_over_positions": 4.0395 + } + }, + "jlens": { + "2": { + "kl_mean": 0.005569, + "dose_mean_per_position": 0.4819, + "dose_sum_over_positions": 4.5459 + }, + "3": { + "kl_mean": 0.003834, + "dose_mean_per_position": 0.5463, + "dose_sum_over_positions": 5.1639 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.000689, + "dose_mean_per_position": 0.2659, + "dose_sum_over_positions": 2.4532 + }, + "3": { + "kl_mean": 0.000802, + "dose_mean_per_position": 0.3342, + "dose_sum_over_positions": 3.1046 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.002213, + "dose_mean_per_position": 0.4331, + "dose_sum_over_positions": 4.0239 + }, + "3": { + "kl_mean": 0.002031, + "dose_mean_per_position": 0.5105, + "dose_sum_over_positions": 4.6388 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.2968, + 0.0493, + 0.3018, + 0.3017, + 0.2819, + 0.188, + 0.0994, + 0.2883 + ], + "3": [ + 0.0, + 0.5021, + 0.0337, + 0.0088, + 0.1413, + 0.5405, + 0.094, + 0.3283, + 0.4194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.6207, + 0.4901, + 0.5225, + 0.4222, + 0.4963, + 1.0822, + 0.4581, + 0.5865 + ], + "3": [ + 0.0, + 1.0701, + 0.8578, + 0.3044, + 0.5196, + 0.52, + 1.0648, + 0.6179, + 0.7518 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.009, + 0.0525, + 0.0318, + 0.1488, + 0.2331, + 0.2281, + 0.0698, + 0.2284 + ], + "3": [ + 0.0, + 0.0159, + 0.1166, + 0.2434, + 0.0348, + 0.1753, + 0.101, + 0.5352, + 0.1461 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.0945, + 0.2806, + 0.0689, + 0.3952, + 0.1473, + 0.2637, + 0.2711, + 0.2763 + ], + "3": [ + 0.0, + 0.4252, + 0.0526, + 0.3179, + 0.3876, + 0.5907, + 0.3064, + 0.4023, + 0.0867 + ] + } + } + }, + "poetry": { + "n_items_scored": 98, + "n_intermediate_pairs_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.011893, + "jlens": 0.0069, + "shuffled": 0.003331, + "rand_match": 0.00537 + }, + "PRIMARY_ratio_jlens_over_logit": 0.5802, + "ratio_jlens_over_shuffled": 2.0715, + "ratio_jlens_over_rand_match": 1.285, + "ratio_by_layer_jlens_over_logit": { + "2": 0.8662, + "3": 0.2922 + }, + "band_max_ratio_SECONDARY": 0.8662, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.011934, + "dose_mean_per_position": 0.941, + "dose_sum_over_positions": 25.7454 + }, + "3": { + "kl_mean": 0.011851, + "dose_mean_per_position": 1.1137, + "dose_sum_over_positions": 30.4835 + } + }, + "jlens": { + "2": { + "kl_mean": 0.010338, + "dose_mean_per_position": 0.7029, + "dose_sum_over_positions": 19.2247 + }, + "3": { + "kl_mean": 0.003463, + "dose_mean_per_position": 0.6808, + "dose_sum_over_positions": 18.639 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.004287, + "dose_mean_per_position": 0.6004, + "dose_sum_over_positions": 16.4011 + }, + "3": { + "kl_mean": 0.002375, + "dose_mean_per_position": 0.9737, + "dose_sum_over_positions": 26.629 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.006514, + "dose_mean_per_position": 0.5916, + "dose_sum_over_positions": 16.1283 + }, + "3": { + "kl_mean": 0.004226, + "dose_mean_per_position": 0.6559, + "dose_sum_over_positions": 17.9902 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.1594, + 0.5944, + 0.2455, + 0.5992, + 0.2572, + 0.9429, + 0.2139, + 11.1563, + 0.0825, + 0.2446, + 0.1937, + 0.292, + 0.0767, + 0.7736, + 0.4319, + 0.6853, + 0.9142, + 0.684, + 0.1378, + 0.081, + 0.7477, + 0.352, + 1.0318, + 0.4303, + 0.3018, + 0.195, + 0.7437, + 0.2196 + ], + "3": [ + 0.0, + 0.3802, + 1.2164, + 1.0455, + 0.1044, + 0.4311, + 0.8451, + 0.2006, + 11.3244, + 0.1434, + 0.5504, + 0.3074, + 0.2068, + 0.2864, + 1.1519, + 0.7277, + 0.8769, + 1.4379, + 1.5382, + 0.4364, + 0.248, + 0.3947, + 0.1714, + 1.1808, + 0.2771, + 0.5054, + 0.6367, + 1.1076, + 0.0233 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.5232, + 1.0828, + 0.8704, + 0.3034, + 0.897, + 0.0041, + 0.2248, + 6.48, + 0.3582, + 0.1645, + 0.1157, + 0.4922, + 0.2448, + 0.3903, + 0.2939, + 0.2943, + 0.1232, + 0.5895, + 0.0879, + 0.1795, + 0.4567, + 0.6953, + 0.1521, + 0.7204, + 0.4913, + 1.0272, + 0.7017, + 0.9337 + ], + "3": [ + 0.0, + 0.9131, + 1.9302, + 1.5304, + 0.1688, + 0.4072, + 0.4286, + 0.0711, + 3.7635, + 0.1654, + 0.181, + 0.1108, + 0.5179, + 0.5478, + 0.5932, + 0.2815, + 0.7199, + 0.6105, + 0.7896, + 0.2288, + 0.2128, + 0.5225, + 0.6035, + 0.6911, + 0.7133, + 0.8136, + 1.1916, + 0.8024, + 0.7102 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.0078, + 0.5977, + 0.1532, + 0.4569, + 0.8812, + 0.2626, + 0.5231, + 7.5545, + 0.5123, + 0.0304, + 0.1152, + 0.2398, + 0.2044, + 0.4046, + 0.0219, + 0.2345, + 0.3873, + 0.1053, + 0.0919, + 0.1387, + 0.2313, + 0.412, + 0.342, + 0.635, + 0.3461, + 0.3783, + 0.3592, + 0.1797 + ], + "3": [ + 0.0, + 0.4078, + 0.6218, + 0.5292, + 0.6402, + 1.3448, + 0.4804, + 0.8425, + 8.1509, + 1.1975, + 0.3938, + 0.3309, + 0.5679, + 0.483, + 0.6788, + 0.4012, + 0.2808, + 0.5529, + 0.65, + 0.0141, + 0.5379, + 0.883, + 0.8933, + 0.9444, + 1.2075, + 1.2923, + 1.3032, + 1.2736, + 1.118 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5663, + 0.0349, + 0.3271, + 0.0591, + 0.224, + 0.5588, + 0.0436, + 4.7702, + 0.2856, + 0.5625, + 0.8871, + 0.5497, + 1.1638, + 0.4989, + 0.2201, + 0.4187, + 0.339, + 0.6105, + 0.1274, + 0.2521, + 0.149, + 0.1882, + 0.2987, + 0.2137, + 0.1715, + 0.3117, + 0.0553, + 0.1175 + ], + "3": [ + 0.0, + 0.1785, + 0.1694, + 0.2338, + 0.1877, + 0.4362, + 0.2648, + 0.0485, + 5.4696, + 0.0288, + 0.8456, + 0.4643, + 0.385, + 0.61, + 0.0647, + 0.5367, + 0.7664, + 0.5221, + 0.2946, + 0.5209, + 0.0216, + 0.6684, + 0.4557, + 0.4048, + 0.2026, + 0.1609, + 0.202, + 0.3115, + 0.2607 + ] + } + } + }, + "typo": { + "n_items_scored": 96, + "n_intermediate_pairs_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "band_mean_kl": { + "logit": 0.019955, + "jlens": 0.018257, + "shuffled": 0.010415, + "rand_match": 0.009576 + }, + "PRIMARY_ratio_jlens_over_logit": 0.9149, + "ratio_jlens_over_shuffled": 1.753, + "ratio_jlens_over_rand_match": 1.9067, + "ratio_by_layer_jlens_over_logit": { + "2": 0.9734, + "3": 0.8566 + }, + "band_max_ratio_SECONDARY": 0.9734, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.019932, + "dose_mean_per_position": 0.486, + "dose_sum_over_positions": 4.7988 + }, + "3": { + "kl_mean": 0.019977, + "dose_mean_per_position": 0.566, + "dose_sum_over_positions": 5.5739 + } + }, + "jlens": { + "2": { + "kl_mean": 0.019402, + "dose_mean_per_position": 0.4049, + "dose_sum_over_positions": 3.9896 + }, + "3": { + "kl_mean": 0.017113, + "dose_mean_per_position": 0.4574, + "dose_sum_over_positions": 4.5191 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.005628, + "dose_mean_per_position": 0.3469, + "dose_sum_over_positions": 3.3905 + }, + "3": { + "kl_mean": 0.015201, + "dose_mean_per_position": 0.7493, + "dose_sum_over_positions": 7.3432 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.011462, + "dose_mean_per_position": 0.3907, + "dose_sum_over_positions": 3.742 + }, + "3": { + "kl_mean": 0.007689, + "dose_mean_per_position": 0.4858, + "dose_sum_over_positions": 4.6135 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.0693, + 0.62, + 0.4626, + 0.3928, + 0.1845, + 1.245, + 1.0045, + 0.4486, + 0.5024, + 1.0145, + 0.4979 + ], + "3": [ + 0.0, + 0.2168, + 0.838, + 0.2187, + 0.25, + 0.1403, + 0.75, + 0.7185, + 0.1199, + 0.3759, + 0.8352, + 0.6194 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.3415, + 0.1907, + 0.4512, + 0.0095, + 0.4673, + 0.4062, + 0.1638, + 0.2159, + 0.069, + 0.0905, + 0.5906 + ], + "3": [ + 0.0, + 0.2507, + 0.2726, + 0.2922, + 0.2146, + 0.1179, + 0.2139, + 0.0425, + 0.127, + 0.0711, + 0.2289, + 0.3851 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.3901, + 0.3789, + 1.0118, + 0.0418, + 0.3103, + 0.3189, + 0.0585, + 0.0384, + 0.2086, + 0.304, + 0.4321 + ], + "3": [ + 0.0, + 0.9631, + 0.7146, + 0.1764, + 0.3445, + 1.0474, + 0.7126, + 0.2163, + 0.8303, + 0.4904, + 0.7628, + 0.3206 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.4129, + 0.2709, + 0.1557, + 0.1544, + 0.5417, + 0.0378, + 0.2531, + 0.5354, + 0.2084, + 0.4157, + 0.5005 + ], + "3": [ + 0.0, + 0.5479, + 1.0458, + 0.7085, + 1.2307, + 0.8921, + 1.3775, + 0.9829, + 0.4017, + 0.3321, + 0.5386, + 0.2979 + ] + } + } + }, + "association": { + "n_items_scored": 102, + "n_intermediate_pairs_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "band_mean_kl": { + "logit": 0.761053, + "jlens": 0.404057, + "shuffled": 0.462795, + "rand_match": 0.331446 + }, + "PRIMARY_ratio_jlens_over_logit": 0.5309, + "ratio_jlens_over_shuffled": 0.8731, + "ratio_jlens_over_rand_match": 1.2191, + "ratio_by_layer_jlens_over_logit": { + "2": 0.8266, + "3": 0.2395 + }, + "band_max_ratio_SECONDARY": 0.8266, + "per_layer": { + "logit": { + "2": { + "kl_mean": 0.755566, + "dose_mean_per_position": 1.0786, + "dose_sum_over_positions": 31.3456 + }, + "3": { + "kl_mean": 0.766541, + "dose_mean_per_position": 1.1292, + "dose_sum_over_positions": 32.8565 + } + }, + "jlens": { + "2": { + "kl_mean": 0.624546, + "dose_mean_per_position": 0.71, + "dose_sum_over_positions": 20.6605 + }, + "3": { + "kl_mean": 0.183568, + "dose_mean_per_position": 0.6139, + "dose_sum_over_positions": 17.9753 + } + }, + "shuffled": { + "2": { + "kl_mean": 0.393159, + "dose_mean_per_position": 0.5761, + "dose_sum_over_positions": 16.6468 + }, + "3": { + "kl_mean": 0.532431, + "dose_mean_per_position": 1.0258, + "dose_sum_over_positions": 29.7293 + } + }, + "rand_match": { + "2": { + "kl_mean": 0.367696, + "dose_mean_per_position": 0.6371, + "dose_sum_over_positions": 18.6053 + }, + "3": { + "kl_mean": 0.295195, + "dose_mean_per_position": 0.7068, + "dose_sum_over_positions": 20.5207 + } + } + }, + "dose_per_position_first_pair": { + "logit": { + "2": [ + 0.0, + 0.261, + 0.7127, + 0.4662, + 0.3029, + 0.3495, + 0.053, + 0.077, + 0.9719, + 0.7852, + 0.6237, + 1.3751, + 0.6093, + 2.0121, + 0.6116, + 1.0531, + 1.7732, + 1.1089, + 0.6423, + 0.3501, + 0.7945, + 1.2471, + 1.3578, + 1.1713, + 1.1945, + 0.3451, + 0.3122, + 1.3023, + 12.4952 + ], + "3": [ + 0.0, + 0.4806, + 0.662, + 0.7729, + 0.4281, + 0.7686, + 0.5767, + 0.175, + 1.3067, + 1.3245, + 0.9131, + 1.5172, + 0.5693, + 1.5528, + 0.9205, + 0.2771, + 1.0986, + 0.5817, + 1.0954, + 0.5864, + 0.5879, + 0.9137, + 1.0181, + 0.8501, + 2.2945, + 0.7854, + 0.5198, + 1.2149, + 12.637 + ] + }, + "jlens": { + "2": [ + 0.0, + 0.41, + 0.9062, + 0.8852, + 0.2574, + 0.2678, + 0.5199, + 0.4107, + 0.4827, + 0.8476, + 0.0157, + 0.6066, + 0.6578, + 0.4655, + 0.7064, + 0.1958, + 0.7013, + 0.6126, + 1.1319, + 0.9383, + 0.4477, + 0.7227, + 0.8952, + 0.7045, + 0.757, + 0.8819, + 0.1927, + 0.9131, + 7.5798 + ], + "3": [ + 0.0, + 0.3809, + 0.6018, + 1.0735, + 0.2232, + 0.6388, + 0.8937, + 0.4471, + 0.6495, + 0.9372, + 0.4727, + 0.7284, + 0.481, + 0.5509, + 0.7011, + 0.3516, + 0.7043, + 0.3951, + 0.8504, + 1.0335, + 0.56, + 0.6048, + 0.6116, + 0.2914, + 1.5153, + 1.2448, + 0.6934, + 0.8503, + 3.433 + ] + }, + "shuffled": { + "2": [ + 0.0, + 0.2914, + 0.452, + 0.4997, + 0.2397, + 0.1058, + 0.9353, + 0.3255, + 0.4937, + 0.471, + 0.5076, + 0.5503, + 0.5488, + 0.0931, + 0.0518, + 0.1805, + 0.5031, + 0.6641, + 0.7724, + 0.4423, + 0.3042, + 0.3444, + 0.1121, + 0.2363, + 0.6173, + 0.0076, + 0.167, + 0.404, + 9.291 + ], + "3": [ + 0.0, + 1.0216, + 1.1059, + 1.1182, + 0.9986, + 0.6347, + 1.0365, + 1.1086, + 1.2365, + 0.9504, + 0.3517, + 0.7791, + 1.1422, + 0.7006, + 0.4317, + 0.6847, + 1.2544, + 1.0703, + 1.3194, + 0.84, + 0.9978, + 1.1323, + 0.4856, + 0.4727, + 0.8505, + 0.9799, + 0.6478, + 0.9878, + 8.973 + ] + }, + "rand_match": { + "2": [ + 0.0, + 0.5339, + 0.0311, + 0.0312, + 0.4897, + 0.5893, + 0.0718, + 0.1459, + 0.131, + 0.2004, + 0.8931, + 0.3031, + 0.3423, + 0.4818, + 0.5813, + 0.1457, + 0.071, + 0.0664, + 0.512, + 0.5542, + 0.1682, + 0.216, + 0.1395, + 0.2062, + 0.1179, + 0.0305, + 0.5685, + 0.2558, + 0.7061 + ], + "3": [ + 0.0, + 1.4369, + 0.589, + 0.4872, + 0.274, + 0.3277, + 0.2342, + 0.3975, + 0.6986, + 0.3046, + 0.1022, + 0.5765, + 0.6712, + 0.5935, + 0.3667, + 0.3788, + 0.7403, + 0.1303, + 0.1983, + 0.0263, + 0.2366, + 0.1407, + 0.0562, + 0.1564, + 0.1818, + 0.1215, + 0.3484, + 0.4221, + 7.1495 + ] + } + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "argv": [ + "t20_ablation_kl.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--lens", + "results/plad_70m_wikitext_A.pt", + "--evals", + "all", + "--out", + "results/t20_ablation_kl_70m_Pwikitext.json" + ], + "runtime_s": 185.8, + "provenance": { + "payload_sha256_before_redaction": "9df0cbe7aa4f60e9782e3824c90fc2d215f974a8fb309882cbee3decec3653b7", + "payload_sha256": "c4a2df98f3f61e0e896d5c3ea7a050bead9d0d0a719cd4f2ca1283cae2e046a8", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t21_conditioning_160m.json b/results/t21_conditioning_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..f911b1000b819c0ce5efae48bda2e0c54e79156e --- /dev/null +++ b/results/t21_conditioning_160m.json @@ -0,0 +1,116 @@ +{ + "experiment": "T21 / E31 \u2014 swap-frame conditioning across lenses", + "status": "EXPLORATORY", + "prereg": "in-file header; decision rule fixed before the first number", + "decision_rule": "delta_med = median|rho|(jlens) - median|rho|(logit) on SEMANTIC pairs; <=-0.05 confound confirmed / |.|<0.05 absent / >=+0.05 inverted", + "primary_outcome": "delta_med_semantic", + "VERDICT": "CONFOUND ABSENT \u2014 median|rho| differs by only +0.0144; the swap benchmark is fair on this axis", + "delta_med_semantic": 0.01444154977798462, + "delta_med_random_C1": 0.013673603534698486, + "delta_med_shuffled_C2": 0.012129127979278564, + "C1_interpretation": "if delta_med_random ~ delta_med_semantic the operator is GLOBALLY whitening and the effect is not about concepts", + "C2_interpretation": "if delta_med_shuffled ~ delta_med_semantic, conditioning is GENERIC to any fitted Jacobian, not a property of layer identity", + "model": "EleutherAI/pythia-160m-deduped", + "lens": "results/lens_160m_n200_db128_pen.pt", + "band": [ + 5, + 6, + 7, + 8, + 9 + ], + "n_layers": 10, + "n_semantic_pairs": 300, + "n_random_pairs": 300, + "seed": 0, + "derangement_layer_map": { + "0": 2, + "1": 9, + "2": 1, + "3": 7, + "4": 6, + "5": 8, + "6": 4, + "7": 5, + "8": 0, + "9": 3 + }, + "anisotropy": { + "raw_mean_cos": 0.895441472530365, + "mean_removed_cos_DIAGNOSTIC_ONLY": 0.0023443244863301516, + "isotropic_control_mean_cos": -3.5552511690184474e-05, + "common_component_frac_of_row_norm": 0.9534543752670288, + "d_model": 768, + "vocab": 50304, + "note": "W_U is ANISOTROPIC: a single common direction carries most of every row's norm, so ALL lenses inherit near-collinear concept pairs. The mean-removed figure is a DIAGNOSTIC and is never applied to a lens vector or a write (repo discipline 1: no centering, ever)." + }, + "results": { + "semantic": { + "logit": { + "median_abs_rho": 0.9375708699226379, + "mean_abs_rho": 0.9376603364944458, + "median_kappa_V": 5.571025848388672, + "frac_rho_above_0.9": 0.95, + "median_norm_ratio": 1.0, + "n": 1500 + }, + "jlens": { + "median_abs_rho": 0.9520124197006226, + "mean_abs_rho": 0.9484120011329651, + "median_kappa_V": 6.377887725830078, + "frac_rho_above_0.9": 0.9633333333333334, + "median_norm_ratio": 1.1543515920639038, + "n": 1500 + }, + "shuffled": { + "median_abs_rho": 0.9496999979019165, + "mean_abs_rho": 0.9451411366462708, + "median_kappa_V": 6.225867748260498, + "frac_rho_above_0.9": 0.9606666666666667, + "median_norm_ratio": 1.010024070739746, + "n": 1500 + } + }, + "random": { + "logit": { + "median_abs_rho": 0.9174813628196716, + "mean_abs_rho": 0.8926994800567627, + "median_kappa_V": 4.82047176361084, + "frac_rho_above_0.9": 0.8366666666666667, + "median_norm_ratio": 1.0, + "n": 1500 + }, + "jlens": { + "median_abs_rho": 0.9311549663543701, + "mean_abs_rho": 0.8978090882301331, + "median_kappa_V": 5.296296119689941, + "frac_rho_above_0.9": 0.8846666666666667, + "median_norm_ratio": 1.1285361051559448, + "n": 1500 + }, + "shuffled": { + "median_abs_rho": 0.9278445243835449, + "mean_abs_rho": 0.8948506712913513, + "median_kappa_V": 5.168938159942627, + "frac_rho_above_0.9": 0.8573333333333333, + "median_norm_ratio": 0.9971628785133362, + "n": 1500 + } + } + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t21_conditioning.py", + "--model", + "EleutherAI/pythia-160m-deduped", + "--lens", + "results/lens_160m_n200_db128_pen.pt", + "--max-pairs", + "300", + "--out", + "results/t21_conditioning_160m.json" + ], + "runtime_s": 10.3 +} \ No newline at end of file diff --git a/results/t21_conditioning_410m.json b/results/t21_conditioning_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..ee8a07f950b991d4f06eeab1d550831d6d3faf0e --- /dev/null +++ b/results/t21_conditioning_410m.json @@ -0,0 +1,136 @@ +{ + "experiment": "T21 / E31 \u2014 swap-frame conditioning across lenses", + "status": "EXPLORATORY", + "prereg": "in-file header; decision rule fixed before the first number", + "decision_rule": "delta_med = median|rho|(jlens) - median|rho|(logit) on SEMANTIC pairs; <=-0.05 confound confirmed / |.|<0.05 absent / >=+0.05 inverted", + "primary_outcome": "delta_med_semantic", + "VERDICT": "CONFOUND INVERTED \u2014 median|rho| is +0.0832 HIGHER for the J-lens; any swap advantage is achieved despite a geometric handicap", + "delta_med_semantic": 0.08318433910608292, + "delta_med_random_C1": 0.004343532025814056, + "delta_med_shuffled_C2": 0.0935654416680336, + "C1_interpretation": "if delta_med_random ~ delta_med_semantic the operator is GLOBALLY whitening and the effect is not about concepts", + "C2_interpretation": "if delta_med_shuffled ~ delta_med_semantic, conditioning is GENERIC to any fitted Jacobian, not a property of layer identity", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/lens_410m_n200_db128_pen.pt", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_layers": 22, + "n_semantic_pairs": 300, + "n_random_pairs": 300, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "anisotropy": { + "raw_mean_cos": 0.07338223606348038, + "mean_removed_cos_DIAGNOSTIC_ONLY": 4.713865200756118e-05, + "isotropic_control_mean_cos": 2.2777026970288716e-05, + "common_component_frac_of_row_norm": 0.285045325756073, + "d_model": 1024, + "vocab": 50304, + "note": "W_U is ANISOTROPIC: a single common direction carries most of every row's norm, so ALL lenses inherit near-collinear concept pairs. The mean-removed figure is a DIAGNOSTIC and is never applied to a lens vector or a write (repo discipline 1: no centering, ever)." + }, + "results": { + "semantic": { + "logit": { + "median_abs_rho": 0.1024174615740776, + "mean_abs_rho": 0.21441972255706787, + "median_kappa_V": 1.1082451343536377, + "frac_rho_above_0.9": 0.0, + "median_norm_ratio": 1.0, + "n": 3900 + }, + "jlens": { + "median_abs_rho": 0.18560180068016052, + "mean_abs_rho": 0.2865291237831116, + "median_kappa_V": 1.2065658569335938, + "frac_rho_above_0.9": 0.005897435897435898, + "median_norm_ratio": 0.9134708046913147, + "n": 3900 + }, + "shuffled": { + "median_abs_rho": 0.1959829032421112, + "mean_abs_rho": 0.2969144284725189, + "median_kappa_V": 1.2196348905563354, + "frac_rho_above_0.9": 0.010769230769230769, + "median_norm_ratio": 0.6364051699638367, + "n": 3900 + } + }, + "random": { + "logit": { + "median_abs_rho": 0.07385557889938354, + "mean_abs_rho": 0.08059753477573395, + "median_kappa_V": 1.0767964124679565, + "frac_rho_above_0.9": 0.0, + "median_norm_ratio": 1.0, + "n": 3900 + }, + "jlens": { + "median_abs_rho": 0.0781991109251976, + "mean_abs_rho": 0.08387891203165054, + "median_kappa_V": 1.0815109014511108, + "frac_rho_above_0.9": 0.0, + "median_norm_ratio": 0.876052975654602, + "n": 3900 + }, + "shuffled": { + "median_abs_rho": 0.07930446416139603, + "mean_abs_rho": 0.08688609302043915, + "median_kappa_V": 1.082714557647705, + "frac_rho_above_0.9": 0.0, + "median_norm_ratio": 0.5923254489898682, + "n": 3900 + } + } + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t21_conditioning.py", + "--model", + "EleutherAI/pythia-410m-deduped", + "--lens", + "results/lens_410m_n200_db128_pen.pt", + "--max-pairs", + "300", + "--out", + "results/t21_conditioning_410m.json" + ], + "runtime_s": 12.1 +} \ No newline at end of file diff --git a/results/t21_conditioning_70m.json b/results/t21_conditioning_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..daf294c87dbd6af3104d9935a90deef840fa7acf --- /dev/null +++ b/results/t21_conditioning_70m.json @@ -0,0 +1,107 @@ +{ + "experiment": "T21 / E31 \u2014 swap-frame conditioning across lenses", + "status": "EXPLORATORY", + "prereg": "in-file header; decision rule fixed before the first number", + "decision_rule": "delta_med = median|rho|(jlens) - median|rho|(logit) on SEMANTIC pairs; <=-0.05 confound confirmed / |.|<0.05 absent / >=+0.05 inverted", + "primary_outcome": "delta_med_semantic", + "VERDICT": "CONFOUND ABSENT \u2014 median|rho| differs by only +0.0046; the swap benchmark is fair on this axis", + "delta_med_semantic": 0.004590392112731934, + "delta_med_random_C1": 0.005445301532745361, + "delta_med_shuffled_C2": 0.0008323192596435547, + "C1_interpretation": "if delta_med_random ~ delta_med_semantic the operator is GLOBALLY whitening and the effect is not about concepts", + "C2_interpretation": "if delta_med_shuffled ~ delta_med_semantic, conditioning is GENERIC to any fitted Jacobian, not a property of layer identity", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "results/lens_70m_n200_db128_pen.pt", + "band": [ + 2, + 3 + ], + "n_layers": 4, + "n_semantic_pairs": 300, + "n_random_pairs": 300, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "anisotropy": { + "raw_mean_cos": 0.9522607326507568, + "mean_removed_cos_DIAGNOSTIC_ONLY": 0.0007652565254829824, + "isotropic_control_mean_cos": -1.3925437997386325e-05, + "common_component_frac_of_row_norm": 0.9769587516784668, + "d_model": 512, + "vocab": 50304, + "note": "W_U is ANISOTROPIC: a single common direction carries most of every row's norm, so ALL lenses inherit near-collinear concept pairs. The mean-removed figure is a DIAGNOSTIC and is never applied to a lens vector or a write (repo discipline 1: no centering, ever)." + }, + "results": { + "semantic": { + "logit": { + "median_abs_rho": 0.9685631394386292, + "mean_abs_rho": 0.9710940718650818, + "median_kappa_V": 7.9132537841796875, + "frac_rho_above_0.9": 1.0, + "median_norm_ratio": 1.0, + "n": 600 + }, + "jlens": { + "median_abs_rho": 0.9731535315513611, + "mean_abs_rho": 0.9749072194099426, + "median_kappa_V": 8.573080062866211, + "frac_rho_above_0.9": 1.0, + "median_norm_ratio": 1.1095771789550781, + "n": 600 + }, + "shuffled": { + "median_abs_rho": 0.9693954586982727, + "mean_abs_rho": 0.9713549017906189, + "median_kappa_V": 8.021831512451172, + "frac_rho_above_0.9": 1.0, + "median_norm_ratio": 1.408603549003601, + "n": 600 + } + }, + "random": { + "logit": { + "median_abs_rho": 0.9556737542152405, + "mean_abs_rho": 0.9514764547348022, + "median_kappa_V": 6.6422882080078125, + "frac_rho_above_0.9": 0.99, + "median_norm_ratio": 1.0, + "n": 600 + }, + "jlens": { + "median_abs_rho": 0.9611190557479858, + "mean_abs_rho": 0.9567757248878479, + "median_kappa_V": 7.102047443389893, + "frac_rho_above_0.9": 0.99, + "median_norm_ratio": 1.1087945699691772, + "n": 600 + }, + "shuffled": { + "median_abs_rho": 0.9548503756523132, + "mean_abs_rho": 0.9496533274650574, + "median_kappa_V": 6.580057621002197, + "frac_rho_above_0.9": 0.9866666666666667, + "median_norm_ratio": 1.3966573476791382, + "n": 600 + } + } + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t21_conditioning.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--lens", + "results/lens_70m_n200_db128_pen.pt", + "--max-pairs", + "300", + "--out", + "results/t21_conditioning_70m.json" + ], + "runtime_s": 10.1 +} \ No newline at end of file diff --git a/results/t22_bootstrap_ci_160m.json b/results/t22_bootstrap_ci_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..88d37a2b26b5bd93ca2759541303f483ddb3a2df --- /dev/null +++ b/results/t22_bootstrap_ci_160m.json @@ -0,0 +1,361 @@ +{ + "experiment": "T22 / E26 \u2014 confidence intervals on every read headline", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "per set: CI excludes 0 -> contribution survives; includes 0 -> not supported and n_win must be reported without it. Hierarchical CI including 0 while per-set CIs exclude it -> does not generalise.", + "VERDICT": "vs LOGIT: 3/6 sets survive their own CI, hierarchical EXCLUDES 0. vs SHUFFLED: 3/6 survive, hierarchical INCLUDES 0.", + "C1_self_test_all_sets_fired": true, + "model": "EleutherAI/pythia-160m-deduped", + "lens": "results/lens_160m_n200_db128_pen.pt", + "band": [ + 5, + 6, + 7, + 8, + 9 + ], + "n_boot": 10000, + "seed": 0, + "derangement_layer_map": { + "0": 2, + "1": 9, + "2": 1, + "3": 7, + "4": 6, + "5": 8, + "6": 4, + "7": 5, + "8": 0, + "9": 3 + }, + "per_set": { + "multihop": { + "n_pairs": 103, + "pooled_auc": { + "logit": 0.10058043897151947, + "jlens": 0.08168338984251022, + "shuffled": 0.03334992751479149, + "scaled_id": 0.10058043897151947 + }, + "jlens_minus_logit": { + "point": -0.0188970398157835, + "ci_lo": -0.04778803512454033, + "ci_hi": 0.007096334360539913, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.16359999775886536, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.04833347350358963, + "ci_lo": 0.02309575118124485, + "ci_hi": 0.07651881128549576, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 103, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": true + }, + "multilingual": { + "n_pairs": 394, + "pooled_auc": { + "logit": 0.036329708993434906, + "jlens": 0.022875316441059113, + "shuffled": 0.024403387680649757, + "scaled_id": 0.036329708993434906 + }, + "jlens_minus_logit": { + "point": -0.013454391621053219, + "ci_lo": -0.025701401755213737, + "ci_hi": -0.0017189139034599066, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.021199999377131462, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": -0.0015280712395906448, + "ci_lo": -0.009660355746746063, + "ci_hi": 0.007225826382637024, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.704200029373169, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 394, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": false + }, + "order-ops": { + "n_pairs": 110, + "pooled_auc": { + "logit": 0.37858468294143677, + "jlens": 0.3255011737346649, + "shuffled": 0.2320522665977478, + "scaled_id": 0.37858468294143677 + }, + "jlens_minus_logit": { + "point": -0.053083501756191254, + "ci_lo": -0.09719312936067581, + "ci_hi": -0.009217520244419575, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.01860000006854534, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.09344888478517532, + "ci_lo": 0.04141981899738312, + "ci_hi": 0.14481781423091888, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0006000000284984708, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 110, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "poetry": { + "n_pairs": 98, + "pooled_auc": { + "logit": 0.035246122628450394, + "jlens": 0.0, + "shuffled": 0.03242160752415657, + "scaled_id": 0.035246122628450394 + }, + "jlens_minus_logit": { + "point": -0.035246122628450394, + "ci_lo": -0.06613186001777649, + "ci_hi": -0.012013443745672703, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": -0.03242160752415657, + "ci_lo": -0.06125076487660408, + "ci_hi": -0.00918892864137888, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.00139999995008111, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "typo": { + "n_pairs": 96, + "pooled_auc": { + "logit": 0.17853295803070068, + "jlens": 0.17481200397014618, + "shuffled": 0.1714775562286377, + "scaled_id": 0.17853295803070068 + }, + "jlens_minus_logit": { + "point": -0.0037209372967481613, + "ci_lo": -0.05135304853320122, + "ci_hi": 0.04575822874903679, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.8604000210762024, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.0033344507683068514, + "ci_lo": -0.03259299322962761, + "ci_hi": 0.03979278355836868, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.8917999863624573, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 96, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + }, + "association": { + "n_pairs": 92, + "pooled_auc": { + "logit": 0.0, + "jlens": 0.0, + "shuffled": 0.0016360327135771513, + "scaled_id": 0.0 + }, + "jlens_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": -0.0016360327135771513, + "ci_lo": -0.00490809790790081, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.7107999920845032, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + } + }, + "hierarchical_jlens_minus_logit": { + "point": -0.0207336638122797, + "ci_lo": -0.04180724173784256, + "ci_hi": -0.0023532172199338675, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "hierarchical_jlens_minus_shuffled": { + "point": 0.018255183473229408, + "ci_lo": -0.013131593354046345, + "ci_hi": 0.05649133026599884, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "nwin_vs_logit": { + "n_win": 0, + "n_sets": 6, + "ci_lo": 0.0, + "ci_hi": 2.0, + "win_rate": 0.0, + "win_rate_ci": [ + 0.0, + 0.3333333333333333 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "nwin_vs_shuffled": { + "n_win": 3, + "n_sets": 6, + "ci_lo": 1.0, + "ci_hi": 5.0, + "win_rate": 0.5, + "win_rate_ci": [ + 0.16666666666666666, + 0.8333333333333334 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "C2_width_ratio": 1.8897781858627603, + "C2_detail": { + "width_item": 0.02019879687577486, + "width_hierarchical": 0.0381712457165122, + "verdict": "the item interval UNDERSTATES uncertainty by 1.9x" + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t22_bootstrap_ci.py", + "--model", + "EleutherAI/pythia-160m-deduped", + "--lens", + "results/lens_160m_n200_db128_pen.pt", + "--evals", + "all", + "--n-boot", + "10000", + "--out", + "results/t22_bootstrap_ci_160m.json" + ], + "runtime_s": 80.2, + "provenance": { + "payload_sha256_before_redaction": "2df8ff4b2c93baf5a1e1c0aea2cef34dee6b92e2623a7750f8fc85e4dc06580d", + "payload_sha256": "bd439d27e9cb8af1e1a1092ae68748f0ff41b0286857f34f621e52968a53f4f4", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t22_bootstrap_ci_410m.json b/results/t22_bootstrap_ci_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..128a1c67c300f2c3b21b8e54862987e2165ad7fa --- /dev/null +++ b/results/t22_bootstrap_ci_410m.json @@ -0,0 +1,381 @@ +{ + "experiment": "T22 / E26 \u2014 confidence intervals on every read headline", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "per set: CI excludes 0 -> contribution survives; includes 0 -> not supported and n_win must be reported without it. Hierarchical CI including 0 while per-set CIs exclude it -> does not generalise.", + "VERDICT": "vs LOGIT: 3/6 sets survive their own CI, hierarchical EXCLUDES 0. vs SHUFFLED: 4/6 survive, hierarchical INCLUDES 0.", + "C1_self_test_all_sets_fired": true, + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/lens_410m_n200_db128_pen.pt", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_boot": 10000, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "per_set": { + "multihop": { + "n_pairs": 103, + "pooled_auc": { + "logit": 0.18678297102451324, + "jlens": 0.28686800599098206, + "shuffled": 0.21214570105075836, + "scaled_id": 0.18678297102451324 + }, + "jlens_minus_logit": { + "point": 0.10008499026298523, + "ci_lo": 0.06028417497873306, + "ci_hi": 0.14213250577449799, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.0747222751379013, + "ci_lo": 0.038314513862133026, + "ci_hi": 0.11118068546056747, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 103, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "multilingual": { + "n_pairs": 394, + "pooled_auc": { + "logit": 0.10726350545883179, + "jlens": 0.15270936489105225, + "shuffled": 0.08893052488565445, + "scaled_id": 0.10726350545883179 + }, + "jlens_minus_logit": { + "point": 0.045445848256349564, + "ci_lo": 0.02779579907655716, + "ci_hi": 0.06288545578718185, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.0637788400053978, + "ci_lo": 0.04645584896206856, + "ci_hi": 0.0818534567952156, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 394, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "order-ops": { + "n_pairs": 110, + "pooled_auc": { + "logit": 0.500210702419281, + "jlens": 0.5070522427558899, + "shuffled": 0.5747234225273132, + "scaled_id": 0.500210702419281 + }, + "jlens_minus_logit": { + "point": 0.006841591093689203, + "ci_lo": -0.025637339800596237, + "ci_hi": 0.03954077139496803, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.6642000079154968, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": -0.06767109036445618, + "ci_lo": -0.10606379061937332, + "ci_hi": -0.03113342635333538, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 110, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": true + }, + "poetry": { + "n_pairs": 98, + "pooled_auc": { + "logit": 0.019966276362538338, + "jlens": 0.01944556087255478, + "shuffled": 0.017909694463014603, + "scaled_id": 0.019966276362538338 + }, + "jlens_minus_logit": { + "point": -0.0005207143840380013, + "ci_lo": -0.012807652354240417, + "ci_hi": 0.011766224168241024, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.9305999875068665, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.001535867340862751, + "ci_lo": -0.010751072317361832, + "ci_hi": 0.013796530663967133, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.8406000137329102, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + }, + "typo": { + "n_pairs": 96, + "pooled_auc": { + "logit": 0.15853719413280487, + "jlens": 0.3433793783187866, + "shuffled": 0.29678353667259216, + "scaled_id": 0.15853719413280487 + }, + "jlens_minus_logit": { + "point": 0.18484216928482056, + "ci_lo": 0.13329042494297028, + "ci_hi": 0.24185535311698914, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.04659580811858177, + "ci_lo": 0.019013099372386932, + "ci_hi": 0.07342645525932312, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0007999999797903001, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 96, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "association": { + "n_pairs": 92, + "pooled_auc": { + "logit": 0.00762548903003335, + "jlens": 0.00490809790790081, + "shuffled": 0.010897553525865078, + "scaled_id": 0.00762548903003335 + }, + "jlens_minus_logit": { + "point": -0.0027173911221325397, + "ci_lo": -0.014696303755044937, + "ci_hi": 0.007098805159330368, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.7360000014305115, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": -0.005989456083625555, + "ci_lo": -0.015805652365088463, + "ci_hi": 0.0022047902457416058, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.23399999737739563, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + } + }, + "hierarchical_jlens_minus_logit": { + "point": 0.05566275119781494, + "ci_lo": 0.006518487352877855, + "ci_hi": 0.11540429294109344, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "hierarchical_jlens_minus_shuffled": { + "point": 0.018828706815838814, + "ci_lo": -0.023924075067043304, + "ci_hi": 0.05630238726735115, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "nwin_vs_logit": { + "n_win": 4, + "n_sets": 6, + "ci_lo": 2.0, + "ci_hi": 6.0, + "win_rate": 0.6666666666666666, + "win_rate_ci": [ + 0.3333333333333333, + 1.0 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "nwin_vs_shuffled": { + "n_win": 4, + "n_sets": 6, + "ci_lo": 1.0, + "ci_hi": 6.0, + "win_rate": 0.6666666666666666, + "win_rate_ci": [ + 0.16666666666666666, + 1.0 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "C2_width_ratio": 4.33581252376634, + "C2_detail": { + "width_item": 0.025383133441209793, + "width_hierarchical": 0.11005650786682963, + "verdict": "the item interval UNDERSTATES uncertainty by 4.3x" + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t22_bootstrap_ci.py", + "--model", + "EleutherAI/pythia-410m-deduped", + "--lens", + "results/lens_410m_n200_db128_pen.pt", + "--evals", + "all", + "--n-boot", + "10000", + "--out", + "results/t22_bootstrap_ci_410m.json" + ], + "runtime_s": 181.3, + "provenance": { + "payload_sha256_before_redaction": "260e36728215258ab34047ff6b4a312245b03c8990b3f733717626848a675090", + "payload_sha256": "bcb1dfcb3de566c33dcf1a5a1a20decf248001ad1b67ee32f93df0844e25e682", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t22_bootstrap_ci_persist_160m.json b/results/t22_bootstrap_ci_persist_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..f9435286d22ff89cb3a0239f7dd0c70eb729ad28 --- /dev/null +++ b/results/t22_bootstrap_ci_persist_160m.json @@ -0,0 +1,366 @@ +{ + "experiment": "T22 / E26 \u2014 confidence intervals on every read headline", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "per set: CI excludes 0 -> contribution survives; includes 0 -> not supported and n_win must be reported without it. Hierarchical CI including 0 while per-set CIs exclude it -> does not generalise.", + "VERDICT": "vs LOGIT: 3/6 sets survive their own CI, hierarchical INCLUDES 0. vs SHUFFLED: 2/6 survive, hierarchical EXCLUDES 0.", + "C1_self_test_all_sets_fired": true, + "aggregation": "persist", + "persist_kfrac": 0.5, + "aggregation_note": "min is EXISTENTIAL and sec F1 shows it launders the J-vs-shuffled difference; persist is the non-existential fix. Any J-vs-shuffled claim must be read off the persist run.", + "model": "EleutherAI/pythia-160m-deduped", + "lens": "results/lens_160m_n200_db128_pen.pt", + "band": [ + 5, + 6, + 7, + 8, + 9 + ], + "n_boot": 10000, + "seed": 0, + "derangement_layer_map": { + "0": 2, + "1": 9, + "2": 1, + "3": 7, + "4": 6, + "5": 8, + "6": 4, + "7": 5, + "8": 0, + "9": 3 + }, + "per_set": { + "multihop": { + "n_pairs": 103, + "pooled_auc": { + "logit": 0.033560145646333694, + "jlens": 0.028210338205099106, + "shuffled": 0.008272427134215832, + "scaled_id": 0.033560145646333694 + }, + "jlens_minus_logit": { + "point": -0.005349805112928152, + "ci_lo": -0.024061625823378563, + "ci_hi": 0.0119007034227252, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.546999990940094, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.019937913864850998, + "ci_lo": 0.006315679755061865, + "ci_hi": 0.034996457397937775, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0015999999595806003, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 103, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": true + }, + "multilingual": { + "n_pairs": 394, + "pooled_auc": { + "logit": 0.013412507250905037, + "jlens": 0.007054257672280073, + "shuffled": 0.006610749289393425, + "scaled_id": 0.013412507250905037 + }, + "jlens_minus_logit": { + "point": -0.0063582491129636765, + "ci_lo": -0.012580444104969501, + "ci_hi": -0.00025250017642974854, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0406000018119812, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.00044350879034027457, + "ci_lo": -0.002988115418702364, + "ci_hi": 0.004257150925695896, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.8519999980926514, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 394, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": false + }, + "order-ops": { + "n_pairs": 110, + "pooled_auc": { + "logit": 0.27044495940208435, + "jlens": 0.15993893146514893, + "shuffled": 0.061944592744112015, + "scaled_id": 0.27044495940208435 + }, + "jlens_minus_logit": { + "point": -0.11050605028867722, + "ci_lo": -0.15916547179222107, + "ci_hi": -0.06432052701711655, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.09799433499574661, + "ci_lo": 0.056156259030103683, + "ci_hi": 0.14415645599365234, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 110, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "poetry": { + "n_pairs": 98, + "pooled_auc": { + "logit": 0.014317245222628117, + "jlens": 0.0, + "shuffled": 0.003071734681725502, + "scaled_id": 0.014317245222628117 + }, + "jlens_minus_logit": { + "point": -0.014317245222628117, + "ci_lo": -0.028113773092627525, + "ci_hi": -0.003071734681725502, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.004999999888241291, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": -0.003071734681725502, + "ci_lo": -0.007679337169975042, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.27059999108314514, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": false + }, + "typo": { + "n_pairs": 96, + "pooled_auc": { + "logit": 0.10273291915655136, + "jlens": 0.13314534723758698, + "shuffled": 0.11648403853178024, + "scaled_id": 0.10273291915655136 + }, + "jlens_minus_logit": { + "point": 0.030412420630455017, + "ci_lo": -0.006829844322055578, + "ci_hi": 0.07261638343334198, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.11800000071525574, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.016661301255226135, + "ci_lo": -0.007056062575429678, + "ci_hi": 0.0416666679084301, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.17800000309944153, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 96, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + }, + "association": { + "n_pairs": 92, + "pooled_auc": { + "logit": 0.0, + "jlens": 0.0, + "shuffled": 0.0, + "scaled_id": 0.0 + }, + "jlens_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + } + }, + "hierarchical_jlens_minus_logit": { + "point": -0.01768648810684681, + "ci_lo": -0.06020132079720497, + "ci_hi": 0.012619917280972004, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "hierarchical_jlens_minus_shuffled": { + "point": 0.021994220092892647, + "ci_lo": 0.0002949030895251781, + "ci_hi": 0.05621275305747986, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "nwin_vs_logit": { + "n_win": 1, + "n_sets": 6, + "ci_lo": 0.0, + "ci_hi": 3.0, + "win_rate": 0.16666666666666666, + "win_rate_ci": [ + 0.0, + 0.5 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "nwin_vs_shuffled": { + "n_win": 4, + "n_sets": 6, + "ci_lo": 1.0, + "ci_hi": 6.0, + "win_rate": 0.6666666666666666, + "win_rate_ci": [ + 0.16666666666666666, + 1.0 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "C2_width_ratio": 4.062665148563666, + "C2_detail": { + "width_item": 0.017102944664657116, + "width_hierarchical": 0.06948353722691536, + "verdict": "the item interval UNDERSTATES uncertainty by 4.1x" + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t22_bootstrap_ci.py", + "--model", + "EleutherAI/pythia-160m-deduped", + "--lens", + "results/lens_160m_n200_db128_pen.pt", + "--evals", + "all", + "--n-boot", + "10000", + "--aggregation", + "persist", + "--out", + "results/t22_bootstrap_ci_persist_160m.json" + ], + "runtime_s": 80.1, + "provenance": { + "payload_sha256_before_redaction": "b2b39e545aa6d2240411712d98c833a7a169ecdef86c340634728aad3f56fd4c", + "payload_sha256": "51534ecebdc5b07a087b61160c5d699ea338d35d583386ed8824f1deb958f41d", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t22_bootstrap_ci_persist_410m.json b/results/t22_bootstrap_ci_persist_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..07279569a86ca981013bfb5d375535a814a94e11 --- /dev/null +++ b/results/t22_bootstrap_ci_persist_410m.json @@ -0,0 +1,386 @@ +{ + "experiment": "T22 / E26 \u2014 confidence intervals on every read headline", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "per set: CI excludes 0 -> contribution survives; includes 0 -> not supported and n_win must be reported without it. Hierarchical CI including 0 while per-set CIs exclude it -> does not generalise.", + "VERDICT": "vs LOGIT: 2/6 sets survive their own CI, hierarchical INCLUDES 0. vs SHUFFLED: 4/6 survive, hierarchical EXCLUDES 0.", + "C1_self_test_all_sets_fired": true, + "aggregation": "persist", + "persist_kfrac": 0.5, + "aggregation_note": "min is EXISTENTIAL and sec F1 shows it launders the J-vs-shuffled difference; persist is the non-existential fix. Any J-vs-shuffled claim must be read off the persist run.", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "results/lens_410m_n200_db128_pen.pt", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_boot": 10000, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "per_set": { + "multihop": { + "n_pairs": 103, + "pooled_auc": { + "logit": 0.04762781783938408, + "jlens": 0.04352910444140434, + "shuffled": 0.0029226215556263924, + "scaled_id": 0.04762781783938408 + }, + "jlens_minus_logit": { + "point": -0.004098713397979736, + "ci_lo": -0.02497749961912632, + "ci_hi": 0.016049416735768318, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.694599986076355, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.040606480091810226, + "ci_lo": 0.021894661709666252, + "ci_hi": 0.06221592053771019, + "n_items": 103, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 103, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": true + }, + "multilingual": { + "n_pairs": 394, + "pooled_auc": { + "logit": 0.021230800077319145, + "jlens": 0.0347466841340065, + "shuffled": 0.0038201778661459684, + "scaled_id": 0.021230800077319145 + }, + "jlens_minus_logit": { + "point": 0.013515881262719631, + "ci_lo": 0.0052656144835054874, + "ci_hi": 0.02251574397087097, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0010000000474974513, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.030926503241062164, + "ci_lo": 0.02007167600095272, + "ci_hi": 0.04335798695683479, + "n_items": 394, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 394, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "order-ops": { + "n_pairs": 110, + "pooled_auc": { + "logit": 0.29151344299316406, + "jlens": 0.30111509561538696, + "shuffled": 0.22755365073680878, + "scaled_id": 0.29151344299316406 + }, + "jlens_minus_logit": { + "point": 0.009601636789739132, + "ci_lo": -0.02339177392423153, + "ci_hi": 0.04139640927314758, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.5508000254631042, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.07356145232915878, + "ci_lo": 0.0425976999104023, + "ci_hi": 0.10515938699245453, + "n_items": 110, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 110, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": true + }, + "poetry": { + "n_pairs": 98, + "pooled_auc": { + "logit": 0.001535867340862751, + "jlens": 0.001535867340862751, + "shuffled": 0.0, + "scaled_id": 0.001535867340862751 + }, + "jlens_minus_logit": { + "point": 0.0, + "ci_lo": -0.004607602022588253, + "ci_hi": 0.004607602022588253, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 1.3037999868392944, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.001535867340862751, + "ci_lo": 0.0, + "ci_hi": 0.004607602022588253, + "n_items": 98, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.7383999824523926, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 98, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + }, + "typo": { + "n_pairs": 96, + "pooled_auc": { + "logit": 0.047487031668424606, + "jlens": 0.15641094744205475, + "shuffled": 0.05451560020446777, + "scaled_id": 0.047487031668424606 + }, + "jlens_minus_logit": { + "point": 0.10892391204833984, + "ci_lo": 0.06750960648059845, + "ci_hi": 0.15428973734378815, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.10189533233642578, + "ci_lo": 0.07193400710821152, + "ci_hi": 0.13549713790416718, + "n_items": 96, + "n_boot": 10000, + "excludes_zero": true, + "p_two_sided": 0.0, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 96, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": true, + "survives_vs_shuffled": true + }, + "association": { + "n_pairs": 92, + "pooled_auc": { + "logit": 0.0, + "jlens": 0.0016360327135771513, + "shuffled": 0.0, + "scaled_id": 0.0 + }, + "jlens_minus_logit": { + "point": 0.0016360327135771513, + "ci_lo": 0.0, + "ci_hi": 0.00490809790790081, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.7185999751091003, + "unit": "item (paired)" + }, + "jlens_minus_shuffled": { + "point": 0.0016360327135771513, + "ci_lo": 0.0, + "ci_hi": 0.00490809790790081, + "n_items": 92, + "n_boot": 10000, + "excludes_zero": false, + "p_two_sided": 0.7185999751091003, + "unit": "item (paired)" + }, + "C1_self_test_scaled_id_minus_logit": { + "point": 0.0, + "ci_lo": 0.0, + "ci_hi": 0.0, + "n_items": 92, + "n_boot": 500, + "excludes_zero": false, + "p_two_sided": 2.0, + "unit": "item (paired)" + }, + "C1_fires": true, + "survives_vs_logit": false, + "survives_vs_shuffled": false + } + }, + "hierarchical_jlens_minus_logit": { + "point": 0.021596459671854973, + "ci_lo": -0.002151099033653736, + "ci_hi": 0.05984563007950783, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": false, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "hierarchical_jlens_minus_shuffled": { + "point": 0.041693612933158875, + "ci_lo": 0.013362868689000607, + "ci_hi": 0.07319377362728119, + "n_sets": 6, + "n_boot": 10000, + "excludes_zero": true, + "unit": "eval set (hierarchical)", + "note": "the eval set is the replication unit; six shared-model sets are closer to one replicate than six (external review Prompt 8 \u00a75A)" + }, + "nwin_vs_logit": { + "n_win": 4, + "n_sets": 6, + "ci_lo": 2.0, + "ci_hi": 6.0, + "win_rate": 0.6666666666666666, + "win_rate_ci": [ + 0.3333333333333333, + 1.0 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "nwin_vs_shuffled": { + "n_win": 6, + "n_sets": 6, + "ci_lo": 3.0, + "ci_hi": 6.0, + "win_rate": 1.0, + "win_rate_ci": [ + 0.5, + 1.0 + ], + "n_boot": 2000, + "unit": "eval set", + "note": "n_win is a sign statistic over non-independent sets; the interval is wide by construction and that is the honest reading" + }, + "C2_width_ratio": 3.8656573036728514, + "C2_detail": { + "width_item": 0.015848972834646702, + "width_hierarchical": 0.061266697593964636, + "verdict": "the item interval UNDERSTATES uncertainty by 3.9x" + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t22_bootstrap_ci.py", + "--model", + "EleutherAI/pythia-410m-deduped", + "--lens", + "results/lens_410m_n200_db128_pen.pt", + "--evals", + "all", + "--n-boot", + "10000", + "--aggregation", + "persist", + "--out", + "results/t22_bootstrap_ci_persist_410m.json" + ], + "runtime_s": 183.6, + "provenance": { + "payload_sha256_before_redaction": "48c290e65a814867ce569edf3339a663970012e3b78f7df497cb73f2d485bfbd", + "payload_sha256": "00bb7043b615caf07f9e25ff770094c9e33d040907618b8e6d6ee1a95dcd0e0b", + "redaction": "identity strings redacted from PAYLOAD keys for double-blind review; see docs/reproducibility/PAYLOAD_REDACTIONS.md. The pre-redaction hash is retained so the original file remains verifiable." + } +} \ No newline at end of file diff --git a/results/t23_pq_ladder_410m.json b/results/t23_pq_ladder_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..d3c04eb501aa967d8c0bdb700e5c86f094b4b3f7 --- /dev/null +++ b/results/t23_pq_ladder_410m.json @@ -0,0 +1,292 @@ +{ + "experiment": "T23 / E27 \u2014 P-ladder: distributional bias vs the sampling floor", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "R = median over band layers of D_act(between)/D_act(within, matched n); >=3.0 variant / [1.5,3.0) weakly variant / <1.5 invariant", + "primary_outcome": "R", + "VERDICT": "WEAKLY VARIANT \u2014 R=2.60 in [1.5,3.0); bias present but comparable to noise, E30 needs a larger n and must be re-priced", + "R": 2.596775770187378, + "R_per_pair": { + "wikitext|pile": 2.575831890106201, + "wikitext|code": 3.570662260055542, + "pile|code": 1.3876919746398926 + }, + "C1_self_distance_max": 0.0, + "C1_fires": true, + "C2_shuffle_floor_median": { + "wikitext": 394.62603759765625, + "pile": 359.3499755859375, + "code": 302.6611022949219 + }, + "C2_between_over_shuffle": { + "wikitext|pile": 0.16278919928099533, + "wikitext|code": 0.30299700058451645, + "pile|code": 0.04913622175589071 + }, + "within_corpus": { + "wikitext": { + "d_act": [ + 9.64004898071289, + 11.268477439880371, + 15.097331047058105, + 17.590763092041016, + 28.41539764404297, + 31.29226303100586, + 47.09077453613281, + 61.59066390991211, + 66.41817474365234, + 66.95723724365234, + 60.47273635864258, + 50.6255989074707, + 26.789854049682617 + ], + "d_act_norm": [ + 0.07539705105836822, + 0.06689105297410691, + 0.06760837515557354, + 0.07115497281944176, + 0.08245505850260569, + 0.05754390607555659, + 0.05113338616322874, + 0.0488461434110272, + 0.04198827946332253, + 0.03286473371931605, + 0.026454990297379005, + 0.020924266587199255, + 0.010212903273881128 + ], + "median": 31.29226303100586 + }, + "pile": { + "d_act": [ + 3.0189573764801025, + 3.8523690700531006, + 4.2848801612854, + 5.327935218811035, + 7.745048999786377, + 8.907552719116211, + 10.888928413391113, + 12.99198055267334, + 17.51521110534668, + 18.54574966430664, + 15.71556568145752, + 12.56242847442627, + 6.597986221313477 + ], + "d_act_norm": [ + 0.03212091640217224, + 0.028020544353169443, + 0.024292263872375736, + 0.021940188543901104, + 0.023721534004386324, + 0.017509773420651038, + 0.013201016715171614, + 0.011725435947166392, + 0.01096327720972317, + 0.009090801350802147, + 0.0068848034561958885, + 0.0050206626414020855, + 0.0025093687199860422 + ], + "median": 8.907552719116211 + }, + "code": { + "d_act": [ + 4.175287246704102, + 4.580672264099121, + 5.0611348152160645, + 6.681285381317139, + 8.494763374328613, + 8.943028450012207, + 11.658864974975586, + 14.872122764587402, + 16.608333587646484, + 18.345401763916016, + 17.452695846557617, + 11.740030288696289, + 7.756032943725586 + ], + "d_act_norm": [ + 0.04513933137009834, + 0.033636496590735164, + 0.030327386435692494, + 0.026820668877094903, + 0.026687593718420968, + 0.018010400555962383, + 0.014315102918434144, + 0.013194139688692524, + 0.010012516338217082, + 0.008699241075878461, + 0.0072753676714169884, + 0.004384154850525145, + 0.002976797832321615 + ], + "median": 8.943028450012207 + } + }, + "between_corpus": { + "wikitext|pile": { + "d_act": [ + 21.81629753112793, + 24.211458206176758, + 32.53272247314453, + 32.33386993408203, + 43.60354995727539, + 44.788082122802734, + 74.67298126220703, + 88.00695037841797, + 87.84504699707031, + 97.76435852050781, + 94.22798919677734, + 82.04257202148438, + 49.26995849609375 + ], + "d_act_norm": [ + 0.17063030511047925, + 0.14372216140914051, + 0.145686975992032, + 0.13079112169658727, + 0.12652764207936965, + 0.08236161086928047, + 0.08108344839200304, + 0.06979629454293851, + 0.05553393174409991, + 0.047985845030080514, + 0.0412218908891166, + 0.03390934004780154, + 0.01878283172038052 + ], + "median": 49.26995849609375 + }, + "wikitext|code": { + "d_act": [ + 27.37684440612793, + 30.734821319580078, + 41.52711486816406, + 43.33364486694336, + 56.89814376831055, + 63.24809646606445, + 101.22335052490234, + 112.15347290039062, + 126.98149108886719, + 143.80844116210938, + 146.8401641845703, + 152.5424041748047, + 91.70540618896484 + ], + "d_act_norm": [ + 0.21412062735735035, + 0.18244563846414083, + 0.18596537046079561, + 0.17528542147610363, + 0.16510554706558028, + 0.11630806371832948, + 0.10991309278963388, + 0.08894634792945602, + 0.08027523121626155, + 0.07058573979365776, + 0.06423812370139531, + 0.06304790461125583, + 0.03496019206983972 + ], + "median": 91.70540618896484 + }, + "pile|code": { + "d_act": [ + 4.315675735473633, + 5.418194770812988, + 6.616461753845215, + 7.902200222015381, + 11.267929077148438, + 13.085437774658203, + 15.507658958435059, + 14.871623039245605, + 23.2187557220459, + 29.722959518432617, + 27.752567291259766, + 31.966707229614258, + 17.936023712158203 + ], + "d_act_norm": [ + 0.04591766038103433, + 0.03940971493875674, + 0.037510695463108035, + 0.0325408916704818, + 0.034511410162797226, + 0.025722334491782427, + 0.01880046018777412, + 0.013421838392545715, + 0.014533290744519225, + 0.014569673668142161, + 0.012158071499177088, + 0.012775718730107403, + 0.006821489975051489 + ], + "median": 14.871623039245605 + } + }, + "model": "EleutherAI/pythia-410m-deduped", + "corpora": [ + "wikitext", + "pile", + "code" + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_eval_prompts": 60, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "env": { + "torch": "2.6.0+cu124" + }, + "argv": [ + "t23_pq_ladder.py", + "--model", + "EleutherAI/pythia-410m-deduped", + "--corpora", + "wikitext,pile,code", + "--tag", + "410m", + "--n-eval-prompts", + "60", + "--out", + "results/t23_pq_ladder_410m.json" + ], + "runtime_s": 24.5 +} \ No newline at end of file diff --git a/results/t23_pq_ladder_410m_LENGTHMATCHED.json b/results/t23_pq_ladder_410m_LENGTHMATCHED.json new file mode 100644 index 0000000000000000000000000000000000000000..2c9ec48022ef6d0fcfd5ea2591937cc45fa9a78f --- /dev/null +++ b/results/t23_pq_ladder_410m_LENGTHMATCHED.json @@ -0,0 +1,292 @@ +{ + "experiment": "T23 / E27 \u2014 P-ladder: distributional bias vs the sampling floor", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "R = median over band layers of D_act(between)/D_act(within, matched n); >=3.0 variant / [1.5,3.0) weakly variant / <1.5 invariant", + "primary_outcome": "R", + "VERDICT": "CORPUS-VARIANT \u2014 R=3.46 >= 3.0; the distributional bias term dominates the sampling floor, S3's premise holds", + "R": 3.459711790084839, + "R_per_pair": { + "wikitext|pile": 3.459711790084839, + "wikitext|code": 5.063662528991699, + "pile|code": 1.3125115633010864 + }, + "C1_self_distance_max": 0.0, + "C1_fires": true, + "C2_shuffle_floor_median": { + "wikitext": 415.9534912109375, + "pile": 356.54461669921875, + "code": 305.40826416015625 + }, + "C2_between_over_shuffle": { + "wikitext|pile": 0.12386691318553526, + "wikitext|code": 0.23756999417843486, + "pile|code": 0.042824776085578005 + }, + "within_corpus": { + "wikitext": { + "d_act": [ + 5.380553245544434, + 6.478600025177002, + 8.088250160217285, + 9.043845176696777, + 12.745851516723633, + 14.201894760131836, + 24.752262115478516, + 34.89788055419922, + 36.99176025390625, + 35.576271057128906, + 30.415172576904297, + 23.354036331176758, + 13.61760425567627 + ], + "d_act_norm": [ + 0.04263583858326435, + 0.03781016806932225, + 0.03573307133292855, + 0.03594557169807991, + 0.03810624453425338, + 0.026243777948688103, + 0.026805785900764652, + 0.027981248589967766, + 0.02301992829890154, + 0.0169969534865411, + 0.013185484692500563, + 0.009619560652483622, + 0.00525504325708214 + ], + "median": 14.201894760131836 + }, + "pile": { + "d_act": [ + 2.8151540756225586, + 3.672285795211792, + 4.241593360900879, + 5.349687099456787, + 7.6959919929504395, + 8.895952224731445, + 10.262511253356934, + 11.861963272094727, + 17.1911678314209, + 18.864383697509766, + 15.482970237731934, + 12.769309043884277, + 6.3693342208862305 + ], + "d_act_norm": [ + 0.029792551843367737, + 0.02657315449637724, + 0.024019582111193056, + 0.021946725338023686, + 0.023643316854820202, + 0.01753751837813493, + 0.012477964571589546, + 0.010743279908550804, + 0.010785355331504248, + 0.009260336418413694, + 0.00680009526977674, + 0.005105385311195361, + 0.002425862750139733 + ], + "median": 8.895952224731445 + }, + "code": { + "d_act": [ + 4.2459797859191895, + 4.623242378234863, + 5.069534778594971, + 6.423276901245117, + 8.317120552062988, + 9.115818977355957, + 11.846217155456543, + 14.072928428649902, + 15.478501319885254, + 16.567955017089844, + 15.68135929107666, + 10.77337646484375, + 6.979499816894531 + ], + "d_act_norm": [ + 0.045638635060302586, + 0.03387552337100513, + 0.03038960224792133, + 0.025978846373880093, + 0.02619009697533588, + 0.018458690893669055, + 0.01463885921072609, + 0.012571476424357637, + 0.009380363730042944, + 0.007846752838828453, + 0.006538762669894368, + 0.004033516732269879, + 0.002673648676326507 + ], + "median": 9.115818977355957 + } + }, + "between_corpus": { + "wikitext|pile": { + "d_act": [ + 18.703468322753906, + 20.947376251220703, + 28.923614501953125, + 28.75986099243164, + 35.36144256591797, + 37.829978942871094, + 62.16440963745117, + 69.4806900024414, + 65.58814239501953, + 75.73932647705078, + 72.12184143066406, + 57.98270034790039, + 36.46123123168945 + ], + "d_act_norm": [ + 0.14820744632840158, + 0.12225230969531034, + 0.12778160414571704, + 0.11430864031087383, + 0.10572002786418809, + 0.06990627546172856, + 0.06732176023406386, + 0.055709872011893945, + 0.04081542280843634, + 0.036185293482983556, + 0.03126602138371836, + 0.023883156422378756, + 0.01407041530444938 + ], + "median": 37.829978942871094 + }, + "wikitext|code": { + "d_act": [ + 24.67207145690918, + 28.10799217224121, + 39.582908630371094, + 39.55224609375, + 49.09362030029297, + 55.42308807373047, + 89.28923797607422, + 94.6415023803711, + 103.22756958007812, + 116.73688507080078, + 117.85627746582031, + 116.61583709716797, + 72.55583953857422 + ], + "d_act_norm": [ + 0.19550302880519213, + 0.1640428339446065, + 0.17487328774903346, + 0.15720394036005284, + 0.1467750897443091, + 0.10241670151793633, + 0.09669694774815529, + 0.07588390363478782, + 0.06423839346634869, + 0.0557723265185911, + 0.051092662338555785, + 0.04803422852000985, + 0.027999350559051995 + ], + "median": 72.55583953857422 + }, + "pile|code": { + "d_act": [ + 4.010067462921143, + 4.998417854309082, + 6.273706912994385, + 7.349700450897217, + 10.512392044067383, + 11.820328712463379, + 14.287734985351562, + 13.07904052734375, + 20.410911560058594, + 26.618173599243164, + 24.8208065032959, + 28.550750732421875, + 15.820362091064453 + ], + "d_act_norm": [ + 0.04243822525346503, + 0.036169224642916255, + 0.03552717234219323, + 0.030151643285636055, + 0.03229574773825739, + 0.023302646731185716, + 0.01737214669529937, + 0.011845576495004655, + 0.012805350746025759, + 0.01306659397546437, + 0.01090125772403674, + 0.011415072100766348, + 0.00602543778666651 + ], + "median": 13.07904052734375 + } + }, + "model": "EleutherAI/pythia-410m-deduped", + "corpora": [ + "wikitext", + "pile", + "code" + ], + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "n_eval_prompts": 60, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 12, + "2": 16, + "3": 17, + "4": 9, + "5": 1, + "6": 15, + "7": 2, + "8": 5, + "9": 10, + "10": 4, + "11": 19, + "12": 8, + "13": 14, + "14": 11, + "15": 18, + "16": 21, + "17": 6, + "18": 0, + "19": 7, + "20": 13, + "21": 20 + }, + "env": { + "torch": "2.6.0+cu124" + }, + "argv": [ + "t23_pq_ladder.py", + "--model", + "EleutherAI/pythia-410m-deduped", + "--corpora", + "wikitext,pile,code", + "--tag", + "410mLM", + "--n-eval-prompts", + "60", + "--out", + "results/t23_pq_ladder_410m_LENGTHMATCHED.json" + ], + "runtime_s": 24.6 +} \ No newline at end of file diff --git a/results/t23_pq_ladder_70m.json b/results/t23_pq_ladder_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..0d63ea7db4f4c541324ef8f0db5c9737872b600d --- /dev/null +++ b/results/t23_pq_ladder_70m.json @@ -0,0 +1,131 @@ +{ + "experiment": "T23 / E27 \u2014 P-ladder: distributional bias vs the sampling floor", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "R = median over band layers of D_act(between)/D_act(within, matched n); >=3.0 variant / [1.5,3.0) weakly variant / <1.5 invariant", + "primary_outcome": "R", + "VERDICT": "CORPUS-VARIANT \u2014 R=5.35 >= 3.0; the distributional bias term dominates the sampling floor, S3's premise holds", + "R": 5.349780559539795, + "R_per_pair": { + "wikitext|pile": 5.349780559539795, + "wikitext|code": 7.975103378295898, + "pile|code": 1.4292482137680054 + }, + "C1_self_distance_max": 0.0, + "C1_fires": true, + "C2_shuffle_floor_median": { + "wikitext": 3790.42529296875, + "pile": 2808.5869140625, + "code": 2642.093017578125 + }, + "C2_between_over_shuffle": { + "wikitext|pile": 0.0773357738527943, + "wikitext|code": 0.11301013268049115, + "pile|code": 0.018988018044794438 + }, + "within_corpus": { + "wikitext": { + "d_act": [ + 44.36017608642578, + 41.26363754272461 + ], + "d_act_norm": [ + 0.016913383794121696, + 0.029155127358001548 + ], + "median": 41.26363754272461 + }, + "pile": { + "d_act": [ + 40.18149185180664, + 25.234033584594727 + ], + "d_act_norm": [ + 0.015461547040209998, + 0.01770190074718583 + ], + "median": 25.234033584594727 + }, + "code": { + "d_act": [ + 30.518674850463867, + 26.264249801635742 + ], + "d_act_norm": [ + 0.011827176549226394, + 0.016654648012728816 + ], + "median": 26.264249801635742 + } + }, + "between_corpus": { + "wikitext|pile": { + "d_act": [ + 226.13967895507812, + 204.32830810546875 + ], + "d_act_norm": [ + 0.08622119023592169, + 0.1443696726807418 + ], + "median": 204.32830810546875 + }, + "wikitext|code": { + "d_act": [ + 298.5832824707031, + 321.24920654296875 + ], + "d_act_norm": [ + 0.1138420560165668, + 0.22698099557314896 + ], + "median": 298.5832824707031 + }, + "pile|code": { + "d_act": [ + 50.524044036865234, + 50.16810989379883 + ], + "d_act_norm": [ + 0.019441286212535432, + 0.03519337877699037 + ], + "median": 50.16810989379883 + } + }, + "model": "EleutherAI/pythia-70m-deduped", + "corpora": [ + "wikitext", + "pile", + "code" + ], + "band": [ + 2, + 3 + ], + "n_eval_prompts": 60, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t23_pq_ladder.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--corpora", + "wikitext,pile,code", + "--tag", + "70m", + "--n-eval-prompts", + "60", + "--out", + "results/t23_pq_ladder_70m.json" + ], + "runtime_s": 10.8 +} \ No newline at end of file diff --git a/results/t23_pq_ladder_70m_LENGTHMATCHED.json b/results/t23_pq_ladder_70m_LENGTHMATCHED.json new file mode 100644 index 0000000000000000000000000000000000000000..2bb1ad115e0aec465b7c791db9184c86c18f37df --- /dev/null +++ b/results/t23_pq_ladder_70m_LENGTHMATCHED.json @@ -0,0 +1,131 @@ +{ + "experiment": "T23 / E27 \u2014 P-ladder: distributional bias vs the sampling floor", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "R = median over band layers of D_act(between)/D_act(within, matched n); >=3.0 variant / [1.5,3.0) weakly variant / <1.5 invariant", + "primary_outcome": "R", + "VERDICT": "CORPUS-VARIANT \u2014 R=3.97 >= 3.0; the distributional bias term dominates the sampling floor, S3's premise holds", + "R": 3.967928886413574, + "R_per_pair": { + "wikitext|pile": 3.967928886413574, + "wikitext|code": 5.61839485168457, + "pile|code": 1.2303110361099243 + }, + "C1_self_distance_max": 0.0, + "C1_fires": true, + "C2_shuffle_floor_median": { + "wikitext": 3790.321044921875, + "pile": 2819.725341796875, + "code": 2628.9208984375 + }, + "C2_between_over_shuffle": { + "wikitext|pile": 0.06611201321315074, + "wikitext|code": 0.10484769308491966, + "pile|code": 0.01760330290460863 + }, + "within_corpus": { + "wikitext": { + "d_act": [ + 58.47085189819336, + 33.18638229370117 + ], + "d_act_norm": [ + 0.022766969249041563, + 0.023382404425873193 + ], + "median": 33.18638229370117 + }, + "pile": { + "d_act": [ + 42.227359771728516, + 22.759355545043945 + ], + "d_act_norm": [ + 0.016176640947359384, + 0.015940345453758886 + ], + "median": 22.759355545043945 + }, + "code": { + "d_act": [ + 39.64837646484375, + 29.41654396057129 + ], + "d_act_norm": [ + 0.015512327400018167, + 0.018699357995144134 + ], + "median": 29.41654396057129 + } + }, + "between_corpus": { + "wikitext|pile": { + "d_act": [ + 199.78167724609375, + 173.80325317382812 + ], + "d_act_norm": [ + 0.07778958497651546, + 0.12245799859342367 + ], + "median": 173.80325317382812 + }, + "wikitext|code": { + "d_act": [ + 275.63629150390625, + 279.3912658691406 + ], + "d_act_norm": [ + 0.10732532140143469, + 0.19685301982581124 + ], + "median": 275.63629150390625 + }, + "pile|code": { + "d_act": [ + 50.366310119628906, + 46.27769088745117 + ], + "d_act_norm": [ + 0.01929454550445457, + 0.032412270114075284 + ], + "median": 46.27769088745117 + } + }, + "model": "EleutherAI/pythia-70m-deduped", + "corpora": [ + "wikitext", + "pile", + "code" + ], + "band": [ + 2, + 3 + ], + "n_eval_prompts": 60, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "env": { + "torch": "2.6.0+cu124" + }, + "argv": [ + "t23_pq_ladder.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--corpora", + "wikitext,pile,code", + "--tag", + "70mLM", + "--n-eval-prompts", + "60", + "--out", + "results/t23_pq_ladder_70m_LENGTHMATCHED.json" + ], + "runtime_s": 5.0 +} \ No newline at end of file diff --git a/results/t23_pq_ladder_70m_wiki_pile.json b/results/t23_pq_ladder_70m_wiki_pile.json new file mode 100644 index 0000000000000000000000000000000000000000..3f424d88c7afbe07c0539afee2a52e35149370f0 --- /dev/null +++ b/results/t23_pq_ladder_70m_wiki_pile.json @@ -0,0 +1,92 @@ +{ + "experiment": "T23 / E27 \u2014 P-ladder: distributional bias vs the sampling floor", + "status": "EXPLORATORY", + "prereg": "in-file header, fixed before the first number", + "decision_rule": "R = median over band layers of D_act(between)/D_act(within, matched n); >=3.0 variant / [1.5,3.0) weakly variant / <1.5 invariant", + "primary_outcome": "R", + "VERDICT": "CORPUS-VARIANT \u2014 R=5.35 >= 3.0; the distributional bias term dominates the sampling floor, S3's premise holds", + "R": 5.349780559539795, + "R_per_pair": { + "wikitext|pile": 5.349780559539795 + }, + "C1_self_distance_max": 0.0, + "C1_fires": true, + "C2_shuffle_floor_median": { + "wikitext": 3790.42529296875, + "pile": 2808.5869140625 + }, + "C2_between_over_shuffle": { + "wikitext|pile": 0.07275128538212715 + }, + "within_corpus": { + "wikitext": { + "d_act": [ + 44.36017608642578, + 41.26363754272461 + ], + "d_act_norm": [ + 0.016913383794121696, + 0.029155127358001548 + ], + "median": 41.26363754272461 + }, + "pile": { + "d_act": [ + 40.18149185180664, + 25.234033584594727 + ], + "d_act_norm": [ + 0.015461547040209998, + 0.01770190074718583 + ], + "median": 25.234033584594727 + } + }, + "between_corpus": { + "wikitext|pile": { + "d_act": [ + 226.13967895507812, + 204.32830810546875 + ], + "d_act_norm": [ + 0.08622119023592169, + 0.1443696726807418 + ], + "median": 204.32830810546875 + } + }, + "model": "EleutherAI/pythia-70m-deduped", + "corpora": [ + "wikitext", + "pile" + ], + "band": [ + 2, + 3 + ], + "n_eval_prompts": 60, + "seed": 0, + "derangement_layer_map": { + "0": 3, + "1": 2, + "2": 0, + "3": 1 + }, + "env": { + "torch": "2.13.0" + }, + "argv": [ + "t23_pq_ladder.py", + "--model", + "EleutherAI/pythia-70m-deduped", + "--corpora", + "wikitext,pile", + "--tag", + "70m", + "--n-eval-prompts", + "60", + "--out", + "results/t23_pq_ladder_70m_wiki_pile.json" + ], + "runtime_s": 10.5 +} \ No newline at end of file diff --git a/results/t3_160m_db128.json b/results/t3_160m_db128.json new file mode 100644 index 0000000000000000000000000000000000000000..1591aee22d9fb1dd4c434b8cbcb53046ae3a5d93 --- /dev/null +++ b/results/t3_160m_db128.json @@ -0,0 +1,227 @@ +{ + "experiment": "T3 read audit (Claim 1a)", + "status": "EXPLORATORY unless a committed PREREG_PYTHIA_READ.md is cited here", + "prereg": null, + "model": "EleutherAI/pythia-160m-deduped", + "lens": "pythia/results/lens_160m_n200_db128.pt", + "lens_n_prompts": 200, + "d_model": 768, + "n_layers": 12, + "n_concepts": 250, + "vocab_size": 50304, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "band_peak_kurtosis_excess": { + "layer": 10, + "depth_pct": 90.9, + "excess": 4.252 + }, + "any_layer_beats_all_nulls": true, + "rows": [ + { + "layer": 0, + "stable_rank": 7.531, + "d_model": 768, + "depth_pct": 0.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 5.5166, + "R3_kurt_jlens": 28.964, + "R3_kurt_logitlens": 25.779, + "R3_kurt_null_randrot": 29.403, + "R3_kurt_null_randJ_normmatched": 25.777, + "R3_kurt_excess_over_max_null": -0.439 + }, + { + "layer": 1, + "stable_rank": 7.581, + "d_model": 768, + "depth_pct": 9.1, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 1.0, + "R2_kl_lens_vs_final": 4.7482, + "R3_kurt_jlens": 29.272, + "R3_kurt_logitlens": 26.412, + "R3_kurt_null_randrot": 28.199, + "R3_kurt_null_randJ_normmatched": 27.618, + "R3_kurt_excess_over_max_null": 1.073 + }, + { + "layer": 2, + "stable_rank": 6.506, + "d_model": 768, + "depth_pct": 18.2, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.9605, + "R3_kurt_jlens": 29.189, + "R3_kurt_logitlens": 26.062, + "R3_kurt_null_randrot": 24.535, + "R3_kurt_null_randJ_normmatched": 24.933, + "R3_kurt_excess_over_max_null": 3.127 + }, + { + "layer": 3, + "stable_rank": 6.09, + "d_model": 768, + "depth_pct": 27.3, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.0331, + "R3_kurt_jlens": 28.181, + "R3_kurt_logitlens": 26.032, + "R3_kurt_null_randrot": 27.4, + "R3_kurt_null_randJ_normmatched": 27.105, + "R3_kurt_excess_over_max_null": 0.781 + }, + { + "layer": 4, + "stable_rank": 5.586, + "d_model": 768, + "depth_pct": 36.4, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 18.1921, + "R3_kurt_jlens": 27.348, + "R3_kurt_logitlens": 26.014, + "R3_kurt_null_randrot": 26.111, + "R3_kurt_null_randJ_normmatched": 26.77, + "R3_kurt_excess_over_max_null": 0.578 + }, + { + "layer": 5, + "stable_rank": 5.27, + "d_model": 768, + "depth_pct": 45.5, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 11.6382, + "R3_kurt_jlens": 25.705, + "R3_kurt_logitlens": 25.842, + "R3_kurt_null_randrot": 27.609, + "R3_kurt_null_randJ_normmatched": 25.973, + "R3_kurt_excess_over_max_null": -1.903 + }, + { + "layer": 6, + "stable_rank": 7.242, + "d_model": 768, + "depth_pct": 54.5, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 201.5437, + "R3_kurt_jlens": 28.426, + "R3_kurt_logitlens": 25.538, + "R3_kurt_null_randrot": 24.898, + "R3_kurt_null_randJ_normmatched": 24.822, + "R3_kurt_excess_over_max_null": 2.888 + }, + { + "layer": 7, + "stable_rank": 6.329, + "d_model": 768, + "depth_pct": 63.6, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 189.0828, + "R3_kurt_jlens": 25.495, + "R3_kurt_logitlens": 25.384, + "R3_kurt_null_randrot": 25.197, + "R3_kurt_null_randJ_normmatched": 27.148, + "R3_kurt_excess_over_max_null": -1.653 + }, + { + "layer": 8, + "stable_rank": 7.05, + "d_model": 768, + "depth_pct": 72.7, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 221.7665, + "R3_kurt_jlens": 21.318, + "R3_kurt_logitlens": 25.253, + "R3_kurt_null_randrot": 24.281, + "R3_kurt_null_randJ_normmatched": 26.422, + "R3_kurt_excess_over_max_null": -5.104 + }, + { + "layer": 9, + "stable_rank": 7.265, + "d_model": 768, + "depth_pct": 81.8, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 27.7284, + "R3_kurt_jlens": 27.413, + "R3_kurt_logitlens": 25.1, + "R3_kurt_null_randrot": 26.696, + "R3_kurt_null_randJ_normmatched": 25.901, + "R3_kurt_excess_over_max_null": 0.717 + }, + { + "layer": 10, + "stable_rank": 5.0, + "d_model": 768, + "depth_pct": 90.9, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 7.3445, + "R3_kurt_jlens": 30.873, + "R3_kurt_logitlens": 26.493, + "R3_kurt_null_randrot": 26.62, + "R3_kurt_null_randJ_normmatched": 24.319, + "R3_kurt_excess_over_max_null": 4.252 + } + ], + "runtime_s": 12.5 +} \ No newline at end of file diff --git a/results/t3_410m_FIXED.json b/results/t3_410m_FIXED.json new file mode 100644 index 0000000000000000000000000000000000000000..88f2378417f3b569e4e34590235ba00a999ef4cf --- /dev/null +++ b/results/t3_410m_FIXED.json @@ -0,0 +1,443 @@ +{ + "experiment": "T3 read audit (Claim 1a)", + "status": "EXPLORATORY unless a committed PREREG_PYTHIA_READ.md is cited here", + "prereg": null, + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "lens_n_prompts": 200, + "d_model": 1024, + "n_layers": 24, + "n_concepts": 250, + "vocab_size": 50304, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "band_peak_kurtosis_excess": { + "layer": 20, + "depth_pct": 87.0, + "excess": 1.119 + }, + "any_layer_beats_all_nulls": true, + "rows": [ + { + "layer": 0, + "stable_rank": 16.157, + "d_model": 1024, + "depth_pct": 0.0, + "R1_self_top1": 0.612, + "R1_self_top3": 0.752, + "R1_ctrl_randrot_top1": 0.98, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 15.2086, + "R3_kurt_jlens": 1.143, + "R3_kurt_logitlens": 0.196, + "R3_kurt_null_randrot": 0.155, + "R3_kurt_null_randJ_normmatched": 0.154, + "R3_kurt_excess_over_max_null": 0.947 + }, + { + "layer": 1, + "stable_rank": 15.746, + "d_model": 1024, + "depth_pct": 4.3, + "R1_self_top1": 0.516, + "R1_self_top3": 0.692, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 14.4887, + "R3_kurt_jlens": 0.486, + "R3_kurt_logitlens": 0.179, + "R3_kurt_null_randrot": 0.133, + "R3_kurt_null_randJ_normmatched": 0.226, + "R3_kurt_excess_over_max_null": 0.261 + }, + { + "layer": 2, + "stable_rank": 15.569, + "d_model": 1024, + "depth_pct": 8.7, + "R1_self_top1": 0.44, + "R1_self_top3": 0.652, + "R1_ctrl_randrot_top1": 0.992, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 13.7097, + "R3_kurt_jlens": 0.997, + "R3_kurt_logitlens": 0.158, + "R3_kurt_null_randrot": 0.229, + "R3_kurt_null_randJ_normmatched": 0.166, + "R3_kurt_excess_over_max_null": 0.769 + }, + { + "layer": 3, + "stable_rank": 14.288, + "d_model": 1024, + "depth_pct": 13.0, + "R1_self_top1": 0.392, + "R1_self_top3": 0.616, + "R1_ctrl_randrot_top1": 0.988, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 14.2248, + "R3_kurt_jlens": 0.599, + "R3_kurt_logitlens": 0.127, + "R3_kurt_null_randrot": 0.231, + "R3_kurt_null_randJ_normmatched": 0.21, + "R3_kurt_excess_over_max_null": 0.368 + }, + { + "layer": 4, + "stable_rank": 14.059, + "d_model": 1024, + "depth_pct": 17.4, + "R1_self_top1": 0.444, + "R1_self_top3": 0.672, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 13.9751, + "R3_kurt_jlens": 0.365, + "R3_kurt_logitlens": 0.127, + "R3_kurt_null_randrot": 0.184, + "R3_kurt_null_randJ_normmatched": 0.247, + "R3_kurt_excess_over_max_null": 0.118 + }, + { + "layer": 5, + "stable_rank": 12.038, + "d_model": 1024, + "depth_pct": 21.7, + "R1_self_top1": 0.336, + "R1_self_top3": 0.492, + "R1_ctrl_randrot_top1": 0.992, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 11.3456, + "R3_kurt_jlens": 0.086, + "R3_kurt_logitlens": 0.143, + "R3_kurt_null_randrot": 0.177, + "R3_kurt_null_randJ_normmatched": 0.152, + "R3_kurt_excess_over_max_null": -0.091 + }, + { + "layer": 6, + "stable_rank": 12.621, + "d_model": 1024, + "depth_pct": 26.1, + "R1_self_top1": 0.428, + "R1_self_top3": 0.568, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.6679, + "R3_kurt_jlens": 0.151, + "R3_kurt_logitlens": 0.152, + "R3_kurt_null_randrot": 0.187, + "R3_kurt_null_randJ_normmatched": 0.26, + "R3_kurt_excess_over_max_null": -0.109 + }, + { + "layer": 7, + "stable_rank": 12.278, + "d_model": 1024, + "depth_pct": 30.4, + "R1_self_top1": 0.432, + "R1_self_top3": 0.584, + "R1_ctrl_randrot_top1": 0.988, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 9.3189, + "R3_kurt_jlens": 0.018, + "R3_kurt_logitlens": 0.184, + "R3_kurt_null_randrot": 0.182, + "R3_kurt_null_randJ_normmatched": 0.202, + "R3_kurt_excess_over_max_null": -0.184 + }, + { + "layer": 8, + "stable_rank": 10.851, + "d_model": 1024, + "depth_pct": 34.8, + "R1_self_top1": 0.432, + "R1_self_top3": 0.572, + "R1_ctrl_randrot_top1": 0.972, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 7.9787, + "R3_kurt_jlens": 0.298, + "R3_kurt_logitlens": 0.188, + "R3_kurt_null_randrot": 0.16, + "R3_kurt_null_randJ_normmatched": 0.321, + "R3_kurt_excess_over_max_null": -0.023 + }, + { + "layer": 9, + "stable_rank": 12.805, + "d_model": 1024, + "depth_pct": 39.1, + "R1_self_top1": 0.84, + "R1_self_top3": 0.92, + "R1_ctrl_randrot_top1": 0.948, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.2754, + "R3_kurt_jlens": -0.01, + "R3_kurt_logitlens": 0.177, + "R3_kurt_null_randrot": 0.218, + "R3_kurt_null_randJ_normmatched": 0.195, + "R3_kurt_excess_over_max_null": -0.227 + }, + { + "layer": 10, + "stable_rank": 12.427, + "d_model": 1024, + "depth_pct": 43.5, + "R1_self_top1": 0.804, + "R1_self_top3": 0.88, + "R1_ctrl_randrot_top1": 0.988, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.2, + "R2_kl_lens_vs_final": 9.1859, + "R3_kurt_jlens": 0.247, + "R3_kurt_logitlens": 0.21, + "R3_kurt_null_randrot": 0.235, + "R3_kurt_null_randJ_normmatched": 0.186, + "R3_kurt_excess_over_max_null": 0.012 + }, + { + "layer": 11, + "stable_rank": 13.625, + "d_model": 1024, + "depth_pct": 47.8, + "R1_self_top1": 0.912, + "R1_self_top3": 0.964, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 7.4603, + "R3_kurt_jlens": 0.192, + "R3_kurt_logitlens": 0.272, + "R3_kurt_null_randrot": 0.179, + "R3_kurt_null_randJ_normmatched": 0.204, + "R3_kurt_excess_over_max_null": -0.081 + }, + { + "layer": 12, + "stable_rank": 23.339, + "d_model": 1024, + "depth_pct": 52.2, + "R1_self_top1": 0.988, + "R1_self_top3": 0.996, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.3745, + "R3_kurt_jlens": 0.335, + "R3_kurt_logitlens": 0.344, + "R3_kurt_null_randrot": 0.205, + "R3_kurt_null_randJ_normmatched": 0.19, + "R3_kurt_excess_over_max_null": -0.009 + }, + { + "layer": 13, + "stable_rank": 25.415, + "d_model": 1024, + "depth_pct": 56.5, + "R1_self_top1": 0.988, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 6.1409, + "R3_kurt_jlens": 0.207, + "R3_kurt_logitlens": 0.367, + "R3_kurt_null_randrot": 0.159, + "R3_kurt_null_randJ_normmatched": 0.201, + "R3_kurt_excess_over_max_null": -0.16 + }, + { + "layer": 14, + "stable_rank": 29.531, + "d_model": 1024, + "depth_pct": 60.9, + "R1_self_top1": 0.992, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.6, + "R2_kl_lens_vs_final": 4.3747, + "R3_kurt_jlens": 0.212, + "R3_kurt_logitlens": 0.306, + "R3_kurt_null_randrot": 0.229, + "R3_kurt_null_randJ_normmatched": 0.23, + "R3_kurt_excess_over_max_null": -0.095 + }, + { + "layer": 15, + "stable_rank": 29.335, + "d_model": 1024, + "depth_pct": 65.2, + "R1_self_top1": 0.996, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 4.3136, + "R3_kurt_jlens": 0.354, + "R3_kurt_logitlens": 0.479, + "R3_kurt_null_randrot": 0.197, + "R3_kurt_null_randJ_normmatched": 0.217, + "R3_kurt_excess_over_max_null": -0.125 + }, + { + "layer": 16, + "stable_rank": 39.035, + "d_model": 1024, + "depth_pct": 69.6, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 4.2176, + "R3_kurt_jlens": 0.344, + "R3_kurt_logitlens": 0.838, + "R3_kurt_null_randrot": 0.189, + "R3_kurt_null_randJ_normmatched": 0.161, + "R3_kurt_excess_over_max_null": -0.494 + }, + { + "layer": 17, + "stable_rank": 44.076, + "d_model": 1024, + "depth_pct": 73.9, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.6811, + "R3_kurt_jlens": 0.55, + "R3_kurt_logitlens": 0.891, + "R3_kurt_null_randrot": 0.238, + "R3_kurt_null_randJ_normmatched": 0.236, + "R3_kurt_excess_over_max_null": -0.341 + }, + { + "layer": 18, + "stable_rank": 51.466, + "d_model": 1024, + "depth_pct": 78.3, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 3.396, + "R3_kurt_jlens": 0.848, + "R3_kurt_logitlens": 0.404, + "R3_kurt_null_randrot": 0.149, + "R3_kurt_null_randJ_normmatched": 0.179, + "R3_kurt_excess_over_max_null": 0.444 + }, + { + "layer": 19, + "stable_rank": 62.118, + "d_model": 1024, + "depth_pct": 82.6, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 3.2072, + "R3_kurt_jlens": 1.087, + "R3_kurt_logitlens": 0.58, + "R3_kurt_null_randrot": 0.238, + "R3_kurt_null_randJ_normmatched": 0.194, + "R3_kurt_excess_over_max_null": 0.508 + }, + { + "layer": 20, + "stable_rank": 61.323, + "d_model": 1024, + "depth_pct": 87.0, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.1207, + "R3_kurt_jlens": 1.639, + "R3_kurt_logitlens": 0.52, + "R3_kurt_null_randrot": 0.209, + "R3_kurt_null_randJ_normmatched": 0.169, + "R3_kurt_excess_over_max_null": 1.119 + }, + { + "layer": 21, + "stable_rank": 66.852, + "d_model": 1024, + "depth_pct": 91.3, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 1.8666, + "R3_kurt_jlens": 1.342, + "R3_kurt_logitlens": 0.479, + "R3_kurt_null_randrot": 0.19, + "R3_kurt_null_randJ_normmatched": 0.211, + "R3_kurt_excess_over_max_null": 0.863 + }, + { + "layer": 22, + "stable_rank": 39.961, + "d_model": 1024, + "depth_pct": 95.7, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.996, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 1.4115, + "R3_kurt_jlens": 0.72, + "R3_kurt_logitlens": 0.497, + "R3_kurt_null_randrot": 0.161, + "R3_kurt_null_randJ_normmatched": 0.204, + "R3_kurt_excess_over_max_null": 0.223 + } + ], + "runtime_s": 30.0 +} \ No newline at end of file diff --git a/results/t3_410m_n200.json b/results/t3_410m_n200.json new file mode 100644 index 0000000000000000000000000000000000000000..aebb871d7453b1ca1bb343529835308bcc32ff3f --- /dev/null +++ b/results/t3_410m_n200.json @@ -0,0 +1,443 @@ +{ + "experiment": "T3 read audit (Claim 1a)", + "status": "EXPLORATORY unless a committed PREREG_PYTHIA_READ.md is cited here", + "prereg": null, + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "lens_n_prompts": 200, + "d_model": 1024, + "n_layers": 24, + "n_concepts": 250, + "vocab_size": 50304, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "band_peak_kurtosis_excess": { + "layer": 20, + "depth_pct": 87.0, + "excess": 1.119 + }, + "any_layer_beats_all_nulls": true, + "rows": [ + { + "layer": 0, + "stable_rank": 16.157, + "d_model": 1024, + "depth_pct": 0.0, + "R1_self_top1": 0.612, + "R1_self_top3": 0.752, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 17.2774, + "R3_kurt_jlens": 1.143, + "R3_kurt_logitlens": 0.196, + "R3_kurt_null_randrot": 0.155, + "R3_kurt_null_randJ_normmatched": 0.154, + "R3_kurt_excess_over_max_null": 0.947 + }, + { + "layer": 1, + "stable_rank": 15.746, + "d_model": 1024, + "depth_pct": 4.3, + "R1_self_top1": 0.516, + "R1_self_top3": 0.692, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.9039, + "R3_kurt_jlens": 0.486, + "R3_kurt_logitlens": 0.179, + "R3_kurt_null_randrot": 0.133, + "R3_kurt_null_randJ_normmatched": 0.226, + "R3_kurt_excess_over_max_null": 0.261 + }, + { + "layer": 2, + "stable_rank": 15.569, + "d_model": 1024, + "depth_pct": 8.7, + "R1_self_top1": 0.44, + "R1_self_top3": 0.652, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 13.229, + "R3_kurt_jlens": 0.997, + "R3_kurt_logitlens": 0.158, + "R3_kurt_null_randrot": 0.229, + "R3_kurt_null_randJ_normmatched": 0.166, + "R3_kurt_excess_over_max_null": 0.769 + }, + { + "layer": 3, + "stable_rank": 14.288, + "d_model": 1024, + "depth_pct": 13.0, + "R1_self_top1": 0.392, + "R1_self_top3": 0.616, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 7.7944, + "R3_kurt_jlens": 0.599, + "R3_kurt_logitlens": 0.127, + "R3_kurt_null_randrot": 0.231, + "R3_kurt_null_randJ_normmatched": 0.21, + "R3_kurt_excess_over_max_null": 0.368 + }, + { + "layer": 4, + "stable_rank": 14.059, + "d_model": 1024, + "depth_pct": 17.4, + "R1_self_top1": 0.444, + "R1_self_top3": 0.672, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 10.2291, + "R3_kurt_jlens": 0.365, + "R3_kurt_logitlens": 0.127, + "R3_kurt_null_randrot": 0.184, + "R3_kurt_null_randJ_normmatched": 0.247, + "R3_kurt_excess_over_max_null": 0.118 + }, + { + "layer": 5, + "stable_rank": 12.038, + "d_model": 1024, + "depth_pct": 21.7, + "R1_self_top1": 0.336, + "R1_self_top3": 0.492, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 5.2671, + "R3_kurt_jlens": 0.086, + "R3_kurt_logitlens": 0.143, + "R3_kurt_null_randrot": 0.177, + "R3_kurt_null_randJ_normmatched": 0.152, + "R3_kurt_excess_over_max_null": -0.091 + }, + { + "layer": 6, + "stable_rank": 12.621, + "d_model": 1024, + "depth_pct": 26.1, + "R1_self_top1": 0.428, + "R1_self_top3": 0.568, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 5.593, + "R3_kurt_jlens": 0.151, + "R3_kurt_logitlens": 0.152, + "R3_kurt_null_randrot": 0.187, + "R3_kurt_null_randJ_normmatched": 0.26, + "R3_kurt_excess_over_max_null": -0.109 + }, + { + "layer": 7, + "stable_rank": 12.278, + "d_model": 1024, + "depth_pct": 30.4, + "R1_self_top1": 0.432, + "R1_self_top3": 0.584, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 7.5809, + "R3_kurt_jlens": 0.018, + "R3_kurt_logitlens": 0.184, + "R3_kurt_null_randrot": 0.182, + "R3_kurt_null_randJ_normmatched": 0.202, + "R3_kurt_excess_over_max_null": -0.184 + }, + { + "layer": 8, + "stable_rank": 10.851, + "d_model": 1024, + "depth_pct": 34.8, + "R1_self_top1": 0.432, + "R1_self_top3": 0.572, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 5.7289, + "R3_kurt_jlens": 0.298, + "R3_kurt_logitlens": 0.188, + "R3_kurt_null_randrot": 0.16, + "R3_kurt_null_randJ_normmatched": 0.321, + "R3_kurt_excess_over_max_null": -0.023 + }, + { + "layer": 9, + "stable_rank": 12.805, + "d_model": 1024, + "depth_pct": 39.1, + "R1_self_top1": 0.84, + "R1_self_top3": 0.92, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 5.8167, + "R3_kurt_jlens": -0.01, + "R3_kurt_logitlens": 0.177, + "R3_kurt_null_randrot": 0.218, + "R3_kurt_null_randJ_normmatched": 0.195, + "R3_kurt_excess_over_max_null": -0.227 + }, + { + "layer": 10, + "stable_rank": 12.427, + "d_model": 1024, + "depth_pct": 43.5, + "R1_self_top1": 0.804, + "R1_self_top3": 0.88, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.2, + "R2_kl_lens_vs_final": 5.4674, + "R3_kurt_jlens": 0.247, + "R3_kurt_logitlens": 0.21, + "R3_kurt_null_randrot": 0.235, + "R3_kurt_null_randJ_normmatched": 0.186, + "R3_kurt_excess_over_max_null": 0.012 + }, + { + "layer": 11, + "stable_rank": 13.625, + "d_model": 1024, + "depth_pct": 47.8, + "R1_self_top1": 0.912, + "R1_self_top3": 0.964, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 4.6282, + "R3_kurt_jlens": 0.192, + "R3_kurt_logitlens": 0.272, + "R3_kurt_null_randrot": 0.179, + "R3_kurt_null_randJ_normmatched": 0.204, + "R3_kurt_excess_over_max_null": -0.081 + }, + { + "layer": 12, + "stable_rank": 23.339, + "d_model": 1024, + "depth_pct": 52.2, + "R1_self_top1": 0.988, + "R1_self_top3": 0.996, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 4.0473, + "R3_kurt_jlens": 0.335, + "R3_kurt_logitlens": 0.344, + "R3_kurt_null_randrot": 0.205, + "R3_kurt_null_randJ_normmatched": 0.19, + "R3_kurt_excess_over_max_null": -0.009 + }, + { + "layer": 13, + "stable_rank": 25.415, + "d_model": 1024, + "depth_pct": 56.5, + "R1_self_top1": 0.988, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.8637, + "R3_kurt_jlens": 0.207, + "R3_kurt_logitlens": 0.367, + "R3_kurt_null_randrot": 0.159, + "R3_kurt_null_randJ_normmatched": 0.201, + "R3_kurt_excess_over_max_null": -0.16 + }, + { + "layer": 14, + "stable_rank": 29.531, + "d_model": 1024, + "depth_pct": 60.9, + "R1_self_top1": 0.992, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.6, + "R2_kl_lens_vs_final": 2.226, + "R3_kurt_jlens": 0.212, + "R3_kurt_logitlens": 0.306, + "R3_kurt_null_randrot": 0.229, + "R3_kurt_null_randJ_normmatched": 0.23, + "R3_kurt_excess_over_max_null": -0.095 + }, + { + "layer": 15, + "stable_rank": 29.335, + "d_model": 1024, + "depth_pct": 65.2, + "R1_self_top1": 0.996, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.8907, + "R3_kurt_jlens": 0.354, + "R3_kurt_logitlens": 0.479, + "R3_kurt_null_randrot": 0.197, + "R3_kurt_null_randJ_normmatched": 0.217, + "R3_kurt_excess_over_max_null": -0.125 + }, + { + "layer": 16, + "stable_rank": 39.035, + "d_model": 1024, + "depth_pct": 69.6, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 3.2274, + "R3_kurt_jlens": 0.344, + "R3_kurt_logitlens": 0.838, + "R3_kurt_null_randrot": 0.189, + "R3_kurt_null_randJ_normmatched": 0.161, + "R3_kurt_excess_over_max_null": -0.494 + }, + { + "layer": 17, + "stable_rank": 44.076, + "d_model": 1024, + "depth_pct": 73.9, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.9754, + "R3_kurt_jlens": 0.55, + "R3_kurt_logitlens": 0.891, + "R3_kurt_null_randrot": 0.238, + "R3_kurt_null_randJ_normmatched": 0.236, + "R3_kurt_excess_over_max_null": -0.341 + }, + { + "layer": 18, + "stable_rank": 51.466, + "d_model": 1024, + "depth_pct": 78.3, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 3.4499, + "R3_kurt_jlens": 0.848, + "R3_kurt_logitlens": 0.404, + "R3_kurt_null_randrot": 0.149, + "R3_kurt_null_randJ_normmatched": 0.179, + "R3_kurt_excess_over_max_null": 0.444 + }, + { + "layer": 19, + "stable_rank": 62.118, + "d_model": 1024, + "depth_pct": 82.6, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 3.1326, + "R3_kurt_jlens": 1.087, + "R3_kurt_logitlens": 0.58, + "R3_kurt_null_randrot": 0.238, + "R3_kurt_null_randJ_normmatched": 0.194, + "R3_kurt_excess_over_max_null": 0.508 + }, + { + "layer": 20, + "stable_rank": 61.323, + "d_model": 1024, + "depth_pct": 87.0, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.7437, + "R3_kurt_jlens": 1.639, + "R3_kurt_logitlens": 0.52, + "R3_kurt_null_randrot": 0.209, + "R3_kurt_null_randJ_normmatched": 0.169, + "R3_kurt_excess_over_max_null": 1.119 + }, + { + "layer": 21, + "stable_rank": 66.852, + "d_model": 1024, + "depth_pct": 91.3, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.5935, + "R3_kurt_jlens": 1.342, + "R3_kurt_logitlens": 0.479, + "R3_kurt_null_randrot": 0.19, + "R3_kurt_null_randJ_normmatched": 0.211, + "R3_kurt_excess_over_max_null": 0.863 + }, + { + "layer": 22, + "stable_rank": 39.961, + "d_model": 1024, + "depth_pct": 95.7, + "R1_self_top1": 1.0, + "R1_self_top3": 1.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": true, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 2.0716, + "R3_kurt_jlens": 0.72, + "R3_kurt_logitlens": 0.497, + "R3_kurt_null_randrot": 0.161, + "R3_kurt_null_randJ_normmatched": 0.204, + "R3_kurt_excess_over_max_null": 0.223 + } + ], + "runtime_s": 45.2 +} \ No newline at end of file diff --git a/results/t3_70m_db128.json b/results/t3_70m_db128.json new file mode 100644 index 0000000000000000000000000000000000000000..e7fe270e00b302a73b448c5bf1d53608f642769e --- /dev/null +++ b/results/t3_70m_db128.json @@ -0,0 +1,119 @@ +{ + "experiment": "T3 read audit (Claim 1a)", + "status": "EXPLORATORY unless a committed PREREG_PYTHIA_READ.md is cited here", + "prereg": null, + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200_db128.pt", + "lens_n_prompts": 200, + "d_model": 512, + "n_layers": 6, + "n_concepts": 250, + "vocab_size": 50304, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "band_peak_kurtosis_excess": { + "layer": 0, + "depth_pct": 0.0, + "excess": 10.683 + }, + "any_layer_beats_all_nulls": true, + "rows": [ + { + "layer": 0, + "stable_rank": 8.517, + "d_model": 512, + "depth_pct": 0.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.5222, + "R3_kurt_jlens": 149.839, + "R3_kurt_logitlens": 130.032, + "R3_kurt_null_randrot": 137.89, + "R3_kurt_null_randJ_normmatched": 139.155, + "R3_kurt_excess_over_max_null": 10.683 + }, + { + "layer": 1, + "stable_rank": 5.107, + "d_model": 512, + "depth_pct": 20.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 5.0979, + "R3_kurt_jlens": 150.501, + "R3_kurt_logitlens": 142.94, + "R3_kurt_null_randrot": 139.782, + "R3_kurt_null_randJ_normmatched": 142.806, + "R3_kurt_excess_over_max_null": 7.561 + }, + { + "layer": 2, + "stable_rank": 5.442, + "d_model": 512, + "depth_pct": 40.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 21.3939, + "R3_kurt_jlens": 148.109, + "R3_kurt_logitlens": 146.746, + "R3_kurt_null_randrot": 138.94, + "R3_kurt_null_randJ_normmatched": 142.393, + "R3_kurt_excess_over_max_null": 1.362 + }, + { + "layer": 3, + "stable_rank": 4.864, + "d_model": 512, + "depth_pct": 60.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 22.0878, + "R3_kurt_jlens": 138.769, + "R3_kurt_logitlens": 147.05, + "R3_kurt_null_randrot": 141.144, + "R3_kurt_null_randJ_normmatched": 140.26, + "R3_kurt_excess_over_max_null": -8.282 + }, + { + "layer": 4, + "stable_rank": 3.339, + "d_model": 512, + "depth_pct": 80.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 36.0285, + "R3_kurt_jlens": 6.466, + "R3_kurt_logitlens": 147.426, + "R3_kurt_null_randrot": 139.501, + "R3_kurt_null_randJ_normmatched": 138.254, + "R3_kurt_excess_over_max_null": -140.959 + } + ], + "runtime_s": 10.6 +} \ No newline at end of file diff --git a/results/t3_70m_n200.json b/results/t3_70m_n200.json new file mode 100644 index 0000000000000000000000000000000000000000..c7eab9feb1d31decb7d3ebf6ef2df896541f059b --- /dev/null +++ b/results/t3_70m_n200.json @@ -0,0 +1,119 @@ +{ + "experiment": "T3 read audit (Claim 1a)", + "status": "EXPLORATORY unless a committed PREREG_PYTHIA_READ.md is cited here", + "prereg": null, + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200.pt", + "lens_n_prompts": 200, + "d_model": 512, + "n_layers": 6, + "n_concepts": 250, + "vocab_size": 50304, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "band_peak_kurtosis_excess": { + "layer": 0, + "depth_pct": 0.0, + "excess": 10.773 + }, + "any_layer_beats_all_nulls": true, + "rows": [ + { + "layer": 0, + "stable_rank": 8.571, + "d_model": 512, + "depth_pct": 0.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 8.3081, + "R3_kurt_jlens": 149.929, + "R3_kurt_logitlens": 130.032, + "R3_kurt_null_randrot": 137.817, + "R3_kurt_null_randJ_normmatched": 139.155, + "R3_kurt_excess_over_max_null": 10.773 + }, + { + "layer": 1, + "stable_rank": 5.116, + "d_model": 512, + "depth_pct": 20.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 5.132, + "R3_kurt_jlens": 150.493, + "R3_kurt_logitlens": 142.94, + "R3_kurt_null_randrot": 139.713, + "R3_kurt_null_randJ_normmatched": 142.806, + "R3_kurt_excess_over_max_null": 7.554 + }, + { + "layer": 2, + "stable_rank": 5.435, + "d_model": 512, + "depth_pct": 40.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 21.6302, + "R3_kurt_jlens": 147.966, + "R3_kurt_logitlens": 146.746, + "R3_kurt_null_randrot": 138.978, + "R3_kurt_null_randJ_normmatched": 142.393, + "R3_kurt_excess_over_max_null": 1.219 + }, + { + "layer": 3, + "stable_rank": 4.871, + "d_model": 512, + "depth_pct": 60.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 22.3815, + "R3_kurt_jlens": 138.768, + "R3_kurt_logitlens": 147.05, + "R3_kurt_null_randrot": 141.154, + "R3_kurt_null_randJ_normmatched": 140.26, + "R3_kurt_excess_over_max_null": -8.283 + }, + { + "layer": 4, + "stable_rank": 3.339, + "d_model": 512, + "depth_pct": 80.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 34.7123, + "R3_kurt_jlens": 5.918, + "R3_kurt_logitlens": 147.426, + "R3_kurt_null_randrot": 139.499, + "R3_kurt_null_randJ_normmatched": 138.254, + "R3_kurt_excess_over_max_null": -141.507 + } + ], + "runtime_s": 6.5 +} \ No newline at end of file diff --git a/results/t3_smoke_70m.json b/results/t3_smoke_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..a0e7ed250bdb05c086d9539f03facd12dc119e16 --- /dev/null +++ b/results/t3_smoke_70m.json @@ -0,0 +1,109 @@ +{ + "experiment": "T3 read audit (Claim 1a)", + "status": "EXPLORATORY unless a committed PREREG_PYTHIA_READ.md is cited here", + "prereg": null, + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/t0_smoke_70m_lens.pt", + "lens_n_prompts": 3, + "d_model": 512, + "n_layers": 6, + "n_concepts": 250, + "vocab_size": 50304, + "centered": false, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "band_peak_kurtosis_excess": { + "layer": 0, + "depth_pct": 0.0, + "excess": -0.0 + }, + "any_layer_beats_all_nulls": false, + "rows": [ + { + "layer": 0, + "depth_pct": 0.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 24.6968, + "R3_kurt_jlens": 147.325, + "R3_kurt_logitlens": 130.032, + "R3_kurt_null_randrot": 141.273, + "R3_kurt_null_rowshuffle": 147.325, + "R3_kurt_excess_over_max_null": -0.0 + }, + { + "layer": 1, + "depth_pct": 20.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 14.924, + "R3_kurt_jlens": 150.022, + "R3_kurt_logitlens": 142.94, + "R3_kurt_null_randrot": 141.021, + "R3_kurt_null_rowshuffle": 150.022, + "R3_kurt_excess_over_max_null": -0.0 + }, + { + "layer": 2, + "depth_pct": 40.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 32.3891, + "R3_kurt_jlens": 143.398, + "R3_kurt_logitlens": 146.746, + "R3_kurt_null_randrot": 137.824, + "R3_kurt_null_rowshuffle": 143.398, + "R3_kurt_excess_over_max_null": -3.348 + }, + { + "layer": 3, + "depth_pct": 60.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 25.4828, + "R3_kurt_jlens": 132.8, + "R3_kurt_logitlens": 147.05, + "R3_kurt_null_randrot": 141.658, + "R3_kurt_null_rowshuffle": 132.8, + "R3_kurt_excess_over_max_null": -14.25 + }, + { + "layer": 4, + "depth_pct": 80.0, + "R1_self_top1": 0.0, + "R1_self_top3": 0.0, + "R1_ctrl_randrot_top1": 0.0, + "R1_ctrl_rowshuffle_top1": 0.0, + "R1_controls_fire": false, + "R2_top1_agreement": 0.0, + "R2_kl_lens_vs_final": 44.4073, + "R3_kurt_jlens": 4.647, + "R3_kurt_logitlens": 147.426, + "R3_kurt_null_randrot": 139.566, + "R3_kurt_null_rowshuffle": 4.647, + "R3_kurt_excess_over_max_null": -142.779 + } + ], + "runtime_s": 8.1 +} \ No newline at end of file diff --git a/results/t4_160m_db128.json b/results/t4_160m_db128.json new file mode 100644 index 0000000000000000000000000000000000000000..a0416cf5c817d764bed04aa8d54e77fe177fcda5 --- /dev/null +++ b/results/t4_160m_db128.json @@ -0,0 +1,225 @@ +{ + "experiment": "T4 geometry \u2014 Claim 2 predictors from J alone", + "status": "EXPLORATORY unless a committed prereg is cited here", + "prereg": null, + "model": "EleutherAI/pythia-160m-deduped", + "lens": "pythia/results/lens_160m_n200_db128.pt", + "lens_n_prompts": 200, + "d_model": 768, + "n_heldout_tokens": 1500, + "centered": false, + "tied_embeddings": false, + "rogue_coordinate_test": { + "note": "Gemma-2 (TIED) had PR(u1) 1.2-1.4 of 2304 \u2014 near one-hot. Pythia is UNTIED.", + "min_PR": 584.4196, + "max_PR": 713.5575, + "median_PR": 666.2647705078125, + "near_one_hot_layers": [] + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "layer": 0, + "depth_pct": 0.0, + "d_model": 768, + "P1_stable_rank": 7.5313, + "P2_effective_rank": 130.0834, + "P3_top_sv_share": 0.132778, + "P4_dist_from_identity": 1.3105, + "P5_top_u_participation": 584.4196, + "P5_top_u_max_abs": 0.0852, + "P5_top_u_argmax_dim": 573, + "fro_norm": 26.1499, + "top_sv": 9.5287, + "dict_anisotropy_raw_meanabscos": 0.9954, + "dict_anisotropy_raw_median": 0.9987, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1019 + }, + { + "layer": 1, + "depth_pct": 9.1, + "d_model": 768, + "P1_stable_rank": 7.5814, + "P2_effective_rank": 134.1697, + "P3_top_sv_share": 0.131902, + "P4_dist_from_identity": 1.3069, + "P5_top_u_participation": 605.0953, + "P5_top_u_max_abs": 0.0779, + "P5_top_u_argmax_dim": 573, + "fro_norm": 25.8198, + "top_sv": 9.3774, + "dict_anisotropy_raw_meanabscos": 0.9954, + "dict_anisotropy_raw_median": 0.9986, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1019 + }, + { + "layer": 2, + "depth_pct": 18.2, + "d_model": 768, + "P1_stable_rank": 6.5058, + "P2_effective_rank": 131.528, + "P3_top_sv_share": 0.153709, + "P4_dist_from_identity": 1.3009, + "P5_top_u_participation": 622.3431, + "P5_top_u_max_abs": 0.0757, + "P5_top_u_argmax_dim": 573, + "fro_norm": 24.9842, + "top_sv": 9.7953, + "dict_anisotropy_raw_meanabscos": 0.9957, + "dict_anisotropy_raw_median": 0.9987, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1022 + }, + { + "layer": 3, + "depth_pct": 27.3, + "d_model": 768, + "P1_stable_rank": 6.0896, + "P2_effective_rank": 138.8577, + "P3_top_sv_share": 0.164215, + "P4_dist_from_identity": 1.2314, + "P5_top_u_participation": 657.8771, + "P5_top_u_max_abs": 0.0641, + "P5_top_u_argmax_dim": 573, + "fro_norm": 25.7388, + "top_sv": 10.4303, + "dict_anisotropy_raw_meanabscos": 0.9962, + "dict_anisotropy_raw_median": 0.9988, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1027 + }, + { + "layer": 4, + "depth_pct": 36.4, + "d_model": 768, + "P1_stable_rank": 5.5862, + "P2_effective_rank": 143.9211, + "P3_top_sv_share": 0.179012, + "P4_dist_from_identity": 1.1258, + "P5_top_u_participation": 666.2648, + "P5_top_u_max_abs": 0.0636, + "P5_top_u_argmax_dim": 573, + "fro_norm": 28.824, + "top_sv": 12.1954, + "dict_anisotropy_raw_meanabscos": 0.9966, + "dict_anisotropy_raw_median": 0.9989, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1031 + }, + { + "layer": 5, + "depth_pct": 45.5, + "d_model": 768, + "P1_stable_rank": 5.2699, + "P2_effective_rank": 155.5179, + "P3_top_sv_share": 0.189759, + "P4_dist_from_identity": 1.0325, + "P5_top_u_participation": 670.7905, + "P5_top_u_max_abs": 0.0682, + "P5_top_u_argmax_dim": 573, + "fro_norm": 30.8919, + "top_sv": 13.457, + "dict_anisotropy_raw_meanabscos": 0.9962, + "dict_anisotropy_raw_median": 0.999, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1027 + }, + { + "layer": 6, + "depth_pct": 54.5, + "d_model": 768, + "P1_stable_rank": 7.2423, + "P2_effective_rank": 210.8613, + "P3_top_sv_share": 0.138077, + "P4_dist_from_identity": 0.9393, + "P5_top_u_participation": 659.8798, + "P5_top_u_max_abs": 0.0736, + "P5_top_u_argmax_dim": 94, + "fro_norm": 32.5112, + "top_sv": 12.0808, + "dict_anisotropy_raw_meanabscos": 0.9949, + "dict_anisotropy_raw_median": 0.9987, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1014 + }, + { + "layer": 7, + "depth_pct": 63.6, + "d_model": 768, + "P1_stable_rank": 6.3295, + "P2_effective_rank": 243.6529, + "P3_top_sv_share": 0.15799, + "P4_dist_from_identity": 0.8354, + "P5_top_u_participation": 681.423, + "P5_top_u_max_abs": 0.0704, + "P5_top_u_argmax_dim": 94, + "fro_norm": 30.7643, + "top_sv": 12.2283, + "dict_anisotropy_raw_meanabscos": 0.9959, + "dict_anisotropy_raw_median": 0.9989, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1024 + }, + { + "layer": 8, + "depth_pct": 72.7, + "d_model": 768, + "P1_stable_rank": 7.05, + "P2_effective_rank": 296.5206, + "P3_top_sv_share": 0.141843, + "P4_dist_from_identity": 0.72, + "P5_top_u_participation": 695.0559, + "P5_top_u_max_abs": 0.0676, + "P5_top_u_argmax_dim": 94, + "fro_norm": 33.5036, + "top_sv": 12.6184, + "dict_anisotropy_raw_meanabscos": 0.996, + "dict_anisotropy_raw_median": 0.9989, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1025 + }, + { + "layer": 9, + "depth_pct": 81.8, + "d_model": 768, + "P1_stable_rank": 7.2653, + "P2_effective_rank": 349.2321, + "P3_top_sv_share": 0.13764, + "P4_dist_from_identity": 0.6182, + "P5_top_u_participation": 713.5575, + "P5_top_u_max_abs": 0.0574, + "P5_top_u_argmax_dim": 94, + "fro_norm": 33.9328, + "top_sv": 12.5893, + "dict_anisotropy_raw_meanabscos": 0.9963, + "dict_anisotropy_raw_median": 0.9988, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1028 + }, + { + "layer": 10, + "depth_pct": 90.9, + "d_model": 768, + "P1_stable_rank": 5.0004, + "P2_effective_rank": 297.4037, + "P3_top_sv_share": 0.199985, + "P4_dist_from_identity": 0.5567, + "P5_top_u_participation": 704.3879, + "P5_top_u_max_abs": 0.0581, + "P5_top_u_argmax_dim": 621, + "fro_norm": 34.244, + "top_sv": 15.3142, + "dict_anisotropy_raw_meanabscos": 0.998, + "dict_anisotropy_raw_median": 0.9994, + "untransported_baseline_meanabscos": 0.8935, + "transport_amplification": 0.1045 + } + ], + "runtime_s": 9.5 +} \ No newline at end of file diff --git a/results/t4_410m_db128.json b/results/t4_410m_db128.json new file mode 100644 index 0000000000000000000000000000000000000000..1e3026dd65e8bd321e3944010bca5e0052e23efa --- /dev/null +++ b/results/t4_410m_db128.json @@ -0,0 +1,441 @@ +{ + "experiment": "T4 geometry \u2014 Claim 2 predictors from J alone", + "status": "EXPLORATORY unless a committed prereg is cited here", + "prereg": null, + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "lens_n_prompts": 200, + "d_model": 1024, + "n_heldout_tokens": 1500, + "centered": false, + "tied_embeddings": false, + "rogue_coordinate_test": { + "note": "Gemma-2 (TIED) had PR(u1) 1.2-1.4 of 2304 \u2014 near one-hot. Pythia is UNTIED.", + "min_PR": 36.4668, + "max_PR": 152.5645, + "median_PR": 86.37329864501953, + "near_one_hot_layers": [] + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "layer": 0, + "depth_pct": 0.0, + "d_model": 1024, + "P1_stable_rank": 16.1584, + "P2_effective_rank": 159.8268, + "P3_top_sv_share": 0.061887, + "P4_dist_from_identity": 3.5052, + "P5_top_u_participation": 133.879, + "P5_top_u_max_abs": 0.2084, + "P5_top_u_argmax_dim": 737, + "fro_norm": 9.0597, + "top_sv": 2.2539, + "dict_anisotropy_raw_meanabscos": 0.2841, + "dict_anisotropy_raw_median": 0.2812, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2014 + }, + { + "layer": 1, + "depth_pct": 4.3, + "d_model": 1024, + "P1_stable_rank": 15.7467, + "P2_effective_rank": 163.0762, + "P3_top_sv_share": 0.063505, + "P4_dist_from_identity": 3.7025, + "P5_top_u_participation": 134.1913, + "P5_top_u_max_abs": 0.2095, + "P5_top_u_argmax_dim": 737, + "fro_norm": 8.5295, + "top_sv": 2.1495, + "dict_anisotropy_raw_meanabscos": 0.2937, + "dict_anisotropy_raw_median": 0.2909, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2111 + }, + { + "layer": 2, + "depth_pct": 8.7, + "d_model": 1024, + "P1_stable_rank": 15.57, + "P2_effective_rank": 163.2325, + "P3_top_sv_share": 0.064226, + "P4_dist_from_identity": 3.4259, + "P5_top_u_participation": 149.2065, + "P5_top_u_max_abs": 0.2022, + "P5_top_u_argmax_dim": 737, + "fro_norm": 9.217, + "top_sv": 2.3359, + "dict_anisotropy_raw_meanabscos": 0.2991, + "dict_anisotropy_raw_median": 0.2966, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2165 + }, + { + "layer": 3, + "depth_pct": 13.0, + "d_model": 1024, + "P1_stable_rank": 14.2892, + "P2_effective_rank": 170.6733, + "P3_top_sv_share": 0.069983, + "P4_dist_from_identity": 3.4833, + "P5_top_u_participation": 98.9524, + "P5_top_u_max_abs": 0.231, + "P5_top_u_argmax_dim": 453, + "fro_norm": 8.9813, + "top_sv": 2.376, + "dict_anisotropy_raw_meanabscos": 0.3017, + "dict_anisotropy_raw_median": 0.2974, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2191 + }, + { + "layer": 4, + "depth_pct": 17.4, + "d_model": 1024, + "P1_stable_rank": 14.06, + "P2_effective_rank": 175.9493, + "P3_top_sv_share": 0.071124, + "P4_dist_from_identity": 3.0372, + "P5_top_u_participation": 122.2919, + "P5_top_u_max_abs": 0.2178, + "P5_top_u_argmax_dim": 737, + "fro_norm": 10.2777, + "top_sv": 2.741, + "dict_anisotropy_raw_meanabscos": 0.3174, + "dict_anisotropy_raw_median": 0.3138, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2348 + }, + { + "layer": 5, + "depth_pct": 21.7, + "d_model": 1024, + "P1_stable_rank": 12.039, + "P2_effective_rank": 184.904, + "P3_top_sv_share": 0.083064, + "P4_dist_from_identity": 2.9333, + "P5_top_u_participation": 108.7094, + "P5_top_u_max_abs": 0.2231, + "P5_top_u_argmax_dim": 453, + "fro_norm": 10.4906, + "top_sv": 3.0235, + "dict_anisotropy_raw_meanabscos": 0.3426, + "dict_anisotropy_raw_median": 0.3397, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.26 + }, + { + "layer": 6, + "depth_pct": 26.1, + "d_model": 1024, + "P1_stable_rank": 12.6213, + "P2_effective_rank": 199.5389, + "P3_top_sv_share": 0.079231, + "P4_dist_from_identity": 2.8082, + "P5_top_u_participation": 104.2538, + "P5_top_u_max_abs": 0.2258, + "P5_top_u_argmax_dim": 737, + "fro_norm": 10.8761, + "top_sv": 3.0615, + "dict_anisotropy_raw_meanabscos": 0.3353, + "dict_anisotropy_raw_median": 0.33, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2527 + }, + { + "layer": 7, + "depth_pct": 30.4, + "d_model": 1024, + "P1_stable_rank": 12.279, + "P2_effective_rank": 209.5655, + "P3_top_sv_share": 0.08144, + "P4_dist_from_identity": 2.514, + "P5_top_u_participation": 91.5043, + "P5_top_u_max_abs": 0.2438, + "P5_top_u_argmax_dim": 453, + "fro_norm": 12.0562, + "top_sv": 3.4407, + "dict_anisotropy_raw_meanabscos": 0.3412, + "dict_anisotropy_raw_median": 0.3353, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2586 + }, + { + "layer": 8, + "depth_pct": 34.8, + "d_model": 1024, + "P1_stable_rank": 10.8515, + "P2_effective_rank": 211.3335, + "P3_top_sv_share": 0.092153, + "P4_dist_from_identity": 2.1756, + "P5_top_u_participation": 86.3733, + "P5_top_u_max_abs": 0.2471, + "P5_top_u_argmax_dim": 453, + "fro_norm": 13.8437, + "top_sv": 4.2026, + "dict_anisotropy_raw_meanabscos": 0.3565, + "dict_anisotropy_raw_median": 0.3517, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2739 + }, + { + "layer": 9, + "depth_pct": 39.1, + "d_model": 1024, + "P1_stable_rank": 12.8053, + "P2_effective_rank": 246.3744, + "P3_top_sv_share": 0.078093, + "P4_dist_from_identity": 1.9423, + "P5_top_u_participation": 87.278, + "P5_top_u_max_abs": 0.2524, + "P5_top_u_argmax_dim": 453, + "fro_norm": 15.2358, + "top_sv": 4.2578, + "dict_anisotropy_raw_meanabscos": 0.3436, + "dict_anisotropy_raw_median": 0.3378, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.261 + }, + { + "layer": 10, + "depth_pct": 43.5, + "d_model": 1024, + "P1_stable_rank": 12.428, + "P2_effective_rank": 260.8736, + "P3_top_sv_share": 0.080464, + "P4_dist_from_identity": 1.6837, + "P5_top_u_participation": 86.3016, + "P5_top_u_max_abs": 0.2525, + "P5_top_u_argmax_dim": 453, + "fro_norm": 17.3087, + "top_sv": 4.91, + "dict_anisotropy_raw_meanabscos": 0.3545, + "dict_anisotropy_raw_median": 0.3486, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2718 + }, + { + "layer": 11, + "depth_pct": 47.8, + "d_model": 1024, + "P1_stable_rank": 13.626, + "P2_effective_rank": 298.9375, + "P3_top_sv_share": 0.073389, + "P4_dist_from_identity": 1.4108, + "P5_top_u_participation": 109.3893, + "P5_top_u_max_abs": 0.2266, + "P5_top_u_argmax_dim": 453, + "fro_norm": 20.2149, + "top_sv": 5.4765, + "dict_anisotropy_raw_meanabscos": 0.3609, + "dict_anisotropy_raw_median": 0.3555, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.2782 + }, + { + "layer": 12, + "depth_pct": 52.2, + "d_model": 1024, + "P1_stable_rank": 23.3402, + "P2_effective_rank": 379.3736, + "P3_top_sv_share": 0.042845, + "P4_dist_from_identity": 1.2571, + "P5_top_u_participation": 68.274, + "P5_top_u_max_abs": 0.2889, + "P5_top_u_argmax_dim": 453, + "fro_norm": 21.5507, + "top_sv": 4.4609, + "dict_anisotropy_raw_meanabscos": 0.2577, + "dict_anisotropy_raw_median": 0.2482, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.1751 + }, + { + "layer": 13, + "depth_pct": 56.5, + "d_model": 1024, + "P1_stable_rank": 25.4164, + "P2_effective_rank": 427.1878, + "P3_top_sv_share": 0.039345, + "P4_dist_from_identity": 1.1132, + "P5_top_u_participation": 82.3189, + "P5_top_u_max_abs": 0.2697, + "P5_top_u_argmax_dim": 453, + "fro_norm": 23.2668, + "top_sv": 4.6153, + "dict_anisotropy_raw_meanabscos": 0.2489, + "dict_anisotropy_raw_median": 0.2393, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.1663 + }, + { + "layer": 14, + "depth_pct": 60.9, + "d_model": 1024, + "P1_stable_rank": 29.5334, + "P2_effective_rank": 462.7931, + "P3_top_sv_share": 0.03386, + "P4_dist_from_identity": 0.948, + "P5_top_u_participation": 85.4189, + "P5_top_u_max_abs": 0.2685, + "P5_top_u_argmax_dim": 453, + "fro_norm": 26.6931, + "top_sv": 4.912, + "dict_anisotropy_raw_meanabscos": 0.2288, + "dict_anisotropy_raw_median": 0.2201, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.1462 + }, + { + "layer": 15, + "depth_pct": 65.2, + "d_model": 1024, + "P1_stable_rank": 29.3374, + "P2_effective_rank": 484.1944, + "P3_top_sv_share": 0.034086, + "P4_dist_from_identity": 0.8408, + "P5_top_u_participation": 68.281, + "P5_top_u_max_abs": 0.2896, + "P5_top_u_argmax_dim": 453, + "fro_norm": 29.704, + "top_sv": 5.4843, + "dict_anisotropy_raw_meanabscos": 0.2228, + "dict_anisotropy_raw_median": 0.2136, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.1402 + }, + { + "layer": 16, + "depth_pct": 69.6, + "d_model": 1024, + "P1_stable_rank": 39.0386, + "P2_effective_rank": 583.4448, + "P3_top_sv_share": 0.025616, + "P4_dist_from_identity": 0.6902, + "P5_top_u_participation": 63.6312, + "P5_top_u_max_abs": 0.305, + "P5_top_u_argmax_dim": 453, + "fro_norm": 32.7859, + "top_sv": 5.2476, + "dict_anisotropy_raw_meanabscos": 0.1883, + "dict_anisotropy_raw_median": 0.1784, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.1057 + }, + { + "layer": 17, + "depth_pct": 73.9, + "d_model": 1024, + "P1_stable_rank": 44.0812, + "P2_effective_rank": 658.7575, + "P3_top_sv_share": 0.022685, + "P4_dist_from_identity": 0.584, + "P5_top_u_participation": 52.1725, + "P5_top_u_max_abs": 0.3315, + "P5_top_u_argmax_dim": 453, + "fro_norm": 34.8828, + "top_sv": 5.2543, + "dict_anisotropy_raw_meanabscos": 0.1719, + "dict_anisotropy_raw_median": 0.1615, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.0893 + }, + { + "layer": 18, + "depth_pct": 78.3, + "d_model": 1024, + "P1_stable_rank": 51.4734, + "P2_effective_rank": 701.3936, + "P3_top_sv_share": 0.019428, + "P4_dist_from_identity": 0.5322, + "P5_top_u_participation": 56.91, + "P5_top_u_max_abs": 0.3064, + "P5_top_u_argmax_dim": 453, + "fro_norm": 36.4578, + "top_sv": 5.0819, + "dict_anisotropy_raw_meanabscos": 0.1585, + "dict_anisotropy_raw_median": 0.1484, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.0759 + }, + { + "layer": 19, + "depth_pct": 82.6, + "d_model": 1024, + "P1_stable_rank": 62.1284, + "P2_effective_rank": 768.2194, + "P3_top_sv_share": 0.016096, + "P4_dist_from_identity": 0.4659, + "P5_top_u_participation": 73.0953, + "P5_top_u_max_abs": 0.2761, + "P5_top_u_argmax_dim": 453, + "fro_norm": 36.4498, + "top_sv": 4.6247, + "dict_anisotropy_raw_meanabscos": 0.1462, + "dict_anisotropy_raw_median": 0.1376, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.0636 + }, + { + "layer": 20, + "depth_pct": 87.0, + "d_model": 1024, + "P1_stable_rank": 61.3349, + "P2_effective_rank": 805.1058, + "P3_top_sv_share": 0.016304, + "P4_dist_from_identity": 0.4288, + "P5_top_u_participation": 36.4668, + "P5_top_u_max_abs": 0.3703, + "P5_top_u_argmax_dim": 453, + "fro_norm": 36.6373, + "top_sv": 4.6786, + "dict_anisotropy_raw_meanabscos": 0.145, + "dict_anisotropy_raw_median": 0.1365, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.0624 + }, + { + "layer": 21, + "depth_pct": 91.3, + "d_model": 1024, + "P1_stable_rank": 66.8664, + "P2_effective_rank": 838.1406, + "P3_top_sv_share": 0.014955, + "P4_dist_from_identity": 0.3867, + "P5_top_u_participation": 45.0857, + "P5_top_u_max_abs": 0.344, + "P5_top_u_argmax_dim": 453, + "fro_norm": 36.5132, + "top_sv": 4.4657, + "dict_anisotropy_raw_meanabscos": 0.1428, + "dict_anisotropy_raw_median": 0.1343, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.0602 + }, + { + "layer": 22, + "depth_pct": 95.7, + "d_model": 1024, + "P1_stable_rank": 39.9711, + "P2_effective_rank": 878.2449, + "P3_top_sv_share": 0.025018, + "P4_dist_from_identity": 0.3204, + "P5_top_u_participation": 152.5645, + "P5_top_u_max_abs": 0.1871, + "P5_top_u_argmax_dim": 180, + "fro_norm": 34.4461, + "top_sv": 5.4491, + "dict_anisotropy_raw_meanabscos": 0.2091, + "dict_anisotropy_raw_median": 0.2012, + "untransported_baseline_meanabscos": 0.0826, + "transport_amplification": 0.1264 + } + ], + "runtime_s": 12.6 +} \ No newline at end of file diff --git a/results/t4_70m_db128.json b/results/t4_70m_db128.json new file mode 100644 index 0000000000000000000000000000000000000000..9722815aa128958b481681e22917516330925f8a --- /dev/null +++ b/results/t4_70m_db128.json @@ -0,0 +1,117 @@ +{ + "experiment": "T4 geometry \u2014 Claim 2 predictors from J alone", + "status": "EXPLORATORY unless a committed prereg is cited here", + "prereg": null, + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200_db128.pt", + "lens_n_prompts": 200, + "d_model": 512, + "n_heldout_tokens": 1500, + "centered": false, + "tied_embeddings": false, + "rogue_coordinate_test": { + "note": "Gemma-2 (TIED) had PR(u1) 1.2-1.4 of 2304 \u2014 near one-hot. Pythia is UNTIED.", + "min_PR": 383.53, + "max_PR": 411.6075, + "median_PR": 400.8973083496094, + "near_one_hot_layers": [] + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "layer": 0, + "depth_pct": 0.0, + "d_model": 512, + "P1_stable_rank": 8.5168, + "P2_effective_rank": 115.1231, + "P3_top_sv_share": 0.117415, + "P4_dist_from_identity": 1.0047, + "P5_top_u_participation": 383.53, + "P5_top_u_max_abs": 0.0846, + "P5_top_u_argmax_dim": 313, + "fro_norm": 50.5124, + "top_sv": 17.3085, + "dict_anisotropy_raw_meanabscos": 0.9989, + "dict_anisotropy_raw_median": 0.999, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0486 + }, + { + "layer": 1, + "depth_pct": 20.0, + "d_model": 512, + "P1_stable_rank": 5.1067, + "P2_effective_rank": 99.9115, + "P3_top_sv_share": 0.195822, + "P4_dist_from_identity": 1.0089, + "P5_top_u_participation": 395.4596, + "P5_top_u_max_abs": 0.0823, + "P5_top_u_argmax_dim": 280, + "fro_norm": 40.2718, + "top_sv": 17.821, + "dict_anisotropy_raw_meanabscos": 0.9994, + "dict_anisotropy_raw_median": 0.9995, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0491 + }, + { + "layer": 2, + "depth_pct": 40.0, + "d_model": 512, + "P1_stable_rank": 5.4425, + "P2_effective_rank": 122.6625, + "P3_top_sv_share": 0.18374, + "P4_dist_from_identity": 0.9613, + "P5_top_u_participation": 400.8973, + "P5_top_u_max_abs": 0.0841, + "P5_top_u_argmax_dim": 280, + "fro_norm": 37.1095, + "top_sv": 15.9069, + "dict_anisotropy_raw_meanabscos": 0.9994, + "dict_anisotropy_raw_median": 0.9995, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0491 + }, + { + "layer": 3, + "depth_pct": 60.0, + "d_model": 512, + "P1_stable_rank": 4.8645, + "P2_effective_rank": 138.4242, + "P3_top_sv_share": 0.205572, + "P4_dist_from_identity": 0.8362, + "P5_top_u_participation": 407.299, + "P5_top_u_max_abs": 0.0862, + "P5_top_u_argmax_dim": 271, + "fro_norm": 34.2093, + "top_sv": 15.5105, + "dict_anisotropy_raw_meanabscos": 0.9995, + "dict_anisotropy_raw_median": 0.9995, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0492 + }, + { + "layer": 4, + "depth_pct": 80.0, + "d_model": 512, + "P1_stable_rank": 3.3387, + "P2_effective_rank": 116.3922, + "P3_top_sv_share": 0.299514, + "P4_dist_from_identity": 0.7235, + "P5_top_u_participation": 411.6075, + "P5_top_u_max_abs": 0.0779, + "P5_top_u_argmax_dim": 11, + "fro_norm": 34.163, + "top_sv": 18.6967, + "dict_anisotropy_raw_meanabscos": 0.9997, + "dict_anisotropy_raw_median": 0.9997, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0494 + } + ], + "runtime_s": 9.2 +} \ No newline at end of file diff --git a/results/t4_70m_n200.json b/results/t4_70m_n200.json new file mode 100644 index 0000000000000000000000000000000000000000..5d3d0121d4da726686c871fb8f4a0e95231761be --- /dev/null +++ b/results/t4_70m_n200.json @@ -0,0 +1,117 @@ +{ + "experiment": "T4 geometry \u2014 Claim 2 predictors from J alone", + "status": "EXPLORATORY unless a committed prereg is cited here", + "prereg": null, + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200.pt", + "lens_n_prompts": 200, + "d_model": 512, + "n_heldout_tokens": 1500, + "centered": false, + "tied_embeddings": false, + "rogue_coordinate_test": { + "note": "Gemma-2 (TIED) had PR(u1) 1.2-1.4 of 2304 \u2014 near one-hot. Pythia is UNTIED.", + "min_PR": 383.769, + "max_PR": 411.6081, + "median_PR": 401.0574035644531, + "near_one_hot_layers": [] + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "layer": 0, + "depth_pct": 0.0, + "d_model": 512, + "P1_stable_rank": 8.5712, + "P2_effective_rank": 115.2678, + "P3_top_sv_share": 0.11667, + "P4_dist_from_identity": 1.0047, + "P5_top_u_participation": 383.769, + "P5_top_u_max_abs": 0.0847, + "P5_top_u_argmax_dim": 313, + "fro_norm": 50.471, + "top_sv": 17.2394, + "dict_anisotropy_raw_meanabscos": 0.9989, + "dict_anisotropy_raw_median": 0.999, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0486 + }, + { + "layer": 1, + "depth_pct": 20.0, + "d_model": 512, + "P1_stable_rank": 5.1159, + "P2_effective_rank": 99.9971, + "P3_top_sv_share": 0.195469, + "P4_dist_from_identity": 1.009, + "P5_top_u_participation": 395.5214, + "P5_top_u_max_abs": 0.0818, + "P5_top_u_argmax_dim": 280, + "fro_norm": 40.2482, + "top_sv": 17.7945, + "dict_anisotropy_raw_meanabscos": 0.9994, + "dict_anisotropy_raw_median": 0.9995, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0491 + }, + { + "layer": 2, + "depth_pct": 40.0, + "d_model": 512, + "P1_stable_rank": 5.4347, + "P2_effective_rank": 122.5161, + "P3_top_sv_share": 0.184004, + "P4_dist_from_identity": 0.9613, + "P5_top_u_participation": 401.0574, + "P5_top_u_max_abs": 0.0837, + "P5_top_u_argmax_dim": 280, + "fro_norm": 37.1048, + "top_sv": 15.9163, + "dict_anisotropy_raw_meanabscos": 0.9994, + "dict_anisotropy_raw_median": 0.9995, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0491 + }, + { + "layer": 3, + "depth_pct": 60.0, + "d_model": 512, + "P1_stable_rank": 4.8713, + "P2_effective_rank": 138.5889, + "P3_top_sv_share": 0.205284, + "P4_dist_from_identity": 0.8361, + "P5_top_u_participation": 407.2848, + "P5_top_u_max_abs": 0.0862, + "P5_top_u_argmax_dim": 271, + "fro_norm": 34.2011, + "top_sv": 15.4959, + "dict_anisotropy_raw_meanabscos": 0.9995, + "dict_anisotropy_raw_median": 0.9995, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0492 + }, + { + "layer": 4, + "depth_pct": 80.0, + "d_model": 512, + "P1_stable_rank": 3.3388, + "P2_effective_rank": 116.393, + "P3_top_sv_share": 0.299511, + "P4_dist_from_identity": 0.7235, + "P5_top_u_participation": 411.6081, + "P5_top_u_max_abs": 0.0779, + "P5_top_u_argmax_dim": 11, + "fro_norm": 34.1619, + "top_sv": 18.6961, + "dict_anisotropy_raw_meanabscos": 0.9997, + "dict_anisotropy_raw_median": 0.9997, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0494 + } + ], + "runtime_s": 5.5 +} \ No newline at end of file diff --git a/results/t4_smoke_70m.json b/results/t4_smoke_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..351a84bb97e88d4df9c1e5bb1eea64296e4188df --- /dev/null +++ b/results/t4_smoke_70m.json @@ -0,0 +1,117 @@ +{ + "experiment": "T4 geometry \u2014 Claim 2 predictors from J alone", + "status": "EXPLORATORY unless a committed prereg is cited here", + "prereg": null, + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/t0_smoke_70m_lens.pt", + "lens_n_prompts": 3, + "d_model": 512, + "n_heldout_tokens": 1500, + "centered": false, + "tied_embeddings": false, + "rogue_coordinate_test": { + "note": "Gemma-2 (TIED) had PR(u1) 1.2-1.4 of 2304 \u2014 near one-hot. Pythia is UNTIED.", + "min_PR": 371.6033, + "max_PR": 406.4131, + "median_PR": 390.5718994140625, + "near_one_hot_layers": [] + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "layer": 0, + "depth_pct": 0.0, + "d_model": 512, + "P1_stable_rank": 7.3839, + "P2_effective_rank": 126.0745, + "P3_top_sv_share": 0.135429, + "P4_dist_from_identity": 1.0297, + "P5_top_u_participation": 371.6033, + "P5_top_u_max_abs": 0.0928, + "P5_top_u_argmax_dim": 336, + "fro_norm": 33.3809, + "top_sv": 12.2844, + "dict_anisotropy_raw_meanabscos": 0.9991, + "dict_anisotropy_raw_median": 0.9992, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0488 + }, + { + "layer": 1, + "depth_pct": 20.0, + "d_model": 512, + "P1_stable_rank": 6.2656, + "P2_effective_rank": 131.2069, + "P3_top_sv_share": 0.159602, + "P4_dist_from_identity": 1.0292, + "P5_top_u_participation": 388.4529, + "P5_top_u_max_abs": 0.0886, + "P5_top_u_argmax_dim": 336, + "fro_norm": 28.4985, + "top_sv": 11.3852, + "dict_anisotropy_raw_meanabscos": 0.9993, + "dict_anisotropy_raw_median": 0.9994, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.049 + }, + { + "layer": 2, + "depth_pct": 40.0, + "d_model": 512, + "P1_stable_rank": 5.6284, + "P2_effective_rank": 140.3499, + "P3_top_sv_share": 0.177669, + "P4_dist_from_identity": 0.9485, + "P5_top_u_participation": 390.5719, + "P5_top_u_max_abs": 0.0889, + "P5_top_u_argmax_dim": 336, + "fro_norm": 29.1157, + "top_sv": 12.2725, + "dict_anisotropy_raw_meanabscos": 0.9994, + "dict_anisotropy_raw_median": 0.9994, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0491 + }, + { + "layer": 3, + "depth_pct": 60.0, + "d_model": 512, + "P1_stable_rank": 4.8151, + "P2_effective_rank": 147.9312, + "P3_top_sv_share": 0.20768, + "P4_dist_from_identity": 0.8189, + "P5_top_u_participation": 401.2303, + "P5_top_u_max_abs": 0.0866, + "P5_top_u_argmax_dim": 390, + "fro_norm": 30.7395, + "top_sv": 14.0086, + "dict_anisotropy_raw_meanabscos": 0.9995, + "dict_anisotropy_raw_median": 0.9996, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0492 + }, + { + "layer": 4, + "depth_pct": 80.0, + "d_model": 512, + "P1_stable_rank": 3.4823, + "P2_effective_rank": 122.753, + "P3_top_sv_share": 0.287163, + "P4_dist_from_identity": 0.7303, + "P5_top_u_participation": 406.4131, + "P5_top_u_max_abs": 0.083, + "P5_top_u_argmax_dim": 390, + "fro_norm": 33.6005, + "top_sv": 18.0057, + "dict_anisotropy_raw_meanabscos": 0.9996, + "dict_anisotropy_raw_median": 0.9997, + "untransported_baseline_meanabscos": 0.9503, + "transport_amplification": 0.0494 + } + ], + "runtime_s": 5.7 +} \ No newline at end of file diff --git a/results/t5_write_410m.json b/results/t5_write_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..a10d030c6ff957888fbe665d964191b5ef963d6b --- /dev/null +++ b/results/t5_write_410m.json @@ -0,0 +1,8812 @@ +{ + "experiment": "T5 write battery \u2014 anchor \u00a72.5 swap on Pythia", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration; not a confirmatory result", + "prereg": null, + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "lens_n_prompts": 200, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "band_lo": 9, + "alphas": [ + 1.0, + 2.0, + 4.0 + ], + "positions": "all", + "clamp": "band, every forward", + "centered": false, + "n_cells_run": 39, + "n_cells_void_no_oracle": 20, + "n_cells_valid": 19, + "median_clean_rank_tgt": 63.0, + "summary": { + "identity": { + "n": 19, + "flips": 1, + "median_rank_tgt": 63.0, + "median_dlogp_tgt": -6.7442, + "median_kl_vs_clean": 0.0, + "median_delta_norm": null + }, + "swap@a1": { + "n": 19, + "flips": 1, + "median_rank_tgt": 4.0, + "median_dlogp_tgt": -3.9653, + "median_kl_vs_clean": 0.2812, + "median_delta_norm": 30.142 + }, + "rand_target@a1": { + "n": 19, + "flips": 1, + "median_rank_tgt": 44.0, + "median_dlogp_tgt": -5.803, + "median_kl_vs_clean": 2.3417, + "median_delta_norm": 30.142 + }, + "orth@a1": { + "n": 19, + "flips": 0, + "median_rank_tgt": 79.0, + "median_dlogp_tgt": -6.9788, + "median_kl_vs_clean": 0.3202, + "median_delta_norm": 30.142 + }, + "oracle@a1": { + "n": 19, + "flips": 4, + "median_rank_tgt": 2.0, + "median_dlogp_tgt": -3.0018, + "median_kl_vs_clean": 0.4342, + "median_delta_norm": 26.416 + }, + "swap@a2": { + "n": 19, + "flips": 0, + "median_rank_tgt": 18.0, + "median_dlogp_tgt": -25.7759, + "median_kl_vs_clean": 31.331, + "median_delta_norm": 20618.637 + }, + "rand_target@a2": { + "n": 19, + "flips": 0, + "median_rank_tgt": 44746.0, + "median_dlogp_tgt": -21.9855, + "median_kl_vs_clean": 20.4331, + "median_delta_norm": 20618.639 + }, + "orth@a2": { + "n": 19, + "flips": 0, + "median_rank_tgt": 401.0, + "median_dlogp_tgt": -33.158, + "median_kl_vs_clean": 23.8779, + "median_delta_norm": 20618.639 + }, + "oracle@a2": { + "n": 19, + "flips": 15, + "median_rank_tgt": 0.0, + "median_dlogp_tgt": -0.0001, + "median_kl_vs_clean": 23.3937, + "median_delta_norm": 21354.232 + }, + "swap@a4": { + "n": 19, + "flips": 0, + "median_rank_tgt": 11.0, + "median_dlogp_tgt": -26.3437, + "median_kl_vs_clean": 33.1498, + "median_delta_norm": 206850416.0 + }, + "rand_target@a4": { + "n": 19, + "flips": 0, + "median_rank_tgt": 37897.0, + "median_dlogp_tgt": -23.391, + "median_kl_vs_clean": 20.835, + "median_delta_norm": 206850432.0 + }, + "orth@a4": { + "n": 19, + "flips": 0, + "median_rank_tgt": 359.0, + "median_dlogp_tgt": -34.5077, + "median_kl_vs_clean": 24.1777, + "median_delta_norm": 206850400.0 + }, + "oracle@a4": { + "n": 19, + "flips": 19, + "median_rank_tgt": 0.0, + "median_dlogp_tgt": 0.0, + "median_kl_vs_clean": 26.4107, + "median_delta_norm": 211710224.0 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "relation": "capital", + "src": "France", + "tgt": "China", + "prompt": "The capital of France is the city of", + "answer_src": " Paris", + "answer_tgt": " Beijing", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Paris", + "rank_tgt": 1265, + "rank_src": 0, + "logp_tgt": -11.8642, + "logp_src": -0.7679, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Paris", + "rank_tgt": 1265, + "rank_src": 0, + "logp_tgt": -11.8642, + "logp_src": -0.7679, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Paris", + "rank_tgt": 73, + "rank_src": 0, + "logp_tgt": -7.1616, + "logp_src": -0.4691, + "flip_to_tgt": false, + "kl_vs_clean": 0.4169, + "delta_norm_band_sum": 49.3557, + "delta_norm_per_layer": { + "9": 1.5889, + "10": 1.2984, + "11": 0.5589, + "12": 0.3861, + "13": 0.5054, + "14": 0.1799, + "15": 0.501, + "16": 0.6888, + "17": 5.9919, + "18": 5.8124, + "19": 8.9641, + "20": 8.8459, + "21": 6.7692, + "22": 7.2648 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a1": { + "top1": " Los", + "rank_tgt": 595, + "rank_src": 1290, + "logp_tgt": -9.4569, + "logp_src": -10.6516, + "flip_to_tgt": false, + "kl_vs_clean": 6.7266, + "delta_norm_band_sum": 49.3557, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Paris", + "rank_tgt": 626, + "rank_src": 0, + "logp_tgt": -9.9644, + "logp_src": -1.1612, + "flip_to_tgt": false, + "kl_vs_clean": 0.3202, + "delta_norm_band_sum": 49.3557, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Paris", + "rank_tgt": 320, + "rank_src": 0, + "logp_tgt": -9.7229, + "logp_src": -0.4102, + "flip_to_tgt": false, + "kl_vs_clean": 0.3482, + "delta_norm_band_sum": 54.3414 + }, + "swap@a2": { + "top1": " China", + "rank_tgt": 7, + "rank_src": 173, + "logp_tgt": -28.2795, + "logp_src": -46.1955, + "flip_to_tgt": false, + "kl_vs_clean": 46.8633, + "delta_norm_band_sum": 22660.8039, + "delta_norm_per_layer": { + "9": 2.6301, + "10": 4.1425, + "11": 6.9239, + "12": 16.6892, + "13": 33.7068, + "14": 59.0159, + "15": 113.9353, + "16": 211.0388, + "17": 407.8275, + "18": 762.2454, + "19": 1486.323, + "20": 2774.8054, + "21": 5791.3521, + "22": 10990.168 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a2": { + "top1": " invari", + "rank_tgt": 44746, + "rank_src": 50291, + "logp_tgt": -21.2893, + "logp_src": -37.3697, + "flip_to_tgt": false, + "kl_vs_clean": 28.8405, + "delta_norm_band_sum": 22660.8042, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " France", + "rank_tgt": 2022, + "rank_src": 5, + "logp_tgt": -39.8271, + "logp_src": -19.7769, + "flip_to_tgt": false, + "kl_vs_clean": 25.48, + "delta_norm_band_sum": 22660.8039, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Beijing", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0, + "logp_src": -11.8916, + "flip_to_tgt": true, + "kl_vs_clean": 24.6404, + "delta_norm_band_sum": 11252.1898 + }, + "swap@a4": { + "top1": " China", + "rank_tgt": 4, + "rank_src": 1327, + "logp_tgt": -27.6583, + "logp_src": -56.0404, + "flip_to_tgt": false, + "kl_vs_clean": 53.8342, + "delta_norm_band_sum": 193272812.0985, + "delta_norm_per_layer": { + "9": 5.0107, + "10": 16.0553, + "11": 60.0538, + "12": 227.0775, + "13": 914.3281, + "14": 3332.782, + "15": 13042.2754, + "16": 46059.2188, + "17": 167146.7969, + "18": 620796.0, + "19": 2406925.5, + "20": 9065991.0, + "21": 37631896.0, + "22": 143316400.0 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a4": { + "top1": " France", + "rank_tgt": 1919, + "rank_src": 4, + "logp_tgt": -24.7525, + "logp_src": -11.2099, + "flip_to_tgt": false, + "kl_vs_clean": 14.7321, + "delta_norm_band_sum": 193272787.8472, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " France", + "rank_tgt": 1678, + "rank_src": 5, + "logp_tgt": -41.8188, + "logp_src": -20.3305, + "flip_to_tgt": false, + "kl_vs_clean": 26.8203, + "delta_norm_band_sum": 193272821.033, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Beijing", + "rank_tgt": 0, + "rank_src": 5, + "logp_tgt": 0.0, + "logp_src": -24.4725, + "flip_to_tgt": true, + "kl_vs_clean": 33.173, + "delta_norm_band_sum": 105250781.3089 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "capital", + "src": "China", + "tgt": "Japan", + "prompt": "The capital of China is the city of", + "answer_src": " Beijing", + "answer_tgt": " Tokyo", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Beijing", + "rank_tgt": 63, + "rank_src": 0, + "logp_tgt": -7.4638, + "logp_src": -1.3493, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Beijing", + "rank_tgt": 63, + "rank_src": 0, + "logp_tgt": -7.4638, + "logp_src": -1.3493, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Shanghai", + "rank_tgt": 2, + "rank_src": 1, + "logp_tgt": -2.9131, + "logp_src": -2.0393, + "flip_to_tgt": false, + "kl_vs_clean": 0.2796, + "delta_norm_band_sum": 29.3938, + "delta_norm_per_layer": { + "9": 1.0262, + "10": 1.4564, + "11": 1.2939, + "12": 1.1825, + "13": 0.8234, + "14": 0.0144, + "15": 1.0301, + "16": 1.3822, + "17": 4.567, + "18": 4.5905, + "19": 4.4527, + "20": 3.0133, + "21": 2.3035, + "22": 2.2578 + }, + "gram_cond_max": 9.11 + }, + "rand_target@a1": { + "top1": " Shanghai", + "rank_tgt": 68, + "rank_src": 1, + "logp_tgt": -6.5116, + "logp_src": -2.4018, + "flip_to_tgt": false, + "kl_vs_clean": 0.4867, + "delta_norm_band_sum": 29.3938, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Shanghai", + "rank_tgt": 112, + "rank_src": 1, + "logp_tgt": -7.5645, + "logp_src": -2.3056, + "flip_to_tgt": false, + "kl_vs_clean": 0.3343, + "delta_norm_band_sum": 29.3938, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Shanghai", + "rank_tgt": 2, + "rank_src": 1, + "logp_tgt": -2.7752, + "logp_src": -2.0485, + "flip_to_tgt": false, + "kl_vs_clean": 0.2822, + "delta_norm_band_sum": 14.6171 + }, + "swap@a2": { + "top1": " China", + "rank_tgt": 21, + "rank_src": 8, + "logp_tgt": -34.2861, + "logp_src": -26.5263, + "flip_to_tgt": false, + "kl_vs_clean": 33.7101, + "delta_norm_band_sum": 15589.6899, + "delta_norm_per_layer": { + "9": 1.8697, + "10": 4.1942, + "11": 6.619, + "12": 12.0107, + "13": 23.1814, + "14": 40.504, + "15": 77.018, + "16": 146.4074, + "17": 283.6694, + "18": 535.4216, + "19": 1053.7601, + "20": 1959.2035, + "21": 4008.9883, + "22": 7436.8428 + }, + "gram_cond_max": 9.11 + }, + "rand_target@a2": { + "top1": " \\%", + "rank_tgt": 50282, + "rank_src": 50299, + "logp_tgt": -27.4546, + "logp_src": -33.3064, + "flip_to_tgt": false, + "kl_vs_clean": 24.6165, + "delta_norm_band_sum": 15589.6887, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " China", + "rank_tgt": 27, + "rank_src": 4, + "logp_tgt": -32.0166, + "logp_src": -22.1962, + "flip_to_tgt": false, + "kl_vs_clean": 27.3743, + "delta_norm_band_sum": 15589.6896, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Tokyo", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0013, + "logp_src": -6.647, + "flip_to_tgt": true, + "kl_vs_clean": 22.9515, + "delta_norm_band_sum": 8819.9379 + }, + "swap@a4": { + "top1": " China", + "rank_tgt": 49, + "rank_src": 5, + "logp_tgt": -43.3712, + "logp_src": -28.2317, + "flip_to_tgt": false, + "kl_vs_clean": 35.8785, + "delta_norm_band_sum": 169100496.2117, + "delta_norm_per_layer": { + "9": 4.062, + "10": 16.3558, + "11": 54.794, + "12": 207.6783, + "13": 793.5613, + "14": 3000.0386, + "15": 11298.0928, + "16": 40678.9258, + "17": 147933.0781, + "18": 557865.625, + "19": 2166638.5, + "20": 8112011.5, + "21": 33072338.0, + "22": 124987656.0 + }, + "gram_cond_max": 9.11 + }, + "rand_target@a4": { + "top1": " China", + "rank_tgt": 26, + "rank_src": 5, + "logp_tgt": -25.9362, + "logp_src": -18.9641, + "flip_to_tgt": false, + "kl_vs_clean": 22.5328, + "delta_norm_band_sum": 169100477.3963, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " China", + "rank_tgt": 29, + "rank_src": 5, + "logp_tgt": -33.4853, + "logp_src": -23.6554, + "flip_to_tgt": false, + "kl_vs_clean": 29.2621, + "delta_norm_band_sum": 169100502.4308, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Tokyo", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0, + "logp_src": -12.2262, + "flip_to_tgt": true, + "kl_vs_clean": 26.4107, + "delta_norm_band_sum": 99477793.9672 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "capital", + "src": "Japan", + "tgt": "Germany", + "prompt": "The capital of Japan is the city of", + "answer_src": " Tokyo", + "answer_tgt": " Berlin", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Tokyo", + "rank_tgt": 1647, + "rank_src": 0, + "logp_tgt": -13.1172, + "logp_src": -0.9752, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Tokyo", + "rank_tgt": 1647, + "rank_src": 0, + "logp_tgt": -13.1172, + "logp_src": -0.9752, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Tokyo", + "rank_tgt": 122, + "rank_src": 0, + "logp_tgt": -7.7672, + "logp_src": -0.6732, + "flip_to_tgt": false, + "kl_vs_clean": 0.2899, + "delta_norm_band_sum": 29.579, + "delta_norm_per_layer": { + "9": 0.3456, + "10": 0.2034, + "11": 0.1119, + "12": 0.5326, + "13": 0.6298, + "14": 0.1382, + "15": 0.2695, + "16": 0.2296, + "17": 6.0962, + "18": 3.7395, + "19": 3.4327, + "20": 3.8624, + "21": 4.6078, + "22": 5.3798 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a1": { + "top1": " New", + "rank_tgt": 279, + "rank_src": 17, + "logp_tgt": -8.3072, + "logp_src": -4.3193, + "flip_to_tgt": false, + "kl_vs_clean": 3.1108, + "delta_norm_band_sum": 29.579, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Os", + "rank_tgt": 1571, + "rank_src": 1, + "logp_tgt": -12.1961, + "logp_src": -1.8771, + "flip_to_tgt": false, + "kl_vs_clean": 0.29, + "delta_norm_band_sum": 29.579, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Tokyo", + "rank_tgt": 85, + "rank_src": 0, + "logp_tgt": -7.2404, + "logp_src": -0.8202, + "flip_to_tgt": false, + "kl_vs_clean": 0.21, + "delta_norm_band_sum": 27.4024 + }, + "swap@a2": { + "top1": " Japan", + "rank_tgt": 56, + "rank_src": 18, + "logp_tgt": -33.7508, + "logp_src": -25.6792, + "flip_to_tgt": false, + "kl_vs_clean": 32.8874, + "delta_norm_band_sum": 14023.3189, + "delta_norm_per_layer": { + "9": 1.106, + "10": 2.4527, + "11": 4.0471, + "12": 10.3863, + "13": 20.8637, + "14": 37.5846, + "15": 69.6539, + "16": 132.8197, + "17": 253.8847, + "18": 485.4539, + "19": 909.8417, + "20": 1783.3701, + "21": 3488.7188, + "22": 6823.1357 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a2": { + "top1": "apest", + "rank_tgt": 49954, + "rank_src": 50299, + "logp_tgt": -24.6741, + "logp_src": -44.0123, + "flip_to_tgt": false, + "kl_vs_clean": 29.8558, + "delta_norm_band_sum": 14023.3184, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Japan", + "rank_tgt": 276, + "rank_src": 4, + "logp_tgt": -36.5572, + "logp_src": -18.5813, + "flip_to_tgt": false, + "kl_vs_clean": 26.0367, + "delta_norm_band_sum": 14023.3181, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Tokyo", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -9.6158, + "logp_src": -0.0001, + "flip_to_tgt": false, + "kl_vs_clean": 20.6759, + "delta_norm_band_sum": 11838.548 + }, + "swap@a4": { + "top1": " Germany", + "rank_tgt": 36, + "rank_src": 31, + "logp_tgt": -28.4031, + "logp_src": -26.9219, + "flip_to_tgt": false, + "kl_vs_clean": 32.0871, + "delta_norm_band_sum": 138969458.9013, + "delta_norm_per_layer": { + "9": 3.0741, + "10": 12.9151, + "11": 44.9172, + "12": 180.5937, + "13": 688.0101, + "14": 2594.0571, + "15": 9549.5723, + "16": 33981.4336, + "17": 124055.8281, + "18": 473548.125, + "19": 1779281.375, + "20": 6835765.0, + "21": 26913642.0, + "22": 102796112.0 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a4": { + "top1": "mathchoice", + "rank_tgt": 49770, + "rank_src": 50299, + "logp_tgt": -22.6239, + "logp_src": -38.1035, + "flip_to_tgt": false, + "kl_vs_clean": 25.6282, + "delta_norm_band_sum": 138969458.6941, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Japan", + "rank_tgt": 313, + "rank_src": 4, + "logp_tgt": -39.1279, + "logp_src": -19.9387, + "flip_to_tgt": false, + "kl_vs_clean": 27.9952, + "delta_norm_band_sum": 138969469.851, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Tokyo", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -8.3612, + "logp_src": -0.0002, + "flip_to_tgt": false, + "kl_vs_clean": 19.6429, + "delta_norm_band_sum": 117550828.6219 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "capital", + "src": "Italy", + "tgt": "Spain", + "prompt": "The capital of Italy is the city of", + "answer_src": " Rome", + "answer_tgt": " Madrid", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Rome", + "rank_tgt": 299, + "rank_src": 0, + "logp_tgt": -9.7823, + "logp_src": -1.192, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Rome", + "rank_tgt": 299, + "rank_src": 0, + "logp_tgt": -9.7823, + "logp_src": -1.192, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Rome", + "rank_tgt": 12, + "rank_src": 0, + "logp_tgt": -4.5823, + "logp_src": -1.1347, + "flip_to_tgt": false, + "kl_vs_clean": 0.1326, + "delta_norm_band_sum": 22.0602, + "delta_norm_per_layer": { + "9": 1.1142, + "10": 1.5057, + "11": 1.2614, + "12": 0.7747, + "13": 0.0869, + "14": 0.2086, + "15": 0.0523, + "16": 0.0915, + "17": 3.0129, + "18": 2.4145, + "19": 3.2973, + "20": 2.2841, + "21": 2.4594, + "22": 3.4966 + }, + "gram_cond_max": 10.5 + }, + "rand_target@a1": { + "top1": " Pal", + "rank_tgt": 2233, + "rank_src": 11, + "logp_tgt": -11.6577, + "logp_src": -4.3227, + "flip_to_tgt": false, + "kl_vs_clean": 1.6436, + "delta_norm_band_sum": 22.0602, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Rome", + "rank_tgt": 325, + "rank_src": 0, + "logp_tgt": -9.5157, + "logp_src": -1.4334, + "flip_to_tgt": false, + "kl_vs_clean": 0.1567, + "delta_norm_band_sum": 22.0602, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Milan", + "rank_tgt": 81, + "rank_src": 4, + "logp_tgt": -7.3769, + "logp_src": -2.875, + "flip_to_tgt": false, + "kl_vs_clean": 0.3324, + "delta_norm_band_sum": 29.9116 + }, + "swap@a2": { + "top1": " Italy", + "rank_tgt": 66, + "rank_src": 19, + "logp_tgt": -39.2902, + "logp_src": -30.4612, + "flip_to_tgt": false, + "kl_vs_clean": 36.9991, + "delta_norm_band_sum": 16556.659, + "delta_norm_per_layer": { + "9": 1.7907, + "10": 3.997, + "11": 6.7245, + "12": 12.2885, + "13": 24.2483, + "14": 41.4036, + "15": 76.233, + "16": 148.0996, + "17": 287.4161, + "18": 545.4855, + "19": 1080.0935, + "20": 2055.9128, + "21": 4147.1504, + "22": 8125.8154 + }, + "gram_cond_max": 10.5 + }, + "rand_target@a2": { + "top1": " Italy", + "rank_tgt": 49, + "rank_src": 21, + "logp_tgt": -20.5301, + "logp_src": -17.9192, + "flip_to_tgt": false, + "kl_vs_clean": 20.1159, + "delta_norm_band_sum": 16556.6577, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Italy", + "rank_tgt": 74, + "rank_src": 17, + "logp_tgt": -31.0478, + "logp_src": -24.2698, + "flip_to_tgt": false, + "kl_vs_clean": 28.1882, + "delta_norm_band_sum": 16556.6595, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Rome", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -35.6927, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 29.6564, + "delta_norm_band_sum": 18119.9641 + }, + "swap@a4": { + "top1": " Italy", + "rank_tgt": 518, + "rank_src": 9, + "logp_tgt": -49.6773, + "logp_src": -31.9454, + "flip_to_tgt": false, + "kl_vs_clean": 37.9101, + "delta_norm_band_sum": 162553429.3836, + "delta_norm_per_layer": { + "9": 3.3518, + "10": 13.6621, + "11": 47.2581, + "12": 174.4185, + "13": 702.5422, + "14": 2566.7046, + "15": 9827.7861, + "16": 36713.7852, + "17": 135292.8125, + "18": 511236.3125, + "19": 2009046.75, + "20": 7669076.0, + "21": 31011952.0, + "22": 121166776.0 + }, + "gram_cond_max": 10.5 + }, + "rand_target@a4": { + "top1": " Italy", + "rank_tgt": 64, + "rank_src": 23, + "logp_tgt": -22.5663, + "logp_src": -18.9343, + "flip_to_tgt": false, + "kl_vs_clean": 21.5444, + "delta_norm_band_sum": 162553425.5233, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Italy", + "rank_tgt": 111, + "rank_src": 16, + "logp_tgt": -33.4807, + "logp_src": -25.1368, + "flip_to_tgt": false, + "kl_vs_clean": 29.4437, + "delta_norm_band_sum": 162553422.7703, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Rome", + "rank_tgt": 628, + "rank_src": 0, + "logp_tgt": -56.0905, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 32.3396, + "delta_norm_band_sum": 159673209.4572 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "capital", + "src": "Spain", + "tgt": "Russia", + "prompt": "The capital of Spain is the city of", + "answer_src": " Madrid", + "answer_tgt": " Moscow", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Madrid", + "rank_tgt": 533, + "rank_src": 0, + "logp_tgt": -10.9212, + "logp_src": -1.0043, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Madrid", + "rank_tgt": 533, + "rank_src": 0, + "logp_tgt": -10.9212, + "logp_src": -1.0043, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Madrid", + "rank_tgt": 23, + "rank_src": 0, + "logp_tgt": -6.2677, + "logp_src": -0.377, + "flip_to_tgt": false, + "kl_vs_clean": 0.5887, + "delta_norm_band_sum": 47.8946, + "delta_norm_per_layer": { + "9": 0.4884, + "10": 0.3154, + "11": 0.7078, + "12": 0.2875, + "13": 0.0651, + "14": 0.3268, + "15": 0.8559, + "16": 0.4994, + "17": 8.3467, + "18": 7.4808, + "19": 8.8574, + "20": 8.4634, + "21": 5.9156, + "22": 5.2844 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a1": { + "top1": " Santa", + "rank_tgt": 60, + "rank_src": 40, + "logp_tgt": -5.9432, + "logp_src": -5.547, + "flip_to_tgt": false, + "kl_vs_clean": 3.0712, + "delta_norm_band_sum": 47.8946, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Madrid", + "rank_tgt": 249, + "rank_src": 0, + "logp_tgt": -8.7391, + "logp_src": -0.8701, + "flip_to_tgt": false, + "kl_vs_clean": 0.4207, + "delta_norm_band_sum": 47.8946, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Barcelona", + "rank_tgt": 2, + "rank_src": 1, + "logp_tgt": -2.5671, + "logp_src": -1.9655, + "flip_to_tgt": false, + "kl_vs_clean": 0.5239, + "delta_norm_band_sum": 31.2965 + }, + "swap@a2": { + "top1": " Russia", + "rank_tgt": 11, + "rank_src": 182, + "logp_tgt": -24.743, + "logp_src": -40.3379, + "flip_to_tgt": false, + "kl_vs_clean": 40.4999, + "delta_norm_band_sum": 15516.954, + "delta_norm_per_layer": { + "9": 0.9678, + "10": 1.9896, + "11": 3.8485, + "12": 11.8338, + "13": 22.7085, + "14": 41.3866, + "15": 74.4563, + "16": 141.5087, + "17": 275.1152, + "18": 522.311, + "19": 1023.0485, + "20": 1955.6108, + "21": 3872.2361, + "22": 7569.9326 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 31973, + "rank_src": 50289, + "logp_tgt": -15.5839, + "logp_src": -24.06, + "flip_to_tgt": false, + "kl_vs_clean": 17.364, + "delta_norm_band_sum": 15516.9549, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Spain", + "rank_tgt": 1094, + "rank_src": 4, + "logp_tgt": -36.7767, + "logp_src": -18.7511, + "flip_to_tgt": false, + "kl_vs_clean": 24.5697, + "delta_norm_band_sum": 15516.9535, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Moscow", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0001, + "logp_src": -9.6286, + "flip_to_tgt": true, + "kl_vs_clean": 25.4766, + "delta_norm_band_sum": 8993.5652 + }, + "swap@a4": { + "top1": " Russia", + "rank_tgt": 9, + "rank_src": 471, + "logp_tgt": -25.0216, + "logp_src": -45.7209, + "flip_to_tgt": false, + "kl_vs_clean": 44.2283, + "delta_norm_band_sum": 127609800.0317, + "delta_norm_per_layer": { + "9": 2.2524, + "10": 9.8257, + "11": 35.8177, + "12": 156.0691, + "13": 595.0185, + "14": 2265.4614, + "15": 8361.8408, + "16": 30158.5664, + "17": 110079.6172, + "18": 416391.5625, + "19": 1598620.5, + "20": 6122909.5, + "21": 24338094.0, + "22": 94982120.0 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a4": { + "top1": " Spain", + "rank_tgt": 3585, + "rank_src": 4, + "logp_tgt": -26.6838, + "logp_src": -13.399, + "flip_to_tgt": false, + "kl_vs_clean": 16.2445, + "delta_norm_band_sum": 127609802.1576, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Spain", + "rank_tgt": 1038, + "rank_src": 5, + "logp_tgt": -39.0535, + "logp_src": -21.3968, + "flip_to_tgt": false, + "kl_vs_clean": 26.8827, + "delta_norm_band_sum": 127609794.679, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Moscow", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": 0.0, + "logp_src": -17.0401, + "flip_to_tgt": true, + "kl_vs_clean": 30.0877, + "delta_norm_band_sum": 69800179.9745 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "capital", + "src": "Russia", + "tgt": "Egypt", + "prompt": "The capital of Russia is the city of", + "answer_src": " Moscow", + "answer_tgt": " Cairo", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Moscow", + "rank_tgt": 4339, + "rank_src": 0, + "logp_tgt": -15.2301, + "logp_src": -1.4361, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Moscow", + "rank_tgt": 4339, + "rank_src": 0, + "logp_tgt": -15.2301, + "logp_src": -1.4361, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Moscow", + "rank_tgt": 78, + "rank_src": 0, + "logp_tgt": -6.9174, + "logp_src": -1.381, + "flip_to_tgt": false, + "kl_vs_clean": 0.7647, + "delta_norm_band_sum": 29.2999, + "delta_norm_per_layer": { + "9": 1.3356, + "10": 1.2297, + "11": 1.4027, + "12": 0.3649, + "13": 0.2545, + "14": 0.7422, + "15": 0.4974, + "16": 0.2889, + "17": 4.5951, + "18": 3.9661, + "19": 4.1825, + "20": 3.8152, + "21": 3.7373, + "22": 2.8879 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a1": { + "top1": " New", + "rank_tgt": 35, + "rank_src": 41, + "logp_tgt": -5.2835, + "logp_src": -5.4106, + "flip_to_tgt": false, + "kl_vs_clean": 2.6153, + "delta_norm_band_sum": 29.2999, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " St", + "rank_tgt": 2155, + "rank_src": 2, + "logp_tgt": -12.8858, + "logp_src": -2.4426, + "flip_to_tgt": false, + "kl_vs_clean": 0.2266, + "delta_norm_band_sum": 29.2999, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Moscow", + "rank_tgt": 59, + "rank_src": 0, + "logp_tgt": -6.3532, + "logp_src": -2.1425, + "flip_to_tgt": false, + "kl_vs_clean": 0.8512, + "delta_norm_band_sum": 30.5888 + }, + "swap@a2": { + "top1": " Egypt", + "rank_tgt": 18, + "rank_src": 20, + "logp_tgt": -25.7759, + "logp_src": -26.7814, + "flip_to_tgt": false, + "kl_vs_clean": 33.0976, + "delta_norm_band_sum": 19175.8193, + "delta_norm_per_layer": { + "9": 2.3949, + "10": 4.1734, + "11": 8.2792, + "12": 15.5925, + "13": 31.4647, + "14": 55.0169, + "15": 107.0524, + "16": 190.5943, + "17": 350.9136, + "18": 661.0923, + "19": 1252.0789, + "20": 2408.7175, + "21": 4794.0591, + "22": 9294.3896 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a2": { + "top1": " answ", + "rank_tgt": 48758, + "rank_src": 50300, + "logp_tgt": -24.8586, + "logp_src": -49.5251, + "flip_to_tgt": false, + "kl_vs_clean": 31.1811, + "delta_norm_band_sum": 19175.8183, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Russia", + "rank_tgt": 721, + "rank_src": 5, + "logp_tgt": -34.7951, + "logp_src": -18.1392, + "flip_to_tgt": false, + "kl_vs_clean": 26.7877, + "delta_norm_band_sum": 19175.8207, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Cairo", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0003, + "logp_src": -8.1764, + "flip_to_tgt": true, + "kl_vs_clean": 28.7406, + "delta_norm_band_sum": 10831.0581 + }, + "swap@a4": { + "top1": " Egypt", + "rank_tgt": 2, + "rank_src": 98, + "logp_tgt": -29.2622, + "logp_src": -41.0843, + "flip_to_tgt": false, + "kl_vs_clean": 42.4527, + "delta_norm_band_sum": 196970949.4137, + "delta_norm_per_layer": { + "9": 5.0297, + "10": 17.6441, + "11": 67.9051, + "12": 240.1335, + "13": 964.739, + "14": 3391.9409, + "15": 13699.0605, + "16": 46638.0859, + "17": 170640.0, + "18": 641039.875, + "19": 2461850.0, + "20": 9429519.0, + "21": 37627788.0, + "22": 146575088.0 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a4": { + "top1": " Russia", + "rank_tgt": 1559, + "rank_src": 9, + "logp_tgt": -20.9878, + "logp_src": -12.1707, + "flip_to_tgt": false, + "kl_vs_clean": 15.4618, + "delta_norm_band_sum": 196970973.6994, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Russia", + "rank_tgt": 813, + "rank_src": 6, + "logp_tgt": -36.8568, + "logp_src": -19.4983, + "flip_to_tgt": false, + "kl_vs_clean": 28.1923, + "delta_norm_band_sum": 196970949.3078, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Cairo", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": 0.0, + "logp_src": -28.2464, + "flip_to_tgt": true, + "kl_vs_clean": 38.6615, + "delta_norm_band_sum": 105892506.2397 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "capital", + "src": "Egypt", + "tgt": "India", + "prompt": "The capital of Egypt is the city of", + "answer_src": " Cairo", + "answer_tgt": " Delhi", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Cairo", + "rank_tgt": 1084, + "rank_src": 0, + "logp_tgt": -12.9154, + "logp_src": -0.5466, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Cairo", + "rank_tgt": 1084, + "rank_src": 0, + "logp_tgt": -12.9154, + "logp_src": -0.5466, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Cairo", + "rank_tgt": 206, + "rank_src": 0, + "logp_tgt": -8.9129, + "logp_src": -0.5548, + "flip_to_tgt": false, + "kl_vs_clean": 0.2703, + "delta_norm_band_sum": 57.8807, + "delta_norm_per_layer": { + "9": 0.9423, + "10": 0.5993, + "11": 1.1646, + "12": 0.9967, + "13": 1.3813, + "14": 1.3993, + "15": 1.3065, + "16": 0.2732, + "17": 8.2329, + "18": 7.2326, + "19": 9.1154, + "20": 9.4548, + "21": 7.5881, + "22": 8.1936 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a1": { + "top1": " Dam", + "rank_tgt": 662, + "rank_src": 455, + "logp_tgt": -8.6387, + "logp_src": -8.2064, + "flip_to_tgt": false, + "kl_vs_clean": 5.7967, + "delta_norm_band_sum": 57.8807, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Cairo", + "rank_tgt": 1574, + "rank_src": 0, + "logp_tgt": -11.4655, + "logp_src": -1.8592, + "flip_to_tgt": false, + "kl_vs_clean": 1.3238, + "delta_norm_band_sum": 57.8807, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Cairo", + "rank_tgt": 90, + "rank_src": 0, + "logp_tgt": -7.1682, + "logp_src": -1.4786, + "flip_to_tgt": false, + "kl_vs_clean": 0.4906, + "delta_norm_band_sum": 42.2476 + }, + "swap@a2": { + "top1": " Egypt", + "rank_tgt": 190, + "rank_src": 6, + "logp_tgt": -39.9028, + "logp_src": -28.3674, + "flip_to_tgt": false, + "kl_vs_clean": 31.104, + "delta_norm_band_sum": 22842.458, + "delta_norm_per_layer": { + "9": 2.0079, + "10": 3.2722, + "11": 6.8985, + "12": 18.9287, + "13": 39.2038, + "14": 68.2245, + "15": 130.6401, + "16": 231.1508, + "17": 428.6426, + "18": 781.1595, + "19": 1499.7535, + "20": 2873.4929, + "21": 5784.1943, + "22": 10974.8887 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 50035, + "rank_src": 50301, + "logp_tgt": -23.1096, + "logp_src": -32.2797, + "flip_to_tgt": false, + "kl_vs_clean": 26.6148, + "delta_norm_band_sum": 22842.4582, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Egypt", + "rank_tgt": 401, + "rank_src": 2, + "logp_tgt": -33.2651, + "logp_src": -18.1487, + "flip_to_tgt": false, + "kl_vs_clean": 21.4937, + "delta_norm_band_sum": 22842.458, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Cairo", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -11.9964, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 12.6546, + "delta_norm_band_sum": 13948.6748 + }, + "swap@a4": { + "top1": " Egypt", + "rank_tgt": 356, + "rank_src": 6, + "logp_tgt": -43.503, + "logp_src": -29.2072, + "flip_to_tgt": false, + "kl_vs_clean": 32.1161, + "delta_norm_band_sum": 192559478.3365, + "delta_norm_per_layer": { + "9": 4.7195, + "10": 16.5403, + "11": 64.5708, + "12": 241.9766, + "13": 962.5889, + "14": 3489.7627, + "15": 13680.0215, + "16": 46680.875, + "17": 171342.9062, + "18": 635329.875, + "19": 2437382.5, + "20": 9326786.0, + "21": 37352440.0, + "22": 142571056.0 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a4": { + "top1": "ningen", + "rank_tgt": 50160, + "rank_src": 50302, + "logp_tgt": -23.8954, + "logp_src": -31.9461, + "flip_to_tgt": false, + "kl_vs_clean": 25.8586, + "delta_norm_band_sum": 192559477.7586, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Egypt", + "rank_tgt": 623, + "rank_src": 2, + "logp_tgt": -35.925, + "logp_src": -19.6673, + "flip_to_tgt": false, + "kl_vs_clean": 23.1322, + "delta_norm_band_sum": 192559465.4676, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Delhi", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": 0.0, + "logp_src": -18.3981, + "flip_to_tgt": true, + "kl_vs_clean": 24.9082, + "delta_norm_band_sum": 128910546.2638 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "capital", + "src": "Mexico", + "tgt": "Greece", + "prompt": "The capital of Mexico is the city of", + "answer_src": " Mexico", + "answer_tgt": " Athens", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Mexico", + "rank_tgt": 6424, + "rank_src": 0, + "logp_tgt": -15.6948, + "logp_src": -1.1937, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Mexico", + "rank_tgt": 6424, + "rank_src": 0, + "logp_tgt": -15.6948, + "logp_src": -1.1937, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " O", + "rank_tgt": 8, + "rank_src": 34, + "logp_tgt": -3.8768, + "logp_src": -5.0801, + "flip_to_tgt": false, + "kl_vs_clean": 1.6074, + "delta_norm_band_sum": 22.3643, + "delta_norm_per_layer": { + "9": 0.005, + "10": 0.145, + "11": 0.3505, + "12": 0.918, + "13": 1.0991, + "14": 0.3531, + "15": 0.0982, + "16": 0.5784, + "17": 7.2847, + "18": 4.0238, + "19": 3.6385, + "20": 1.7919, + "21": 1.7034, + "22": 0.3746 + }, + "gram_cond_max": 4.02 + }, + "rand_target@a1": { + "top1": " Yang", + "rank_tgt": 97, + "rank_src": 1330, + "logp_tgt": -6.3346, + "logp_src": -10.5818, + "flip_to_tgt": false, + "kl_vs_clean": 4.1891, + "delta_norm_band_sum": 22.3643, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " T", + "rank_tgt": 3585, + "rank_src": 1, + "logp_tgt": -14.0697, + "logp_src": -2.0165, + "flip_to_tgt": false, + "kl_vs_clean": 0.2587, + "delta_norm_band_sum": 22.3643, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Athens", + "rank_tgt": 0, + "rank_src": 31, + "logp_tgt": -2.0499, + "logp_src": -5.4214, + "flip_to_tgt": true, + "kl_vs_clean": 1.6486, + "delta_norm_band_sum": 19.6964 + }, + "swap@a2": { + "top1": " Mexico", + "rank_tgt": 80, + "rank_src": 0, + "logp_tgt": -26.1706, + "logp_src": -0.1762, + "flip_to_tgt": false, + "kl_vs_clean": 18.3553, + "delta_norm_band_sum": 14913.8037, + "delta_norm_per_layer": { + "9": 1.1698, + "10": 2.4562, + "11": 4.0531, + "12": 11.8494, + "13": 23.4857, + "14": 41.4575, + "15": 75.2551, + "16": 142.748, + "17": 270.2561, + "18": 521.2962, + "19": 985.3654, + "20": 1912.01, + "21": 3743.1663, + "22": 7179.2349 + }, + "gram_cond_max": 4.02 + }, + "rand_target@a2": { + "top1": " answ", + "rank_tgt": 44926, + "rank_src": 50303, + "logp_tgt": -21.9855, + "logp_src": -63.7937, + "flip_to_tgt": false, + "kl_vs_clean": 37.1439, + "delta_norm_band_sum": 14913.8047, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Mexico", + "rank_tgt": 5254, + "rank_src": 0, + "logp_tgt": -37.5418, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 17.3872, + "delta_norm_band_sum": 14913.8046, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Athens", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0002, + "logp_src": -8.3415, + "flip_to_tgt": true, + "kl_vs_clean": 20.3044, + "delta_norm_band_sum": 15468.8537 + }, + "swap@a4": { + "top1": " Greece", + "rank_tgt": 71, + "rank_src": 1, + "logp_tgt": -24.8044, + "logp_src": -1.8182, + "flip_to_tgt": false, + "kl_vs_clean": 17.9405, + "delta_norm_band_sum": 150988023.1142, + "delta_norm_per_layer": { + "9": 3.5093, + "10": 13.882, + "11": 46.884, + "12": 189.7583, + "13": 720.608, + "14": 2713.1416, + "15": 10151.3818, + "16": 36144.0742, + "17": 132612.2812, + "18": 508289.2188, + "19": 1910890.375, + "20": 7419056.0, + "21": 29052680.0, + "22": 111914512.0 + }, + "gram_cond_max": 4.02 + }, + "rand_target@a4": { + "top1": " Mexico", + "rank_tgt": 8968, + "rank_src": 0, + "logp_tgt": -23.4117, + "logp_src": -0.0031, + "flip_to_tgt": false, + "kl_vs_clean": 8.426, + "delta_norm_band_sum": 150988009.9619, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Mexico", + "rank_tgt": 4695, + "rank_src": 0, + "logp_tgt": -39.8174, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 18.2212, + "delta_norm_band_sum": 150988039.181, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Athens", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.051, + "logp_src": -3.0008, + "flip_to_tgt": true, + "kl_vs_clean": 15.7887, + "delta_norm_band_sum": 149492430.3262 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "capital", + "src": "Norway", + "tgt": "Turkey", + "prompt": "The capital of Norway is the city of", + "answer_src": " Oslo", + "answer_tgt": " Ankara", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Oslo", + "rank_tgt": 2092, + "rank_src": 0, + "logp_tgt": -12.8728, + "logp_src": -1.1597, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Oslo", + "rank_tgt": 2092, + "rank_src": 0, + "logp_tgt": -12.8728, + "logp_src": -1.1597, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Oslo", + "rank_tgt": 367, + "rank_src": 0, + "logp_tgt": -9.5987, + "logp_src": -0.7433, + "flip_to_tgt": false, + "kl_vs_clean": 0.5058, + "delta_norm_band_sum": 77.8204, + "delta_norm_per_layer": { + "9": 0.9707, + "10": 0.7821, + "11": 0.6169, + "12": 1.4024, + "13": 1.6366, + "14": 1.3473, + "15": 1.3383, + "16": 1.7616, + "17": 11.7659, + "18": 10.855, + "19": 13.2739, + "20": 11.7134, + "21": 10.2225, + "22": 10.1337 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 412, + "rank_src": 58, + "logp_tgt": -8.7497, + "logp_src": -5.9048, + "flip_to_tgt": false, + "kl_vs_clean": 3.1543, + "delta_norm_band_sum": 77.8204, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " St", + "rank_tgt": 667, + "rank_src": 5, + "logp_tgt": -9.4928, + "logp_src": -3.4876, + "flip_to_tgt": false, + "kl_vs_clean": 1.6275, + "delta_norm_band_sum": 77.8204, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " N", + "rank_tgt": 495, + "rank_src": 3, + "logp_tgt": -9.7364, + "logp_src": -3.0166, + "flip_to_tgt": false, + "kl_vs_clean": 0.5843, + "delta_norm_band_sum": 42.3065 + }, + "swap@a2": { + "top1": " Norway", + "rank_tgt": 1850, + "rank_src": 25, + "logp_tgt": -50.6127, + "logp_src": -33.1253, + "flip_to_tgt": false, + "kl_vs_clean": 36.4724, + "delta_norm_band_sum": 20836.0338, + "delta_norm_per_layer": { + "9": 1.6663, + "10": 3.1358, + "11": 5.6648, + "12": 14.4532, + "13": 28.101, + "14": 52.7302, + "15": 98.0719, + "16": 201.2715, + "17": 367.9024, + "18": 733.6795, + "19": 1368.6162, + "20": 2713.1565, + "21": 5158.5493, + "22": 10089.0352 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 41559, + "rank_src": 49672, + "logp_tgt": -17.0625, + "logp_src": -20.6367, + "flip_to_tgt": false, + "kl_vs_clean": 12.7819, + "delta_norm_band_sum": 20836.0325, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Norway", + "rank_tgt": 4828, + "rank_src": 7, + "logp_tgt": -38.2118, + "logp_src": -21.3108, + "flip_to_tgt": false, + "kl_vs_clean": 25.4085, + "delta_norm_band_sum": 20836.0336, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Oslo", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -7.9069, + "logp_src": -0.0004, + "flip_to_tgt": false, + "kl_vs_clean": 17.1091, + "delta_norm_band_sum": 9053.4682 + }, + "swap@a4": { + "top1": " Norway", + "rank_tgt": 5206, + "rank_src": 16, + "logp_tgt": -56.1528, + "logp_src": -34.1028, + "flip_to_tgt": false, + "kl_vs_clean": 37.9407, + "delta_norm_band_sum": 174052880.057, + "delta_norm_per_layer": { + "9": 3.4856, + "10": 13.7897, + "11": 49.6322, + "12": 201.9051, + "13": 765.5312, + "14": 2975.5715, + "15": 10987.6924, + "16": 42423.3555, + "17": 147408.0312, + "18": 586315.3125, + "19": 2166245.75, + "20": 8630874.0, + "21": 33056072.0, + "22": 129408544.0 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a4": { + "top1": " Norway", + "rank_tgt": 7455, + "rank_src": 24, + "logp_tgt": -22.6332, + "logp_src": -12.7745, + "flip_to_tgt": false, + "kl_vs_clean": 13.2422, + "delta_norm_band_sum": 174052882.2972, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Norway", + "rank_tgt": 6474, + "rank_src": 13, + "logp_tgt": -41.1975, + "logp_src": -23.685, + "flip_to_tgt": false, + "kl_vs_clean": 26.7232, + "delta_norm_band_sum": 174052879.5885, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Oslo", + "rank_tgt": 16, + "rank_src": 0, + "logp_tgt": -27.9714, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 20.8854, + "delta_norm_band_sum": 97311186.7181 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "capital", + "src": "Korea", + "tgt": "Thailand", + "prompt": "The capital of Korea is the city of", + "answer_src": " Seoul", + "answer_tgt": " Bangkok", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Seoul", + "rank_tgt": 167, + "rank_src": 0, + "logp_tgt": -8.9432, + "logp_src": -1.4599, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Seoul", + "rank_tgt": 167, + "rank_src": 0, + "logp_tgt": -8.9432, + "logp_src": -1.4599, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Seoul", + "rank_tgt": 22, + "rank_src": 0, + "logp_tgt": -4.8053, + "logp_src": -1.6534, + "flip_to_tgt": false, + "kl_vs_clean": 0.3203, + "delta_norm_band_sum": 31.6006, + "delta_norm_per_layer": { + "9": 0.2698, + "10": 0.5068, + "11": 0.0914, + "12": 1.0165, + "13": 1.7298, + "14": 2.0376, + "15": 1.3625, + "16": 0.9275, + "17": 4.15, + "18": 3.4332, + "19": 4.9681, + "20": 4.613, + "21": 3.3981, + "22": 3.0962 + }, + "gram_cond_max": 6.66 + }, + "rand_target@a1": { + "top1": " Santiago", + "rank_tgt": 1198, + "rank_src": 458, + "logp_tgt": -10.7514, + "logp_src": -8.9404, + "flip_to_tgt": false, + "kl_vs_clean": 5.0841, + "delta_norm_band_sum": 31.6006, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Seoul", + "rank_tgt": 194, + "rank_src": 0, + "logp_tgt": -8.6091, + "logp_src": -1.3686, + "flip_to_tgt": false, + "kl_vs_clean": 0.3128, + "delta_norm_band_sum": 31.6006, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Seoul", + "rank_tgt": 102, + "rank_src": 0, + "logp_tgt": -8.2733, + "logp_src": -0.3786, + "flip_to_tgt": false, + "kl_vs_clean": 0.7119, + "delta_norm_band_sum": 62.3547 + }, + "swap@a2": { + "top1": " Korea", + "rank_tgt": 28, + "rank_src": 9, + "logp_tgt": -28.5476, + "logp_src": -23.6109, + "flip_to_tgt": false, + "kl_vs_clean": 33.8283, + "delta_norm_band_sum": 12180.5225, + "delta_norm_per_layer": { + "9": 0.945, + "10": 2.1646, + "11": 3.7258, + "12": 9.8657, + "13": 19.7088, + "14": 34.8413, + "15": 61.9828, + "16": 120.1979, + "17": 222.079, + "18": 429.8769, + "19": 799.8683, + "20": 1570.574, + "21": 3045.0322, + "22": 5859.6602 + }, + "gram_cond_max": 6.66 + }, + "rand_target@a2": { + "top1": " deleter", + "rank_tgt": 50177, + "rank_src": 50300, + "logp_tgt": -28.7538, + "logp_src": -47.0519, + "flip_to_tgt": false, + "kl_vs_clean": 27.4802, + "delta_norm_band_sum": 12180.5232, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Korea", + "rank_tgt": 79, + "rank_src": 4, + "logp_tgt": -35.3733, + "logp_src": -19.788, + "flip_to_tgt": false, + "kl_vs_clean": 31.3395, + "delta_norm_band_sum": 12180.5225, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Seoul", + "rank_tgt": 23, + "rank_src": 0, + "logp_tgt": -44.646, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 36.6237, + "delta_norm_band_sum": 23093.8695 + }, + "swap@a4": { + "top1": " Korea", + "rank_tgt": 24, + "rank_src": 12, + "logp_tgt": -25.854, + "logp_src": -22.676, + "flip_to_tgt": false, + "kl_vs_clean": 31.121, + "delta_norm_band_sum": 125668704.4617, + "delta_norm_per_layer": { + "9": 2.6537, + "10": 11.4379, + "11": 39.7972, + "12": 162.1539, + "13": 617.6088, + "14": 2350.1296, + "15": 8716.1826, + "16": 31578.2012, + "17": 111796.7656, + "18": 426085.0312, + "19": 1596419.5, + "20": 6184231.0, + "21": 24421614.0, + "22": 92885080.0 + }, + "gram_cond_max": 6.66 + }, + "rand_target@a4": { + "top1": " Korea", + "rank_tgt": 90, + "rank_src": 4, + "logp_tgt": -22.5513, + "logp_src": -12.2763, + "flip_to_tgt": false, + "kl_vs_clean": 18.9685, + "delta_norm_band_sum": 125668704.6257, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Korea", + "rank_tgt": 93, + "rank_src": 4, + "logp_tgt": -37.0703, + "logp_src": -21.3655, + "flip_to_tgt": false, + "kl_vs_clean": 32.3801, + "delta_norm_band_sum": 125668724.5145, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Seoul", + "rank_tgt": 574, + "rank_src": 0, + "logp_tgt": -55.2923, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 36.8795, + "delta_norm_band_sum": 217162346.0251 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "capital", + "src": "Thailand", + "tgt": "Kenya", + "prompt": "The capital of Thailand is the city of", + "answer_src": " Bangkok", + "answer_tgt": " Nairobi", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Bangkok", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -4.1987, + "logp_src": -0.6799, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Bangkok", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -4.1987, + "logp_src": -0.6799, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Bangkok", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -2.4006, + "logp_src": -1.8314, + "flip_to_tgt": false, + "kl_vs_clean": 0.9956, + "delta_norm_band_sum": 41.1633, + "delta_norm_per_layer": { + "9": 0.7221, + "10": 1.0603, + "11": 1.1026, + "12": 0.9717, + "13": 0.3744, + "14": 0.8204, + "15": 0.4649, + "16": 1.1038, + "17": 7.571, + "18": 6.3459, + "19": 5.8479, + "20": 5.2042, + "21": 4.7223, + "22": 4.8519 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a1": { + "top1": "\n", + "rank_tgt": 42, + "rank_src": 305, + "logp_tgt": -5.3852, + "logp_src": -7.7969, + "flip_to_tgt": false, + "kl_vs_clean": 4.7836, + "delta_norm_band_sum": 41.1633, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Bangkok", + "rank_tgt": 11, + "rank_src": 0, + "logp_tgt": -4.1404, + "logp_src": -1.3194, + "flip_to_tgt": false, + "kl_vs_clean": 0.4228, + "delta_norm_band_sum": 41.1633, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Chi", + "rank_tgt": 1, + "rank_src": 7, + "logp_tgt": -3.0018, + "logp_src": -3.7443, + "flip_to_tgt": false, + "kl_vs_clean": 1.5298, + "delta_norm_band_sum": 41.5427 + }, + "swap@a2": { + "top1": " Kenya", + "rank_tgt": 34, + "rank_src": 157, + "logp_tgt": -23.9838, + "logp_src": -30.5263, + "flip_to_tgt": false, + "kl_vs_clean": 29.5964, + "delta_norm_band_sum": 12253.0418, + "delta_norm_per_layer": { + "9": 1.5466, + "10": 3.1741, + "11": 5.5392, + "12": 11.2998, + "13": 20.8961, + "14": 36.921, + "15": 66.7761, + "16": 121.8358, + "17": 217.1953, + "18": 427.5516, + "19": 787.1, + "20": 1567.5415, + "21": 2998.6538, + "22": 5987.0107 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a2": { + "top1": " Thailand", + "rank_tgt": 479, + "rank_src": 240, + "logp_tgt": -12.5403, + "logp_src": -11.6143, + "flip_to_tgt": false, + "kl_vs_clean": 10.2011, + "delta_norm_band_sum": 12253.0423, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Thailand", + "rank_tgt": 90, + "rank_src": 4, + "logp_tgt": -23.6084, + "logp_src": -12.037, + "flip_to_tgt": false, + "kl_vs_clean": 16.4077, + "delta_norm_band_sum": 12253.0428, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " N", + "rank_tgt": 0, + "rank_src": 73, + "logp_tgt": 0.0, + "logp_src": -40.5887, + "flip_to_tgt": true, + "kl_vs_clean": 38.1387, + "delta_norm_band_sum": 21354.2328 + }, + "swap@a4": { + "top1": " Kenya", + "rank_tgt": 17, + "rank_src": 673, + "logp_tgt": -24.5182, + "logp_src": -37.9635, + "flip_to_tgt": false, + "kl_vs_clean": 34.7843, + "delta_norm_band_sum": 146739072.0831, + "delta_norm_per_layer": { + "9": 3.6997, + "10": 13.7056, + "11": 48.5041, + "12": 185.1443, + "13": 708.6155, + "14": 2654.4382, + "15": 9960.9092, + "16": 35436.168, + "17": 125486.6172, + "18": 489130.9062, + "19": 1797473.375, + "20": 7164514.0, + "21": 27465072.0, + "22": 109648384.0 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 50054, + "rank_src": 50301, + "logp_tgt": -23.391, + "logp_src": -32.816, + "flip_to_tgt": false, + "kl_vs_clean": 24.7343, + "delta_norm_band_sum": 146739082.392, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Thailand", + "rank_tgt": 74, + "rank_src": 4, + "logp_tgt": -23.0962, + "logp_src": -12.612, + "flip_to_tgt": false, + "kl_vs_clean": 17.1596, + "delta_norm_band_sum": 146739052.9563, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " N", + "rank_tgt": 0, + "rank_src": 46881, + "logp_tgt": 0.0, + "logp_src": -65.7718, + "flip_to_tgt": true, + "kl_vs_clean": 54.2179, + "delta_norm_band_sum": 248010404.3009 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "capital", + "src": "Kenya", + "tgt": "France", + "prompt": "The capital of Kenya is the city of", + "answer_src": " Nairobi", + "answer_tgt": " Paris", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " N", + "rank_tgt": 4457, + "rank_src": 0, + "logp_tgt": -16.1265, + "logp_src": -0.3248, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " N", + "rank_tgt": 4457, + "rank_src": 0, + "logp_tgt": -16.1265, + "logp_src": -0.3248, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " N", + "rank_tgt": 138, + "rank_src": 0, + "logp_tgt": -8.3013, + "logp_src": -0.4481, + "flip_to_tgt": false, + "kl_vs_clean": 0.3592, + "delta_norm_band_sum": 52.4051, + "delta_norm_per_layer": { + "9": 1.8163, + "10": 1.5646, + "11": 1.7092, + "12": 1.675, + "13": 0.3831, + "14": 0.4036, + "15": 0.0333, + "16": 1.532, + "17": 8.4409, + "18": 6.4784, + "19": 7.7491, + "20": 7.4913, + "21": 5.5493, + "22": 7.579 + }, + "gram_cond_max": 3.14 + }, + "rand_target@a1": { + "top1": " New", + "rank_tgt": 265, + "rank_src": 2, + "logp_tgt": -7.592, + "logp_src": -3.8677, + "flip_to_tgt": false, + "kl_vs_clean": 3.0627, + "delta_norm_band_sum": 52.4051, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " K", + "rank_tgt": 1135, + "rank_src": 1, + "logp_tgt": -11.0718, + "logp_src": -2.0655, + "flip_to_tgt": false, + "kl_vs_clean": 1.2401, + "delta_norm_band_sum": 52.4051, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " N", + "rank_tgt": 885, + "rank_src": 0, + "logp_tgt": -13.8482, + "logp_src": -0.1093, + "flip_to_tgt": false, + "kl_vs_clean": 0.1539, + "delta_norm_band_sum": 44.9048 + }, + "swap@a2": { + "top1": " Kenya", + "rank_tgt": 1388, + "rank_src": 14, + "logp_tgt": -43.7778, + "logp_src": -28.5484, + "flip_to_tgt": false, + "kl_vs_clean": 27.5787, + "delta_norm_band_sum": 23524.8914, + "delta_norm_per_layer": { + "9": 2.988, + "10": 4.7778, + "11": 9.182, + "12": 18.9388, + "13": 37.0932, + "14": 63.6161, + "15": 122.3645, + "16": 215.9981, + "17": 409.363, + "18": 766.476, + "19": 1496.7552, + "20": 2873.6721, + "21": 5894.5112, + "22": 11609.1553 + }, + "gram_cond_max": 3.14 + }, + "rand_target@a2": { + "top1": " arrang", + "rank_tgt": 5926, + "rank_src": 50281, + "logp_tgt": -13.8769, + "logp_src": -29.7278, + "flip_to_tgt": false, + "kl_vs_clean": 28.4497, + "delta_norm_band_sum": 23524.8897, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Kenya", + "rank_tgt": 2640, + "rank_src": 14, + "logp_tgt": -28.9428, + "logp_src": -18.1914, + "flip_to_tgt": false, + "kl_vs_clean": 17.4298, + "delta_norm_band_sum": 23524.8912, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " N", + "rank_tgt": 1031, + "rank_src": 0, + "logp_tgt": -51.6743, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 9.0661, + "delta_norm_band_sum": 30740.9268 + }, + "swap@a4": { + "top1": " Kenya", + "rank_tgt": 9512, + "rank_src": 11, + "logp_tgt": -53.3591, + "logp_src": -30.9371, + "flip_to_tgt": false, + "kl_vs_clean": 30.0975, + "delta_norm_band_sum": 213004992.0488, + "delta_norm_per_layer": { + "9": 5.6114, + "10": 17.7392, + "11": 67.9792, + "12": 248.2552, + "13": 1024.0322, + "14": 3585.793, + "15": 14352.0723, + "16": 48225.832, + "17": 178357.2344, + "18": 666161.25, + "19": 2606395.25, + "20": 9958879.0, + "21": 40838104.0, + "22": 158689568.0 + }, + "gram_cond_max": 3.14 + }, + "rand_target@a4": { + "top1": " Kenya", + "rank_tgt": 996, + "rank_src": 7, + "logp_tgt": -22.2172, + "logp_src": -12.2812, + "flip_to_tgt": false, + "kl_vs_clean": 11.7175, + "delta_norm_band_sum": 213005006.9823, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Kenya", + "rank_tgt": 1579, + "rank_src": 8, + "logp_tgt": -29.8611, + "logp_src": -17.7448, + "flip_to_tgt": false, + "kl_vs_clean": 17.1523, + "delta_norm_band_sum": 213004956.4233, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " N", + "rank_tgt": 46940, + "rank_src": 0, + "logp_tgt": -66.2599, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 9.7184, + "delta_norm_band_sum": 272328835.4221 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "France", + "tgt": "China", + "prompt": "The main language spoken in France is", + "answer_src": " French", + "answer_tgt": " Chinese", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " French", + "rank_tgt": 1074, + "rank_src": 0, + "logp_tgt": -10.9048, + "logp_src": -0.7296, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " French", + "rank_tgt": 1074, + "rank_src": 0, + "logp_tgt": -10.9048, + "logp_src": -0.7296, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " French", + "rank_tgt": 24, + "rank_src": 0, + "logp_tgt": -6.0988, + "logp_src": -0.3524, + "flip_to_tgt": false, + "kl_vs_clean": 0.3627, + "delta_norm_band_sum": 44.9773, + "delta_norm_per_layer": { + "9": 1.9992, + "10": 1.4417, + "11": 1.3284, + "12": 0.247, + "13": 0.5518, + "14": 0.7234, + "15": 0.075, + "16": 1.0493, + "17": 5.994, + "18": 5.968, + "19": 7.0691, + "20": 7.0615, + "21": 5.8567, + "22": 5.6121 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 107, + "rank_src": 16, + "logp_tgt": -6.8355, + "logp_src": -4.6107, + "flip_to_tgt": false, + "kl_vs_clean": 2.0397, + "delta_norm_band_sum": 44.9773, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " French", + "rank_tgt": 466, + "rank_src": 0, + "logp_tgt": -9.2672, + "logp_src": -1.3558, + "flip_to_tgt": false, + "kl_vs_clean": 0.3874, + "delta_norm_band_sum": 44.9773, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " French", + "rank_tgt": 18, + "rank_src": 0, + "logp_tgt": -5.8985, + "logp_src": -0.3546, + "flip_to_tgt": false, + "kl_vs_clean": 0.4482, + "delta_norm_band_sum": 49.5215 + }, + "swap@a2": { + "top1": " China", + "rank_tgt": 3, + "rank_src": 54, + "logp_tgt": -21.0588, + "logp_src": -40.0586, + "flip_to_tgt": false, + "kl_vs_clean": 41.0295, + "delta_norm_band_sum": 31397.7655, + "delta_norm_per_layer": { + "9": 3.8336, + "10": 5.6856, + "11": 10.54, + "12": 23.3723, + "13": 47.7047, + "14": 85.0959, + "15": 164.7859, + "16": 301.9756, + "17": 570.4111, + "18": 1063.3719, + "19": 2058.5728, + "20": 3838.3608, + "21": 8025.7046, + "22": 15198.3506 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 48137, + "rank_src": 50300, + "logp_tgt": -19.5396, + "logp_src": -29.0356, + "flip_to_tgt": false, + "kl_vs_clean": 20.4331, + "delta_norm_band_sum": 31397.7671, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " France", + "rank_tgt": 507, + "rank_src": 2, + "logp_tgt": -37.0171, + "logp_src": -16.6841, + "flip_to_tgt": false, + "kl_vs_clean": 23.2625, + "delta_norm_band_sum": 31397.7632, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Chinese", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0, + "logp_src": -9.9605, + "flip_to_tgt": true, + "kl_vs_clean": 23.3937, + "delta_norm_band_sum": 26104.951 + }, + "swap@a4": { + "top1": " China", + "rank_tgt": 2, + "rank_src": 158, + "logp_tgt": -20.7067, + "logp_src": -47.0359, + "flip_to_tgt": false, + "kl_vs_clean": 45.5349, + "delta_norm_band_sum": 304897372.1501, + "delta_norm_per_layer": { + "9": 8.4233, + "10": 26.6122, + "11": 100.4781, + "12": 365.992, + "13": 1458.1644, + "14": 5324.8843, + "15": 20692.9707, + "16": 72401.6562, + "17": 266335.3438, + "18": 986567.375, + "19": 3813216.25, + "20": 14347502.0, + "21": 59473020.0, + "22": 225910352.0 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 49765, + "rank_src": 50300, + "logp_tgt": -22.0657, + "logp_src": -33.903, + "flip_to_tgt": false, + "kl_vs_clean": 23.711, + "delta_norm_band_sum": 304897404.952, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " France", + "rank_tgt": 359, + "rank_src": 2, + "logp_tgt": -38.0789, + "logp_src": -16.532, + "flip_to_tgt": false, + "kl_vs_clean": 23.8144, + "delta_norm_band_sum": 304897370.3052, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Chinese", + "rank_tgt": 0, + "rank_src": 2, + "logp_tgt": -0.0, + "logp_src": -16.2772, + "flip_to_tgt": true, + "kl_vs_clean": 27.2298, + "delta_norm_band_sum": 259621811.2603 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "language", + "src": "Japan", + "tgt": "Germany", + "prompt": "The main language spoken in Japan is", + "answer_src": " Japanese", + "answer_tgt": " German", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Japanese", + "rank_tgt": 69, + "rank_src": 0, + "logp_tgt": -6.7442, + "logp_src": -1.7233, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Japanese", + "rank_tgt": 69, + "rank_src": 0, + "logp_tgt": -6.7442, + "logp_src": -1.7233, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Japanese", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -3.1383, + "logp_src": -1.3976, + "flip_to_tgt": false, + "kl_vs_clean": 0.2812, + "delta_norm_band_sum": 29.0148, + "delta_norm_per_layer": { + "9": 0.3631, + "10": 0.0529, + "11": 0.2799, + "12": 0.2463, + "13": 0.4118, + "14": 0.5302, + "15": 0.6229, + "16": 0.3458, + "17": 5.8525, + "18": 4.6356, + "19": 4.2803, + "20": 4.004, + "21": 2.8291, + "22": 4.5603 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 28, + "rank_src": 63, + "logp_tgt": -5.2022, + "logp_src": -6.355, + "flip_to_tgt": false, + "kl_vs_clean": 2.0797, + "delta_norm_band_sum": 29.0148, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Japanese", + "rank_tgt": 67, + "rank_src": 0, + "logp_tgt": -6.4037, + "logp_src": -1.8252, + "flip_to_tgt": false, + "kl_vs_clean": 0.2003, + "delta_norm_band_sum": 29.0148, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Japanese", + "rank_tgt": 7, + "rank_src": 0, + "logp_tgt": -3.795, + "logp_src": -1.3379, + "flip_to_tgt": false, + "kl_vs_clean": 0.1981, + "delta_norm_band_sum": 26.2107 + }, + "swap@a2": { + "top1": " Japan", + "rank_tgt": 18, + "rank_src": 4, + "logp_tgt": -29.4052, + "logp_src": -19.1629, + "flip_to_tgt": false, + "kl_vs_clean": 35.0063, + "delta_norm_band_sum": 22162.4194, + "delta_norm_per_layer": { + "9": 2.1141, + "10": 3.9631, + "11": 6.8647, + "12": 16.3709, + "13": 33.0085, + "14": 59.9969, + "15": 113.9133, + "16": 212.553, + "17": 398.1974, + "18": 762.3251, + "19": 1403.8762, + "20": 2798.45, + "21": 5463.583, + "22": 10887.2031 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 39521, + "rank_src": 50300, + "logp_tgt": -16.6832, + "logp_src": -27.2188, + "flip_to_tgt": false, + "kl_vs_clean": 13.8832, + "delta_norm_band_sum": 22162.4219, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Japan", + "rank_tgt": 95, + "rank_src": 2, + "logp_tgt": -33.158, + "logp_src": -14.2606, + "flip_to_tgt": false, + "kl_vs_clean": 28.8206, + "delta_norm_band_sum": 22162.4214, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Japanese", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -4.7835, + "logp_src": -0.0084, + "flip_to_tgt": false, + "kl_vs_clean": 29.6095, + "delta_norm_band_sum": 23058.1093 + }, + "swap@a4": { + "top1": " Japan", + "rank_tgt": 15, + "rank_src": 4, + "logp_tgt": -26.691, + "logp_src": -18.6357, + "flip_to_tgt": false, + "kl_vs_clean": 33.1498, + "delta_norm_band_sum": 239203961.7581, + "delta_norm_per_layer": { + "9": 6.1754, + "10": 22.8351, + "11": 80.4332, + "12": 303.0876, + "13": 1148.762, + "14": 4319.0098, + "15": 16315.8535, + "16": 57857.1328, + "17": 209455.2188, + "18": 798812.0, + "19": 2968173.25, + "20": 11629392.0, + "21": 45511788.0, + "22": 178006288.0 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a4": { + "top1": "mathchoice", + "rank_tgt": 50099, + "rank_src": 50300, + "logp_tgt": -24.2397, + "logp_src": -40.4517, + "flip_to_tgt": false, + "kl_vs_clean": 20.7917, + "delta_norm_band_sum": 239203955.3913, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Japan", + "rank_tgt": 79, + "rank_src": 2, + "logp_tgt": -34.5077, + "logp_src": -14.7739, + "flip_to_tgt": false, + "kl_vs_clean": 30.1595, + "delta_norm_band_sum": 239203972.3426, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " German", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0665, + "logp_src": -2.7442, + "flip_to_tgt": true, + "kl_vs_clean": 28.1061, + "delta_norm_band_sum": 253658833.9918 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "language", + "src": "Germany", + "tgt": "Italy", + "prompt": "The main language spoken in Germany is", + "answer_src": " German", + "answer_tgt": " Italian", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " German", + "rank_tgt": 42, + "rank_src": 0, + "logp_tgt": -6.5972, + "logp_src": -0.4806, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " German", + "rank_tgt": 42, + "rank_src": 0, + "logp_tgt": -6.5972, + "logp_src": -0.4806, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " German", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -1.6663, + "logp_src": -1.1025, + "flip_to_tgt": false, + "kl_vs_clean": 0.3633, + "delta_norm_band_sum": 22.4578, + "delta_norm_per_layer": { + "9": 1.0953, + "10": 0.4203, + "11": 0.7367, + "12": 1.499, + "13": 1.3744, + "14": 1.1447, + "15": 0.8782, + "16": 0.1861, + "17": 3.168, + "18": 3.1834, + "19": 3.0903, + "20": 2.5966, + "21": 2.2576, + "22": 0.8271 + }, + "gram_cond_max": 8.82 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 8, + "rank_src": 6, + "logp_tgt": -3.8171, + "logp_src": -3.7114, + "flip_to_tgt": false, + "kl_vs_clean": 1.9634, + "delta_norm_band_sum": 22.4578, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " German", + "rank_tgt": 29, + "rank_src": 0, + "logp_tgt": -6.1635, + "logp_src": -0.5893, + "flip_to_tgt": false, + "kl_vs_clean": 0.053, + "delta_norm_band_sum": 22.4578, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " German", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -1.6913, + "logp_src": -1.0103, + "flip_to_tgt": false, + "kl_vs_clean": 0.3354, + "delta_norm_band_sum": 24.4319 + }, + "swap@a2": { + "top1": " Germany", + "rank_tgt": 46, + "rank_src": 5, + "logp_tgt": -35.0505, + "logp_src": -20.9445, + "flip_to_tgt": false, + "kl_vs_clean": 27.3328, + "delta_norm_band_sum": 24927.5023, + "delta_norm_per_layer": { + "9": 2.4206, + "10": 4.1207, + "11": 7.1108, + "12": 20.8797, + "13": 38.6385, + "14": 72.6666, + "15": 130.8792, + "16": 242.8225, + "17": 457.1577, + "18": 888.1027, + "19": 1677.7207, + "20": 3236.3113, + "21": 6226.9185, + "22": 11921.7529 + }, + "gram_cond_max": 8.82 + }, + "rand_target@a2": { + "top1": "enance", + "rank_tgt": 48345, + "rank_src": 50297, + "logp_tgt": -19.9752, + "logp_src": -31.9221, + "flip_to_tgt": false, + "kl_vs_clean": 25.0168, + "delta_norm_band_sum": 24927.5035, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Germany", + "rank_tgt": 169, + "rank_src": 4, + "logp_tgt": -34.2683, + "logp_src": -16.8895, + "flip_to_tgt": false, + "kl_vs_clean": 21.3542, + "delta_norm_band_sum": 24927.5, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " German", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -19.4013, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 14.0455, + "delta_norm_band_sum": 30188.2045 + }, + "swap@a4": { + "top1": " Germany", + "rank_tgt": 64, + "rank_src": 4, + "logp_tgt": -38.673, + "logp_src": -20.2999, + "flip_to_tgt": false, + "kl_vs_clean": 27.0145, + "delta_norm_band_sum": 238668280.2447, + "delta_norm_per_layer": { + "9": 6.0147, + "10": 21.9105, + "11": 76.6031, + "12": 310.7428, + "13": 1149.9238, + "14": 4506.498, + "15": 16155.5752, + "16": 58052.2266, + "17": 210454.125, + "18": 814307.125, + "19": 3059973.5, + "20": 11864222.0, + "21": 45964756.0, + "22": 176674288.0 + }, + "gram_cond_max": 8.82 + }, + "rand_target@a4": { + "top1": " Germany", + "rank_tgt": 355, + "rank_src": 5, + "logp_tgt": -27.3538, + "logp_src": -12.8575, + "flip_to_tgt": false, + "kl_vs_clean": 15.5713, + "delta_norm_band_sum": 238668244.4805, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Germany", + "rank_tgt": 150, + "rank_src": 4, + "logp_tgt": -34.7971, + "logp_src": -16.7561, + "flip_to_tgt": false, + "kl_vs_clean": 21.5207, + "delta_norm_band_sum": 238668285.545, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " German", + "rank_tgt": 8, + "rank_src": 0, + "logp_tgt": -26.8267, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 14.3458, + "delta_norm_band_sum": 266039528.1426 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Italy", + "tgt": "Spain", + "prompt": "The main language spoken in Italy is", + "answer_src": " Italian", + "answer_tgt": " Spanish", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Italian", + "rank_tgt": 32, + "rank_src": 0, + "logp_tgt": -5.7337, + "logp_src": -0.7296, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Italian", + "rank_tgt": 32, + "rank_src": 0, + "logp_tgt": -5.7337, + "logp_src": -0.7296, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Italian", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -2.8444, + "logp_src": -0.8439, + "flip_to_tgt": false, + "kl_vs_clean": 0.1304, + "delta_norm_band_sum": 18.4742, + "delta_norm_per_layer": { + "9": 0.0651, + "10": 0.149, + "11": 0.2827, + "12": 0.4925, + "13": 1.0121, + "14": 0.8356, + "15": 0.4213, + "16": 0.2233, + "17": 2.4167, + "18": 2.6716, + "19": 2.3902, + "20": 2.4822, + "21": 2.17, + "22": 2.8615 + }, + "gram_cond_max": 10.5 + }, + "rand_target@a1": { + "top1": " Italian", + "rank_tgt": 44, + "rank_src": 0, + "logp_tgt": -5.803, + "logp_src": -1.4783, + "flip_to_tgt": false, + "kl_vs_clean": 0.3729, + "delta_norm_band_sum": 18.4742, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Italian", + "rank_tgt": 24, + "rank_src": 0, + "logp_tgt": -5.4878, + "logp_src": -0.6682, + "flip_to_tgt": false, + "kl_vs_clean": 0.0811, + "delta_norm_band_sum": 18.4742, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Italian", + "rank_tgt": 4, + "rank_src": 0, + "logp_tgt": -4.0248, + "logp_src": -0.7253, + "flip_to_tgt": false, + "kl_vs_clean": 0.0681, + "delta_norm_band_sum": 28.3817 + }, + "swap@a2": { + "top1": " Spain", + "rank_tgt": 13, + "rank_src": 8, + "logp_tgt": -22.8795, + "logp_src": -21.3204, + "flip_to_tgt": false, + "kl_vs_clean": 28.586, + "delta_norm_band_sum": 18360.0974, + "delta_norm_per_layer": { + "9": 1.5417, + "10": 3.0526, + "11": 5.2966, + "12": 14.5161, + "13": 28.8969, + "14": 52.5895, + "15": 96.8208, + "16": 179.7659, + "17": 334.4159, + "18": 643.0988, + "19": 1201.2289, + "20": 2340.05, + "21": 4593.0532, + "22": 8865.7705 + }, + "gram_cond_max": 10.5 + }, + "rand_target@a2": { + "top1": "andum", + "rank_tgt": 50259, + "rank_src": 50298, + "logp_tgt": -30.9154, + "logp_src": -38.2842, + "flip_to_tgt": false, + "kl_vs_clean": 26.7679, + "delta_norm_band_sum": 18360.0982, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Italy", + "rank_tgt": 56, + "rank_src": 4, + "logp_tgt": -29.8744, + "logp_src": -17.2564, + "flip_to_tgt": false, + "kl_vs_clean": 23.8779, + "delta_norm_band_sum": 18360.097, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Spanish", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0, + "logp_src": -11.0468, + "flip_to_tgt": true, + "kl_vs_clean": 26.0723, + "delta_norm_band_sum": 22696.3571 + }, + "swap@a4": { + "top1": " Spain", + "rank_tgt": 11, + "rank_src": 8, + "logp_tgt": -22.4428, + "logp_src": -21.5088, + "flip_to_tgt": false, + "kl_vs_clean": 28.2925, + "delta_norm_band_sum": 186623700.7249, + "delta_norm_per_layer": { + "9": 4.6226, + "10": 17.4469, + "11": 61.0998, + "12": 239.6521, + "13": 904.2989, + "14": 3411.4924, + "15": 12769.8779, + "16": 45171.7656, + "17": 163349.6562, + "18": 626393.5625, + "19": 2340505.25, + "20": 9130116.0, + "21": 35803316.0, + "22": 138497440.0 + }, + "gram_cond_max": 10.5 + }, + "rand_target@a4": { + "top1": " Italy", + "rank_tgt": 31, + "rank_src": 5, + "logp_tgt": -19.9988, + "logp_src": -13.675, + "flip_to_tgt": false, + "kl_vs_clean": 17.5655, + "delta_norm_band_sum": 186623696.4399, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Italy", + "rank_tgt": 51, + "rank_src": 3, + "logp_tgt": -30.5024, + "logp_src": -17.177, + "flip_to_tgt": false, + "kl_vs_clean": 24.1777, + "delta_norm_band_sum": 186623681.1924, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Spanish", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0, + "logp_src": -13.8212, + "flip_to_tgt": true, + "kl_vs_clean": 27.3597, + "delta_norm_band_sum": 211103325.1962 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "language", + "src": "Spain", + "tgt": "Russia", + "prompt": "The main language spoken in Spain is", + "answer_src": " Spanish", + "answer_tgt": " Russian", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Spanish", + "rank_tgt": 997, + "rank_src": 0, + "logp_tgt": -11.0697, + "logp_src": -0.9243, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Spanish", + "rank_tgt": 997, + "rank_src": 0, + "logp_tgt": -11.0697, + "logp_src": -0.9243, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Spanish", + "rank_tgt": 47, + "rank_src": 0, + "logp_tgt": -7.178, + "logp_src": -0.2893, + "flip_to_tgt": false, + "kl_vs_clean": 0.479, + "delta_norm_band_sum": 50.1065, + "delta_norm_per_layer": { + "9": 0.5403, + "10": 0.095, + "11": 0.6935, + "12": 0.4273, + "13": 0.3552, + "14": 0.5001, + "15": 1.5823, + "16": 1.9571, + "17": 8.5123, + "18": 5.8989, + "19": 7.8241, + "20": 7.598, + "21": 6.6481, + "22": 7.4742 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a1": { + "top1": " called", + "rank_tgt": 541, + "rank_src": 86, + "logp_tgt": -9.0816, + "logp_src": -6.5661, + "flip_to_tgt": false, + "kl_vs_clean": 3.1752, + "delta_norm_band_sum": 50.1065, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Spanish", + "rank_tgt": 555, + "rank_src": 0, + "logp_tgt": -9.562, + "logp_src": -1.3927, + "flip_to_tgt": false, + "kl_vs_clean": 0.4857, + "delta_norm_band_sum": 50.1065, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Spanish", + "rank_tgt": 12, + "rank_src": 0, + "logp_tgt": -5.357, + "logp_src": -0.4168, + "flip_to_tgt": false, + "kl_vs_clean": 0.4627, + "delta_norm_band_sum": 38.3494 + }, + "swap@a2": { + "top1": " Russia", + "rank_tgt": 7, + "rank_src": 49, + "logp_tgt": -18.5575, + "logp_src": -29.4386, + "flip_to_tgt": false, + "kl_vs_clean": 33.2396, + "delta_norm_band_sum": 20605.6062, + "delta_norm_per_layer": { + "9": 2.0373, + "10": 3.6606, + "11": 6.4748, + "12": 16.1288, + "13": 31.1666, + "14": 57.676, + "15": 105.3039, + "16": 196.5682, + "17": 373.7943, + "18": 709.7169, + "19": 1356.3766, + "20": 2607.7759, + "21": 5145.4282, + "22": 9993.498 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a2": { + "top1": " Spain", + "rank_tgt": 4687, + "rank_src": 7, + "logp_tgt": -21.9813, + "logp_src": -10.2957, + "flip_to_tgt": false, + "kl_vs_clean": 12.4287, + "delta_norm_band_sum": 20605.606, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Spain", + "rank_tgt": 272, + "rank_src": 3, + "logp_tgt": -33.9123, + "logp_src": -16.2242, + "flip_to_tgt": false, + "kl_vs_clean": 23.9685, + "delta_norm_band_sum": 20605.6068, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Spanish", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -7.5067, + "logp_src": -0.0006, + "flip_to_tgt": false, + "kl_vs_clean": 21.2648, + "delta_norm_band_sum": 22843.3351 + }, + "swap@a4": { + "top1": " Russia", + "rank_tgt": 6, + "rank_src": 59, + "logp_tgt": -18.179, + "logp_src": -31.0351, + "flip_to_tgt": false, + "kl_vs_clean": 34.1024, + "delta_norm_band_sum": 227327920.0967, + "delta_norm_per_layer": { + "9": 5.7632, + "10": 21.1412, + "11": 74.3618, + "12": 285.6572, + "13": 1080.1874, + "14": 4084.436, + "15": 15138.0459, + "16": 53972.3008, + "17": 198918.3281, + "18": 755615.375, + "19": 2856925.5, + "20": 11015911.0, + "21": 43520816.0, + "22": 168905072.0 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 49726, + "rank_src": 50300, + "logp_tgt": -21.7837, + "logp_src": -34.2832, + "flip_to_tgt": false, + "kl_vs_clean": 22.5764, + "delta_norm_band_sum": 227327918.0155, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Spain", + "rank_tgt": 201, + "rank_src": 3, + "logp_tgt": -34.9085, + "logp_src": -16.955, + "flip_to_tgt": false, + "kl_vs_clean": 25.2782, + "delta_norm_band_sum": 227327916.0514, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Spanish", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -6.0434, + "logp_src": -0.0024, + "flip_to_tgt": false, + "kl_vs_clean": 20.5221, + "delta_norm_band_sum": 238441962.5396 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Russia", + "tgt": "Egypt", + "prompt": "The main language spoken in Russia is", + "answer_src": " Russian", + "answer_tgt": " Arabic", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Russian", + "rank_tgt": 239, + "rank_src": 0, + "logp_tgt": -8.994, + "logp_src": -0.4226, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Russian", + "rank_tgt": 239, + "rank_src": 0, + "logp_tgt": -8.994, + "logp_src": -0.4226, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Russian", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -3.8149, + "logp_src": -1.0128, + "flip_to_tgt": false, + "kl_vs_clean": 0.2745, + "delta_norm_band_sum": 24.6962, + "delta_norm_per_layer": { + "9": 0.0314, + "10": 0.1305, + "11": 0.2794, + "12": 1.2075, + "13": 0.8833, + "14": 0.1147, + "15": 0.5722, + "16": 0.6351, + "17": 3.5098, + "18": 3.7762, + "19": 3.9642, + "20": 3.6498, + "21": 3.8844, + "22": 2.0576 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 32, + "rank_src": 25, + "logp_tgt": -5.0995, + "logp_src": -4.9991, + "flip_to_tgt": false, + "kl_vs_clean": 2.9424, + "delta_norm_band_sum": 24.6962, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Russian", + "rank_tgt": 57, + "rank_src": 0, + "logp_tgt": -6.6537, + "logp_src": -0.7673, + "flip_to_tgt": false, + "kl_vs_clean": 0.1652, + "delta_norm_band_sum": 24.6962, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Russian", + "rank_tgt": 33, + "rank_src": 0, + "logp_tgt": -6.656, + "logp_src": -0.3007, + "flip_to_tgt": false, + "kl_vs_clean": 0.0907, + "delta_norm_band_sum": 44.6322 + }, + "swap@a2": { + "top1": " Russia", + "rank_tgt": 480, + "rank_src": 3, + "logp_tgt": -39.0807, + "logp_src": -17.1359, + "flip_to_tgt": false, + "kl_vs_clean": 21.2738, + "delta_norm_band_sum": 23805.3969, + "delta_norm_per_layer": { + "9": 2.0459, + "10": 3.9215, + "11": 7.0261, + "12": 18.5756, + "13": 35.8927, + "14": 67.8587, + "15": 122.1888, + "16": 229.0579, + "17": 434.3582, + "18": 843.5316, + "19": 1565.1493, + "20": 3083.178, + "21": 5962.563, + "22": 11430.0498 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a2": { + "top1": " Russia", + "rank_tgt": 3714, + "rank_src": 1, + "logp_tgt": -47.3487, + "logp_src": -17.1413, + "flip_to_tgt": false, + "kl_vs_clean": 22.3721, + "delta_norm_band_sum": 23805.3986, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Russia", + "rank_tgt": 2418, + "rank_src": 2, + "logp_tgt": -37.2621, + "logp_src": -13.8475, + "flip_to_tgt": false, + "kl_vs_clean": 18.1638, + "delta_norm_band_sum": 23805.3953, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Russian", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -18.7222, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 9.8835, + "delta_norm_band_sum": 23594.4838 + }, + "swap@a4": { + "top1": " Russia", + "rank_tgt": 559, + "rank_src": 3, + "logp_tgt": -38.0805, + "logp_src": -16.3416, + "flip_to_tgt": false, + "kl_vs_clean": 20.0069, + "delta_norm_band_sum": 242312743.1683, + "delta_norm_per_layer": { + "9": 6.1411, + "10": 22.7166, + "11": 80.3146, + "12": 309.8742, + "13": 1159.3943, + "14": 4428.3252, + "15": 16384.4844, + "16": 58310.0586, + "17": 213874.6094, + "18": 822574.75, + "19": 3048957.5, + "20": 11956383.0, + "21": 46636956.0, + "22": 179553296.0 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 46190, + "rank_src": 50300, + "logp_tgt": -19.0971, + "logp_src": -33.1134, + "flip_to_tgt": false, + "kl_vs_clean": 26.225, + "delta_norm_band_sum": 242312744.8778, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Russia", + "rank_tgt": 2900, + "rank_src": 1, + "logp_tgt": -39.8662, + "logp_src": -13.9057, + "flip_to_tgt": false, + "kl_vs_clean": 18.469, + "delta_norm_band_sum": 242312722.1267, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Russian", + "rank_tgt": 33, + "rank_src": 0, + "logp_tgt": -32.9324, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 11.1835, + "delta_norm_band_sum": 245717949.3186 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Egypt", + "tgt": "India", + "prompt": "The main language spoken in Egypt is", + "answer_src": " Arabic", + "answer_tgt": " Hindi", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Arabic", + "rank_tgt": 167, + "rank_src": 0, + "logp_tgt": -7.705, + "logp_src": -1.9369, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Arabic", + "rank_tgt": 167, + "rank_src": 0, + "logp_tgt": -7.705, + "logp_src": -1.9369, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Arabic", + "rank_tgt": 14, + "rank_src": 0, + "logp_tgt": -4.737, + "logp_src": -2.0508, + "flip_to_tgt": false, + "kl_vs_clean": 0.2176, + "delta_norm_band_sum": 45.6803, + "delta_norm_per_layer": { + "9": 0.7031, + "10": 0.6807, + "11": 0.7778, + "12": 0.9711, + "13": 0.7593, + "14": 0.4589, + "15": 0.9854, + "16": 1.0645, + "17": 6.7563, + "18": 4.9644, + "19": 6.0006, + "20": 6.8288, + "21": 6.2016, + "22": 8.5278 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 54, + "rank_src": 9, + "logp_tgt": -6.3754, + "logp_src": -4.4373, + "flip_to_tgt": false, + "kl_vs_clean": 1.0103, + "delta_norm_band_sum": 45.6803, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " C", + "rank_tgt": 195, + "rank_src": 1, + "logp_tgt": -7.9924, + "logp_src": -1.992, + "flip_to_tgt": false, + "kl_vs_clean": 0.4981, + "delta_norm_band_sum": 45.6803, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " the", + "rank_tgt": 48, + "rank_src": 4, + "logp_tgt": -5.7137, + "logp_src": -3.4684, + "flip_to_tgt": false, + "kl_vs_clean": 0.2608, + "delta_norm_band_sum": 19.9876 + }, + "swap@a2": { + "top1": " Egypt", + "rank_tgt": 241, + "rank_src": 178, + "logp_tgt": -37.4294, + "logp_src": -36.3598, + "flip_to_tgt": false, + "kl_vs_clean": 31.4352, + "delta_norm_band_sum": 24929.4296, + "delta_norm_per_layer": { + "9": 2.3717, + "10": 4.2568, + "11": 8.277, + "12": 20.9801, + "13": 42.99, + "14": 75.2381, + "15": 143.4856, + "16": 252.7803, + "17": 468.7535, + "18": 862.8579, + "19": 1637.1257, + "20": 3149.2046, + "21": 6299.1309, + "22": 11961.9775 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a2": { + "top1": " Egypt", + "rank_tgt": 2307, + "rank_src": 263, + "logp_tgt": -23.0744, + "logp_src": -19.8943, + "flip_to_tgt": false, + "kl_vs_clean": 15.5543, + "delta_norm_band_sum": 24929.4308, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Egypt", + "rank_tgt": 3643, + "rank_src": 26, + "logp_tgt": -37.3821, + "logp_src": -27.2476, + "flip_to_tgt": false, + "kl_vs_clean": 25.9845, + "delta_norm_band_sum": 24929.4299, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Arabic", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -3.201, + "logp_src": -0.0416, + "flip_to_tgt": false, + "kl_vs_clean": 21.4346, + "delta_norm_band_sum": 21991.1031 + }, + "swap@a4": { + "top1": " Egypt", + "rank_tgt": 298, + "rank_src": 217, + "logp_tgt": -37.8621, + "logp_src": -36.6192, + "flip_to_tgt": false, + "kl_vs_clean": 30.743, + "delta_norm_band_sum": 243544062.7394, + "delta_norm_per_layer": { + "9": 6.4692, + "10": 22.9441, + "11": 85.9465, + "12": 313.5377, + "13": 1216.8766, + "14": 4473.5396, + "15": 17213.9766, + "16": 59010.4648, + "17": 216777.1719, + "18": 813370.3125, + "19": 3077138.5, + "20": 11863425.0, + "21": 47115920.0, + "22": 180375088.0 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a4": { + "top1": "ningen", + "rank_tgt": 37897, + "rank_src": 50268, + "logp_tgt": -17.3744, + "logp_src": -25.4235, + "flip_to_tgt": false, + "kl_vs_clean": 18.0866, + "delta_norm_band_sum": 243544050.034, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Egypt", + "rank_tgt": 3711, + "rank_src": 44, + "logp_tgt": -39.9549, + "logp_src": -29.6827, + "flip_to_tgt": false, + "kl_vs_clean": 27.1873, + "delta_norm_band_sum": 243544091.8039, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Hind", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0002, + "logp_src": -8.4353, + "flip_to_tgt": true, + "kl_vs_clean": 24.472, + "delta_norm_band_sum": 195401735.4885 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "language", + "src": "India", + "tgt": "Brazil", + "prompt": "The main language spoken in India is", + "answer_src": " Hindi", + "answer_tgt": " Portuguese", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Hind", + "rank_tgt": 91, + "rank_src": 0, + "logp_tgt": -7.6046, + "logp_src": -1.0568, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Hind", + "rank_tgt": 91, + "rank_src": 0, + "logp_tgt": -7.6046, + "logp_src": -1.0568, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " the", + "rank_tgt": 4, + "rank_src": 1, + "logp_tgt": -2.9884, + "logp_src": -2.1713, + "flip_to_tgt": false, + "kl_vs_clean": 0.374, + "delta_norm_band_sum": 14.5676, + "delta_norm_per_layer": { + "9": 1.325, + "10": 0.6491, + "11": 1.2685, + "12": 1.21, + "13": 1.3463, + "14": 1.3971, + "15": 1.2617, + "16": 0.244, + "17": 1.3534, + "18": 1.6, + "19": 0.9519, + "20": 0.1689, + "21": 1.1548, + "22": 0.6369 + }, + "gram_cond_max": 5.28 + }, + "rand_target@a1": { + "top1": " Spanish", + "rank_tgt": 10, + "rank_src": 29, + "logp_tgt": -4.1801, + "logp_src": -5.2446, + "flip_to_tgt": false, + "kl_vs_clean": 2.1042, + "delta_norm_band_sum": 14.5676, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Hind", + "rank_tgt": 87, + "rank_src": 0, + "logp_tgt": -7.2425, + "logp_src": -1.564, + "flip_to_tgt": false, + "kl_vs_clean": 0.1118, + "delta_norm_band_sum": 14.5676, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " English", + "rank_tgt": 7, + "rank_src": 4, + "logp_tgt": -3.3766, + "logp_src": -2.6441, + "flip_to_tgt": false, + "kl_vs_clean": 0.3956, + "delta_norm_band_sum": 13.8205 + }, + "swap@a2": { + "top1": " India", + "rank_tgt": 582, + "rank_src": 107, + "logp_tgt": -49.9505, + "logp_src": -42.7703, + "flip_to_tgt": false, + "kl_vs_clean": 42.5763, + "delta_norm_band_sum": 19182.2503, + "delta_norm_per_layer": { + "9": 2.2263, + "10": 4.0513, + "11": 6.8742, + "12": 15.595, + "13": 26.3924, + "14": 51.2954, + "15": 89.946, + "16": 182.6373, + "17": 322.7949, + "18": 671.4594, + "19": 1184.3151, + "20": 2439.1304, + "21": 4587.2856, + "22": 9598.2471 + }, + "gram_cond_max": 5.28 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 39657, + "rank_src": 39845, + "logp_tgt": -17.2327, + "logp_src": -17.2651, + "flip_to_tgt": false, + "kl_vs_clean": 13.1656, + "delta_norm_band_sum": 19182.2486, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " India", + "rank_tgt": 4597, + "rank_src": 85, + "logp_tgt": -42.1196, + "logp_src": -32.603, + "flip_to_tgt": false, + "kl_vs_clean": 32.1308, + "delta_norm_band_sum": 19182.2502, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Hind", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -12.2678, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 18.5616, + "delta_norm_band_sum": 19664.1027 + }, + "swap@a4": { + "top1": " India", + "rank_tgt": 2138, + "rank_src": 81, + "logp_tgt": -57.03, + "logp_src": -44.5292, + "flip_to_tgt": false, + "kl_vs_clean": 44.8897, + "delta_norm_band_sum": 198369182.7203, + "delta_norm_per_layer": { + "9": 4.8336, + "10": 19.9579, + "11": 61.5766, + "12": 257.3708, + "13": 894.3665, + "14": 3602.2507, + "15": 12411.4229, + "16": 47742.832, + "17": 160340.2344, + "18": 660871.875, + "19": 2321560.0, + "20": 9572132.0, + "21": 35919860.0, + "22": 149669424.0 + }, + "gram_cond_max": 5.28 + }, + "rand_target@a4": { + "top1": "ningen", + "rank_tgt": 47003, + "rank_src": 49985, + "logp_tgt": -19.9009, + "logp_src": -23.4043, + "flip_to_tgt": false, + "kl_vs_clean": 18.6131, + "delta_norm_band_sum": 198369186.6699, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " India", + "rank_tgt": 3557, + "rank_src": 105, + "logp_tgt": -44.6879, + "logp_src": -35.3836, + "flip_to_tgt": false, + "kl_vs_clean": 34.4121, + "delta_norm_band_sum": 198369178.7823, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Hind", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -30.2403, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 23.0737, + "delta_norm_band_sum": 196756134.9473 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Brazil", + "tgt": "Canada", + "prompt": "The main language spoken in Brazil is", + "answer_src": " Portuguese", + "answer_tgt": " English", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Portuguese", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -3.118, + "logp_src": -1.07, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Portuguese", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -3.118, + "logp_src": -1.07, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " French", + "rank_tgt": 2, + "rank_src": 1, + "logp_tgt": -2.5719, + "logp_src": -1.7981, + "flip_to_tgt": false, + "kl_vs_clean": 0.4029, + "delta_norm_band_sum": 44.3828, + "delta_norm_per_layer": { + "9": 1.7758, + "10": 0.8161, + "11": 0.6479, + "12": 0.1846, + "13": 0.175, + "14": 0.0138, + "15": 0.8887, + "16": 0.271, + "17": 5.8999, + "18": 4.4686, + "19": 7.0032, + "20": 7.5435, + "21": 6.9637, + "22": 7.731 + }, + "gram_cond_max": 4.81 + }, + "rand_target@a1": { + "top1": " called", + "rank_tgt": 4, + "rank_src": 59, + "logp_tgt": -3.4317, + "logp_src": -6.0872, + "flip_to_tgt": false, + "kl_vs_clean": 2.312, + "delta_norm_band_sum": 44.3828, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " C", + "rank_tgt": 11, + "rank_src": 2, + "logp_tgt": -4.5037, + "logp_src": -3.098, + "flip_to_tgt": false, + "kl_vs_clean": 0.8437, + "delta_norm_band_sum": 44.3828, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " the", + "rank_tgt": 1, + "rank_src": 2, + "logp_tgt": -2.0319, + "logp_src": -2.6254, + "flip_to_tgt": false, + "kl_vs_clean": 0.3876, + "delta_norm_band_sum": 26.4163 + }, + "swap@a2": { + "top1": " Canada", + "rank_tgt": 288, + "rank_src": 777, + "logp_tgt": -47.6066, + "logp_src": -50.9703, + "flip_to_tgt": false, + "kl_vs_clean": 43.9932, + "delta_norm_band_sum": 21979.2515, + "delta_norm_per_layer": { + "9": 3.0812, + "10": 5.1582, + "11": 7.9851, + "12": 19.8599, + "13": 34.8069, + "14": 70.902, + "15": 120.4815, + "16": 230.9441, + "17": 380.3842, + "18": 778.7027, + "19": 1397.4111, + "20": 2838.865, + "21": 5235.8755, + "22": 10854.7939 + }, + "gram_cond_max": 4.81 + }, + "rand_target@a2": { + "top1": " Brazil", + "rank_tgt": 330, + "rank_src": 28, + "logp_tgt": -36.188, + "logp_src": -27.7836, + "flip_to_tgt": false, + "kl_vs_clean": 27.439, + "delta_norm_band_sum": 21979.2503, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Brazil", + "rank_tgt": 302, + "rank_src": 20, + "logp_tgt": -29.1466, + "logp_src": -20.8573, + "flip_to_tgt": false, + "kl_vs_clean": 21.8871, + "delta_norm_band_sum": 21979.2517, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " English", + "rank_tgt": 0, + "rank_src": 3, + "logp_tgt": -0.0, + "logp_src": -20.9859, + "flip_to_tgt": true, + "kl_vs_clean": 34.7013, + "delta_norm_band_sum": 23640.2529 + }, + "swap@a4": { + "top1": " Canada", + "rank_tgt": 358, + "rank_src": 4192, + "logp_tgt": -53.7407, + "logp_src": -61.821, + "flip_to_tgt": false, + "kl_vs_clean": 51.523, + "delta_norm_band_sum": 257079850.9993, + "delta_norm_per_layer": { + "9": 6.6817, + "10": 25.3338, + "11": 82.6923, + "12": 346.6964, + "13": 1211.1093, + "14": 4952.3257, + "15": 17195.7148, + "16": 64210.5391, + "17": 209722.4688, + "18": 864827.9375, + "19": 3080231.5, + "20": 12624522.0, + "21": 46578980.0, + "22": 193633536.0 + }, + "gram_cond_max": 4.81 + }, + "rand_target@a4": { + "top1": "ningen", + "rank_tgt": 49591, + "rank_src": 50287, + "logp_tgt": -22.2786, + "logp_src": -28.756, + "flip_to_tgt": false, + "kl_vs_clean": 20.835, + "delta_norm_band_sum": 257079878.7264, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Brazil", + "rank_tgt": 275, + "rank_src": 18, + "logp_tgt": -30.3032, + "logp_src": -22.1356, + "flip_to_tgt": false, + "kl_vs_clean": 22.8617, + "delta_norm_band_sum": 257079878.7602, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " English", + "rank_tgt": 0, + "rank_src": 7, + "logp_tgt": -0.0, + "logp_src": -38.8327, + "flip_to_tgt": true, + "kl_vs_clean": 44.6566, + "delta_norm_band_sum": 269313773.6555 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "language", + "src": "Canada", + "tgt": "Mexico", + "prompt": "The main language spoken in Canada is", + "answer_src": " English", + "answer_tgt": " Spanish", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " English", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -3.951, + "logp_src": -0.8963, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " English", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -3.951, + "logp_src": -0.8963, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " English", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -1.9834, + "logp_src": -0.8056, + "flip_to_tgt": false, + "kl_vs_clean": 0.2322, + "delta_norm_band_sum": 33.2157, + "delta_norm_per_layer": { + "9": 0.7976, + "10": 0.9539, + "11": 0.7198, + "12": 0.4611, + "13": 0.505, + "14": 0.2368, + "15": 0.004, + "16": 0.642, + "17": 5.1354, + "18": 6.2223, + "19": 5.4062, + "20": 4.6858, + "21": 3.9903, + "22": 3.4555 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 2, + "rank_src": 3, + "logp_tgt": -2.5395, + "logp_src": -2.9404, + "flip_to_tgt": false, + "kl_vs_clean": 1.0989, + "delta_norm_band_sum": 33.2157, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " French", + "rank_tgt": 4, + "rank_src": 1, + "logp_tgt": -3.8346, + "logp_src": -1.729, + "flip_to_tgt": false, + "kl_vs_clean": 0.267, + "delta_norm_band_sum": 33.2157, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " French", + "rank_tgt": 2, + "rank_src": 1, + "logp_tgt": -1.8757, + "logp_src": -1.6359, + "flip_to_tgt": false, + "kl_vs_clean": 0.218, + "delta_norm_band_sum": 7.7239 + }, + "swap@a2": { + "top1": " Canada", + "rank_tgt": 413, + "rank_src": 317, + "logp_tgt": -49.6655, + "logp_src": -48.5996, + "flip_to_tgt": false, + "kl_vs_clean": 44.7794, + "delta_norm_band_sum": 24245.1235, + "delta_norm_per_layer": { + "9": 2.4015, + "10": 5.0235, + "11": 8.0749, + "12": 20.1083, + "13": 37.2718, + "14": 71.702, + "15": 129.4879, + "16": 246.2786, + "17": 430.3999, + "18": 860.877, + "19": 1624.7393, + "20": 3161.832, + "21": 5926.334, + "22": 11720.5928 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a2": { + "top1": " Canada", + "rank_tgt": 2414, + "rank_src": 1419, + "logp_tgt": -22.4073, + "logp_src": -21.642, + "flip_to_tgt": false, + "kl_vs_clean": 18.5719, + "delta_norm_band_sum": 24245.1244, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Canada", + "rank_tgt": 1417, + "rank_src": 440, + "logp_tgt": -40.9477, + "logp_src": -38.4151, + "flip_to_tgt": false, + "kl_vs_clean": 35.0783, + "delta_norm_band_sum": 24245.1256, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " English", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -15.5317, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 19.5435, + "delta_norm_band_sum": 24382.9679 + }, + "swap@a4": { + "top1": " Canada", + "rank_tgt": 645, + "rank_src": 333, + "logp_tgt": -52.5277, + "logp_src": -50.2375, + "flip_to_tgt": false, + "kl_vs_clean": 46.2682, + "delta_norm_band_sum": 268194965.3856, + "delta_norm_per_layer": { + "9": 6.6538, + "10": 25.3439, + "11": 85.6799, + "12": 344.1767, + "13": 1264.124, + "14": 5015.3467, + "15": 18272.8105, + "16": 66589.1406, + "17": 229793.1719, + "18": 905228.9375, + "19": 3386104.0, + "20": 13240744.0, + "21": 50566260.0, + "22": 199775232.0 + }, + "gram_cond_max": 4.71 + }, + "rand_target@a4": { + "top1": "ningen", + "rank_tgt": 49022, + "rank_src": 48872, + "logp_tgt": -20.7372, + "logp_src": -20.5449, + "flip_to_tgt": false, + "kl_vs_clean": 18.0319, + "delta_norm_band_sum": 268194934.3856, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Canada", + "rank_tgt": 958, + "rank_src": 401, + "logp_tgt": -42.9171, + "logp_src": -40.6398, + "flip_to_tgt": false, + "kl_vs_clean": 37.0336, + "delta_norm_band_sum": 268194958.1212, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " English", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -16.659, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 19.5442, + "delta_norm_band_sum": 250157012.3686 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Poland", + "tgt": "Sweden", + "prompt": "The main language spoken in Poland is", + "answer_src": " Polish", + "answer_tgt": " Swedish", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Polish", + "rank_tgt": 89, + "rank_src": 0, + "logp_tgt": -7.1121, + "logp_src": -0.7416, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Polish", + "rank_tgt": 89, + "rank_src": 0, + "logp_tgt": -7.1121, + "logp_src": -0.7416, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Polish", + "rank_tgt": 7, + "rank_src": 0, + "logp_tgt": -3.8038, + "logp_src": -1.8532, + "flip_to_tgt": false, + "kl_vs_clean": 0.3228, + "delta_norm_band_sum": 17.0579, + "delta_norm_per_layer": { + "9": 0.168, + "10": 0.1215, + "11": 0.349, + "12": 1.0765, + "13": 0.2283, + "14": 0.3719, + "15": 0.3805, + "16": 0.415, + "17": 3.0581, + "18": 3.0024, + "19": 2.5685, + "20": 2.1015, + "21": 2.2076, + "22": 1.0091 + }, + "gram_cond_max": 6.96 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 157, + "rank_src": 39, + "logp_tgt": -7.3862, + "logp_src": -5.5128, + "flip_to_tgt": false, + "kl_vs_clean": 2.324, + "delta_norm_band_sum": 17.0579, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Polish", + "rank_tgt": 44, + "rank_src": 0, + "logp_tgt": -6.2806, + "logp_src": -0.564, + "flip_to_tgt": false, + "kl_vs_clean": 0.0934, + "delta_norm_band_sum": 17.0579, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Polish", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -3.3147, + "logp_src": -2.2049, + "flip_to_tgt": false, + "kl_vs_clean": 0.4841, + "delta_norm_band_sum": 15.6497 + }, + "swap@a2": { + "top1": " Poland", + "rank_tgt": 24, + "rank_src": 9, + "logp_tgt": -24.1859, + "logp_src": -18.9126, + "flip_to_tgt": false, + "kl_vs_clean": 23.9928, + "delta_norm_band_sum": 17237.7855, + "delta_norm_per_layer": { + "9": 1.6601, + "10": 3.2812, + "11": 5.7049, + "12": 13.4101, + "13": 27.4612, + "14": 48.9966, + "15": 90.0772, + "16": 167.8968, + "17": 314.4961, + "18": 614.457, + "19": 1120.98, + "20": 2225.5996, + "21": 4296.9502, + "22": 8306.8145 + }, + "gram_cond_max": 6.96 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 35863, + "rank_src": 50231, + "logp_tgt": -16.3144, + "logp_src": -23.1444, + "flip_to_tgt": false, + "kl_vs_clean": 16.1912, + "delta_norm_band_sum": 17237.7864, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Poland", + "rank_tgt": 286, + "rank_src": 1, + "logp_tgt": -32.0905, + "logp_src": -14.611, + "flip_to_tgt": false, + "kl_vs_clean": 20.0545, + "delta_norm_band_sum": 17237.7848, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Polish", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -4.5218, + "logp_src": -0.0109, + "flip_to_tgt": false, + "kl_vs_clean": 15.9886, + "delta_norm_band_sum": 17406.057 + }, + "swap@a4": { + "top1": " Poland", + "rank_tgt": 20, + "rank_src": 10, + "logp_tgt": -20.8433, + "logp_src": -17.9235, + "flip_to_tgt": false, + "kl_vs_clean": 21.801, + "delta_norm_band_sum": 197213068.9301, + "delta_norm_per_layer": { + "9": 4.9293, + "10": 18.6574, + "11": 65.667, + "12": 248.279, + "13": 947.6785, + "14": 3567.3936, + "15": 13351.7197, + "16": 48162.7305, + "17": 173431.375, + "18": 665142.5, + "19": 2477329.0, + "20": 9710767.0, + "21": 37889296.0, + "22": 146230736.0 + }, + "gram_cond_max": 6.96 + }, + "rand_target@a4": { + "top1": " Poland", + "rank_tgt": 781, + "rank_src": 1, + "logp_tgt": -21.5188, + "logp_src": -8.7824, + "flip_to_tgt": false, + "kl_vs_clean": 11.1099, + "delta_norm_band_sum": 197213056.8998, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Poland", + "rank_tgt": 305, + "rank_src": 1, + "logp_tgt": -33.0379, + "logp_src": -14.6072, + "flip_to_tgt": false, + "kl_vs_clean": 20.1383, + "delta_norm_band_sum": 197213061.8685, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Polish", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -0.6982, + "logp_src": -0.6882, + "flip_to_tgt": false, + "kl_vs_clean": 14.5028, + "delta_norm_band_sum": 210487440.8618 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Sweden", + "tgt": "Norway", + "prompt": "The main language spoken in Sweden is", + "answer_src": " Swedish", + "answer_tgt": " Norwegian", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Swedish", + "rank_tgt": 8, + "rank_src": 0, + "logp_tgt": -4.5295, + "logp_src": -0.4218, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Swedish", + "rank_tgt": 8, + "rank_src": 0, + "logp_tgt": -4.5295, + "logp_src": -0.4218, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Swedish", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -3.5469, + "logp_src": -0.5819, + "flip_to_tgt": false, + "kl_vs_clean": 0.0313, + "delta_norm_band_sum": 12.3684, + "delta_norm_per_layer": { + "9": 0.6948, + "10": 0.9557, + "11": 1.1006, + "12": 0.5829, + "13": 0.115, + "14": 0.0671, + "15": 0.2648, + "16": 0.4719, + "17": 1.7177, + "18": 0.0656, + "19": 0.7256, + "20": 1.8311, + "21": 1.0894, + "22": 2.6861 + }, + "gram_cond_max": 10.39 + }, + "rand_target@a1": { + "top1": " Spanish", + "rank_tgt": 48, + "rank_src": 5, + "logp_tgt": -6.2356, + "logp_src": -3.3011, + "flip_to_tgt": false, + "kl_vs_clean": 1.8051, + "delta_norm_band_sum": 12.3684, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Swedish", + "rank_tgt": 6, + "rank_src": 0, + "logp_tgt": -4.9061, + "logp_src": -0.3064, + "flip_to_tgt": false, + "kl_vs_clean": 0.0515, + "delta_norm_band_sum": 12.3684, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Norwegian", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -1.6226, + "logp_src": -1.7111, + "flip_to_tgt": true, + "kl_vs_clean": 0.6233, + "delta_norm_band_sum": 8.3732 + }, + "swap@a2": { + "top1": " Norway", + "rank_tgt": 4, + "rank_src": 18, + "logp_tgt": -22.4023, + "logp_src": -29.1946, + "flip_to_tgt": false, + "kl_vs_clean": 31.331, + "delta_norm_band_sum": 19745.1251, + "delta_norm_per_layer": { + "9": 1.965, + "10": 4.0047, + "11": 7.3418, + "12": 15.1519, + "13": 30.3821, + "14": 53.8688, + "15": 101.7683, + "16": 192.5405, + "17": 356.3169, + "18": 697.7408, + "19": 1280.4889, + "20": 2539.0229, + "21": 4899.6343, + "22": 9564.8984 + }, + "gram_cond_max": 10.39 + }, + "rand_target@a2": { + "top1": "heta", + "rank_tgt": 50204, + "rank_src": 50299, + "logp_tgt": -27.835, + "logp_src": -39.0254, + "flip_to_tgt": false, + "kl_vs_clean": 32.1486, + "delta_norm_band_sum": 19745.1245, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Sweden", + "rank_tgt": 117, + "rank_src": 2, + "logp_tgt": -30.6071, + "logp_src": -16.2828, + "flip_to_tgt": false, + "kl_vs_clean": 19.4595, + "delta_norm_band_sum": 19745.124, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Norwegian", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0179, + "logp_src": -4.0298, + "flip_to_tgt": true, + "kl_vs_clean": 12.6284, + "delta_norm_band_sum": 17753.0662 + }, + "swap@a4": { + "top1": " Norway", + "rank_tgt": 3, + "rank_src": 20, + "logp_tgt": -22.6802, + "logp_src": -32.21, + "flip_to_tgt": false, + "kl_vs_clean": 33.4722, + "delta_norm_band_sum": 222575031.9613, + "delta_norm_per_layer": { + "9": 5.3085, + "10": 20.169, + "11": 73.2066, + "12": 275.4263, + "13": 1060.6619, + "14": 3933.5005, + "15": 14885.583, + "16": 54068.9336, + "17": 192886.5469, + "18": 747635.125, + "19": 2757759.5, + "20": 10953448.0, + "21": 42352036.0, + "22": 165496944.0 + }, + "gram_cond_max": 10.39 + }, + "rand_target@a4": { + "top1": " Sweden", + "rank_tgt": 328, + "rank_src": 4, + "logp_tgt": -24.3032, + "logp_src": -12.6126, + "flip_to_tgt": false, + "kl_vs_clean": 14.0388, + "delta_norm_band_sum": 222575051.3481, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Sweden", + "rank_tgt": 136, + "rank_src": 2, + "logp_tgt": -31.8812, + "logp_src": -16.7183, + "flip_to_tgt": false, + "kl_vs_clean": 19.9167, + "delta_norm_band_sum": 222575034.9983, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Norwegian", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0129, + "logp_src": -4.3607, + "flip_to_tgt": true, + "kl_vs_clean": 12.2321, + "delta_norm_band_sum": 211710229.1866 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "language", + "src": "Norway", + "tgt": "Turkey", + "prompt": "The main language spoken in Norway is", + "answer_src": " Norwegian", + "answer_tgt": " Turkish", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Norwegian", + "rank_tgt": 1242, + "rank_src": 0, + "logp_tgt": -11.6626, + "logp_src": -0.5487, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Norwegian", + "rank_tgt": 1242, + "rank_src": 0, + "logp_tgt": -11.6626, + "logp_src": -0.5487, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Norwegian", + "rank_tgt": 63, + "rank_src": 0, + "logp_tgt": -6.6914, + "logp_src": -1.1657, + "flip_to_tgt": false, + "kl_vs_clean": 0.2625, + "delta_norm_band_sum": 61.2419, + "delta_norm_per_layer": { + "9": 0.0493, + "10": 0.2833, + "11": 0.1562, + "12": 0.6297, + "13": 0.4411, + "14": 0.5638, + "15": 0.6386, + "16": 0.8146, + "17": 9.0622, + "18": 8.7277, + "19": 10.7617, + "20": 8.9679, + "21": 9.7981, + "22": 10.3478 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a1": { + "top1": " called", + "rank_tgt": 126, + "rank_src": 3406, + "logp_tgt": -7.0873, + "logp_src": -11.8414, + "flip_to_tgt": false, + "kl_vs_clean": 7.1546, + "delta_norm_band_sum": 61.2419, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Norwegian", + "rank_tgt": 303, + "rank_src": 0, + "logp_tgt": -8.6297, + "logp_src": -2.027, + "flip_to_tgt": false, + "kl_vs_clean": 0.9579, + "delta_norm_band_sum": 61.2419, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Norwegian", + "rank_tgt": 676, + "rank_src": 0, + "logp_tgt": -11.5839, + "logp_src": -0.1824, + "flip_to_tgt": false, + "kl_vs_clean": 0.2711, + "delta_norm_band_sum": 79.1281 + }, + "swap@a2": { + "top1": " Norway", + "rank_tgt": 38, + "rank_src": 7, + "logp_tgt": -30.2988, + "logp_src": -21.7224, + "flip_to_tgt": false, + "kl_vs_clean": 25.8946, + "delta_norm_band_sum": 20852.0694, + "delta_norm_per_layer": { + "9": 1.8931, + "10": 3.5585, + "11": 6.3087, + "12": 15.1897, + "13": 30.0606, + "14": 55.6379, + "15": 102.7059, + "16": 201.9964, + "17": 375.2986, + "18": 737.3751, + "19": 1372.1887, + "20": 2708.8225, + "21": 5178.0083, + "22": 10063.0254 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a2": { + "top1": "----", + "rank_tgt": 17062, + "rank_src": 6697, + "logp_tgt": -13.3224, + "logp_src": -11.4741, + "flip_to_tgt": false, + "kl_vs_clean": 8.6861, + "delta_norm_band_sum": 20852.0726, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Norway", + "rank_tgt": 1077, + "rank_src": 2, + "logp_tgt": -35.0815, + "logp_src": -17.8505, + "flip_to_tgt": false, + "kl_vs_clean": 21.0562, + "delta_norm_band_sum": 20852.0696, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Norwegian", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -20.0278, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 13.5259, + "delta_norm_band_sum": 22502.3685 + }, + "swap@a4": { + "top1": " Norway", + "rank_tgt": 45, + "rank_src": 10, + "logp_tgt": -28.5628, + "logp_src": -21.3403, + "flip_to_tgt": false, + "kl_vs_clean": 24.7321, + "delta_norm_band_sum": 226877187.1301, + "delta_norm_per_layer": { + "9": 5.6798, + "10": 20.4868, + "11": 73.2949, + "12": 280.5454, + "13": 1062.0223, + "14": 4048.3586, + "15": 15026.1914, + "16": 55102.0195, + "17": 198064.0312, + "18": 768088.0, + "19": 2848841.5, + "20": 11201011.0, + "21": 43421404.0, + "22": 168364160.0 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 47538, + "rank_src": 50295, + "logp_tgt": -19.7332, + "logp_src": -28.4857, + "flip_to_tgt": false, + "kl_vs_clean": 22.6123, + "delta_norm_band_sum": 226877182.5256, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Norway", + "rank_tgt": 883, + "rank_src": 2, + "logp_tgt": -36.2952, + "logp_src": -18.8568, + "flip_to_tgt": false, + "kl_vs_clean": 21.9436, + "delta_norm_band_sum": 226877186.1609, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Norwegian", + "rank_tgt": 11, + "rank_src": 0, + "logp_tgt": -24.3897, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 13.3678, + "delta_norm_band_sum": 229912552.7905 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Turkey", + "tgt": "Korea", + "prompt": "The main language spoken in Turkey is", + "answer_src": " Turkish", + "answer_tgt": " Korean", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Turkish", + "rank_tgt": 281, + "rank_src": 0, + "logp_tgt": -9.3402, + "logp_src": -0.5842, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Turkish", + "rank_tgt": 281, + "rank_src": 0, + "logp_tgt": -9.3402, + "logp_src": -0.5842, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Turkish", + "rank_tgt": 37, + "rank_src": 0, + "logp_tgt": -6.0051, + "logp_src": -0.7626, + "flip_to_tgt": false, + "kl_vs_clean": 0.078, + "delta_norm_band_sum": 30.1421, + "delta_norm_per_layer": { + "9": 0.1014, + "10": 0.3594, + "11": 0.8603, + "12": 0.6695, + "13": 0.2757, + "14": 0.4029, + "15": 0.6697, + "16": 2.1309, + "17": 6.2039, + "18": 4.694, + "19": 4.2896, + "20": 4.6135, + "21": 2.6561, + "22": 2.2152 + }, + "gram_cond_max": 4.9 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 537, + "rank_src": 30, + "logp_tgt": -9.411, + "logp_src": -5.359, + "flip_to_tgt": false, + "kl_vs_clean": 2.7625, + "delta_norm_band_sum": 30.1421, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Turkish", + "rank_tgt": 265, + "rank_src": 0, + "logp_tgt": -8.9845, + "logp_src": -0.7647, + "flip_to_tgt": false, + "kl_vs_clean": 0.1585, + "delta_norm_band_sum": 30.1421, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Turkish", + "rank_tgt": 6, + "rank_src": 0, + "logp_tgt": -3.9919, + "logp_src": -1.4597, + "flip_to_tgt": false, + "kl_vs_clean": 0.39, + "delta_norm_band_sum": 33.9205 + }, + "swap@a2": { + "top1": " Korea", + "rank_tgt": 4, + "rank_src": 13, + "logp_tgt": -14.6328, + "logp_src": -24.3306, + "flip_to_tgt": false, + "kl_vs_clean": 27.0688, + "delta_norm_band_sum": 19830.5477, + "delta_norm_per_layer": { + "9": 1.9012, + "10": 3.6849, + "11": 7.2547, + "12": 15.7947, + "13": 31.249, + "14": 57.0566, + "15": 104.5899, + "16": 195.4961, + "17": 363.8093, + "18": 695.6789, + "19": 1297.2822, + "20": 2538.4136, + "21": 4942.9048, + "22": 9575.4316 + }, + "gram_cond_max": 4.9 + }, + "rand_target@a2": { + "top1": "----", + "rank_tgt": 17451, + "rank_src": 3494, + "logp_tgt": -13.3057, + "logp_src": -10.4344, + "flip_to_tgt": false, + "kl_vs_clean": 8.4631, + "delta_norm_band_sum": 19830.5458, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Turkey", + "rank_tgt": 1627, + "rank_src": 1, + "logp_tgt": -33.5778, + "logp_src": -16.7816, + "flip_to_tgt": false, + "kl_vs_clean": 20.2479, + "delta_norm_band_sum": 19830.5476, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Korean", + "rank_tgt": 0, + "rank_src": 2, + "logp_tgt": -0.0, + "logp_src": -17.1408, + "flip_to_tgt": true, + "kl_vs_clean": 25.7874, + "delta_norm_band_sum": 19249.9083 + }, + "swap@a4": { + "top1": " Korea", + "rank_tgt": 4, + "rank_src": 11, + "logp_tgt": -14.1192, + "logp_src": -21.4762, + "flip_to_tgt": false, + "kl_vs_clean": 23.9135, + "delta_norm_band_sum": 228139408.2772, + "delta_norm_per_layer": { + "9": 5.7, + "10": 21.347, + "11": 76.9882, + "12": 289.1056, + "13": 1098.1975, + "14": 4150.4575, + "15": 15506.415, + "16": 55453.9727, + "17": 202417.2188, + "18": 768388.875, + "19": 2888142.0, + "20": 11202674.0, + "21": 44090016.0, + "22": 168911168.0 + }, + "gram_cond_max": 4.9 + }, + "rand_target@a4": { + "top1": "ningen", + "rank_tgt": 47485, + "rank_src": 50300, + "logp_tgt": -19.6761, + "logp_src": -27.8408, + "flip_to_tgt": false, + "kl_vs_clean": 21.6169, + "delta_norm_band_sum": 228139473.0275, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Turkey", + "rank_tgt": 1252, + "rank_src": 1, + "logp_tgt": -35.2111, + "logp_src": -17.739, + "flip_to_tgt": false, + "kl_vs_clean": 21.1678, + "delta_norm_band_sum": 228139436.7821, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Korean", + "rank_tgt": 0, + "rank_src": 3, + "logp_tgt": -0.0, + "logp_src": -18.5816, + "flip_to_tgt": true, + "kl_vs_clean": 26.0371, + "delta_norm_band_sum": 214330533.0543 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "language", + "src": "Korea", + "tgt": "Thailand", + "prompt": "The main language spoken in Korea is", + "answer_src": " Korean", + "answer_tgt": " Thai", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Korean", + "rank_tgt": 197, + "rank_src": 0, + "logp_tgt": -8.501, + "logp_src": -1.6125, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Korean", + "rank_tgt": 197, + "rank_src": 0, + "logp_tgt": -8.501, + "logp_src": -1.6125, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " the", + "rank_tgt": 19, + "rank_src": 4, + "logp_tgt": -4.8963, + "logp_src": -2.7977, + "flip_to_tgt": false, + "kl_vs_clean": 0.2527, + "delta_norm_band_sum": 22.2347, + "delta_norm_per_layer": { + "9": 1.1955, + "10": 1.1616, + "11": 1.5738, + "12": 1.5271, + "13": 1.4989, + "14": 1.3929, + "15": 1.2298, + "16": 0.0826, + "17": 2.0079, + "18": 1.5228, + "19": 2.3399, + "20": 1.193, + "21": 2.4355, + "22": 3.0735 + }, + "gram_cond_max": 6.66 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 248, + "rank_src": 10, + "logp_tgt": -8.8253, + "logp_src": -4.6709, + "flip_to_tgt": false, + "kl_vs_clean": 1.0844, + "delta_norm_band_sum": 22.2347, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Korean", + "rank_tgt": 140, + "rank_src": 0, + "logp_tgt": -8.0251, + "logp_src": -1.6063, + "flip_to_tgt": false, + "kl_vs_clean": 0.0774, + "delta_norm_band_sum": 22.2347, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " the", + "rank_tgt": 9, + "rank_src": 6, + "logp_tgt": -4.1239, + "logp_src": -3.4061, + "flip_to_tgt": false, + "kl_vs_clean": 0.3484, + "delta_norm_band_sum": 15.4369 + }, + "swap@a2": { + "top1": " Korea", + "rank_tgt": 30, + "rank_src": 1, + "logp_tgt": -36.5872, + "logp_src": -16.2573, + "flip_to_tgt": false, + "kl_vs_clean": 35.4648, + "delta_norm_band_sum": 18195.0714, + "delta_norm_per_layer": { + "9": 2.4479, + "10": 4.5973, + "11": 8.1355, + "12": 16.3479, + "13": 31.3226, + "14": 57.7784, + "15": 99.7919, + "16": 191.8409, + "17": 328.7849, + "18": 654.331, + "19": 1161.3954, + "20": 2380.8542, + "21": 4368.1494, + "22": 8889.2939 + }, + "gram_cond_max": 6.66 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 50009, + "rank_src": 50302, + "logp_tgt": -22.4377, + "logp_src": -36.094, + "flip_to_tgt": false, + "kl_vs_clean": 18.676, + "delta_norm_band_sum": 18195.0692, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Korea", + "rank_tgt": 136, + "rank_src": 1, + "logp_tgt": -36.7791, + "logp_src": -13.5371, + "flip_to_tgt": false, + "kl_vs_clean": 29.5674, + "delta_norm_band_sum": 18195.0704, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Korean", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -14.7161, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 30.812, + "delta_norm_band_sum": 17813.8864 + }, + "swap@a4": { + "top1": " Korea", + "rank_tgt": 41, + "rank_src": 1, + "logp_tgt": -40.1937, + "logp_src": -16.6515, + "flip_to_tgt": false, + "kl_vs_clean": 35.7478, + "delta_norm_band_sum": 216501377.9562, + "delta_norm_per_layer": { + "9": 5.8411, + "10": 21.5791, + "11": 74.3505, + "12": 287.2041, + "13": 1056.6049, + "14": 4145.4009, + "15": 14821.2881, + "16": 55257.1094, + "17": 185967.3281, + "18": 735958.0, + "19": 2625273.25, + "20": 10721142.0, + "21": 39974568.0, + "22": 162182800.0 + }, + "gram_cond_max": 6.66 + }, + "rand_target@a4": { + "top1": " Korea", + "rank_tgt": 380, + "rank_src": 1, + "logp_tgt": -24.5561, + "logp_src": -9.225, + "flip_to_tgt": false, + "kl_vs_clean": 17.7466, + "delta_norm_band_sum": 216501390.9088, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Korea", + "rank_tgt": 164, + "rank_src": 1, + "logp_tgt": -38.3113, + "logp_src": -13.8711, + "flip_to_tgt": false, + "kl_vs_clean": 29.8402, + "delta_norm_band_sum": 216501382.2106, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Korean", + "rank_tgt": 4, + "rank_src": 0, + "logp_tgt": -23.3976, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 31.9657, + "delta_norm_band_sum": 206729049.0678 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "language", + "src": "Thailand", + "tgt": "Kenya", + "prompt": "The main language spoken in Thailand is", + "answer_src": " Thai", + "answer_tgt": " Swahili", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Thai", + "rank_tgt": 51, + "rank_src": 0, + "logp_tgt": -6.6342, + "logp_src": -1.0565, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Thai", + "rank_tgt": 51, + "rank_src": 0, + "logp_tgt": -6.6342, + "logp_src": -1.0565, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " the", + "rank_tgt": 8, + "rank_src": 4, + "logp_tgt": -3.9653, + "logp_src": -2.6596, + "flip_to_tgt": false, + "kl_vs_clean": 0.5058, + "delta_norm_band_sum": 35.6964, + "delta_norm_per_layer": { + "9": 0.7823, + "10": 1.1768, + "11": 1.3366, + "12": 0.8846, + "13": 0.1975, + "14": 0.4555, + "15": 0.7902, + "16": 1.4821, + "17": 5.6075, + "18": 4.37, + "19": 4.7609, + "20": 5.2342, + "21": 4.3676, + "22": 4.2506 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 20, + "rank_src": 202, + "logp_tgt": -5.0867, + "logp_src": -8.1701, + "flip_to_tgt": false, + "kl_vs_clean": 2.9329, + "delta_norm_band_sum": 35.6964, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Thai", + "rank_tgt": 79, + "rank_src": 0, + "logp_tgt": -6.7577, + "logp_src": -1.8126, + "flip_to_tgt": false, + "kl_vs_clean": 0.323, + "delta_norm_band_sum": 35.6964, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " the", + "rank_tgt": 5, + "rank_src": 1, + "logp_tgt": -3.2319, + "logp_src": -2.5654, + "flip_to_tgt": false, + "kl_vs_clean": 0.4553, + "delta_norm_band_sum": 31.2004 + }, + "swap@a2": { + "top1": " Kenya", + "rank_tgt": 548, + "rank_src": 44, + "logp_tgt": -32.1216, + "logp_src": -22.2817, + "flip_to_tgt": false, + "kl_vs_clean": 22.669, + "delta_norm_band_sum": 15441.0777, + "delta_norm_per_layer": { + "9": 2.166, + "10": 4.1469, + "11": 7.2818, + "12": 14.0776, + "13": 26.3533, + "14": 46.7989, + "15": 85.2823, + "16": 155.5744, + "17": 281.0412, + "18": 544.5287, + "19": 1005.1901, + "20": 1976.8267, + "21": 3817.9919, + "22": 7473.8179 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a2": { + "top1": " Thailand", + "rank_tgt": 441, + "rank_src": 3, + "logp_tgt": -27.0216, + "logp_src": -11.2636, + "flip_to_tgt": false, + "kl_vs_clean": 16.3746, + "delta_norm_band_sum": 15441.0776, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Thailand", + "rank_tgt": 746, + "rank_src": 2, + "logp_tgt": -28.4783, + "logp_src": -11.2926, + "flip_to_tgt": false, + "kl_vs_clean": 17.3731, + "delta_norm_band_sum": 15441.0777, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Thai", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -2.6039, + "logp_src": -0.0769, + "flip_to_tgt": false, + "kl_vs_clean": 11.9774, + "delta_norm_band_sum": 22615.0496 + }, + "swap@a4": { + "top1": " Kenya", + "rank_tgt": 544, + "rank_src": 164, + "logp_tgt": -34.9334, + "logp_src": -30.7129, + "flip_to_tgt": false, + "kl_vs_clean": 27.3653, + "delta_norm_band_sum": 200357436.913, + "delta_norm_per_layer": { + "9": 5.7156, + "10": 20.003, + "11": 69.9912, + "12": 257.6141, + "13": 977.7638, + "14": 3661.5039, + "15": 13721.876, + "16": 48757.1641, + "17": 173571.5312, + "18": 672444.25, + "19": 2475068.5, + "20": 9808377.0, + "21": 37750840.0, + "22": 149409664.0 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 47736, + "rank_src": 50299, + "logp_tgt": -20.1935, + "logp_src": -31.0339, + "flip_to_tgt": false, + "kl_vs_clean": 21.1852, + "delta_norm_band_sum": 200357447.9027, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Thailand", + "rank_tgt": 655, + "rank_src": 3, + "logp_tgt": -29.3872, + "logp_src": -11.9508, + "flip_to_tgt": false, + "kl_vs_clean": 17.7678, + "delta_norm_band_sum": 200357456.96, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Sw", + "rank_tgt": 0, + "rank_src": 4, + "logp_tgt": -0.0, + "logp_src": -19.9619, + "flip_to_tgt": true, + "kl_vs_clean": 23.3746, + "delta_norm_band_sum": 216675673.8522 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "continent", + "src": "France", + "tgt": "China", + "prompt": "The country France is located in the continent of", + "answer_src": " Europe", + "answer_tgt": " Asia", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Europe", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -3.3368, + "logp_src": -0.5715, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Europe", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -3.3368, + "logp_src": -0.5715, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Europe", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -3.1775, + "logp_src": -0.3987, + "flip_to_tgt": false, + "kl_vs_clean": 0.066, + "delta_norm_band_sum": 29.7464, + "delta_norm_per_layer": { + "9": 0.7059, + "10": 1.2805, + "11": 0.7594, + "12": 0.1926, + "13": 0.6934, + "14": 0.2726, + "15": 0.3796, + "16": 1.1576, + "17": 3.646, + "18": 4.059, + "19": 4.7926, + "20": 4.9026, + "21": 3.3925, + "22": 3.5122 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a1": { + "top1": " Africa", + "rank_tgt": 4, + "rank_src": 14, + "logp_tgt": -3.2095, + "logp_src": -4.5274, + "flip_to_tgt": false, + "kl_vs_clean": 2.0658, + "delta_norm_band_sum": 29.7464, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Africa", + "rank_tgt": 5, + "rank_src": 1, + "logp_tgt": -4.1752, + "logp_src": -1.092, + "flip_to_tgt": false, + "kl_vs_clean": 0.1687, + "delta_norm_band_sum": 29.7464, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Europe", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -3.1766, + "logp_src": -0.6357, + "flip_to_tgt": false, + "kl_vs_clean": 0.0053, + "delta_norm_band_sum": 28.3784 + }, + "swap@a2": { + "top1": " China", + "rank_tgt": 6, + "rank_src": 8, + "logp_tgt": -23.5429, + "logp_src": -25.6434, + "flip_to_tgt": false, + "kl_vs_clean": 29.1225, + "delta_norm_band_sum": 31420.9875, + "delta_norm_per_layer": { + "9": 2.713, + "10": 4.8692, + "11": 8.9019, + "12": 23.2452, + "13": 47.1261, + "14": 84.9548, + "15": 164.3503, + "16": 300.1636, + "17": 573.2126, + "18": 1068.5902, + "19": 2063.4927, + "20": 3851.1809, + "21": 8035.7896, + "22": 15192.3975 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a2": { + "top1": " France", + "rank_tgt": 15475, + "rank_src": 353, + "logp_tgt": -13.2753, + "logp_src": -7.8797, + "flip_to_tgt": false, + "kl_vs_clean": 8.5306, + "delta_norm_band_sum": 31420.9905, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " France", + "rank_tgt": 105, + "rank_src": 4, + "logp_tgt": -33.1222, + "logp_src": -18.1162, + "flip_to_tgt": false, + "kl_vs_clean": 23.2856, + "delta_norm_band_sum": 31420.9883, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Europe", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -5.9602, + "logp_src": -0.0026, + "flip_to_tgt": false, + "kl_vs_clean": 11.563, + "delta_norm_band_sum": 31619.0444 + }, + "swap@a4": { + "top1": " China", + "rank_tgt": 7, + "rank_src": 8, + "logp_tgt": -23.921, + "logp_src": -25.5113, + "flip_to_tgt": false, + "kl_vs_clean": 29.0726, + "delta_norm_band_sum": 287546962.9225, + "delta_norm_per_layer": { + "9": 7.5569, + "10": 26.1002, + "11": 96.8897, + "12": 356.7734, + "13": 1392.5618, + "14": 5134.978, + "15": 19631.0391, + "16": 68272.5234, + "17": 253795.0, + "18": 945702.25, + "19": 3612723.25, + "20": 13706872.0, + "21": 55969352.0, + "22": 212963600.0 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 50016, + "rank_src": 50298, + "logp_tgt": -22.9067, + "logp_src": -33.1179, + "flip_to_tgt": false, + "kl_vs_clean": 27.3336, + "delta_norm_band_sum": 287546946.7531, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " France", + "rank_tgt": 122, + "rank_src": 4, + "logp_tgt": -34.6286, + "logp_src": -18.4985, + "flip_to_tgt": false, + "kl_vs_clean": 24.2453, + "delta_norm_band_sum": 287546965.2117, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Europe", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -11.5795, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 12.9232, + "delta_norm_band_sum": 361089802.8395 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "continent", + "src": "Japan", + "tgt": "Germany", + "prompt": "The country Japan is located in the continent of", + "answer_src": " Asia", + "answer_tgt": " Europe", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Asia", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -2.7504, + "logp_src": -0.689, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Asia", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -2.7504, + "logp_src": -0.689, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Asia", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -1.7934, + "logp_src": -1.0072, + "flip_to_tgt": false, + "kl_vs_clean": 0.1032, + "delta_norm_band_sum": 9.9757, + "delta_norm_per_layer": { + "9": 1.2553, + "10": 1.4544, + "11": 1.3705, + "12": 0.9756, + "13": 0.9139, + "14": 0.0167, + "15": 0.2799, + "16": 0.8526, + "17": 0.2458, + "18": 0.5814, + "19": 0.8316, + "20": 0.1346, + "21": 0.4839, + "22": 0.5797 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a1": { + "top1": " Africa", + "rank_tgt": 1, + "rank_src": 5, + "logp_tgt": -2.3229, + "logp_src": -4.5183, + "flip_to_tgt": false, + "kl_vs_clean": 2.3417, + "delta_norm_band_sum": 9.9757, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Asia", + "rank_tgt": 4, + "rank_src": 0, + "logp_tgt": -3.0834, + "logp_src": -0.8556, + "flip_to_tgt": false, + "kl_vs_clean": 0.0309, + "delta_norm_band_sum": 9.9757, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Europe", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -1.0187, + "logp_src": -1.7724, + "flip_to_tgt": true, + "kl_vs_clean": 0.4342, + "delta_norm_band_sum": 17.7663 + }, + "swap@a2": { + "top1": " Germany", + "rank_tgt": 2, + "rank_src": 16, + "logp_tgt": -14.2883, + "logp_src": -23.2539, + "flip_to_tgt": false, + "kl_vs_clean": 26.5492, + "delta_norm_band_sum": 25283.3012, + "delta_norm_per_layer": { + "9": 2.9782, + "10": 6.1957, + "11": 10.5688, + "12": 21.7314, + "13": 41.9327, + "14": 76.9405, + "15": 140.1141, + "16": 257.7858, + "17": 476.4055, + "18": 906.4908, + "19": 1710.6404, + "20": 3244.7273, + "21": 6376.8037, + "22": 12009.9863 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 50286, + "rank_src": 50285, + "logp_tgt": -26.1856, + "logp_src": -26.1601, + "flip_to_tgt": false, + "kl_vs_clean": 21.2883, + "delta_norm_band_sum": 25283.3021, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Japan", + "rank_tgt": 12, + "rank_src": 10, + "logp_tgt": -24.4848, + "logp_src": -23.4535, + "flip_to_tgt": false, + "kl_vs_clean": 26.5671, + "delta_norm_band_sum": 25283.3014, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Europe", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0356, + "logp_src": -3.3532, + "flip_to_tgt": true, + "kl_vs_clean": 16.0585, + "delta_norm_band_sum": 30769.2116 + }, + "swap@a4": { + "top1": " Germany", + "rank_tgt": 3, + "rank_src": 29, + "logp_tgt": -17.2187, + "logp_src": -30.0565, + "flip_to_tgt": false, + "kl_vs_clean": 31.909, + "delta_norm_band_sum": 296483580.2506, + "delta_norm_per_layer": { + "9": 7.6646, + "10": 30.2373, + "11": 105.6634, + "12": 395.1044, + "13": 1490.8977, + "14": 5642.5542, + "15": 20450.8633, + "16": 73400.2344, + "17": 266555.5938, + "18": 1020844.9375, + "19": 3870301.5, + "20": 14687243.0, + "21": 58095944.0, + "22": 218441168.0 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a4": { + "top1": "mathchoice", + "rank_tgt": 50289, + "rank_src": 50290, + "logp_tgt": -32.1657, + "logp_src": -32.2563, + "flip_to_tgt": false, + "kl_vs_clean": 26.1078, + "delta_norm_band_sum": 296483578.1235, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Japan", + "rank_tgt": 11, + "rank_src": 10, + "logp_tgt": -25.6457, + "logp_src": -24.9069, + "flip_to_tgt": false, + "kl_vs_clean": 28.2736, + "delta_norm_band_sum": 296483556.2464, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Europe", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0001, + "logp_src": -9.1311, + "flip_to_tgt": true, + "kl_vs_clean": 20.0989, + "delta_norm_band_sum": 352917625.4054 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "continent", + "src": "Russia", + "tgt": "Egypt", + "prompt": "The country Russia is located in the continent of", + "answer_src": " Europe", + "answer_tgt": " Africa", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Europe", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -3.291, + "logp_src": -1.1043, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Europe", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -3.291, + "logp_src": -1.1043, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Africa", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.5069, + "logp_src": -2.5069, + "flip_to_tgt": true, + "kl_vs_clean": 0.8658, + "delta_norm_band_sum": 8.9838, + "delta_norm_per_layer": { + "9": 0.8111, + "10": 0.4993, + "11": 1.1422, + "12": 0.5468, + "13": 0.9664, + "14": 0.0218, + "15": 0.414, + "16": 0.4394, + "17": 0.2167, + "18": 0.3835, + "19": 0.5268, + "20": 0.542, + "21": 0.4824, + "22": 1.9914 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a1": { + "top1": " Africa", + "rank_tgt": 0, + "rank_src": 10, + "logp_tgt": -0.1478, + "logp_src": -5.9217, + "flip_to_tgt": true, + "kl_vs_clean": 3.3846, + "delta_norm_band_sum": 8.9838, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Europe", + "rank_tgt": 7, + "rank_src": 0, + "logp_tgt": -3.2003, + "logp_src": -1.125, + "flip_to_tgt": false, + "kl_vs_clean": 0.0288, + "delta_norm_band_sum": 8.9838, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Europe", + "rank_tgt": 4, + "rank_src": 0, + "logp_tgt": -2.4976, + "logp_src": -1.7265, + "flip_to_tgt": false, + "kl_vs_clean": 0.0926, + "delta_norm_band_sum": 16.2298 + }, + "swap@a2": { + "top1": " Russia", + "rank_tgt": 33, + "rank_src": 9, + "logp_tgt": -25.9189, + "logp_src": -18.2555, + "flip_to_tgt": false, + "kl_vs_clean": 24.3525, + "delta_norm_band_sum": 26225.7899, + "delta_norm_per_layer": { + "9": 2.8178, + "10": 4.7225, + "11": 8.9698, + "12": 20.6895, + "13": 40.9811, + "14": 75.5792, + "15": 141.0195, + "16": 256.4966, + "17": 482.8866, + "18": 926.3219, + "19": 1722.1417, + "20": 3366.5471, + "21": 6565.3267, + "22": 12611.29 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a2": { + "top1": " Russia", + "rank_tgt": 925, + "rank_src": 17, + "logp_tgt": -30.6266, + "logp_src": -21.7843, + "flip_to_tgt": false, + "kl_vs_clean": 22.9844, + "delta_norm_band_sum": 26225.792, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Russia", + "rank_tgt": 137, + "rank_src": 9, + "logp_tgt": -31.2855, + "logp_src": -21.709, + "flip_to_tgt": false, + "kl_vs_clean": 25.9196, + "delta_norm_band_sum": 26225.7923, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Europe", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -17.849, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 23.0457, + "delta_norm_band_sum": 31323.6815 + }, + "swap@a4": { + "top1": " Egypt", + "rank_tgt": 41, + "rank_src": 8, + "logp_tgt": -26.8814, + "logp_src": -20.6405, + "flip_to_tgt": false, + "kl_vs_clean": 26.2273, + "delta_norm_band_sum": 288162630.8203, + "delta_norm_per_layer": { + "9": 7.7807, + "10": 27.1417, + "11": 100.6137, + "12": 365.9174, + "13": 1408.8141, + "14": 5177.4375, + "15": 19906.5215, + "16": 68919.2031, + "17": 254294.0156, + "18": 965603.875, + "19": 3624340.5, + "20": 14050439.0, + "21": 55339976.0, + "22": 213832064.0 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 49830, + "rank_src": 50291, + "logp_tgt": -22.0538, + "logp_src": -28.5401, + "flip_to_tgt": false, + "kl_vs_clean": 20.8222, + "delta_norm_band_sum": 288162638.0469, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Russia", + "rank_tgt": 145, + "rank_src": 9, + "logp_tgt": -32.6555, + "logp_src": -21.8775, + "flip_to_tgt": false, + "kl_vs_clean": 27.0825, + "delta_norm_band_sum": 288162618.8184, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Europe", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -21.2729, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 24.1048, + "delta_norm_band_sum": 347201576.8978 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "continent", + "src": "Egypt", + "tgt": "India", + "prompt": "The country Egypt is located in the continent of", + "answer_src": " Africa", + "answer_tgt": " Asia", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Africa", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -3.8668, + "logp_src": -0.1296, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Africa", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -3.8668, + "logp_src": -0.1296, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Africa", + "rank_tgt": 4, + "rank_src": 0, + "logp_tgt": -4.2139, + "logp_src": -0.1719, + "flip_to_tgt": false, + "kl_vs_clean": 0.0213, + "delta_norm_band_sum": 24.2568, + "delta_norm_per_layer": { + "9": 1.1184, + "10": 0.6808, + "11": 0.7138, + "12": 1.8001, + "13": 1.4997, + "14": 1.5509, + "15": 0.5818, + "16": 0.7777, + "17": 1.7366, + "18": 1.6898, + "19": 2.3231, + "20": 2.7641, + "21": 2.6213, + "22": 4.3985 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a1": { + "top1": " North", + "rank_tgt": 4, + "rank_src": 3, + "logp_tgt": -2.6321, + "logp_src": -2.5362, + "flip_to_tgt": false, + "kl_vs_clean": 1.9495, + "delta_norm_band_sum": 24.2568, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Africa", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -4.3856, + "logp_src": -0.1655, + "flip_to_tgt": false, + "kl_vs_clean": 0.0317, + "delta_norm_band_sum": 24.2568, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Africa", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -2.8421, + "logp_src": -0.2461, + "flip_to_tgt": false, + "kl_vs_clean": 0.036, + "delta_norm_band_sum": 17.9282 + }, + "swap@a2": { + "top1": " Egypt", + "rank_tgt": 6, + "rank_src": 9, + "logp_tgt": -23.0211, + "logp_src": -24.8576, + "flip_to_tgt": false, + "kl_vs_clean": 24.9225, + "delta_norm_band_sum": 27986.5626, + "delta_norm_per_layer": { + "9": 3.1491, + "10": 4.9654, + "11": 9.6662, + "12": 23.4769, + "13": 47.7644, + "14": 84.5155, + "15": 162.1112, + "16": 284.5602, + "17": 526.7292, + "18": 971.4453, + "19": 1837.6042, + "20": 3537.3303, + "21": 7067.354, + "22": 13425.8906 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a2": { + "top1": " Egypt", + "rank_tgt": 1419, + "rank_src": 150, + "logp_tgt": -16.3371, + "logp_src": -13.4938, + "flip_to_tgt": false, + "kl_vs_clean": 13.0083, + "delta_norm_band_sum": 27986.5623, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Egypt", + "rank_tgt": 127, + "rank_src": 20, + "logp_tgt": -30.9557, + "logp_src": -26.6811, + "flip_to_tgt": false, + "kl_vs_clean": 26.4915, + "delta_norm_band_sum": 27986.5612, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Asia", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0, + "logp_src": -12.3417, + "flip_to_tgt": true, + "kl_vs_clean": 14.0641, + "delta_norm_band_sum": 29076.7594 + }, + "swap@a4": { + "top1": " Egypt", + "rank_tgt": 7, + "rank_src": 8, + "logp_tgt": -26.3437, + "logp_src": -27.1858, + "flip_to_tgt": false, + "kl_vs_clean": 27.1777, + "delta_norm_band_sum": 293942925.0096, + "delta_norm_per_layer": { + "9": 8.2521, + "10": 27.7656, + "11": 105.8575, + "12": 377.6288, + "13": 1469.9828, + "14": 5386.019, + "15": 20825.4492, + "16": 71239.1953, + "17": 261788.1719, + "18": 979036.4375, + "19": 3716526.25, + "20": 14299486.0, + "21": 56914296.0, + "22": 217672352.0 + }, + "gram_cond_max": 4.98 + }, + "rand_target@a4": { + "top1": "ningen", + "rank_tgt": 49905, + "rank_src": 50274, + "logp_tgt": -22.5838, + "logp_src": -25.6581, + "flip_to_tgt": false, + "kl_vs_clean": 24.4079, + "delta_norm_band_sum": 293942943.5271, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Egypt", + "rank_tgt": 138, + "rank_src": 20, + "logp_tgt": -32.3525, + "logp_src": -28.1677, + "flip_to_tgt": false, + "kl_vs_clean": 27.9648, + "delta_norm_band_sum": 293942927.7322, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Asia", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0001, + "logp_src": -9.6406, + "flip_to_tgt": true, + "kl_vs_clean": 11.4936, + "delta_norm_band_sum": 322709498.5645 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "continent", + "src": "Norway", + "tgt": "Turkey", + "prompt": "The country Norway is located in the continent of", + "answer_src": " Europe", + "answer_tgt": " Asia", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Europe", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -3.0518, + "logp_src": -0.7176, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Europe", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -3.0518, + "logp_src": -0.7176, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Europe", + "rank_tgt": 4, + "rank_src": 0, + "logp_tgt": -3.5032, + "logp_src": -0.522, + "flip_to_tgt": false, + "kl_vs_clean": 0.1335, + "delta_norm_band_sum": 50.3931, + "delta_norm_per_layer": { + "9": 0.9462, + "10": 1.2948, + "11": 1.2691, + "12": 1.8683, + "13": 1.7694, + "14": 1.4787, + "15": 1.4194, + "16": 1.7698, + "17": 6.2004, + "18": 6.3839, + "19": 7.0228, + "20": 6.1861, + "21": 6.0477, + "22": 6.7365 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a1": { + "top1": " the", + "rank_tgt": 3, + "rank_src": 33, + "logp_tgt": -3.4979, + "logp_src": -6.0935, + "flip_to_tgt": false, + "kl_vs_clean": 3.2083, + "delta_norm_band_sum": 50.3931, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " S", + "rank_tgt": 14, + "rank_src": 1, + "logp_tgt": -4.4794, + "logp_src": -2.2493, + "flip_to_tgt": false, + "kl_vs_clean": 0.7714, + "delta_norm_band_sum": 50.3931, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Europe", + "rank_tgt": 4, + "rank_src": 0, + "logp_tgt": -3.5395, + "logp_src": -0.6157, + "flip_to_tgt": false, + "kl_vs_clean": 0.0272, + "delta_norm_band_sum": 28.2771 + }, + "swap@a2": { + "top1": " Norway", + "rank_tgt": 70, + "rank_src": 9, + "logp_tgt": -37.2296, + "logp_src": -26.6905, + "flip_to_tgt": false, + "kl_vs_clean": 31.2801, + "delta_norm_band_sum": 28096.2089, + "delta_norm_per_layer": { + "9": 2.1843, + "10": 4.7413, + "11": 8.5551, + "12": 20.339, + "13": 39.3087, + "14": 74.1656, + "15": 137.544, + "16": 277.3425, + "17": 499.5915, + "18": 992.8758, + "19": 1845.3635, + "20": 3657.0713, + "21": 6954.0103, + "22": 13583.1162 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a2": { + "top1": " Norway", + "rank_tgt": 395, + "rank_src": 10, + "logp_tgt": -30.452, + "logp_src": -21.0958, + "flip_to_tgt": false, + "kl_vs_clean": 22.9635, + "delta_norm_band_sum": 28096.207, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Norway", + "rank_tgt": 223, + "rank_src": 9, + "logp_tgt": -31.9243, + "logp_src": -22.1305, + "flip_to_tgt": false, + "kl_vs_clean": 24.5601, + "delta_norm_band_sum": 28096.2102, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Europe", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -3.8607, + "logp_src": -0.0213, + "flip_to_tgt": false, + "kl_vs_clean": 16.5457, + "delta_norm_band_sum": 30606.0771 + }, + "swap@a4": { + "top1": " Norway", + "rank_tgt": 99, + "rank_src": 8, + "logp_tgt": -40.1258, + "logp_src": -28.9432, + "flip_to_tgt": false, + "kl_vs_clean": 33.3023, + "delta_norm_band_sum": 263824787.4917, + "delta_norm_per_layer": { + "9": 5.5754, + "10": 22.2581, + "11": 79.7497, + "12": 311.8341, + "13": 1176.8585, + "14": 4560.5381, + "15": 16834.7246, + "16": 64383.5469, + "17": 225057.4688, + "18": 891410.4375, + "19": 3290713.5, + "20": 13085415.0, + "21": 50176736.0, + "22": 196068080.0 + }, + "gram_cond_max": 4.64 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 49382, + "rank_src": 50294, + "logp_tgt": -21.2183, + "logp_src": -28.1864, + "flip_to_tgt": false, + "kl_vs_clean": 21.9445, + "delta_norm_band_sum": 263824789.2578, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Norway", + "rank_tgt": 208, + "rank_src": 8, + "logp_tgt": -32.6773, + "logp_src": -22.3103, + "flip_to_tgt": false, + "kl_vs_clean": 25.3721, + "delta_norm_band_sum": 263824819.8047, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Europe", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -8.7303, + "logp_src": -0.0002, + "flip_to_tgt": false, + "kl_vs_clean": 17.7456, + "delta_norm_band_sum": 350772073.516 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "continent", + "src": "Thailand", + "tgt": "Kenya", + "prompt": "The country Thailand is located in the continent of", + "answer_src": " Asia", + "answer_tgt": " Africa", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Asia", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -3.8037, + "logp_src": -0.6303, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Asia", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -3.8037, + "logp_src": -0.6303, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Africa", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.6101, + "logp_src": -1.6593, + "flip_to_tgt": true, + "kl_vs_clean": 0.7806, + "delta_norm_band_sum": 11.2686, + "delta_norm_per_layer": { + "9": 0.6188, + "10": 0.2479, + "11": 0.4392, + "12": 0.4375, + "13": 0.6657, + "14": 1.0511, + "15": 0.4943, + "16": 0.6931, + "17": 1.0062, + "18": 0.8565, + "19": 1.7285, + "20": 1.6178, + "21": 0.503, + "22": 0.909 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a1": { + "top1": " Africa", + "rank_tgt": 0, + "rank_src": 2, + "logp_tgt": -0.5721, + "logp_src": -2.2181, + "flip_to_tgt": true, + "kl_vs_clean": 1.2954, + "delta_norm_band_sum": 11.2686, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Asia", + "rank_tgt": 5, + "rank_src": 0, + "logp_tgt": -3.4671, + "logp_src": -0.8941, + "flip_to_tgt": false, + "kl_vs_clean": 0.0964, + "delta_norm_band_sum": 11.2686, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Asia", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -2.1038, + "logp_src": -0.725, + "flip_to_tgt": false, + "kl_vs_clean": 0.0803, + "delta_norm_band_sum": 6.1403 + }, + "swap@a2": { + "top1": " Kenya", + "rank_tgt": 13, + "rank_src": 6, + "logp_tgt": -13.7898, + "logp_src": -11.9657, + "flip_to_tgt": false, + "kl_vs_clean": 15.0162, + "delta_norm_band_sum": 16722.7671, + "delta_norm_per_layer": { + "9": 2.0262, + "10": 3.4622, + "11": 5.9333, + "12": 14.1064, + "13": 27.2489, + "14": 47.61, + "15": 87.8291, + "16": 164.3588, + "17": 312.3068, + "18": 591.7996, + "19": 1115.932, + "20": 2144.9724, + "21": 4225.8281, + "22": 7979.3535 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a2": { + "top1": " Thailand", + "rank_tgt": 4929, + "rank_src": 185, + "logp_tgt": -14.0623, + "logp_src": -9.5134, + "flip_to_tgt": false, + "kl_vs_clean": 8.1128, + "delta_norm_band_sum": 16722.7667, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Thailand", + "rank_tgt": 81, + "rank_src": 12, + "logp_tgt": -23.3015, + "logp_src": -16.9977, + "flip_to_tgt": false, + "kl_vs_clean": 18.6854, + "delta_norm_band_sum": 16722.7669, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Asia", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -13.945, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 17.1149, + "delta_norm_band_sum": 29180.7777 + }, + "swap@a4": { + "top1": " Thailand", + "rank_tgt": 25, + "rank_src": 6, + "logp_tgt": -17.2839, + "logp_src": -13.7482, + "flip_to_tgt": false, + "kl_vs_clean": 16.7988, + "delta_norm_band_sum": 205978485.5958, + "delta_norm_per_layer": { + "9": 5.6933, + "10": 20.6918, + "11": 70.69, + "12": 269.783, + "13": 1008.4653, + "14": 3784.489, + "15": 14183.1973, + "16": 51101.6797, + "17": 186999.9688, + "18": 704131.1875, + "19": 2654872.75, + "20": 10180545.0, + "21": 40343652.0, + "22": 151837840.0 + }, + "gram_cond_max": 5.26 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 50119, + "rank_src": 50277, + "logp_tgt": -23.7458, + "logp_src": -26.8411, + "flip_to_tgt": false, + "kl_vs_clean": 22.2533, + "delta_norm_band_sum": 205978484.9762, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Thailand", + "rank_tgt": 96, + "rank_src": 12, + "logp_tgt": -24.1916, + "logp_src": -17.323, + "flip_to_tgt": false, + "kl_vs_clean": 19.3925, + "delta_norm_band_sum": 205978490.2989, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Asia", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -12.1076, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 16.5449, + "delta_norm_band_sum": 324724476.8392 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "continent", + "src": "Kenya", + "tgt": "France", + "prompt": "The country Kenya is located in the continent of", + "answer_src": " Africa", + "answer_tgt": " Europe", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " Africa", + "rank_tgt": 15, + "rank_src": 0, + "logp_tgt": -6.7277, + "logp_src": -0.1325, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " Africa", + "rank_tgt": 15, + "rank_src": 0, + "logp_tgt": -6.7277, + "logp_src": -0.1325, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Africa", + "rank_tgt": 3, + "rank_src": 0, + "logp_tgt": -4.8425, + "logp_src": -0.098, + "flip_to_tgt": false, + "kl_vs_clean": 0.0527, + "delta_norm_band_sum": 10.2413, + "delta_norm_per_layer": { + "9": 0.1019, + "10": 0.844, + "11": 1.0808, + "12": 1.376, + "13": 0.0896, + "14": 0.1179, + "15": 0.9378, + "16": 1.6991, + "17": 1.1394, + "18": 0.1282, + "19": 0.1722, + "20": 0.0203, + "21": 0.9654, + "22": 1.5687 + }, + "gram_cond_max": 3.14 + }, + "rand_target@a1": { + "top1": " Europe", + "rank_tgt": 0, + "rank_src": 2, + "logp_tgt": -0.5725, + "logp_src": -2.9015, + "flip_to_tgt": true, + "kl_vs_clean": 2.5239, + "delta_norm_band_sum": 10.2413, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " Africa", + "rank_tgt": 18, + "rank_src": 0, + "logp_tgt": -6.9788, + "logp_src": -0.098, + "flip_to_tgt": false, + "kl_vs_clean": 0.0183, + "delta_norm_band_sum": 10.2412, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " Africa", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -2.374, + "logp_src": -1.0417, + "flip_to_tgt": false, + "kl_vs_clean": 0.6366, + "delta_norm_band_sum": 17.4537 + }, + "swap@a2": { + "top1": " Kenya", + "rank_tgt": 5, + "rank_src": 3, + "logp_tgt": -15.6648, + "logp_src": -14.7208, + "flip_to_tgt": false, + "kl_vs_clean": 15.3902, + "delta_norm_band_sum": 23456.4915, + "delta_norm_per_layer": { + "9": 2.0003, + "10": 3.7101, + "11": 7.1663, + "12": 18.4104, + "13": 37.6668, + "14": 65.5469, + "15": 125.7638, + "16": 219.6995, + "17": 418.3611, + "18": 782.3038, + "19": 1511.1582, + "20": 2888.802, + "21": 5892.4639, + "22": 11483.4385 + }, + "gram_cond_max": 3.14 + }, + "rand_target@a2": { + "top1": " Kenya", + "rank_tgt": 3229, + "rank_src": 205, + "logp_tgt": -19.1947, + "logp_src": -14.0471, + "flip_to_tgt": false, + "kl_vs_clean": 13.3642, + "delta_norm_band_sum": 23456.4898, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Kenya", + "rank_tgt": 306, + "rank_src": 4, + "logp_tgt": -24.7682, + "logp_src": -16.8466, + "flip_to_tgt": false, + "kl_vs_clean": 16.9759, + "delta_norm_band_sum": 23456.4914, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " Europe", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0001, + "logp_src": -9.8453, + "flip_to_tgt": true, + "kl_vs_clean": 12.7431, + "delta_norm_band_sum": 29270.6266 + }, + "swap@a4": { + "top1": " Kenya", + "rank_tgt": 3, + "rank_src": 8, + "logp_tgt": -11.0638, + "logp_src": -12.3938, + "flip_to_tgt": false, + "kl_vs_clean": 12.8968, + "delta_norm_band_sum": 228927495.52, + "delta_norm_per_layer": { + "9": 5.9616, + "10": 21.4698, + "11": 77.2247, + "12": 289.3229, + "13": 1127.3853, + "14": 4121.9976, + "15": 15704.0098, + "16": 53894.0078, + "17": 201239.6406, + "18": 759485.25, + "19": 2878217.25, + "20": 11081396.0, + "21": 44153564.0, + "22": 169778352.0 + }, + "gram_cond_max": 3.14 + }, + "rand_target@a4": { + "top1": " Kenya", + "rank_tgt": 285, + "rank_src": 28, + "logp_tgt": -17.3873, + "logp_src": -12.7171, + "flip_to_tgt": false, + "kl_vs_clean": 12.5516, + "delta_norm_band_sum": 228927516.2607, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Kenya", + "rank_tgt": 258, + "rank_src": 9, + "logp_tgt": -25.6027, + "logp_src": -17.9993, + "flip_to_tgt": false, + "kl_vs_clean": 18.1296, + "delta_norm_band_sum": 228927478.4625, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " Europe", + "rank_tgt": 0, + "rank_src": 1, + "logp_tgt": -0.0, + "logp_src": -10.8082, + "flip_to_tgt": true, + "kl_vs_clean": 13.609, + "delta_norm_band_sum": 331136995.6401 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "currency", + "src": "China", + "tgt": "Japan", + "prompt": "The currency used in China is called the", + "answer_src": " yuan", + "answer_tgt": " yen", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " y", + "rank_tgt": 0, + "rank_src": 0, + "logp_tgt": -1.1567, + "logp_src": -1.1567, + "flip_to_tgt": true + }, + "arms": { + "identity": { + "top1": " y", + "rank_tgt": 0, + "rank_src": 0, + "logp_tgt": -1.1567, + "logp_src": -1.1567, + "flip_to_tgt": true, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " y", + "rank_tgt": 0, + "rank_src": 0, + "logp_tgt": -0.8088, + "logp_src": -0.8088, + "flip_to_tgt": true, + "kl_vs_clean": 0.4208, + "delta_norm_band_sum": 49.7642, + "delta_norm_per_layer": { + "9": 1.2217, + "10": 1.7682, + "11": 1.2981, + "12": 1.7381, + "13": 2.0789, + "14": 1.3497, + "15": 1.8663, + "16": 1.6648, + "17": 5.2415, + "18": 5.7811, + "19": 7.05, + "20": 6.6447, + "21": 5.6503, + "22": 6.4108 + }, + "gram_cond_max": 9.11 + }, + "rand_target@a1": { + "top1": " Chinese", + "rank_tgt": 1, + "rank_src": 1, + "logp_tgt": -2.783, + "logp_src": -2.783, + "flip_to_tgt": false, + "kl_vs_clean": 1.7589, + "delta_norm_band_sum": 49.7642, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " \"", + "rank_tgt": 2, + "rank_src": 2, + "logp_tgt": -2.7242, + "logp_src": -2.7242, + "flip_to_tgt": false, + "kl_vs_clean": 0.916, + "delta_norm_band_sum": 49.7642, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " y", + "rank_tgt": 0, + "rank_src": 0, + "logp_tgt": -1.1567, + "logp_src": -1.1567, + "flip_to_tgt": true, + "kl_vs_clean": 0.0, + "delta_norm_band_sum": 0.0 + }, + "swap@a2": { + "top1": " China", + "rank_tgt": 804, + "rank_src": 804, + "logp_tgt": -50.9253, + "logp_src": -50.9253, + "flip_to_tgt": false, + "kl_vs_clean": 43.958, + "delta_norm_band_sum": 20618.6367, + "delta_norm_per_layer": { + "9": 2.2668, + "10": 5.2406, + "11": 8.0479, + "12": 15.963, + "13": 31.0891, + "14": 55.4152, + "15": 105.347, + "16": 197.7242, + "17": 377.4768, + "18": 711.5781, + "19": 1392.0167, + "20": 2588.6162, + "21": 5299.0181, + "22": 9828.8369 + }, + "gram_cond_max": 9.11 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 48791, + "rank_src": 48791, + "logp_tgt": -20.2246, + "logp_src": -20.2246, + "flip_to_tgt": false, + "kl_vs_clean": 18.1703, + "delta_norm_band_sum": 20618.6388, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " China", + "rank_tgt": 643, + "rank_src": 643, + "logp_tgt": -41.6103, + "logp_src": -41.6103, + "flip_to_tgt": false, + "kl_vs_clean": 36.0328, + "delta_norm_band_sum": 20618.6384, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " y", + "rank_tgt": 0, + "rank_src": 0, + "logp_tgt": 0.0, + "logp_src": 0.0, + "flip_to_tgt": true, + "kl_vs_clean": 38.4146, + "delta_norm_band_sum": 20671.4601 + }, + "swap@a4": { + "top1": " China", + "rank_tgt": 919, + "rank_src": 919, + "logp_tgt": -53.9658, + "logp_src": -53.9658, + "flip_to_tgt": false, + "kl_vs_clean": 46.6451, + "delta_norm_band_sum": 206850421.623, + "delta_norm_per_layer": { + "9": 5.0023, + "10": 20.4667, + "11": 68.1772, + "12": 255.199, + "13": 972.3379, + "14": 3676.9077, + "15": 13843.208, + "16": 49801.4961, + "17": 181087.5156, + "18": 682804.5625, + "19": 2650432.75, + "20": 9925254.0, + "21": 40452984.0, + "22": 152889216.0 + }, + "gram_cond_max": 9.11 + }, + "rand_target@a4": { + "top1": " China", + "rank_tgt": 309, + "rank_src": 309, + "logp_tgt": -31.8452, + "logp_src": -31.8452, + "flip_to_tgt": false, + "kl_vs_clean": 28.2304, + "delta_norm_band_sum": 206850426.5863, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " China", + "rank_tgt": 441, + "rank_src": 441, + "logp_tgt": -42.3522, + "logp_src": -42.3522, + "flip_to_tgt": false, + "kl_vs_clean": 37.3429, + "delta_norm_band_sum": 206850394.871, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " y", + "rank_tgt": 0, + "rank_src": 0, + "logp_tgt": 0.0, + "logp_src": 0.0, + "flip_to_tgt": true, + "kl_vs_clean": 38.3528, + "delta_norm_band_sum": 204133132.5189 + } + }, + "oracle_any_flip": true, + "VOID_no_oracle": false + }, + { + "relation": "currency", + "src": "Japan", + "tgt": "Germany", + "prompt": "The currency used in Japan is called the", + "answer_src": " yen", + "answer_tgt": " euro", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " y", + "rank_tgt": 15, + "rank_src": 0, + "logp_tgt": -5.6672, + "logp_src": -0.4188, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " y", + "rank_tgt": 15, + "rank_src": 0, + "logp_tgt": -5.6672, + "logp_src": -0.4188, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " y", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -1.8454, + "logp_src": -1.7181, + "flip_to_tgt": false, + "kl_vs_clean": 0.7235, + "delta_norm_band_sum": 27.429, + "delta_norm_per_layer": { + "9": 0.8729, + "10": 0.4634, + "11": 0.4735, + "12": 0.235, + "13": 0.3848, + "14": 0.2342, + "15": 0.7459, + "16": 0.5002, + "17": 4.7079, + "18": 3.1694, + "19": 3.3229, + "20": 3.388, + "21": 3.8974, + "22": 5.0335 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a1": { + "top1": " \"", + "rank_tgt": 2, + "rank_src": 5, + "logp_tgt": -2.8508, + "logp_src": -3.6313, + "flip_to_tgt": false, + "kl_vs_clean": 2.2261, + "delta_norm_band_sum": 27.429, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " y", + "rank_tgt": 44, + "rank_src": 0, + "logp_tgt": -7.0481, + "logp_src": -0.7001, + "flip_to_tgt": false, + "kl_vs_clean": 0.1334, + "delta_norm_band_sum": 27.429, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " y", + "rank_tgt": 98, + "rank_src": 0, + "logp_tgt": -7.7088, + "logp_src": -1.1166, + "flip_to_tgt": false, + "kl_vs_clean": 0.3022, + "delta_norm_band_sum": 28.8776 + }, + "swap@a2": { + "top1": " Japan", + "rank_tgt": 2284, + "rank_src": 487, + "logp_tgt": -54.03, + "logp_src": -49.3259, + "flip_to_tgt": false, + "kl_vs_clean": 45.264, + "delta_norm_band_sum": 20495.796, + "delta_norm_per_layer": { + "9": 1.6634, + "10": 2.9738, + "11": 5.3055, + "12": 14.6336, + "13": 29.73, + "14": 53.5628, + "15": 102.732, + "16": 193.9212, + "17": 360.1717, + "18": 695.1199, + "19": 1273.6595, + "20": 2576.3757, + "21": 5022.2788, + "22": 10163.668 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 14915, + "rank_src": 49392, + "logp_tgt": -13.3186, + "logp_src": -20.3567, + "flip_to_tgt": false, + "kl_vs_clean": 17.8524, + "delta_norm_band_sum": 20495.7958, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Japan", + "rank_tgt": 11318, + "rank_src": 170, + "logp_tgt": -44.8527, + "logp_src": -35.1845, + "flip_to_tgt": false, + "kl_vs_clean": 32.6172, + "delta_norm_band_sum": 20495.7956, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " y", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -16.4522, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 10.7253, + "delta_norm_band_sum": 19190.4336 + }, + "swap@a4": { + "top1": " Japan", + "rank_tgt": 2978, + "rank_src": 366, + "logp_tgt": -55.8937, + "logp_src": -49.0545, + "flip_to_tgt": false, + "kl_vs_clean": 45.1139, + "delta_norm_band_sum": 163904450.0879, + "delta_norm_per_layer": { + "9": 3.7346, + "10": 14.2176, + "11": 50.2264, + "12": 198.359, + "13": 759.9113, + "14": 2850.7004, + "15": 11062.79, + "16": 39478.9297, + "17": 139893.3438, + "18": 534998.375, + "19": 1971335.0, + "20": 7875978.5, + "21": 30675362.0, + "22": 122652464.0 + }, + "gram_cond_max": 6.03 + }, + "rand_target@a4": { + "top1": "mathchoice", + "rank_tgt": 35618, + "rank_src": 50110, + "logp_tgt": -16.7708, + "logp_src": -24.4545, + "flip_to_tgt": false, + "kl_vs_clean": 22.3929, + "delta_norm_band_sum": 163904432.6602, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Japan", + "rank_tgt": 7864, + "rank_src": 163, + "logp_tgt": -47.3016, + "logp_src": -36.9319, + "flip_to_tgt": false, + "kl_vs_clean": 34.2804, + "delta_norm_band_sum": 163904477.1662, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " y", + "rank_tgt": 74, + "rank_src": 0, + "logp_tgt": -47.4849, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 14.4447, + "delta_norm_band_sum": 198861170.4854 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "currency", + "src": "Russia", + "tgt": "Egypt", + "prompt": "The currency used in Russia is called the", + "answer_src": " ruble", + "answer_tgt": " pound", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " rub", + "rank_tgt": 22, + "rank_src": 0, + "logp_tgt": -6.4556, + "logp_src": -0.5553, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " rub", + "rank_tgt": 22, + "rank_src": 0, + "logp_tgt": -6.4556, + "logp_src": -0.5553, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " rub", + "rank_tgt": 9, + "rank_src": 0, + "logp_tgt": -4.3383, + "logp_src": -1.9573, + "flip_to_tgt": false, + "kl_vs_clean": 0.6961, + "delta_norm_band_sum": 28.4832, + "delta_norm_per_layer": { + "9": 1.6758, + "10": 0.8524, + "11": 0.7763, + "12": 0.2199, + "13": 0.0662, + "14": 0.7607, + "15": 0.6338, + "16": 0.5026, + "17": 4.004, + "18": 3.9584, + "19": 4.6325, + "20": 4.965, + "21": 4.0761, + "22": 1.3592 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a1": { + "top1": " \"", + "rank_tgt": 15, + "rank_src": 3, + "logp_tgt": -4.735, + "logp_src": -3.9589, + "flip_to_tgt": false, + "kl_vs_clean": 2.4989, + "delta_norm_band_sum": 28.4832, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " \"", + "rank_tgt": 33, + "rank_src": 1, + "logp_tgt": -5.885, + "logp_src": -1.6838, + "flip_to_tgt": false, + "kl_vs_clean": 0.6371, + "delta_norm_band_sum": 28.4832, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " rub", + "rank_tgt": 26, + "rank_src": 0, + "logp_tgt": -6.5924, + "logp_src": -0.4381, + "flip_to_tgt": false, + "kl_vs_clean": 0.0291, + "delta_norm_band_sum": 17.2678 + }, + "swap@a2": { + "top1": " Egypt", + "rank_tgt": 28890, + "rank_src": 1437, + "logp_tgt": -50.6682, + "logp_src": -40.6332, + "flip_to_tgt": false, + "kl_vs_clean": 36.4468, + "delta_norm_band_sum": 20429.3765, + "delta_norm_per_layer": { + "9": 2.906, + "10": 4.1499, + "11": 7.9444, + "12": 16.5627, + "13": 33.1234, + "14": 58.7197, + "15": 112.4351, + "16": 201.4712, + "17": 375.2526, + "18": 710.8369, + "19": 1336.9005, + "20": 2585.3049, + "21": 5111.0181, + "22": 9872.751 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a2": { + "top1": "ningen", + "rank_tgt": 26076, + "rank_src": 1781, + "logp_tgt": -14.8836, + "logp_src": -9.9219, + "flip_to_tgt": false, + "kl_vs_clean": 10.7424, + "delta_norm_band_sum": 20429.3756, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Russia", + "rank_tgt": 26051, + "rank_src": 8914, + "logp_tgt": -43.1633, + "logp_src": -40.0078, + "flip_to_tgt": false, + "kl_vs_clean": 34.8194, + "delta_norm_band_sum": 20429.3757, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " rub", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -16.2702, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 14.0761, + "delta_norm_band_sum": 22248.7713 + }, + "swap@a4": { + "top1": " Egypt", + "rank_tgt": 27928, + "rank_src": 2843, + "logp_tgt": -59.9316, + "logp_src": -51.9815, + "flip_to_tgt": false, + "kl_vs_clean": 47.3676, + "delta_norm_band_sum": 209087643.8211, + "delta_norm_per_layer": { + "9": 5.9021, + "10": 19.0451, + "11": 72.8661, + "12": 255.4738, + "13": 1024.5768, + "14": 3604.7649, + "15": 14546.3057, + "16": 49517.2773, + "17": 181258.7969, + "18": 680966.0625, + "19": 2613858.75, + "20": 10013082.0, + "21": 39947816.0, + "22": 155581616.0 + }, + "gram_cond_max": 4.36 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 25500, + "rank_src": 9128, + "logp_tgt": -15.3268, + "logp_src": -12.8161, + "flip_to_tgt": false, + "kl_vs_clean": 13.5542, + "delta_norm_band_sum": 209087645.811, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Russia", + "rank_tgt": 24904, + "rank_src": 7365, + "logp_tgt": -45.9443, + "logp_src": -42.1806, + "flip_to_tgt": false, + "kl_vs_clean": 36.5824, + "delta_norm_band_sum": 209087667.7637, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " rub", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -16.3429, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 13.4149, + "delta_norm_band_sum": 202352598.9122 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + }, + { + "relation": "currency", + "src": "Mexico", + "tgt": "Greece", + "prompt": "The currency used in Mexico is called the", + "answer_src": " peso", + "answer_tgt": " euro", + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "clean": { + "top1": " pes", + "rank_tgt": 40, + "rank_src": 0, + "logp_tgt": -5.9561, + "logp_src": -1.3856, + "flip_to_tgt": false + }, + "arms": { + "identity": { + "top1": " pes", + "rank_tgt": 40, + "rank_src": 0, + "logp_tgt": -5.9561, + "logp_src": -1.3856, + "flip_to_tgt": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " pes", + "rank_tgt": 2, + "rank_src": 0, + "logp_tgt": -2.5374, + "logp_src": -1.6157, + "flip_to_tgt": false, + "kl_vs_clean": 0.3533, + "delta_norm_band_sum": 20.7813, + "delta_norm_per_layer": { + "9": 0.4356, + "10": 0.6079, + "11": 0.6023, + "12": 1.7098, + "13": 1.8316, + "14": 0.6624, + "15": 1.0043, + "16": 0.3245, + "17": 5.5697, + "18": 3.1891, + "19": 2.0357, + "20": 1.6004, + "21": 0.8618, + "22": 0.3462 + }, + "gram_cond_max": 4.02 + }, + "rand_target@a1": { + "top1": " _", + "rank_tgt": 80, + "rank_src": 1, + "logp_tgt": -6.8925, + "logp_src": -2.4107, + "flip_to_tgt": false, + "kl_vs_clean": 1.5485, + "delta_norm_band_sum": 20.7813, + "dose_matched_to": "swap@a1" + }, + "orth@a1": { + "top1": " pes", + "rank_tgt": 68, + "rank_src": 0, + "logp_tgt": -6.7187, + "logp_src": -1.7839, + "flip_to_tgt": false, + "kl_vs_clean": 0.1705, + "delta_norm_band_sum": 20.7813, + "dose_matched_to": "swap@a1" + }, + "oracle@a1": { + "top1": " \"", + "rank_tgt": 20, + "rank_src": 1, + "logp_tgt": -4.9286, + "logp_src": -1.8842, + "flip_to_tgt": false, + "kl_vs_clean": 0.0621, + "delta_norm_band_sum": 19.3174 + }, + "swap@a2": { + "top1": " Mexico", + "rank_tgt": 2532, + "rank_src": 2775, + "logp_tgt": -45.5574, + "logp_src": -45.8778, + "flip_to_tgt": false, + "kl_vs_clean": 34.6897, + "delta_norm_band_sum": 17815.559, + "delta_norm_per_layer": { + "9": 1.2616, + "10": 3.0294, + "11": 5.1761, + "12": 15.8032, + "13": 31.0267, + "14": 52.7901, + "15": 94.4346, + "16": 172.3323, + "17": 324.4903, + "18": 623.5466, + "19": 1154.0911, + "20": 2257.4973, + "21": 4472.4771, + "22": 8607.6025 + }, + "gram_cond_max": 4.02 + }, + "rand_target@a2": { + "top1": " atheros", + "rank_tgt": 36286, + "rank_src": 43972, + "logp_tgt": -19.6953, + "logp_src": -21.6473, + "flip_to_tgt": false, + "kl_vs_clean": 22.4307, + "delta_norm_band_sum": 17815.5576, + "dose_matched_to": "swap@a2" + }, + "orth@a2": { + "top1": " Mexico", + "rank_tgt": 20429, + "rank_src": 1493, + "logp_tgt": -41.1731, + "logp_src": -34.695, + "flip_to_tgt": false, + "kl_vs_clean": 26.9302, + "delta_norm_band_sum": 17815.561, + "dose_matched_to": "swap@a2" + }, + "oracle@a2": { + "top1": " pes", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -12.8727, + "logp_src": -0.0, + "flip_to_tgt": false, + "kl_vs_clean": 23.8367, + "delta_norm_band_sum": 14971.3791 + }, + "swap@a4": { + "top1": " Mexico", + "rank_tgt": 3113, + "rank_src": 3446, + "logp_tgt": -46.3455, + "logp_src": -46.7567, + "flip_to_tgt": false, + "kl_vs_clean": 34.0031, + "delta_norm_band_sum": 152540994.3116, + "delta_norm_per_layer": { + "9": 3.4186, + "10": 14.4381, + "11": 48.7206, + "12": 197.0501, + "13": 738.854, + "14": 2771.7581, + "15": 10301.2676, + "16": 36508.4609, + "17": 134539.9062, + "18": 514822.1875, + "19": 1915769.75, + "20": 7480472.5, + "21": 29401846.0, + "22": 113042960.0 + }, + "gram_cond_max": 4.02 + }, + "rand_target@a4": { + "top1": "enance", + "rank_tgt": 25740, + "rank_src": 48394, + "logp_tgt": -15.5752, + "logp_src": -20.7216, + "flip_to_tgt": false, + "kl_vs_clean": 18.156, + "delta_norm_band_sum": 152540998.122, + "dose_matched_to": "swap@a4" + }, + "orth@a4": { + "top1": " Mexico", + "rank_tgt": 14099, + "rank_src": 1594, + "logp_tgt": -42.9957, + "logp_src": -36.9575, + "flip_to_tgt": false, + "kl_vs_clean": 28.1018, + "delta_norm_band_sum": 152540986.8264, + "dose_matched_to": "swap@a4" + }, + "oracle@a4": { + "top1": " pes", + "rank_tgt": 1, + "rank_src": 0, + "logp_tgt": -24.0817, + "logp_src": 0.0, + "flip_to_tgt": false, + "kl_vs_clean": 26.221, + "delta_norm_band_sum": 135521953.3199 + } + }, + "oracle_any_flip": false, + "VOID_no_oracle": true + } + ], + "runtime_s": 55.8, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "retracted": "RETRACTED 2026-08-10: oracle gated at ANY alpha (only 4/19 cells pass at alpha=1) and control_rand_target was deterministic. Superseded by t8_probe_swap_410m.json. Do not cite." +} \ No newline at end of file diff --git a/results/t5a_capability_160m.json b/results/t5a_capability_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..b9e8e2e7ef313c22267e0fd1ebd5f24ea2b0fc8f --- /dev/null +++ b/results/t5a_capability_160m.json @@ -0,0 +1,922 @@ +{ + "experiment": "T5a capability screen (gates the write battery, PLAN.md \u00a73.5)", + "status": "GATE \u2014 pre-declared bar fixed in-script before any model was run", + "model": "EleutherAI/pythia-160m-deduped", + "n_cells": 80, + "n_cells_ok": 20, + "accuracy": 0.25, + "median_rank_overall": 2, + "bar_min_cells": 32, + "WRITE_BATTERY_ADMISSIBLE": false, + "by_relation": { + "capital": { + "n": 20, + "n_ok": 12, + "median_rank": 0 + }, + "language": { + "n": 20, + "n_ok": 4, + "median_rank": 2 + }, + "continent": { + "n": 20, + "n_ok": 4, + "median_rank": 2 + }, + "currency": { + "n": 20, + "n_ok": 0, + "median_rank": 17 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "country": "France", + "relation": "capital", + "prompt": "The capital of France is the city of", + "answer": " Paris", + "answer_token": " Paris", + "top1": " Paris", + "rank_of_answer": 0, + "logp_answer": -1.8163, + "cell_ok": true + }, + { + "country": "France", + "relation": "language", + "prompt": "The main language spoken in France is", + "answer": " French", + "answer_token": " French", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.1686, + "cell_ok": false + }, + { + "country": "France", + "relation": "continent", + "prompt": "The country France is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Africa", + "rank_of_answer": 2, + "logp_answer": -2.1883, + "cell_ok": false + }, + { + "country": "France", + "relation": "currency", + "prompt": "The currency used in France is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 14, + "logp_answer": -4.5386, + "cell_ok": false + }, + { + "country": "China", + "relation": "capital", + "prompt": "The capital of China is the city of", + "answer": " Beijing", + "answer_token": " Beijing", + "top1": " Shanghai", + "rank_of_answer": 6, + "logp_answer": -3.0667, + "cell_ok": false + }, + { + "country": "China", + "relation": "language", + "prompt": "The main language spoken in China is", + "answer": " Chinese", + "answer_token": " Chinese", + "top1": " ", + "rank_of_answer": 6, + "logp_answer": -3.6465, + "cell_ok": false + }, + { + "country": "China", + "relation": "continent", + "prompt": "The country China is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -1.5588, + "cell_ok": true + }, + { + "country": "China", + "relation": "currency", + "prompt": "The currency used in China is called the", + "answer": " yuan", + "answer_token": " y", + "top1": " \u201c", + "rank_of_answer": 2, + "logp_answer": -2.9267, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "capital", + "prompt": "The capital of Japan is the city of", + "answer": " Tokyo", + "answer_token": " Tokyo", + "top1": " Tokyo", + "rank_of_answer": 0, + "logp_answer": -0.9122, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "language", + "prompt": "The main language spoken in Japan is", + "answer": " Japanese", + "answer_token": " Japanese", + "top1": " ", + "rank_of_answer": 4, + "logp_answer": -3.6886, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "continent", + "prompt": "The country Japan is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Japan", + "rank_of_answer": 1, + "logp_answer": -2.1904, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "currency", + "prompt": "The currency used in Japan is called the", + "answer": " yen", + "answer_token": " y", + "top1": " \u201c", + "rank_of_answer": 2, + "logp_answer": -2.6048, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "capital", + "prompt": "The capital of Germany is the city of", + "answer": " Berlin", + "answer_token": " Berlin", + "top1": " Berlin", + "rank_of_answer": 0, + "logp_answer": -2.6405, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "language", + "prompt": "The main language spoken in Germany is", + "answer": " German", + "answer_token": " German", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -1.995, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "continent", + "prompt": "The country Germany is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -1.8341, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "currency", + "prompt": "The currency used in Germany is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 16, + "logp_answer": -4.5007, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "capital", + "prompt": "The capital of Italy is the city of", + "answer": " Rome", + "answer_token": " Rome", + "top1": " Florence", + "rank_of_answer": 4, + "logp_answer": -3.1553, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "language", + "prompt": "The main language spoken in Italy is", + "answer": " Italian", + "answer_token": " Italian", + "top1": " spoken", + "rank_of_answer": 2, + "logp_answer": -3.0261, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "continent", + "prompt": "The country Italy is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Italy", + "rank_of_answer": 2, + "logp_answer": -2.6228, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "currency", + "prompt": "The currency used in Italy is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \u201c", + "rank_of_answer": 3, + "logp_answer": -3.3813, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "capital", + "prompt": "The capital of Spain is the city of", + "answer": " Madrid", + "answer_token": " Madrid", + "top1": " Barcelona", + "rank_of_answer": 1, + "logp_answer": -2.6211, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "language", + "prompt": "The main language spoken in Spain is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " spoken", + "rank_of_answer": 2, + "logp_answer": -2.0382, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "continent", + "prompt": "The country Spain is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -2.2507, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "currency", + "prompt": "The currency used in Spain is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \u201c", + "rank_of_answer": 13, + "logp_answer": -4.5219, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "capital", + "prompt": "The capital of Russia is the city of", + "answer": " Moscow", + "answer_token": " Moscow", + "top1": " Moscow", + "rank_of_answer": 0, + "logp_answer": -1.9403, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "language", + "prompt": "The main language spoken in Russia is", + "answer": " Russian", + "answer_token": " Russian", + "top1": " Russian", + "rank_of_answer": 0, + "logp_answer": -1.7676, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "continent", + "prompt": "The country Russia is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Russia", + "rank_of_answer": 5, + "logp_answer": -3.074, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "currency", + "prompt": "The currency used in Russia is called the", + "answer": " ruble", + "answer_token": " rub", + "top1": " \u201c", + "rank_of_answer": 57, + "logp_answer": -6.4136, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "capital", + "prompt": "The capital of Egypt is the city of", + "answer": " Cairo", + "answer_token": " Cairo", + "top1": " the", + "rank_of_answer": 8, + "logp_answer": -3.6397, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "language", + "prompt": "The main language spoken in Egypt is", + "answer": " Arabic", + "answer_token": " Arabic", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.3991, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "continent", + "prompt": "The country Egypt is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -1.0213, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "currency", + "prompt": "The currency used in Egypt is called the", + "answer": " pound", + "answer_token": " pound", + "top1": " Egyptian", + "rank_of_answer": 249, + "logp_answer": -8.1646, + "cell_ok": false + }, + { + "country": "India", + "relation": "capital", + "prompt": "The capital of India is the city of", + "answer": " Delhi", + "answer_token": " Delhi", + "top1": " J", + "rank_of_answer": 1, + "logp_answer": -3.0334, + "cell_ok": false + }, + { + "country": "India", + "relation": "language", + "prompt": "The main language spoken in India is", + "answer": " Hindi", + "answer_token": " Hind", + "top1": " spoken", + "rank_of_answer": 7, + "logp_answer": -3.7297, + "cell_ok": false + }, + { + "country": "India", + "relation": "continent", + "prompt": "The country India is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " India", + "rank_of_answer": 2, + "logp_answer": -2.3277, + "cell_ok": false + }, + { + "country": "India", + "relation": "currency", + "prompt": "The currency used in India is called the", + "answer": " rupee", + "answer_token": " ru", + "top1": " \u201c", + "rank_of_answer": 1, + "logp_answer": -2.4253, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "capital", + "prompt": "The capital of Brazil is the city of", + "answer": " Brasilia", + "answer_token": " Bras", + "top1": " S\u00e3o", + "rank_of_answer": 3, + "logp_answer": -3.459, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "language", + "prompt": "The main language spoken in Brazil is", + "answer": " Portuguese", + "answer_token": " Portuguese", + "top1": " the", + "rank_of_answer": 7, + "logp_answer": -3.5271, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "continent", + "prompt": "The country Brazil is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Brazil", + "rank_of_answer": 11, + "logp_answer": -5.2995, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "currency", + "prompt": "The currency used in Brazil is called the", + "answer": " real", + "answer_token": " real", + "top1": " Brazilian", + "rank_of_answer": 8606, + "logp_answer": -14.3064, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "capital", + "prompt": "The capital of Canada is the city of", + "answer": " Ottawa", + "answer_token": " Ottawa", + "top1": " Toronto", + "rank_of_answer": 1, + "logp_answer": -1.7289, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "language", + "prompt": "The main language spoken in Canada is", + "answer": " English", + "answer_token": " English", + "top1": " English", + "rank_of_answer": 0, + "logp_answer": -1.5551, + "cell_ok": true + }, + { + "country": "Canada", + "relation": "continent", + "prompt": "The country Canada is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Africa", + "rank_of_answer": 9, + "logp_answer": -4.7133, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "currency", + "prompt": "The currency used in Canada is called the", + "answer": " dollar", + "answer_token": " dollar", + "top1": " \"", + "rank_of_answer": 44, + "logp_answer": -6.0425, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "capital", + "prompt": "The capital of Mexico is the city of", + "answer": " Mexico", + "answer_token": " Mexico", + "top1": " Mexico", + "rank_of_answer": 0, + "logp_answer": -1.9405, + "cell_ok": true + }, + { + "country": "Mexico", + "relation": "language", + "prompt": "The main language spoken in Mexico is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " the", + "rank_of_answer": 8, + "logp_answer": -4.3558, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "continent", + "prompt": "The country Mexico is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Mexico", + "rank_of_answer": 9, + "logp_answer": -4.6569, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "currency", + "prompt": "The currency used in Mexico is called the", + "answer": " peso", + "answer_token": " pes", + "top1": " \u201c", + "rank_of_answer": 17, + "logp_answer": -4.9294, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "capital", + "prompt": "The capital of Greece is the city of", + "answer": " Athens", + "answer_token": " Athens", + "top1": " Athens", + "rank_of_answer": 0, + "logp_answer": -1.0354, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "language", + "prompt": "The main language spoken in Greece is", + "answer": " Greek", + "answer_token": " Greek", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.8525, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "continent", + "prompt": "The country Greece is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Greece", + "rank_of_answer": 3, + "logp_answer": -2.4704, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "currency", + "prompt": "The currency used in Greece is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \u201c", + "rank_of_answer": 11, + "logp_answer": -4.3542, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "capital", + "prompt": "The capital of Poland is the city of", + "answer": " Warsaw", + "answer_token": " Warsaw", + "top1": " K", + "rank_of_answer": 1, + "logp_answer": -2.7945, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "language", + "prompt": "The main language spoken in Poland is", + "answer": " Polish", + "answer_token": " Polish", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -1.8114, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "continent", + "prompt": "The country Poland is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -1.7411, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "currency", + "prompt": "The currency used in Poland is called the", + "answer": " zloty", + "answer_token": " z", + "top1": " \u201c", + "rank_of_answer": 17, + "logp_answer": -4.6702, + "cell_ok": false + }, + { + "country": "Sweden", + "relation": "capital", + "prompt": "The capital of Sweden is the city of", + "answer": " Stockholm", + "answer_token": " Stockholm", + "top1": " Stockholm", + "rank_of_answer": 0, + "logp_answer": -1.1343, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "language", + "prompt": "The main language spoken in Sweden is", + "answer": " Swedish", + "answer_token": " Swedish", + "top1": " Swedish", + "rank_of_answer": 0, + "logp_answer": -1.9047, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "continent", + "prompt": "The country Sweden is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Sweden", + "rank_of_answer": 2, + "logp_answer": -2.7174, + "cell_ok": false + }, + { + "country": "Sweden", + "relation": "currency", + "prompt": "The currency used in Sweden is called the", + "answer": " krona", + "answer_token": " k", + "top1": " \"", + "rank_of_answer": 28, + "logp_answer": -5.4518, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "capital", + "prompt": "The capital of Norway is the city of", + "answer": " Oslo", + "answer_token": " Oslo", + "top1": " Oslo", + "rank_of_answer": 0, + "logp_answer": -1.4257, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "language", + "prompt": "The main language spoken in Norway is", + "answer": " Norwegian", + "answer_token": " Norwegian", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.9579, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "continent", + "prompt": "The country Norway is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Norway", + "rank_of_answer": 3, + "logp_answer": -2.6229, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "currency", + "prompt": "The currency used in Norway is called the", + "answer": " krone", + "answer_token": " k", + "top1": " \"", + "rank_of_answer": 30, + "logp_answer": -5.5224, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "capital", + "prompt": "The capital of Turkey is the city of", + "answer": " Ankara", + "answer_token": " Ank", + "top1": " Ank", + "rank_of_answer": 0, + "logp_answer": -2.5778, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "language", + "prompt": "The main language spoken in Turkey is", + "answer": " Turkish", + "answer_token": " Turkish", + "top1": " Turkish", + "rank_of_answer": 0, + "logp_answer": -0.7356, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "continent", + "prompt": "The country Turkey is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Turkey", + "rank_of_answer": 3, + "logp_answer": -3.4616, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "currency", + "prompt": "The currency used in Turkey is called the", + "answer": " lira", + "answer_token": " l", + "top1": " \u201c", + "rank_of_answer": 1010, + "logp_answer": -10.4683, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "capital", + "prompt": "The capital of Korea is the city of", + "answer": " Seoul", + "answer_token": " Seoul", + "top1": " Seoul", + "rank_of_answer": 0, + "logp_answer": -2.2679, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "language", + "prompt": "The main language spoken in Korea is", + "answer": " Korean", + "answer_token": " Korean", + "top1": " written", + "rank_of_answer": 4, + "logp_answer": -3.0455, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "continent", + "prompt": "The country Korea is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " South", + "rank_of_answer": 2, + "logp_answer": -1.9774, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "currency", + "prompt": "The currency used in Korea is called the", + "answer": " won", + "answer_token": " won", + "top1": " \u201c", + "rank_of_answer": 1430, + "logp_answer": -10.5564, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "capital", + "prompt": "The capital of Thailand is the city of", + "answer": " Bangkok", + "answer_token": " Bangkok", + "top1": " Bangkok", + "rank_of_answer": 0, + "logp_answer": -2.0189, + "cell_ok": true + }, + { + "country": "Thailand", + "relation": "language", + "prompt": "The main language spoken in Thailand is", + "answer": " Thai", + "answer_token": " Thai", + "top1": " the", + "rank_of_answer": 3, + "logp_answer": -3.4, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "continent", + "prompt": "The country Thailand is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Thailand", + "rank_of_answer": 2, + "logp_answer": -2.7871, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "currency", + "prompt": "The currency used in Thailand is called the", + "answer": " baht", + "answer_token": " ba", + "top1": " \"", + "rank_of_answer": 35, + "logp_answer": -5.5204, + "cell_ok": false + }, + { + "country": "Kenya", + "relation": "capital", + "prompt": "The capital of Kenya is the city of", + "answer": " Nairobi", + "answer_token": " N", + "top1": " N", + "rank_of_answer": 0, + "logp_answer": -2.6406, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "language", + "prompt": "The main language spoken in Kenya is", + "answer": " Swahili", + "answer_token": " Sw", + "top1": " the", + "rank_of_answer": 68, + "logp_answer": -6.3598, + "cell_ok": false + }, + { + "country": "Kenya", + "relation": "continent", + "prompt": "The country Kenya is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.742, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "currency", + "prompt": "The currency used in Kenya is called the", + "answer": " shilling", + "answer_token": " sh", + "top1": " Kenya", + "rank_of_answer": 324, + "logp_answer": -8.0137, + "cell_ok": false + } + ], + "runtime_s": 13.8 +} \ No newline at end of file diff --git a/results/t5a_capability_160m_v2.json b/results/t5a_capability_160m_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..0d892f9458dc09e184cda1f2ac43b753abff9530 --- /dev/null +++ b/results/t5a_capability_160m_v2.json @@ -0,0 +1,716 @@ +{ + "experiment": "T5a capability screen (gates the write battery, PLAN.md \u00a73.5)", + "status": "GATE \u2014 pre-declared bar fixed in-script before any model was run", + "model": "EleutherAI/pythia-160m-deduped", + "n_cells": 61, + "n_excluded_multitoken_answer": 16, + "n_excluded_answer_copyable_from_prompt": 3, + "n_distinct_answer_tokens": 40, + "n_cells_ok": 17, + "accuracy": 0.2787, + "median_rank_overall": 2, + "bar_min_cells": 32, + "WRITE_BATTERY_ADMISSIBLE": false, + "by_relation": { + "capital": { + "n": 16, + "n_ok": 9, + "median_rank": 0 + }, + "language": { + "n": 16, + "n_ok": 4, + "median_rank": 2 + }, + "continent": { + "n": 20, + "n_ok": 4, + "median_rank": 2 + }, + "currency": { + "n": 9, + "n_ok": 0, + "median_rank": 16 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "country": "France", + "relation": "capital", + "prompt": "The capital of France is the city of", + "answer": " Paris", + "answer_token": " Paris", + "top1": " Paris", + "rank_of_answer": 0, + "logp_answer": -1.8166, + "cell_ok": true + }, + { + "country": "France", + "relation": "language", + "prompt": "The main language spoken in France is", + "answer": " French", + "answer_token": " French", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.168, + "cell_ok": false + }, + { + "country": "France", + "relation": "continent", + "prompt": "The country France is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Africa", + "rank_of_answer": 2, + "logp_answer": -2.1883, + "cell_ok": false + }, + { + "country": "France", + "relation": "currency", + "prompt": "The currency used in France is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 14, + "logp_answer": -4.5393, + "cell_ok": false + }, + { + "country": "China", + "relation": "capital", + "prompt": "The capital of China is the city of", + "answer": " Beijing", + "answer_token": " Beijing", + "top1": " Shanghai", + "rank_of_answer": 6, + "logp_answer": -3.0666, + "cell_ok": false + }, + { + "country": "China", + "relation": "language", + "prompt": "The main language spoken in China is", + "answer": " Chinese", + "answer_token": " Chinese", + "top1": " ", + "rank_of_answer": 6, + "logp_answer": -3.6462, + "cell_ok": false + }, + { + "country": "China", + "relation": "continent", + "prompt": "The country China is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -1.5588, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "capital", + "prompt": "The capital of Japan is the city of", + "answer": " Tokyo", + "answer_token": " Tokyo", + "top1": " Tokyo", + "rank_of_answer": 0, + "logp_answer": -0.912, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "language", + "prompt": "The main language spoken in Japan is", + "answer": " Japanese", + "answer_token": " Japanese", + "top1": " ", + "rank_of_answer": 4, + "logp_answer": -3.6879, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "continent", + "prompt": "The country Japan is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Japan", + "rank_of_answer": 1, + "logp_answer": -2.1904, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "capital", + "prompt": "The capital of Germany is the city of", + "answer": " Berlin", + "answer_token": " Berlin", + "top1": " Berlin", + "rank_of_answer": 0, + "logp_answer": -2.6407, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "continent", + "prompt": "The country Germany is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -1.8341, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "currency", + "prompt": "The currency used in Germany is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 16, + "logp_answer": -4.5011, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "capital", + "prompt": "The capital of Italy is the city of", + "answer": " Rome", + "answer_token": " Rome", + "top1": " Florence", + "rank_of_answer": 4, + "logp_answer": -3.1559, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "language", + "prompt": "The main language spoken in Italy is", + "answer": " Italian", + "answer_token": " Italian", + "top1": " spoken", + "rank_of_answer": 2, + "logp_answer": -3.0264, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "continent", + "prompt": "The country Italy is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Italy", + "rank_of_answer": 2, + "logp_answer": -2.6228, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "currency", + "prompt": "The currency used in Italy is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \u201c", + "rank_of_answer": 3, + "logp_answer": -3.3819, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "capital", + "prompt": "The capital of Spain is the city of", + "answer": " Madrid", + "answer_token": " Madrid", + "top1": " Barcelona", + "rank_of_answer": 1, + "logp_answer": -2.6204, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "language", + "prompt": "The main language spoken in Spain is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " spoken", + "rank_of_answer": 2, + "logp_answer": -2.0383, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "continent", + "prompt": "The country Spain is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -2.2507, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "currency", + "prompt": "The currency used in Spain is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \u201c", + "rank_of_answer": 13, + "logp_answer": -4.5216, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "capital", + "prompt": "The capital of Russia is the city of", + "answer": " Moscow", + "answer_token": " Moscow", + "top1": " Moscow", + "rank_of_answer": 0, + "logp_answer": -1.9399, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "language", + "prompt": "The main language spoken in Russia is", + "answer": " Russian", + "answer_token": " Russian", + "top1": " Russian", + "rank_of_answer": 0, + "logp_answer": -1.7678, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "continent", + "prompt": "The country Russia is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Russia", + "rank_of_answer": 5, + "logp_answer": -3.074, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "capital", + "prompt": "The capital of Egypt is the city of", + "answer": " Cairo", + "answer_token": " Cairo", + "top1": " the", + "rank_of_answer": 8, + "logp_answer": -3.6394, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "language", + "prompt": "The main language spoken in Egypt is", + "answer": " Arabic", + "answer_token": " Arabic", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.3993, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "continent", + "prompt": "The country Egypt is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -1.0213, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "currency", + "prompt": "The currency used in Egypt is called the", + "answer": " pound", + "answer_token": " pound", + "top1": " Egyptian", + "rank_of_answer": 249, + "logp_answer": -8.1639, + "cell_ok": false + }, + { + "country": "India", + "relation": "capital", + "prompt": "The capital of India is the city of", + "answer": " Delhi", + "answer_token": " Delhi", + "top1": " J", + "rank_of_answer": 1, + "logp_answer": -3.0333, + "cell_ok": false + }, + { + "country": "India", + "relation": "continent", + "prompt": "The country India is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " India", + "rank_of_answer": 2, + "logp_answer": -2.3277, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "language", + "prompt": "The main language spoken in Brazil is", + "answer": " Portuguese", + "answer_token": " Portuguese", + "top1": " the", + "rank_of_answer": 7, + "logp_answer": -3.5268, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "continent", + "prompt": "The country Brazil is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Brazil", + "rank_of_answer": 11, + "logp_answer": -5.2995, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "currency", + "prompt": "The currency used in Brazil is called the", + "answer": " real", + "answer_token": " real", + "top1": " Brazilian", + "rank_of_answer": 8608, + "logp_answer": -14.3076, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "capital", + "prompt": "The capital of Canada is the city of", + "answer": " Ottawa", + "answer_token": " Ottawa", + "top1": " Toronto", + "rank_of_answer": 1, + "logp_answer": -1.7289, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "language", + "prompt": "The main language spoken in Canada is", + "answer": " English", + "answer_token": " English", + "top1": " English", + "rank_of_answer": 0, + "logp_answer": -1.5548, + "cell_ok": true + }, + { + "country": "Canada", + "relation": "continent", + "prompt": "The country Canada is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Africa", + "rank_of_answer": 9, + "logp_answer": -4.7133, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "currency", + "prompt": "The currency used in Canada is called the", + "answer": " dollar", + "answer_token": " dollar", + "top1": " \"", + "rank_of_answer": 44, + "logp_answer": -6.0427, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "language", + "prompt": "The main language spoken in Mexico is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " the", + "rank_of_answer": 8, + "logp_answer": -4.3558, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "continent", + "prompt": "The country Mexico is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Mexico", + "rank_of_answer": 9, + "logp_answer": -4.6569, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "capital", + "prompt": "The capital of Greece is the city of", + "answer": " Athens", + "answer_token": " Athens", + "top1": " Athens", + "rank_of_answer": 0, + "logp_answer": -1.0355, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "language", + "prompt": "The main language spoken in Greece is", + "answer": " Greek", + "answer_token": " Greek", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.8533, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "continent", + "prompt": "The country Greece is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Greece", + "rank_of_answer": 3, + "logp_answer": -2.4704, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "currency", + "prompt": "The currency used in Greece is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \u201c", + "rank_of_answer": 11, + "logp_answer": -4.3539, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "capital", + "prompt": "The capital of Poland is the city of", + "answer": " Warsaw", + "answer_token": " Warsaw", + "top1": " K", + "rank_of_answer": 1, + "logp_answer": -2.794, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "language", + "prompt": "The main language spoken in Poland is", + "answer": " Polish", + "answer_token": " Polish", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -1.8118, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "continent", + "prompt": "The country Poland is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -1.7411, + "cell_ok": false + }, + { + "country": "Sweden", + "relation": "capital", + "prompt": "The capital of Sweden is the city of", + "answer": " Stockholm", + "answer_token": " Stockholm", + "top1": " Stockholm", + "rank_of_answer": 0, + "logp_answer": -1.1344, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "language", + "prompt": "The main language spoken in Sweden is", + "answer": " Swedish", + "answer_token": " Swedish", + "top1": " Swedish", + "rank_of_answer": 0, + "logp_answer": -1.9047, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "continent", + "prompt": "The country Sweden is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Sweden", + "rank_of_answer": 2, + "logp_answer": -2.7174, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "capital", + "prompt": "The capital of Norway is the city of", + "answer": " Oslo", + "answer_token": " Oslo", + "top1": " Oslo", + "rank_of_answer": 0, + "logp_answer": -1.4256, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "language", + "prompt": "The main language spoken in Norway is", + "answer": " Norwegian", + "answer_token": " Norwegian", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.9582, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "continent", + "prompt": "The country Norway is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Norway", + "rank_of_answer": 3, + "logp_answer": -2.6229, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "language", + "prompt": "The main language spoken in Turkey is", + "answer": " Turkish", + "answer_token": " Turkish", + "top1": " Turkish", + "rank_of_answer": 0, + "logp_answer": -0.7358, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "continent", + "prompt": "The country Turkey is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Turkey", + "rank_of_answer": 3, + "logp_answer": -3.4616, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "capital", + "prompt": "The capital of Korea is the city of", + "answer": " Seoul", + "answer_token": " Seoul", + "top1": " Seoul", + "rank_of_answer": 0, + "logp_answer": -2.2676, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "language", + "prompt": "The main language spoken in Korea is", + "answer": " Korean", + "answer_token": " Korean", + "top1": " written", + "rank_of_answer": 4, + "logp_answer": -3.0453, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "continent", + "prompt": "The country Korea is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " South", + "rank_of_answer": 2, + "logp_answer": -1.9774, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "currency", + "prompt": "The currency used in Korea is called the", + "answer": " won", + "answer_token": " won", + "top1": " \u201c", + "rank_of_answer": 1430, + "logp_answer": -10.5564, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "capital", + "prompt": "The capital of Thailand is the city of", + "answer": " Bangkok", + "answer_token": " Bangkok", + "top1": " Bangkok", + "rank_of_answer": 0, + "logp_answer": -2.0187, + "cell_ok": true + }, + { + "country": "Thailand", + "relation": "continent", + "prompt": "The country Thailand is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Thailand", + "rank_of_answer": 2, + "logp_answer": -2.7871, + "cell_ok": false + }, + { + "country": "Kenya", + "relation": "continent", + "prompt": "The country Kenya is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.742, + "cell_ok": true + } + ], + "runtime_s": 19.0 +} \ No newline at end of file diff --git a/results/t5a_capability_1b.json b/results/t5a_capability_1b.json new file mode 100644 index 0000000000000000000000000000000000000000..5cf2427b994d233382de888d06223d851f905c34 --- /dev/null +++ b/results/t5a_capability_1b.json @@ -0,0 +1,922 @@ +{ + "experiment": "T5a capability screen (gates the write battery, PLAN.md \u00a73.5)", + "status": "GATE \u2014 pre-declared bar fixed in-script before any model was run", + "model": "EleutherAI/pythia-1b-deduped", + "n_cells": 80, + "n_cells_ok": 53, + "accuracy": 0.6625, + "median_rank_overall": 0, + "bar_min_cells": 32, + "WRITE_BATTERY_ADMISSIBLE": true, + "by_relation": { + "capital": { + "n": 20, + "n_ok": 13, + "median_rank": 0 + }, + "language": { + "n": 20, + "n_ok": 19, + "median_rank": 0 + }, + "continent": { + "n": 20, + "n_ok": 16, + "median_rank": 0 + }, + "currency": { + "n": 20, + "n_ok": 5, + "median_rank": 1 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "country": "France", + "relation": "capital", + "prompt": "The capital of France is the city of", + "answer": " Paris", + "answer_token": " Paris", + "top1": " Paris", + "rank_of_answer": 0, + "logp_answer": -0.8386, + "cell_ok": true + }, + { + "country": "France", + "relation": "language", + "prompt": "The main language spoken in France is", + "answer": " French", + "answer_token": " French", + "top1": " French", + "rank_of_answer": 0, + "logp_answer": -0.2157, + "cell_ok": true + }, + { + "country": "France", + "relation": "continent", + "prompt": "The country France is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.0623, + "cell_ok": true + }, + { + "country": "France", + "relation": "currency", + "prompt": "The currency used in France is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " Euro", + "rank_of_answer": 1, + "logp_answer": -2.1036, + "cell_ok": false + }, + { + "country": "China", + "relation": "capital", + "prompt": "The capital of China is the city of", + "answer": " Beijing", + "answer_token": " Beijing", + "top1": " Beijing", + "rank_of_answer": 0, + "logp_answer": -2.0291, + "cell_ok": true + }, + { + "country": "China", + "relation": "language", + "prompt": "The main language spoken in China is", + "answer": " Chinese", + "answer_token": " Chinese", + "top1": " Mand", + "rank_of_answer": 1, + "logp_answer": -1.7659, + "cell_ok": false + }, + { + "country": "China", + "relation": "continent", + "prompt": "The country China is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.5877, + "cell_ok": true + }, + { + "country": "China", + "relation": "currency", + "prompt": "The currency used in China is called the", + "answer": " yuan", + "answer_token": " y", + "top1": " Ren", + "rank_of_answer": 1, + "logp_answer": -1.6059, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "capital", + "prompt": "The capital of Japan is the city of", + "answer": " Tokyo", + "answer_token": " Tokyo", + "top1": " Tokyo", + "rank_of_answer": 0, + "logp_answer": -0.8448, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "language", + "prompt": "The main language spoken in Japan is", + "answer": " Japanese", + "answer_token": " Japanese", + "top1": " Japanese", + "rank_of_answer": 0, + "logp_answer": -0.1574, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "continent", + "prompt": "The country Japan is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.2718, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "currency", + "prompt": "The currency used in Japan is called the", + "answer": " yen", + "answer_token": " y", + "top1": " y", + "rank_of_answer": 0, + "logp_answer": -1.117, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "capital", + "prompt": "The capital of Germany is the city of", + "answer": " Berlin", + "answer_token": " Berlin", + "top1": " C", + "rank_of_answer": 1, + "logp_answer": -1.9579, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "language", + "prompt": "The main language spoken in Germany is", + "answer": " German", + "answer_token": " German", + "top1": " German", + "rank_of_answer": 0, + "logp_answer": -0.1572, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "continent", + "prompt": "The country Germany is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.0816, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "currency", + "prompt": "The currency used in Germany is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " Deutsche", + "rank_of_answer": 2, + "logp_answer": -2.4813, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "capital", + "prompt": "The capital of Italy is the city of", + "answer": " Rome", + "answer_token": " Rome", + "top1": " Rome", + "rank_of_answer": 0, + "logp_answer": -1.3317, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "language", + "prompt": "The main language spoken in Italy is", + "answer": " Italian", + "answer_token": " Italian", + "top1": " Italian", + "rank_of_answer": 0, + "logp_answer": -0.1143, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "continent", + "prompt": "The country Italy is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.0699, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "currency", + "prompt": "The currency used in Italy is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " l", + "rank_of_answer": 3, + "logp_answer": -2.4185, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "capital", + "prompt": "The capital of Spain is the city of", + "answer": " Madrid", + "answer_token": " Madrid", + "top1": " Barcelona", + "rank_of_answer": 1, + "logp_answer": -1.8387, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "language", + "prompt": "The main language spoken in Spain is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " Spanish", + "rank_of_answer": 0, + "logp_answer": -0.2586, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "continent", + "prompt": "The country Spain is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.2004, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "currency", + "prompt": "The currency used in Spain is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " Euro", + "rank_of_answer": 1, + "logp_answer": -1.9253, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "capital", + "prompt": "The capital of Russia is the city of", + "answer": " Moscow", + "answer_token": " Moscow", + "top1": " Moscow", + "rank_of_answer": 0, + "logp_answer": -1.1223, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "language", + "prompt": "The main language spoken in Russia is", + "answer": " Russian", + "answer_token": " Russian", + "top1": " Russian", + "rank_of_answer": 0, + "logp_answer": -0.0765, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "continent", + "prompt": "The country Russia is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.4378, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "currency", + "prompt": "The currency used in Russia is called the", + "answer": " ruble", + "answer_token": " rub", + "top1": " rub", + "rank_of_answer": 0, + "logp_answer": -1.1446, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "capital", + "prompt": "The capital of Egypt is the city of", + "answer": " Cairo", + "answer_token": " Cairo", + "top1": " Cairo", + "rank_of_answer": 0, + "logp_answer": -1.2176, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "language", + "prompt": "The main language spoken in Egypt is", + "answer": " Arabic", + "answer_token": " Arabic", + "top1": " Arabic", + "rank_of_answer": 0, + "logp_answer": -0.5473, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "continent", + "prompt": "The country Egypt is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.9345, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "currency", + "prompt": "The currency used in Egypt is called the", + "answer": " pound", + "answer_token": " pound", + "top1": " Egyptian", + "rank_of_answer": 12, + "logp_answer": -4.6191, + "cell_ok": false + }, + { + "country": "India", + "relation": "capital", + "prompt": "The capital of India is the city of", + "answer": " Delhi", + "answer_token": " Delhi", + "top1": " Delhi", + "rank_of_answer": 0, + "logp_answer": -1.4684, + "cell_ok": true + }, + { + "country": "India", + "relation": "language", + "prompt": "The main language spoken in India is", + "answer": " Hindi", + "answer_token": " Hind", + "top1": " Hind", + "rank_of_answer": 0, + "logp_answer": -0.3274, + "cell_ok": true + }, + { + "country": "India", + "relation": "continent", + "prompt": "The country India is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.7523, + "cell_ok": true + }, + { + "country": "India", + "relation": "currency", + "prompt": "The currency used in India is called the", + "answer": " rupee", + "answer_token": " ru", + "top1": " Ru", + "rank_of_answer": 2, + "logp_answer": -1.9446, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "capital", + "prompt": "The capital of Brazil is the city of", + "answer": " Brasilia", + "answer_token": " Bras", + "top1": " Rio", + "rank_of_answer": 3, + "logp_answer": -3.4061, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "language", + "prompt": "The main language spoken in Brazil is", + "answer": " Portuguese", + "answer_token": " Portuguese", + "top1": " Portuguese", + "rank_of_answer": 0, + "logp_answer": -0.2077, + "cell_ok": true + }, + { + "country": "Brazil", + "relation": "continent", + "prompt": "The country Brazil is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " South", + "rank_of_answer": 3, + "logp_answer": -3.2149, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "currency", + "prompt": "The currency used in Brazil is called the", + "answer": " real", + "answer_token": " real", + "top1": " Real", + "rank_of_answer": 1, + "logp_answer": -1.5234, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "capital", + "prompt": "The capital of Canada is the city of", + "answer": " Ottawa", + "answer_token": " Ottawa", + "top1": " Toronto", + "rank_of_answer": 1, + "logp_answer": -1.4793, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "language", + "prompt": "The main language spoken in Canada is", + "answer": " English", + "answer_token": " English", + "top1": " English", + "rank_of_answer": 0, + "logp_answer": -0.2813, + "cell_ok": true + }, + { + "country": "Canada", + "relation": "continent", + "prompt": "The country Canada is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " North", + "rank_of_answer": 6, + "logp_answer": -4.1124, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "currency", + "prompt": "The currency used in Canada is called the", + "answer": " dollar", + "answer_token": " dollar", + "top1": " Canadian", + "rank_of_answer": 4, + "logp_answer": -4.0593, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "capital", + "prompt": "The capital of Mexico is the city of", + "answer": " Mexico", + "answer_token": " Mexico", + "top1": " Mexico", + "rank_of_answer": 0, + "logp_answer": -1.2633, + "cell_ok": true + }, + { + "country": "Mexico", + "relation": "language", + "prompt": "The main language spoken in Mexico is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " Spanish", + "rank_of_answer": 0, + "logp_answer": -0.1114, + "cell_ok": true + }, + { + "country": "Mexico", + "relation": "continent", + "prompt": "The country Mexico is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " North", + "rank_of_answer": 2, + "logp_answer": -2.644, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "currency", + "prompt": "The currency used in Mexico is called the", + "answer": " peso", + "answer_token": " pes", + "top1": " pes", + "rank_of_answer": 0, + "logp_answer": -0.7736, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "capital", + "prompt": "The capital of Greece is the city of", + "answer": " Athens", + "answer_token": " Athens", + "top1": " Athens", + "rank_of_answer": 0, + "logp_answer": -0.9796, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "language", + "prompt": "The main language spoken in Greece is", + "answer": " Greek", + "answer_token": " Greek", + "top1": " Greek", + "rank_of_answer": 0, + "logp_answer": -0.2984, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "continent", + "prompt": "The country Greece is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.1008, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "currency", + "prompt": "The currency used in Greece is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " Euro", + "rank_of_answer": 2, + "logp_answer": -2.3659, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "capital", + "prompt": "The capital of Poland is the city of", + "answer": " Warsaw", + "answer_token": " Warsaw", + "top1": " W", + "rank_of_answer": 3, + "logp_answer": -2.4121, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "language", + "prompt": "The main language spoken in Poland is", + "answer": " Polish", + "answer_token": " Polish", + "top1": " Polish", + "rank_of_answer": 0, + "logp_answer": -0.2705, + "cell_ok": true + }, + { + "country": "Poland", + "relation": "continent", + "prompt": "The country Poland is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.1083, + "cell_ok": true + }, + { + "country": "Poland", + "relation": "currency", + "prompt": "The currency used in Poland is called the", + "answer": " zloty", + "answer_token": " z", + "top1": " z", + "rank_of_answer": 0, + "logp_answer": -0.9478, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "capital", + "prompt": "The capital of Sweden is the city of", + "answer": " Stockholm", + "answer_token": " Stockholm", + "top1": " Stockholm", + "rank_of_answer": 0, + "logp_answer": -1.2726, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "language", + "prompt": "The main language spoken in Sweden is", + "answer": " Swedish", + "answer_token": " Swedish", + "top1": " Swedish", + "rank_of_answer": 0, + "logp_answer": -0.1388, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "continent", + "prompt": "The country Sweden is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.0932, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "currency", + "prompt": "The currency used in Sweden is called the", + "answer": " krona", + "answer_token": " k", + "top1": " Swedish", + "rank_of_answer": 1, + "logp_answer": -1.669, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "capital", + "prompt": "The capital of Norway is the city of", + "answer": " Oslo", + "answer_token": " Oslo", + "top1": " Tr", + "rank_of_answer": 2, + "logp_answer": -1.8818, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "language", + "prompt": "The main language spoken in Norway is", + "answer": " Norwegian", + "answer_token": " Norwegian", + "top1": " Norwegian", + "rank_of_answer": 0, + "logp_answer": -0.2958, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "continent", + "prompt": "The country Norway is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.1172, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "currency", + "prompt": "The currency used in Norway is called the", + "answer": " krone", + "answer_token": " k", + "top1": " Norwegian", + "rank_of_answer": 1, + "logp_answer": -2.2286, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "capital", + "prompt": "The capital of Turkey is the city of", + "answer": " Ankara", + "answer_token": " Ank", + "top1": " Istanbul", + "rank_of_answer": 1, + "logp_answer": -1.9737, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "language", + "prompt": "The main language spoken in Turkey is", + "answer": " Turkish", + "answer_token": " Turkish", + "top1": " Turkish", + "rank_of_answer": 0, + "logp_answer": -0.2496, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "continent", + "prompt": "The country Turkey is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Europe", + "rank_of_answer": 1, + "logp_answer": -1.205, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "currency", + "prompt": "The currency used in Turkey is called the", + "answer": " lira", + "answer_token": " l", + "top1": " Turkish", + "rank_of_answer": 1, + "logp_answer": -1.8281, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "capital", + "prompt": "The capital of Korea is the city of", + "answer": " Seoul", + "answer_token": " Seoul", + "top1": " Seoul", + "rank_of_answer": 0, + "logp_answer": -0.7901, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "language", + "prompt": "The main language spoken in Korea is", + "answer": " Korean", + "answer_token": " Korean", + "top1": " Korean", + "rank_of_answer": 0, + "logp_answer": -0.3206, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "continent", + "prompt": "The country Korea is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.3034, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "currency", + "prompt": "The currency used in Korea is called the", + "answer": " won", + "answer_token": " won", + "top1": " Korean", + "rank_of_answer": 1, + "logp_answer": -1.7947, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "capital", + "prompt": "The capital of Thailand is the city of", + "answer": " Bangkok", + "answer_token": " Bangkok", + "top1": " Bangkok", + "rank_of_answer": 0, + "logp_answer": -1.0666, + "cell_ok": true + }, + { + "country": "Thailand", + "relation": "language", + "prompt": "The main language spoken in Thailand is", + "answer": " Thai", + "answer_token": " Thai", + "top1": " Thai", + "rank_of_answer": 0, + "logp_answer": -0.4572, + "cell_ok": true + }, + { + "country": "Thailand", + "relation": "continent", + "prompt": "The country Thailand is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.4541, + "cell_ok": true + }, + { + "country": "Thailand", + "relation": "currency", + "prompt": "The currency used in Thailand is called the", + "answer": " baht", + "answer_token": " ba", + "top1": " ba", + "rank_of_answer": 0, + "logp_answer": -0.7501, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "capital", + "prompt": "The capital of Kenya is the city of", + "answer": " Nairobi", + "answer_token": " N", + "top1": " N", + "rank_of_answer": 0, + "logp_answer": -0.842, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "language", + "prompt": "The main language spoken in Kenya is", + "answer": " Swahili", + "answer_token": " Sw", + "top1": " Sw", + "rank_of_answer": 0, + "logp_answer": -1.0314, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "continent", + "prompt": "The country Kenya is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.0905, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "currency", + "prompt": "The currency used in Kenya is called the", + "answer": " shilling", + "answer_token": " sh", + "top1": " Ken", + "rank_of_answer": 1, + "logp_answer": -1.8639, + "cell_ok": false + } + ], + "runtime_s": 34.9 +} \ No newline at end of file diff --git a/results/t5a_capability_410m.json b/results/t5a_capability_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..97f0c156e52c0b5444d7c4850f2b10d365b34a54 --- /dev/null +++ b/results/t5a_capability_410m.json @@ -0,0 +1,716 @@ +{ + "experiment": "T5a capability screen (gates the write battery, PLAN.md \u00a73.5)", + "status": "GATE \u2014 pre-declared bar fixed in-script before any model was run", + "model": "EleutherAI/pythia-410m-deduped", + "n_cells": 61, + "n_excluded_multitoken_answer": 16, + "n_excluded_answer_copyable_from_prompt": 3, + "n_distinct_answer_tokens": 40, + "n_cells_ok": 44, + "accuracy": 0.7213, + "median_rank_overall": 0, + "bar_min_cells": 32, + "WRITE_BATTERY_ADMISSIBLE": true, + "by_relation": { + "capital": { + "n": 16, + "n_ok": 11, + "median_rank": 0 + }, + "language": { + "n": 16, + "n_ok": 15, + "median_rank": 0 + }, + "continent": { + "n": 20, + "n_ok": 15, + "median_rank": 0 + }, + "currency": { + "n": 9, + "n_ok": 3, + "median_rank": 2 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "country": "France", + "relation": "capital", + "prompt": "The capital of France is the city of", + "answer": " Paris", + "answer_token": " Paris", + "top1": " Paris", + "rank_of_answer": 0, + "logp_answer": -0.8275, + "cell_ok": true + }, + { + "country": "France", + "relation": "language", + "prompt": "The main language spoken in France is", + "answer": " French", + "answer_token": " French", + "top1": " French", + "rank_of_answer": 0, + "logp_answer": -0.6594, + "cell_ok": true + }, + { + "country": "France", + "relation": "continent", + "prompt": "The country France is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.5479, + "cell_ok": true + }, + { + "country": "France", + "relation": "currency", + "prompt": "The currency used in France is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -1.4684, + "cell_ok": true + }, + { + "country": "China", + "relation": "capital", + "prompt": "The capital of China is the city of", + "answer": " Beijing", + "answer_token": " Beijing", + "top1": " Beijing", + "rank_of_answer": 0, + "logp_answer": -1.3057, + "cell_ok": true + }, + { + "country": "China", + "relation": "language", + "prompt": "The main language spoken in China is", + "answer": " Chinese", + "answer_token": " Chinese", + "top1": " Mand", + "rank_of_answer": 1, + "logp_answer": -2.6818, + "cell_ok": false + }, + { + "country": "China", + "relation": "continent", + "prompt": "The country China is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.8499, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "capital", + "prompt": "The capital of Japan is the city of", + "answer": " Tokyo", + "answer_token": " Tokyo", + "top1": " Tokyo", + "rank_of_answer": 0, + "logp_answer": -0.8976, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "language", + "prompt": "The main language spoken in Japan is", + "answer": " Japanese", + "answer_token": " Japanese", + "top1": " Japanese", + "rank_of_answer": 0, + "logp_answer": -1.4655, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "continent", + "prompt": "The country Japan is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.5154, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "capital", + "prompt": "The capital of Germany is the city of", + "answer": " Berlin", + "answer_token": " Berlin", + "top1": " Hamburg", + "rank_of_answer": 1, + "logp_answer": -2.0265, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "continent", + "prompt": "The country Germany is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.4763, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "currency", + "prompt": "The currency used in Germany is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -1.76, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "capital", + "prompt": "The capital of Italy is the city of", + "answer": " Rome", + "answer_token": " Rome", + "top1": " Rome", + "rank_of_answer": 0, + "logp_answer": -1.3385, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "language", + "prompt": "The main language spoken in Italy is", + "answer": " Italian", + "answer_token": " Italian", + "top1": " Italian", + "rank_of_answer": 0, + "logp_answer": -0.5998, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "continent", + "prompt": "The country Italy is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.7264, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "currency", + "prompt": "The currency used in Italy is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -1.2018, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "capital", + "prompt": "The capital of Spain is the city of", + "answer": " Madrid", + "answer_token": " Madrid", + "top1": " Madrid", + "rank_of_answer": 0, + "logp_answer": -0.9623, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "language", + "prompt": "The main language spoken in Spain is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " Spanish", + "rank_of_answer": 0, + "logp_answer": -0.8753, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "continent", + "prompt": "The country Spain is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.6948, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "currency", + "prompt": "The currency used in Spain is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.3914, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "capital", + "prompt": "The capital of Russia is the city of", + "answer": " Moscow", + "answer_token": " Moscow", + "top1": " Moscow", + "rank_of_answer": 0, + "logp_answer": -1.0556, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "language", + "prompt": "The main language spoken in Russia is", + "answer": " Russian", + "answer_token": " Russian", + "top1": " Russian", + "rank_of_answer": 0, + "logp_answer": -0.3365, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "continent", + "prompt": "The country Russia is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -1.0631, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "capital", + "prompt": "The capital of Egypt is the city of", + "answer": " Cairo", + "answer_token": " Cairo", + "top1": " Cairo", + "rank_of_answer": 0, + "logp_answer": -0.4134, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "language", + "prompt": "The main language spoken in Egypt is", + "answer": " Arabic", + "answer_token": " Arabic", + "top1": " Arabic", + "rank_of_answer": 0, + "logp_answer": -1.5728, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "continent", + "prompt": "The country Egypt is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.1166, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "currency", + "prompt": "The currency used in Egypt is called the", + "answer": " pound", + "answer_token": " pound", + "top1": " Egyptian", + "rank_of_answer": 3, + "logp_answer": -3.7586, + "cell_ok": false + }, + { + "country": "India", + "relation": "capital", + "prompt": "The capital of India is the city of", + "answer": " Delhi", + "answer_token": " Delhi", + "top1": " Mumbai", + "rank_of_answer": 2, + "logp_answer": -2.3012, + "cell_ok": false + }, + { + "country": "India", + "relation": "continent", + "prompt": "The country India is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Africa", + "rank_of_answer": 1, + "logp_answer": -1.7905, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "language", + "prompt": "The main language spoken in Brazil is", + "answer": " Portuguese", + "answer_token": " Portuguese", + "top1": " Portuguese", + "rank_of_answer": 0, + "logp_answer": -0.7208, + "cell_ok": true + }, + { + "country": "Brazil", + "relation": "continent", + "prompt": "The country Brazil is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " South", + "rank_of_answer": 14, + "logp_answer": -5.3646, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "currency", + "prompt": "The currency used in Brazil is called the", + "answer": " real", + "answer_token": " real", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.2547, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "capital", + "prompt": "The capital of Canada is the city of", + "answer": " Ottawa", + "answer_token": " Ottawa", + "top1": " Toronto", + "rank_of_answer": 1, + "logp_answer": -2.3182, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "language", + "prompt": "The main language spoken in Canada is", + "answer": " English", + "answer_token": " English", + "top1": " English", + "rank_of_answer": 0, + "logp_answer": -0.7561, + "cell_ok": true + }, + { + "country": "Canada", + "relation": "continent", + "prompt": "The country Canada is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Africa", + "rank_of_answer": 16, + "logp_answer": -5.3412, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "currency", + "prompt": "The currency used in Canada is called the", + "answer": " dollar", + "answer_token": " dollar", + "top1": " Canadian", + "rank_of_answer": 2, + "logp_answer": -2.5911, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "language", + "prompt": "The main language spoken in Mexico is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " Spanish", + "rank_of_answer": 0, + "logp_answer": -1.7178, + "cell_ok": true + }, + { + "country": "Mexico", + "relation": "continent", + "prompt": "The country Mexico is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " North", + "rank_of_answer": 4, + "logp_answer": -2.9401, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "capital", + "prompt": "The capital of Greece is the city of", + "answer": " Athens", + "answer_token": " Athens", + "top1": " The", + "rank_of_answer": 1, + "logp_answer": -1.4739, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "language", + "prompt": "The main language spoken in Greece is", + "answer": " Greek", + "answer_token": " Greek", + "top1": " Greek", + "rank_of_answer": 0, + "logp_answer": -1.4368, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "continent", + "prompt": "The country Greece is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.4759, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "currency", + "prompt": "The currency used in Greece is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.396, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "capital", + "prompt": "The capital of Poland is the city of", + "answer": " Warsaw", + "answer_token": " Warsaw", + "top1": " K", + "rank_of_answer": 1, + "logp_answer": -1.8418, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "language", + "prompt": "The main language spoken in Poland is", + "answer": " Polish", + "answer_token": " Polish", + "top1": " Polish", + "rank_of_answer": 0, + "logp_answer": -0.6176, + "cell_ok": true + }, + { + "country": "Poland", + "relation": "continent", + "prompt": "The country Poland is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.2106, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "capital", + "prompt": "The capital of Sweden is the city of", + "answer": " Stockholm", + "answer_token": " Stockholm", + "top1": " Stockholm", + "rank_of_answer": 0, + "logp_answer": -1.3623, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "language", + "prompt": "The main language spoken in Sweden is", + "answer": " Swedish", + "answer_token": " Swedish", + "top1": " Swedish", + "rank_of_answer": 0, + "logp_answer": -0.3748, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "continent", + "prompt": "The country Sweden is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.7839, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "capital", + "prompt": "The capital of Norway is the city of", + "answer": " Oslo", + "answer_token": " Oslo", + "top1": " Oslo", + "rank_of_answer": 0, + "logp_answer": -0.7036, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "language", + "prompt": "The main language spoken in Norway is", + "answer": " Norwegian", + "answer_token": " Norwegian", + "top1": " Norwegian", + "rank_of_answer": 0, + "logp_answer": -0.4279, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "continent", + "prompt": "The country Norway is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.6542, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "language", + "prompt": "The main language spoken in Turkey is", + "answer": " Turkish", + "answer_token": " Turkish", + "top1": " Turkish", + "rank_of_answer": 0, + "logp_answer": -0.4299, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "continent", + "prompt": "The country Turkey is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Europe", + "rank_of_answer": 1, + "logp_answer": -1.5055, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "capital", + "prompt": "The capital of Korea is the city of", + "answer": " Seoul", + "answer_token": " Seoul", + "top1": " Seoul", + "rank_of_answer": 0, + "logp_answer": -1.2251, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "language", + "prompt": "The main language spoken in Korea is", + "answer": " Korean", + "answer_token": " Korean", + "top1": " Korean", + "rank_of_answer": 0, + "logp_answer": -1.2347, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "continent", + "prompt": "The country Korea is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.3787, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "currency", + "prompt": "The currency used in Korea is called the", + "answer": " won", + "answer_token": " won", + "top1": " \"", + "rank_of_answer": 7, + "logp_answer": -3.4374, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "capital", + "prompt": "The capital of Thailand is the city of", + "answer": " Bangkok", + "answer_token": " Bangkok", + "top1": " Bangkok", + "rank_of_answer": 0, + "logp_answer": -0.433, + "cell_ok": true + }, + { + "country": "Thailand", + "relation": "continent", + "prompt": "The country Thailand is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.4783, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "continent", + "prompt": "The country Kenya is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.1129, + "cell_ok": true + } + ], + "runtime_s": 16.7 +} \ No newline at end of file diff --git a/results/t5a_capability_410m_v2.json b/results/t5a_capability_410m_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..e183b95a4e4966bf6bacfa09cd0f5b5f8a21a39f --- /dev/null +++ b/results/t5a_capability_410m_v2.json @@ -0,0 +1,716 @@ +{ + "experiment": "T5a capability screen (gates the write battery, PLAN.md \u00a73.5)", + "status": "GATE \u2014 pre-declared bar fixed in-script before any model was run", + "model": "EleutherAI/pythia-410m-deduped", + "n_cells": 61, + "n_excluded_multitoken_answer": 16, + "n_excluded_answer_copyable_from_prompt": 3, + "n_distinct_answer_tokens": 40, + "n_cells_ok": 44, + "accuracy": 0.7213, + "median_rank_overall": 0, + "bar_min_cells": 32, + "WRITE_BATTERY_ADMISSIBLE": true, + "by_relation": { + "capital": { + "n": 16, + "n_ok": 11, + "median_rank": 0 + }, + "language": { + "n": 16, + "n_ok": 15, + "median_rank": 0 + }, + "continent": { + "n": 20, + "n_ok": 15, + "median_rank": 0 + }, + "currency": { + "n": 9, + "n_ok": 3, + "median_rank": 2 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "country": "France", + "relation": "capital", + "prompt": "The capital of France is the city of", + "answer": " Paris", + "answer_token": " Paris", + "top1": " Paris", + "rank_of_answer": 0, + "logp_answer": -0.8275, + "cell_ok": true + }, + { + "country": "France", + "relation": "language", + "prompt": "The main language spoken in France is", + "answer": " French", + "answer_token": " French", + "top1": " French", + "rank_of_answer": 0, + "logp_answer": -0.6593, + "cell_ok": true + }, + { + "country": "France", + "relation": "continent", + "prompt": "The country France is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.5479, + "cell_ok": true + }, + { + "country": "France", + "relation": "currency", + "prompt": "The currency used in France is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -1.4684, + "cell_ok": true + }, + { + "country": "China", + "relation": "capital", + "prompt": "The capital of China is the city of", + "answer": " Beijing", + "answer_token": " Beijing", + "top1": " Beijing", + "rank_of_answer": 0, + "logp_answer": -1.3058, + "cell_ok": true + }, + { + "country": "China", + "relation": "language", + "prompt": "The main language spoken in China is", + "answer": " Chinese", + "answer_token": " Chinese", + "top1": " Mand", + "rank_of_answer": 1, + "logp_answer": -2.6818, + "cell_ok": false + }, + { + "country": "China", + "relation": "continent", + "prompt": "The country China is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.8499, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "capital", + "prompt": "The capital of Japan is the city of", + "answer": " Tokyo", + "answer_token": " Tokyo", + "top1": " Tokyo", + "rank_of_answer": 0, + "logp_answer": -0.8975, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "language", + "prompt": "The main language spoken in Japan is", + "answer": " Japanese", + "answer_token": " Japanese", + "top1": " Japanese", + "rank_of_answer": 0, + "logp_answer": -1.4657, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "continent", + "prompt": "The country Japan is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.5154, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "capital", + "prompt": "The capital of Germany is the city of", + "answer": " Berlin", + "answer_token": " Berlin", + "top1": " Hamburg", + "rank_of_answer": 1, + "logp_answer": -2.0265, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "continent", + "prompt": "The country Germany is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.4763, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "currency", + "prompt": "The currency used in Germany is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -1.7599, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "capital", + "prompt": "The capital of Italy is the city of", + "answer": " Rome", + "answer_token": " Rome", + "top1": " Rome", + "rank_of_answer": 0, + "logp_answer": -1.3385, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "language", + "prompt": "The main language spoken in Italy is", + "answer": " Italian", + "answer_token": " Italian", + "top1": " Italian", + "rank_of_answer": 0, + "logp_answer": -0.5999, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "continent", + "prompt": "The country Italy is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.7264, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "currency", + "prompt": "The currency used in Italy is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -1.2022, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "capital", + "prompt": "The capital of Spain is the city of", + "answer": " Madrid", + "answer_token": " Madrid", + "top1": " Madrid", + "rank_of_answer": 0, + "logp_answer": -0.9625, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "language", + "prompt": "The main language spoken in Spain is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " Spanish", + "rank_of_answer": 0, + "logp_answer": -0.8756, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "continent", + "prompt": "The country Spain is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.6948, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "currency", + "prompt": "The currency used in Spain is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.3915, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "capital", + "prompt": "The capital of Russia is the city of", + "answer": " Moscow", + "answer_token": " Moscow", + "top1": " Moscow", + "rank_of_answer": 0, + "logp_answer": -1.0556, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "language", + "prompt": "The main language spoken in Russia is", + "answer": " Russian", + "answer_token": " Russian", + "top1": " Russian", + "rank_of_answer": 0, + "logp_answer": -0.3365, + "cell_ok": true + }, + { + "country": "Russia", + "relation": "continent", + "prompt": "The country Russia is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -1.0631, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "capital", + "prompt": "The capital of Egypt is the city of", + "answer": " Cairo", + "answer_token": " Cairo", + "top1": " Cairo", + "rank_of_answer": 0, + "logp_answer": -0.4133, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "language", + "prompt": "The main language spoken in Egypt is", + "answer": " Arabic", + "answer_token": " Arabic", + "top1": " Arabic", + "rank_of_answer": 0, + "logp_answer": -1.5727, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "continent", + "prompt": "The country Egypt is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.1166, + "cell_ok": true + }, + { + "country": "Egypt", + "relation": "currency", + "prompt": "The currency used in Egypt is called the", + "answer": " pound", + "answer_token": " pound", + "top1": " Egyptian", + "rank_of_answer": 3, + "logp_answer": -3.7585, + "cell_ok": false + }, + { + "country": "India", + "relation": "capital", + "prompt": "The capital of India is the city of", + "answer": " Delhi", + "answer_token": " Delhi", + "top1": " Mumbai", + "rank_of_answer": 2, + "logp_answer": -2.3012, + "cell_ok": false + }, + { + "country": "India", + "relation": "continent", + "prompt": "The country India is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Africa", + "rank_of_answer": 1, + "logp_answer": -1.7905, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "language", + "prompt": "The main language spoken in Brazil is", + "answer": " Portuguese", + "answer_token": " Portuguese", + "top1": " Portuguese", + "rank_of_answer": 0, + "logp_answer": -0.7209, + "cell_ok": true + }, + { + "country": "Brazil", + "relation": "continent", + "prompt": "The country Brazil is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " South", + "rank_of_answer": 14, + "logp_answer": -5.3646, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "currency", + "prompt": "The currency used in Brazil is called the", + "answer": " real", + "answer_token": " real", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.2549, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "capital", + "prompt": "The capital of Canada is the city of", + "answer": " Ottawa", + "answer_token": " Ottawa", + "top1": " Toronto", + "rank_of_answer": 1, + "logp_answer": -2.3184, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "language", + "prompt": "The main language spoken in Canada is", + "answer": " English", + "answer_token": " English", + "top1": " English", + "rank_of_answer": 0, + "logp_answer": -0.7562, + "cell_ok": true + }, + { + "country": "Canada", + "relation": "continent", + "prompt": "The country Canada is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Africa", + "rank_of_answer": 16, + "logp_answer": -5.3412, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "currency", + "prompt": "The currency used in Canada is called the", + "answer": " dollar", + "answer_token": " dollar", + "top1": " Canadian", + "rank_of_answer": 2, + "logp_answer": -2.5909, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "language", + "prompt": "The main language spoken in Mexico is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " Spanish", + "rank_of_answer": 0, + "logp_answer": -1.7179, + "cell_ok": true + }, + { + "country": "Mexico", + "relation": "continent", + "prompt": "The country Mexico is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " North", + "rank_of_answer": 4, + "logp_answer": -2.9401, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "capital", + "prompt": "The capital of Greece is the city of", + "answer": " Athens", + "answer_token": " Athens", + "top1": " The", + "rank_of_answer": 1, + "logp_answer": -1.4738, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "language", + "prompt": "The main language spoken in Greece is", + "answer": " Greek", + "answer_token": " Greek", + "top1": " Greek", + "rank_of_answer": 0, + "logp_answer": -1.4371, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "continent", + "prompt": "The country Greece is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.4759, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "currency", + "prompt": "The currency used in Greece is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.3961, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "capital", + "prompt": "The capital of Poland is the city of", + "answer": " Warsaw", + "answer_token": " Warsaw", + "top1": " K", + "rank_of_answer": 1, + "logp_answer": -1.8418, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "language", + "prompt": "The main language spoken in Poland is", + "answer": " Polish", + "answer_token": " Polish", + "top1": " Polish", + "rank_of_answer": 0, + "logp_answer": -0.6176, + "cell_ok": true + }, + { + "country": "Poland", + "relation": "continent", + "prompt": "The country Poland is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.2106, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "capital", + "prompt": "The capital of Sweden is the city of", + "answer": " Stockholm", + "answer_token": " Stockholm", + "top1": " Stockholm", + "rank_of_answer": 0, + "logp_answer": -1.3625, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "language", + "prompt": "The main language spoken in Sweden is", + "answer": " Swedish", + "answer_token": " Swedish", + "top1": " Swedish", + "rank_of_answer": 0, + "logp_answer": -0.3747, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "continent", + "prompt": "The country Sweden is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.7839, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "capital", + "prompt": "The capital of Norway is the city of", + "answer": " Oslo", + "answer_token": " Oslo", + "top1": " Oslo", + "rank_of_answer": 0, + "logp_answer": -0.7037, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "language", + "prompt": "The main language spoken in Norway is", + "answer": " Norwegian", + "answer_token": " Norwegian", + "top1": " Norwegian", + "rank_of_answer": 0, + "logp_answer": -0.4279, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "continent", + "prompt": "The country Norway is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Europe", + "rank_of_answer": 0, + "logp_answer": -0.6542, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "language", + "prompt": "The main language spoken in Turkey is", + "answer": " Turkish", + "answer_token": " Turkish", + "top1": " Turkish", + "rank_of_answer": 0, + "logp_answer": -0.43, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "continent", + "prompt": "The country Turkey is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Europe", + "rank_of_answer": 1, + "logp_answer": -1.5055, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "capital", + "prompt": "The capital of Korea is the city of", + "answer": " Seoul", + "answer_token": " Seoul", + "top1": " Seoul", + "rank_of_answer": 0, + "logp_answer": -1.2251, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "language", + "prompt": "The main language spoken in Korea is", + "answer": " Korean", + "answer_token": " Korean", + "top1": " Korean", + "rank_of_answer": 0, + "logp_answer": -1.2345, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "continent", + "prompt": "The country Korea is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.3787, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "currency", + "prompt": "The currency used in Korea is called the", + "answer": " won", + "answer_token": " won", + "top1": " \"", + "rank_of_answer": 7, + "logp_answer": -3.4374, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "capital", + "prompt": "The capital of Thailand is the city of", + "answer": " Bangkok", + "answer_token": " Bangkok", + "top1": " Bangkok", + "rank_of_answer": 0, + "logp_answer": -0.4329, + "cell_ok": true + }, + { + "country": "Thailand", + "relation": "continent", + "prompt": "The country Thailand is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Asia", + "rank_of_answer": 0, + "logp_answer": -0.4783, + "cell_ok": true + }, + { + "country": "Kenya", + "relation": "continent", + "prompt": "The country Kenya is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Africa", + "rank_of_answer": 0, + "logp_answer": -0.1129, + "cell_ok": true + } + ], + "runtime_s": 25.4 +} \ No newline at end of file diff --git a/results/t5a_capability_70m.json b/results/t5a_capability_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..83e2143e0189e3ac80142447f3e6aae063ea25a7 --- /dev/null +++ b/results/t5a_capability_70m.json @@ -0,0 +1,716 @@ +{ + "experiment": "T5a capability screen (gates the write battery, PLAN.md \u00a73.5)", + "status": "GATE \u2014 pre-declared bar fixed in-script before any model was run", + "model": "EleutherAI/pythia-70m-deduped", + "n_cells": 61, + "n_excluded_multitoken_answer": 16, + "n_excluded_answer_copyable_from_prompt": 3, + "n_distinct_answer_tokens": 40, + "n_cells_ok": 13, + "accuracy": 0.2131, + "median_rank_overall": 2, + "bar_min_cells": 32, + "WRITE_BATTERY_ADMISSIBLE": false, + "by_relation": { + "capital": { + "n": 16, + "n_ok": 8, + "median_rank": 0 + }, + "language": { + "n": 16, + "n_ok": 2, + "median_rank": 2 + }, + "continent": { + "n": 20, + "n_ok": 0, + "median_rank": 2 + }, + "currency": { + "n": 9, + "n_ok": 3, + "median_rank": 1 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "country": "France", + "relation": "capital", + "prompt": "The capital of France is the city of", + "answer": " Paris", + "answer_token": " Paris", + "top1": " Paris", + "rank_of_answer": 0, + "logp_answer": -2.302, + "cell_ok": true + }, + { + "country": "France", + "relation": "language", + "prompt": "The main language spoken in France is", + "answer": " French", + "answer_token": " French", + "top1": " French", + "rank_of_answer": 0, + "logp_answer": -2.1175, + "cell_ok": true + }, + { + "country": "France", + "relation": "continent", + "prompt": "The country France is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.1724, + "cell_ok": false + }, + { + "country": "France", + "relation": "currency", + "prompt": "The currency used in France is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " currency", + "rank_of_answer": 1, + "logp_answer": -2.9637, + "cell_ok": false + }, + { + "country": "China", + "relation": "capital", + "prompt": "The capital of China is the city of", + "answer": " Beijing", + "answer_token": " Beijing", + "top1": " Shanghai", + "rank_of_answer": 8, + "logp_answer": -4.2986, + "cell_ok": false + }, + { + "country": "China", + "relation": "language", + "prompt": "The main language spoken in China is", + "answer": " Chinese", + "answer_token": " Chinese", + "top1": " \"", + "rank_of_answer": 4, + "logp_answer": -3.9576, + "cell_ok": false + }, + { + "country": "China", + "relation": "continent", + "prompt": "The country China is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " China", + "rank_of_answer": 2, + "logp_answer": -3.1046, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "capital", + "prompt": "The capital of Japan is the city of", + "answer": " Tokyo", + "answer_token": " Tokyo", + "top1": " Tokyo", + "rank_of_answer": 0, + "logp_answer": -2.4124, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "language", + "prompt": "The main language spoken in Japan is", + "answer": " Japanese", + "answer_token": " Japanese", + "top1": " \"", + "rank_of_answer": 3, + "logp_answer": -3.3984, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "continent", + "prompt": "The country Japan is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Japan", + "rank_of_answer": 2, + "logp_answer": -3.1063, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "capital", + "prompt": "The capital of Germany is the city of", + "answer": " Berlin", + "answer_token": " Berlin", + "top1": " Berlin", + "rank_of_answer": 0, + "logp_answer": -2.4874, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "continent", + "prompt": "The country Germany is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.4144, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "currency", + "prompt": "The currency used in Germany is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -2.8519, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "capital", + "prompt": "The capital of Italy is the city of", + "answer": " Rome", + "answer_token": " Rome", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -3.4809, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "language", + "prompt": "The main language spoken in Italy is", + "answer": " Italian", + "answer_token": " Italian", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.184, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "continent", + "prompt": "The country Italy is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Italy", + "rank_of_answer": 3, + "logp_answer": -3.6073, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "currency", + "prompt": "The currency used in Italy is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -2.807, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "capital", + "prompt": "The capital of Spain is the city of", + "answer": " Madrid", + "answer_token": " Madrid", + "top1": " Madrid", + "rank_of_answer": 0, + "logp_answer": -2.7668, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "language", + "prompt": "The main language spoken in Spain is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.425, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "continent", + "prompt": "The country Spain is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.1864, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "currency", + "prompt": "The currency used in Spain is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " currency", + "rank_of_answer": 1, + "logp_answer": -2.9396, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "capital", + "prompt": "The capital of Russia is the city of", + "answer": " Moscow", + "answer_token": " Moscow", + "top1": " the", + "rank_of_answer": 3, + "logp_answer": -3.7377, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "language", + "prompt": "The main language spoken in Russia is", + "answer": " Russian", + "answer_token": " Russian", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.5828, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "continent", + "prompt": "The country Russia is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 5, + "logp_answer": -3.7641, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "capital", + "prompt": "The capital of Egypt is the city of", + "answer": " Cairo", + "answer_token": " Cairo", + "top1": " the", + "rank_of_answer": 18, + "logp_answer": -4.5837, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "language", + "prompt": "The main language spoken in Egypt is", + "answer": " Arabic", + "answer_token": " Arabic", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.1826, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "continent", + "prompt": "The country Egypt is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Egypt", + "rank_of_answer": 2, + "logp_answer": -2.748, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "currency", + "prompt": "The currency used in Egypt is called the", + "answer": " pound", + "answer_token": " pound", + "top1": " currency", + "rank_of_answer": 62, + "logp_answer": -6.2902, + "cell_ok": false + }, + { + "country": "India", + "relation": "capital", + "prompt": "The capital of India is the city of", + "answer": " Delhi", + "answer_token": " Delhi", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.7117, + "cell_ok": false + }, + { + "country": "India", + "relation": "continent", + "prompt": "The country India is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " India", + "rank_of_answer": 2, + "logp_answer": -3.2643, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "language", + "prompt": "The main language spoken in Brazil is", + "answer": " Portuguese", + "answer_token": " Portuguese", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.3748, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "continent", + "prompt": "The country Brazil is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Brazil", + "rank_of_answer": 69, + "logp_answer": -6.7407, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "currency", + "prompt": "The currency used in Brazil is called the", + "answer": " real", + "answer_token": " real", + "top1": " currency", + "rank_of_answer": 197, + "logp_answer": -7.3145, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "capital", + "prompt": "The capital of Canada is the city of", + "answer": " Ottawa", + "answer_token": " Ottawa", + "top1": " Montreal", + "rank_of_answer": 4, + "logp_answer": -3.1938, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "language", + "prompt": "The main language spoken in Canada is", + "answer": " English", + "answer_token": " English", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.7066, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "continent", + "prompt": "The country Canada is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " the", + "rank_of_answer": 21, + "logp_answer": -5.0984, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "currency", + "prompt": "The currency used in Canada is called the", + "answer": " dollar", + "answer_token": " dollar", + "top1": " currency", + "rank_of_answer": 3, + "logp_answer": -3.6801, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "language", + "prompt": "The main language spoken in Mexico is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.2886, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "continent", + "prompt": "The country Mexico is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Mexico", + "rank_of_answer": 22, + "logp_answer": -5.4717, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "capital", + "prompt": "The capital of Greece is the city of", + "answer": " Athens", + "answer_token": " Athens", + "top1": " Athens", + "rank_of_answer": 0, + "logp_answer": -2.5814, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "language", + "prompt": "The main language spoken in Greece is", + "answer": " Greek", + "answer_token": " Greek", + "top1": " \"", + "rank_of_answer": 5, + "logp_answer": -3.9531, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "continent", + "prompt": "The country Greece is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Greece", + "rank_of_answer": 4, + "logp_answer": -3.6077, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "currency", + "prompt": "The currency used in Greece is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -2.5577, + "cell_ok": true + }, + { + "country": "Poland", + "relation": "capital", + "prompt": "The capital of Poland is the city of", + "answer": " Warsaw", + "answer_token": " Warsaw", + "top1": " the", + "rank_of_answer": 3, + "logp_answer": -3.704, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "language", + "prompt": "The main language spoken in Poland is", + "answer": " Polish", + "answer_token": " Polish", + "top1": " \"", + "rank_of_answer": 4, + "logp_answer": -3.6374, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "continent", + "prompt": "The country Poland is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Poland", + "rank_of_answer": 2, + "logp_answer": -3.5191, + "cell_ok": false + }, + { + "country": "Sweden", + "relation": "capital", + "prompt": "The capital of Sweden is the city of", + "answer": " Stockholm", + "answer_token": " Stockholm", + "top1": " Stockholm", + "rank_of_answer": 0, + "logp_answer": -2.9603, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "language", + "prompt": "The main language spoken in Sweden is", + "answer": " Swedish", + "answer_token": " Swedish", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.1818, + "cell_ok": false + }, + { + "country": "Sweden", + "relation": "continent", + "prompt": "The country Sweden is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.3144, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "capital", + "prompt": "The capital of Norway is the city of", + "answer": " Oslo", + "answer_token": " Oslo", + "top1": " Oslo", + "rank_of_answer": 0, + "logp_answer": -2.5502, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "language", + "prompt": "The main language spoken in Norway is", + "answer": " Norwegian", + "answer_token": " Norwegian", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.7018, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "continent", + "prompt": "The country Norway is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 5, + "logp_answer": -3.8379, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "language", + "prompt": "The main language spoken in Turkey is", + "answer": " Turkish", + "answer_token": " Turkish", + "top1": " Turkish", + "rank_of_answer": 0, + "logp_answer": -2.2832, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "continent", + "prompt": "The country Turkey is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Turkey", + "rank_of_answer": 2, + "logp_answer": -4.0071, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "capital", + "prompt": "The capital of Korea is the city of", + "answer": " Seoul", + "answer_token": " Seoul", + "top1": " Seoul", + "rank_of_answer": 0, + "logp_answer": -2.0525, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "language", + "prompt": "The main language spoken in Korea is", + "answer": " Korean", + "answer_token": " Korean", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.5013, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "continent", + "prompt": "The country Korea is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " South", + "rank_of_answer": 4, + "logp_answer": -2.8881, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "currency", + "prompt": "The currency used in Korea is called the", + "answer": " won", + "answer_token": " won", + "top1": " currency", + "rank_of_answer": 10343, + "logp_answer": -12.307, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "capital", + "prompt": "The capital of Thailand is the city of", + "answer": " Bangkok", + "answer_token": " Bangkok", + "top1": " B", + "rank_of_answer": 4, + "logp_answer": -3.801, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "continent", + "prompt": "The country Thailand is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Thailand", + "rank_of_answer": 2, + "logp_answer": -3.4783, + "cell_ok": false + }, + { + "country": "Kenya", + "relation": "continent", + "prompt": "The country Kenya is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Kenya", + "rank_of_answer": 1, + "logp_answer": -2.6069, + "cell_ok": false + } + ], + "runtime_s": 14.0 +} \ No newline at end of file diff --git a/results/t5a_capability_70m_v2.json b/results/t5a_capability_70m_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..459086bdb15b26ea53cd525c7e71820722fc76a5 --- /dev/null +++ b/results/t5a_capability_70m_v2.json @@ -0,0 +1,716 @@ +{ + "experiment": "T5a capability screen (gates the write battery, PLAN.md \u00a73.5)", + "status": "GATE \u2014 pre-declared bar fixed in-script before any model was run", + "model": "EleutherAI/pythia-70m-deduped", + "n_cells": 61, + "n_excluded_multitoken_answer": 16, + "n_excluded_answer_copyable_from_prompt": 3, + "n_distinct_answer_tokens": 40, + "n_cells_ok": 13, + "accuracy": 0.2131, + "median_rank_overall": 2, + "bar_min_cells": 32, + "WRITE_BATTERY_ADMISSIBLE": false, + "by_relation": { + "capital": { + "n": 16, + "n_ok": 8, + "median_rank": 0 + }, + "language": { + "n": 16, + "n_ok": 2, + "median_rank": 2 + }, + "continent": { + "n": 20, + "n_ok": 0, + "median_rank": 2 + }, + "currency": { + "n": 9, + "n_ok": 3, + "median_rank": 1 + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "rows": [ + { + "country": "France", + "relation": "capital", + "prompt": "The capital of France is the city of", + "answer": " Paris", + "answer_token": " Paris", + "top1": " Paris", + "rank_of_answer": 0, + "logp_answer": -2.302, + "cell_ok": true + }, + { + "country": "France", + "relation": "language", + "prompt": "The main language spoken in France is", + "answer": " French", + "answer_token": " French", + "top1": " French", + "rank_of_answer": 0, + "logp_answer": -2.1175, + "cell_ok": true + }, + { + "country": "France", + "relation": "continent", + "prompt": "The country France is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.1724, + "cell_ok": false + }, + { + "country": "France", + "relation": "currency", + "prompt": "The currency used in France is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " currency", + "rank_of_answer": 1, + "logp_answer": -2.9637, + "cell_ok": false + }, + { + "country": "China", + "relation": "capital", + "prompt": "The capital of China is the city of", + "answer": " Beijing", + "answer_token": " Beijing", + "top1": " Shanghai", + "rank_of_answer": 8, + "logp_answer": -4.2992, + "cell_ok": false + }, + { + "country": "China", + "relation": "language", + "prompt": "The main language spoken in China is", + "answer": " Chinese", + "answer_token": " Chinese", + "top1": " \"", + "rank_of_answer": 4, + "logp_answer": -3.9576, + "cell_ok": false + }, + { + "country": "China", + "relation": "continent", + "prompt": "The country China is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " China", + "rank_of_answer": 2, + "logp_answer": -3.1046, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "capital", + "prompt": "The capital of Japan is the city of", + "answer": " Tokyo", + "answer_token": " Tokyo", + "top1": " Tokyo", + "rank_of_answer": 0, + "logp_answer": -2.4126, + "cell_ok": true + }, + { + "country": "Japan", + "relation": "language", + "prompt": "The main language spoken in Japan is", + "answer": " Japanese", + "answer_token": " Japanese", + "top1": " \"", + "rank_of_answer": 3, + "logp_answer": -3.3984, + "cell_ok": false + }, + { + "country": "Japan", + "relation": "continent", + "prompt": "The country Japan is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Japan", + "rank_of_answer": 2, + "logp_answer": -3.1063, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "capital", + "prompt": "The capital of Germany is the city of", + "answer": " Berlin", + "answer_token": " Berlin", + "top1": " Berlin", + "rank_of_answer": 0, + "logp_answer": -2.4875, + "cell_ok": true + }, + { + "country": "Germany", + "relation": "continent", + "prompt": "The country Germany is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -2.4144, + "cell_ok": false + }, + { + "country": "Germany", + "relation": "currency", + "prompt": "The currency used in Germany is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -2.8524, + "cell_ok": true + }, + { + "country": "Italy", + "relation": "capital", + "prompt": "The capital of Italy is the city of", + "answer": " Rome", + "answer_token": " Rome", + "top1": " the", + "rank_of_answer": 1, + "logp_answer": -3.4811, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "language", + "prompt": "The main language spoken in Italy is", + "answer": " Italian", + "answer_token": " Italian", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.184, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "continent", + "prompt": "The country Italy is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Italy", + "rank_of_answer": 3, + "logp_answer": -3.6073, + "cell_ok": false + }, + { + "country": "Italy", + "relation": "currency", + "prompt": "The currency used in Italy is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -2.807, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "capital", + "prompt": "The capital of Spain is the city of", + "answer": " Madrid", + "answer_token": " Madrid", + "top1": " Madrid", + "rank_of_answer": 0, + "logp_answer": -2.7663, + "cell_ok": true + }, + { + "country": "Spain", + "relation": "language", + "prompt": "The main language spoken in Spain is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.425, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "continent", + "prompt": "The country Spain is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.1864, + "cell_ok": false + }, + { + "country": "Spain", + "relation": "currency", + "prompt": "The currency used in Spain is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " currency", + "rank_of_answer": 1, + "logp_answer": -2.9397, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "capital", + "prompt": "The capital of Russia is the city of", + "answer": " Moscow", + "answer_token": " Moscow", + "top1": " the", + "rank_of_answer": 3, + "logp_answer": -3.7369, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "language", + "prompt": "The main language spoken in Russia is", + "answer": " Russian", + "answer_token": " Russian", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.5828, + "cell_ok": false + }, + { + "country": "Russia", + "relation": "continent", + "prompt": "The country Russia is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 5, + "logp_answer": -3.7641, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "capital", + "prompt": "The capital of Egypt is the city of", + "answer": " Cairo", + "answer_token": " Cairo", + "top1": " the", + "rank_of_answer": 18, + "logp_answer": -4.5839, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "language", + "prompt": "The main language spoken in Egypt is", + "answer": " Arabic", + "answer_token": " Arabic", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.1826, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "continent", + "prompt": "The country Egypt is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Egypt", + "rank_of_answer": 2, + "logp_answer": -2.748, + "cell_ok": false + }, + { + "country": "Egypt", + "relation": "currency", + "prompt": "The currency used in Egypt is called the", + "answer": " pound", + "answer_token": " pound", + "top1": " currency", + "rank_of_answer": 62, + "logp_answer": -6.29, + "cell_ok": false + }, + { + "country": "India", + "relation": "capital", + "prompt": "The capital of India is the city of", + "answer": " Delhi", + "answer_token": " Delhi", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.714, + "cell_ok": false + }, + { + "country": "India", + "relation": "continent", + "prompt": "The country India is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " India", + "rank_of_answer": 2, + "logp_answer": -3.2643, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "language", + "prompt": "The main language spoken in Brazil is", + "answer": " Portuguese", + "answer_token": " Portuguese", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.3748, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "continent", + "prompt": "The country Brazil is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Brazil", + "rank_of_answer": 69, + "logp_answer": -6.7407, + "cell_ok": false + }, + { + "country": "Brazil", + "relation": "currency", + "prompt": "The currency used in Brazil is called the", + "answer": " real", + "answer_token": " real", + "top1": " currency", + "rank_of_answer": 197, + "logp_answer": -7.3146, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "capital", + "prompt": "The capital of Canada is the city of", + "answer": " Ottawa", + "answer_token": " Ottawa", + "top1": " Montreal", + "rank_of_answer": 4, + "logp_answer": -3.1919, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "language", + "prompt": "The main language spoken in Canada is", + "answer": " English", + "answer_token": " English", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -2.7066, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "continent", + "prompt": "The country Canada is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " the", + "rank_of_answer": 21, + "logp_answer": -5.0984, + "cell_ok": false + }, + { + "country": "Canada", + "relation": "currency", + "prompt": "The currency used in Canada is called the", + "answer": " dollar", + "answer_token": " dollar", + "top1": " currency", + "rank_of_answer": 3, + "logp_answer": -3.6804, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "language", + "prompt": "The main language spoken in Mexico is", + "answer": " Spanish", + "answer_token": " Spanish", + "top1": " \"", + "rank_of_answer": 2, + "logp_answer": -3.2886, + "cell_ok": false + }, + { + "country": "Mexico", + "relation": "continent", + "prompt": "The country Mexico is located in the continent of", + "answer": " America", + "answer_token": " America", + "top1": " Mexico", + "rank_of_answer": 22, + "logp_answer": -5.4717, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "capital", + "prompt": "The capital of Greece is the city of", + "answer": " Athens", + "answer_token": " Athens", + "top1": " Athens", + "rank_of_answer": 0, + "logp_answer": -2.5812, + "cell_ok": true + }, + { + "country": "Greece", + "relation": "language", + "prompt": "The main language spoken in Greece is", + "answer": " Greek", + "answer_token": " Greek", + "top1": " \"", + "rank_of_answer": 5, + "logp_answer": -3.9531, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "continent", + "prompt": "The country Greece is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Greece", + "rank_of_answer": 4, + "logp_answer": -3.6077, + "cell_ok": false + }, + { + "country": "Greece", + "relation": "currency", + "prompt": "The currency used in Greece is called the", + "answer": " euro", + "answer_token": " euro", + "top1": " euro", + "rank_of_answer": 0, + "logp_answer": -2.5576, + "cell_ok": true + }, + { + "country": "Poland", + "relation": "capital", + "prompt": "The capital of Poland is the city of", + "answer": " Warsaw", + "answer_token": " Warsaw", + "top1": " the", + "rank_of_answer": 3, + "logp_answer": -3.7036, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "language", + "prompt": "The main language spoken in Poland is", + "answer": " Polish", + "answer_token": " Polish", + "top1": " \"", + "rank_of_answer": 4, + "logp_answer": -3.6374, + "cell_ok": false + }, + { + "country": "Poland", + "relation": "continent", + "prompt": "The country Poland is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " Poland", + "rank_of_answer": 2, + "logp_answer": -3.5191, + "cell_ok": false + }, + { + "country": "Sweden", + "relation": "capital", + "prompt": "The capital of Sweden is the city of", + "answer": " Stockholm", + "answer_token": " Stockholm", + "top1": " Stockholm", + "rank_of_answer": 0, + "logp_answer": -2.9603, + "cell_ok": true + }, + { + "country": "Sweden", + "relation": "language", + "prompt": "The main language spoken in Sweden is", + "answer": " Swedish", + "answer_token": " Swedish", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.1818, + "cell_ok": false + }, + { + "country": "Sweden", + "relation": "continent", + "prompt": "The country Sweden is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 2, + "logp_answer": -3.3144, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "capital", + "prompt": "The capital of Norway is the city of", + "answer": " Oslo", + "answer_token": " Oslo", + "top1": " Oslo", + "rank_of_answer": 0, + "logp_answer": -2.5494, + "cell_ok": true + }, + { + "country": "Norway", + "relation": "language", + "prompt": "The main language spoken in Norway is", + "answer": " Norwegian", + "answer_token": " Norwegian", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.7018, + "cell_ok": false + }, + { + "country": "Norway", + "relation": "continent", + "prompt": "The country Norway is located in the continent of", + "answer": " Europe", + "answer_token": " Europe", + "top1": " the", + "rank_of_answer": 5, + "logp_answer": -3.8379, + "cell_ok": false + }, + { + "country": "Turkey", + "relation": "language", + "prompt": "The main language spoken in Turkey is", + "answer": " Turkish", + "answer_token": " Turkish", + "top1": " Turkish", + "rank_of_answer": 0, + "logp_answer": -2.2832, + "cell_ok": true + }, + { + "country": "Turkey", + "relation": "continent", + "prompt": "The country Turkey is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Turkey", + "rank_of_answer": 2, + "logp_answer": -4.0071, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "capital", + "prompt": "The capital of Korea is the city of", + "answer": " Seoul", + "answer_token": " Seoul", + "top1": " Seoul", + "rank_of_answer": 0, + "logp_answer": -2.0539, + "cell_ok": true + }, + { + "country": "Korea", + "relation": "language", + "prompt": "The main language spoken in Korea is", + "answer": " Korean", + "answer_token": " Korean", + "top1": " \"", + "rank_of_answer": 1, + "logp_answer": -2.5013, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "continent", + "prompt": "The country Korea is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " South", + "rank_of_answer": 4, + "logp_answer": -2.8881, + "cell_ok": false + }, + { + "country": "Korea", + "relation": "currency", + "prompt": "The currency used in Korea is called the", + "answer": " won", + "answer_token": " won", + "top1": " currency", + "rank_of_answer": 10343, + "logp_answer": -12.3072, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "capital", + "prompt": "The capital of Thailand is the city of", + "answer": " Bangkok", + "answer_token": " Bangkok", + "top1": " B", + "rank_of_answer": 4, + "logp_answer": -3.8009, + "cell_ok": false + }, + { + "country": "Thailand", + "relation": "continent", + "prompt": "The country Thailand is located in the continent of", + "answer": " Asia", + "answer_token": " Asia", + "top1": " Thailand", + "rank_of_answer": 2, + "logp_answer": -3.4783, + "cell_ok": false + }, + { + "country": "Kenya", + "relation": "continent", + "prompt": "The country Kenya is located in the continent of", + "answer": " Africa", + "answer_token": " Africa", + "top1": " Kenya", + "rank_of_answer": 1, + "logp_answer": -2.6069, + "cell_ok": false + } + ], + "runtime_s": 15.6 +} \ No newline at end of file diff --git a/results/t7_bootstrap_160m.json b/results/t7_bootstrap_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..33c434df5efb79e7ec6b6ad22f4a3452dc5f11c8 --- /dev/null +++ b/results/t7_bootstrap_160m.json @@ -0,0 +1,124 @@ +{ + "experiment": "T7 paired item-level bootstrap (audit of the n_win headline)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 secondary analysis of PREREG \u00a70 burned cells", + "model": "EleutherAI/pythia-160m-deduped", + "lens": "pythia/results/lens_160m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "n_boot": 10000, + "seed": 0, + "bootstrap": "paired over items; one index draw applied to both transports", + "metric": "normalized pass@k AUC over log k, ks=(1,2,5,10,25,100)", + "n_win_point": 2, + "n_win_bootstrap_hist": { + "0": 62, + "1": 885, + "2": 3726, + "3": 3897, + "4": 1306, + "5": 124, + "6": 0 + }, + "P_n_win_ge_5": 0.0124, + "P_n_win_le_3": 0.857, + "n_sets_ci_excludes_zero": 1, + "per_set": { + "multihop": { + "n_items_scored": 93, + "auc_logit": 0.1345, + "auc_jlens": 0.1202, + "delta": -0.0143, + "ci95": [ + -0.0732, + 0.0406 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3164, + "floor_cell_both_arms_zero_through_k10": false + }, + "multilingual": { + "n_items_scored": 107, + "auc_logit": 0.0388, + "auc_jlens": 0.0277, + "delta": -0.0111, + "ci95": [ + -0.0321, + 0.008 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1401, + "floor_cell_both_arms_zero_through_k10": false + }, + "order-ops": { + "n_items_scored": 55, + "auc_logit": 0.4493, + "auc_jlens": 0.4402, + "delta": -0.0091, + "ci95": [ + -0.0768, + 0.0584 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3905, + "floor_cell_both_arms_zero_through_k10": false + }, + "poetry": { + "n_items_scored": 98, + "auc_logit": 0.0383, + "auc_jlens": 0.0031, + "delta": -0.0352, + "ci95": [ + -0.0649, + -0.0108 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0004, + "floor_cell_both_arms_zero_through_k10": false + }, + "typo": { + "n_items_scored": 96, + "auc_logit": 0.2423, + "auc_jlens": 0.2808, + "delta": 0.0385, + "ci95": [ + -0.0422, + 0.12 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1752, + "floor_cell_both_arms_zero_through_k10": false + }, + "association": { + "n_items_scored": 92, + "auc_logit": 0.0033, + "auc_jlens": 0.0109, + "delta": 0.0076, + "ci95": [ + -0.0016, + 0.0196 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.084, + "floor_cell_both_arms_zero_through_k10": true + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 37.4 +} \ No newline at end of file diff --git a/results/t7_bootstrap_410m.json b/results/t7_bootstrap_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..6b46a4b328240ad6b34e816e5fba90a6239e1295 --- /dev/null +++ b/results/t7_bootstrap_410m.json @@ -0,0 +1,136 @@ +{ + "experiment": "T7 paired item-level bootstrap (audit of the n_win headline)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 secondary analysis of PREREG \u00a70 burned cells", + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "n_boot": 10000, + "seed": 0, + "bootstrap": "paired over items; one index draw applied to both transports", + "metric": "normalized pass@k AUC over log k, ks=(1,2,5,10,25,100)", + "n_win_point": 6, + "n_win_bootstrap_hist": { + "0": 0, + "1": 0, + "2": 0, + "3": 10, + "4": 304, + "5": 3239, + "6": 6447 + }, + "P_n_win_ge_5": 0.9686, + "P_n_win_le_3": 0.001, + "n_sets_ci_excludes_zero": 3, + "per_set": { + "multihop": { + "n_items_scored": 93, + "auc_logit": 0.2112, + "auc_jlens": 0.2757, + "delta": 0.0646, + "ci95": [ + 0.0033, + 0.128 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.019, + "floor_cell_both_arms_zero_through_k10": false + }, + "multilingual": { + "n_items_scored": 107, + "auc_logit": 0.1177, + "auc_jlens": 0.2141, + "delta": 0.0964, + "ci95": [ + 0.0727, + 0.1212 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0, + "floor_cell_both_arms_zero_through_k10": false + }, + "order-ops": { + "n_items_scored": 55, + "auc_logit": 0.5234, + "auc_jlens": 0.5679, + "delta": 0.0446, + "ci95": [ + -0.0007, + 0.0895 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.0259, + "floor_cell_both_arms_zero_through_k10": false + }, + "poetry": { + "n_items_scored": 98, + "auc_logit": 0.0287, + "auc_jlens": 0.0435, + "delta": 0.0148, + "ci95": [ + -0.0046, + 0.0338 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.0725, + "floor_cell_both_arms_zero_through_k10": true + }, + "typo": { + "n_items_scored": 96, + "auc_logit": 0.2034, + "auc_jlens": 0.3241, + "delta": 0.1208, + "ci95": [ + 0.0614, + 0.1828 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0, + "floor_cell_both_arms_zero_through_k10": false + }, + "association": { + "n_items_scored": 92, + "auc_logit": 0.0076, + "auc_jlens": 0.0115, + "delta": 0.0038, + "ci95": [ + -0.0098, + 0.0164 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.2703, + "floor_cell_both_arms_zero_through_k10": true + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 100.7 +} \ No newline at end of file diff --git a/results/t7_bootstrap_70m.json b/results/t7_bootstrap_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..ca9ebc642fd7892a480b500538f037f2bd6a4720 --- /dev/null +++ b/results/t7_bootstrap_70m.json @@ -0,0 +1,118 @@ +{ + "experiment": "T7 paired item-level bootstrap (audit of the n_win headline)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 secondary analysis of PREREG \u00a70 burned cells", + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200_db128.pt", + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "n_boot": 10000, + "seed": 0, + "bootstrap": "paired over items; one index draw applied to both transports", + "metric": "normalized pass@k AUC over log k, ks=(1,2,5,10,25,100)", + "n_win_point": 3, + "n_win_bootstrap_hist": { + "0": 1, + "1": 330, + "2": 2104, + "3": 3710, + "4": 2893, + "5": 870, + "6": 92 + }, + "P_n_win_ge_5": 0.0962, + "P_n_win_le_3": 0.6145, + "n_sets_ci_excludes_zero": 1, + "per_set": { + "multihop": { + "n_items_scored": 93, + "auc_logit": 0.0323, + "auc_jlens": 0.0436, + "delta": 0.0113, + "ci95": [ + -0.0156, + 0.0377 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1912, + "floor_cell_both_arms_zero_through_k10": false + }, + "multilingual": { + "n_items_scored": 107, + "auc_logit": 0.045, + "auc_jlens": 0.0356, + "delta": -0.0094, + "ci95": [ + -0.0302, + 0.01 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.1842, + "floor_cell_both_arms_zero_through_k10": false + }, + "order-ops": { + "n_items_scored": 55, + "auc_logit": 0.3496, + "auc_jlens": 0.3524, + "delta": 0.0028, + "ci95": [ + -0.0549, + 0.0617 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.4635, + "floor_cell_both_arms_zero_through_k10": false + }, + "poetry": { + "n_items_scored": 98, + "auc_logit": 0.0319, + "auc_jlens": 0.0245, + "delta": -0.0074, + "ci95": [ + -0.0362, + 0.0164 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3032, + "floor_cell_both_arms_zero_through_k10": false + }, + "typo": { + "n_items_scored": 96, + "auc_logit": 0.0886, + "auc_jlens": 0.0826, + "delta": -0.006, + "ci95": [ + -0.0493, + 0.038 + ], + "ci_excludes_zero": false, + "p_sign_flip": 0.3915, + "floor_cell_both_arms_zero_through_k10": false + }, + "association": { + "n_items_scored": 92, + "auc_logit": 0.0076, + "auc_jlens": 0.0446, + "delta": 0.037, + "ci95": [ + 0.0082, + 0.0713 + ], + "ci_excludes_zero": true, + "p_sign_flip": 0.0048, + "floor_cell_both_arms_zero_through_k10": false + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "runtime_s": 16.2 +} \ No newline at end of file diff --git a/results/t7_lens_comparison_160m.json b/results/t7_lens_comparison_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..7bd3ee91b662b6d257f271d0f7b903c13bf2f2b3 --- /dev/null +++ b/results/t7_lens_comparison_160m.json @@ -0,0 +1,498 @@ +{ + "experiment": "T7 lens comparison \u2014 anchor \u00a7A.6 pass@k, one metric, three transports", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prereg": null, + "model": "EleutherAI/pythia-160m-deduped", + "lens": "pythia/results/lens_160m_n200_db128.pt", + "lens_n_prompts": 200, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "n_layers_used": 11, + "fold_norm_gain": false, + "eval_data_provenance": "anthropics/jacobian-lens @581d398, Apache-2.0, data/evaluations/*.json \u2014 vendored, not downloaded", + "metric": "pass@k = mean over ITEMS of the fraction of that item's intermediates whose min-over-layers rank <= k; AUC normalized over log k (anchor README)", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "results": { + "multihop": { + "logit_lens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0215, + "2": 0.043, + "5": 0.086, + "10": 0.1075, + "25": 0.172, + "100": 0.3226 + }, + "normalized_auc_over_logk": 0.1345, + "median_best_rank": 406.0 + }, + "jlens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0215, + "2": 0.0323, + "5": 0.086, + "10": 0.1183, + "25": 0.1505, + "100": 0.2634 + }, + "normalized_auc_over_logk": 0.1202, + "median_best_rank": 454.0 + }, + "jlens_minus_logit_auc": -0.0143 + }, + "multilingual": { + "logit_lens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.007, + "2": 0.0156, + "5": 0.0234, + "10": 0.028, + "25": 0.0405, + "100": 0.109 + }, + "normalized_auc_over_logk": 0.0388, + "median_best_rank": 868.0 + }, + "jlens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0023, + "2": 0.0023, + "5": 0.0093, + "10": 0.0257, + "25": 0.0374, + "100": 0.0771 + }, + "normalized_auc_over_logk": 0.0277, + "median_best_rank": 1474.0 + }, + "jlens_minus_logit_auc": -0.0111 + }, + "order-ops": { + "logit_lens": { + "eval": "order-ops", + "n_items": 55, + "n_items_scored": 55, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0727, + "2": 0.1636, + "5": 0.2727, + "10": 0.4545, + "25": 0.6818, + "100": 0.7818 + }, + "normalized_auc_over_logk": 0.4493, + "median_best_rank": 12.0 + }, + "jlens": { + "eval": "order-ops", + "n_items": 55, + "n_items_scored": 55, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0273, + "2": 0.1273, + "5": 0.3, + "10": 0.4545, + "25": 0.6545, + "100": 0.8 + }, + "normalized_auc_over_logk": 0.4402, + "median_best_rank": 11.0 + }, + "jlens_minus_logit_auc": -0.0091 + }, + "poetry": { + "logit_lens": { + "eval": "poetry", + "n_items": 98, + "n_items_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0102, + "2": 0.0102, + "5": 0.0102, + "10": 0.0306, + "25": 0.0408, + "100": 0.1224 + }, + "normalized_auc_over_logk": 0.0383, + "median_best_rank": 712.0 + }, + "jlens": { + "eval": "poetry", + "n_items": 98, + "n_items_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0, + "100": 0.0204 + }, + "normalized_auc_over_logk": 0.0031, + "median_best_rank": 1309.0 + }, + "jlens_minus_logit_auc": -0.0352 + }, + "typo": { + "logit_lens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0729, + "2": 0.125, + "5": 0.1771, + "10": 0.2083, + "25": 0.2708, + "100": 0.5312 + }, + "normalized_auc_over_logk": 0.2423, + "median_best_rank": 80.0 + }, + "jlens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.1354, + "2": 0.1354, + "5": 0.2292, + "10": 0.2917, + "25": 0.3542, + "100": 0.4479 + }, + "normalized_auc_over_logk": 0.2808, + "median_best_rank": 146.0 + }, + "jlens_minus_logit_auc": 0.0385 + }, + "association": { + "logit_lens": { + "eval": "association", + "n_items": 102, + "n_items_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0, + "100": 0.0217 + }, + "normalized_auc_over_logk": 0.0033, + "median_best_rank": 4974.0 + }, + "jlens": { + "eval": "association", + "n_items": 102, + "n_items_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0109, + "100": 0.0543 + }, + "normalized_auc_over_logk": 0.0109, + "median_best_rank": 1266.0 + }, + "jlens_minus_logit_auc": 0.0076 + } + }, + "runtime_s": 37.8, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "centered": false +} \ No newline at end of file diff --git a/results/t7_lens_comparison_410m.json b/results/t7_lens_comparison_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..8b39320b20f440a556186a42159373ab68031aac --- /dev/null +++ b/results/t7_lens_comparison_410m.json @@ -0,0 +1,654 @@ +{ + "experiment": "T7 lens comparison \u2014 anchor \u00a7A.6 pass@k, one metric, three transports", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prereg": null, + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "lens_n_prompts": 200, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "n_layers_used": 23, + "fold_norm_gain": false, + "eval_data_provenance": "anthropics/jacobian-lens @581d398, Apache-2.0, data/evaluations/*.json \u2014 vendored, not downloaded", + "metric": "pass@k = mean over ITEMS of the fraction of that item's intermediates whose min-over-layers rank <= k; AUC normalized over log k (anchor README)", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "results": { + "multihop": { + "logit_lens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.086, + "2": 0.1075, + "5": 0.1398, + "10": 0.1935, + "25": 0.2473, + "100": 0.4373 + }, + "normalized_auc_over_logk": 0.2112, + "median_best_rank": 136.0 + }, + "jlens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0323, + "2": 0.043, + "5": 0.1505, + "10": 0.2796, + "25": 0.3871, + "100": 0.6237 + }, + "normalized_auc_over_logk": 0.2757, + "median_best_rank": 48.0 + }, + "jlens_minus_logit_auc": 0.0645 + }, + "multilingual": { + "logit_lens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0241, + "2": 0.0374, + "5": 0.0693, + "10": 0.0942, + "25": 0.1519, + "100": 0.2843 + }, + "normalized_auc_over_logk": 0.1177, + "median_best_rank": 372.0 + }, + "jlens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0802, + "2": 0.109, + "5": 0.1589, + "10": 0.2048, + "25": 0.2609, + "100": 0.4003 + }, + "normalized_auc_over_logk": 0.2141, + "median_best_rank": 213.0 + }, + "jlens_minus_logit_auc": 0.0964 + }, + "order-ops": { + "logit_lens": { + "eval": "order-ops", + "n_items": 55, + "n_items_scored": 55, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.2, + "2": 0.2727, + "5": 0.4182, + "10": 0.5091, + "25": 0.6636, + "100": 0.8818 + }, + "normalized_auc_over_logk": 0.5234, + "median_best_rank": 9.0 + }, + "jlens": { + "eval": "order-ops", + "n_items": 55, + "n_items_scored": 55, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.1364, + "2": 0.1909, + "5": 0.4, + "10": 0.6182, + "25": 0.8273, + "100": 0.9273 + }, + "normalized_auc_over_logk": 0.5679, + "median_best_rank": 8.0 + }, + "jlens_minus_logit_auc": 0.0445 + }, + "poetry": { + "logit_lens": { + "eval": "poetry", + "n_items": 98, + "n_items_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0102, + "100": 0.1735 + }, + "normalized_auc_over_logk": 0.0287, + "median_best_rank": 414.0 + }, + "jlens": { + "eval": "poetry", + "n_items": 98, + "n_items_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0204, + "100": 0.2551 + }, + "normalized_auc_over_logk": 0.0435, + "median_best_rank": 250.0 + }, + "jlens_minus_logit_auc": 0.0148 + }, + "typo": { + "logit_lens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0312, + "2": 0.0312, + "5": 0.0938, + "10": 0.1146, + "25": 0.2917, + "100": 0.5729 + }, + "normalized_auc_over_logk": 0.2034, + "median_best_rank": 75.0 + }, + "jlens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0729, + "2": 0.1458, + "5": 0.2188, + "10": 0.2917, + "25": 0.4271, + "100": 0.6458 + }, + "normalized_auc_over_logk": 0.3241, + "median_best_rank": 38.0 + }, + "jlens_minus_logit_auc": 0.1207 + }, + "association": { + "logit_lens": { + "eval": "association", + "n_items": 102, + "n_items_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0109, + "100": 0.0326 + }, + "normalized_auc_over_logk": 0.0076, + "median_best_rank": 1659.0 + }, + "jlens": { + "eval": "association", + "n_items": 102, + "n_items_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0, + "100": 0.0761 + }, + "normalized_auc_over_logk": 0.0115, + "median_best_rank": 959.0 + }, + "jlens_minus_logit_auc": 0.0039 + } + }, + "runtime_s": 101.7, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "centered": false +} \ No newline at end of file diff --git a/results/t7_lens_comparison_70m.json b/results/t7_lens_comparison_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..f0cb70521703a3bf2c40db1760595a2e22782cd7 --- /dev/null +++ b/results/t7_lens_comparison_70m.json @@ -0,0 +1,420 @@ +{ + "experiment": "T7 lens comparison \u2014 anchor \u00a7A.6 pass@k, one metric, three transports", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prereg": null, + "model": "EleutherAI/pythia-70m-deduped", + "lens": "pythia/results/lens_70m_n200_db128.pt", + "lens_n_prompts": 200, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "n_layers_used": 5, + "fold_norm_gain": false, + "eval_data_provenance": "anthropics/jacobian-lens @581d398, Apache-2.0, data/evaluations/*.json \u2014 vendored, not downloaded", + "metric": "pass@k = mean over ITEMS of the fraction of that item's intermediates whose min-over-layers rank <= k; AUC normalized over log k (anchor README)", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "results": { + "multihop": { + "logit_lens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0108, + "10": 0.0323, + "25": 0.0538, + "100": 0.0753 + }, + "normalized_auc_over_logk": 0.0323, + "median_best_rank": 837.0 + }, + "jlens": { + "eval": "multihop", + "n_items": 93, + "n_items_scored": 93, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0108, + "5": 0.0108, + "10": 0.0215, + "25": 0.0538, + "100": 0.1505 + }, + "normalized_auc_over_logk": 0.0436, + "median_best_rank": 799.0 + }, + "jlens_minus_logit_auc": 0.0113 + }, + "multilingual": { + "logit_lens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0023, + "2": 0.0023, + "5": 0.0132, + "10": 0.0296, + "25": 0.0514, + "100": 0.1597 + }, + "normalized_auc_over_logk": 0.045, + "median_best_rank": 899.0 + }, + "jlens": { + "eval": "multilingual", + "n_items": 107, + "n_items_scored": 107, + "n_intermediates_unscorable_multitoken": 34, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0117, + "10": 0.0257, + "25": 0.0537, + "100": 0.1036 + }, + "normalized_auc_over_logk": 0.0356, + "median_best_rank": 1277.0 + }, + "jlens_minus_logit_auc": -0.0094 + }, + "order-ops": { + "logit_lens": { + "eval": "order-ops", + "n_items": 55, + "n_items_scored": 55, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0182, + "2": 0.0727, + "5": 0.1545, + "10": 0.3091, + "25": 0.5364, + "100": 0.8 + }, + "normalized_auc_over_logk": 0.3496, + "median_best_rank": 22.0 + }, + "jlens": { + "eval": "order-ops", + "n_items": 55, + "n_items_scored": 55, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0182, + "2": 0.1, + "5": 0.2818, + "10": 0.3818, + "25": 0.5091, + "100": 0.6 + }, + "normalized_auc_over_logk": 0.3524, + "median_best_rank": 25.0 + }, + "jlens_minus_logit_auc": 0.0028 + }, + "poetry": { + "logit_lens": { + "eval": "poetry", + "n_items": 98, + "n_items_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0102, + "5": 0.0102, + "10": 0.0102, + "25": 0.051, + "100": 0.0918 + }, + "normalized_auc_over_logk": 0.0319, + "median_best_rank": 1339.0 + }, + "jlens": { + "eval": "poetry", + "n_items": 98, + "n_items_scored": 98, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0306, + "100": 0.1122 + }, + "normalized_auc_over_logk": 0.0245, + "median_best_rank": 755.0 + }, + "jlens_minus_logit_auc": -0.0074 + }, + "typo": { + "logit_lens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0104, + "2": 0.0208, + "5": 0.0417, + "10": 0.0729, + "25": 0.125, + "100": 0.2188 + }, + "normalized_auc_over_logk": 0.0886, + "median_best_rank": 478.0 + }, + "jlens": { + "eval": "typo", + "n_items": 96, + "n_items_scored": 96, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0208, + "2": 0.0312, + "5": 0.0521, + "10": 0.0729, + "25": 0.1146, + "100": 0.1667 + }, + "normalized_auc_over_logk": 0.0826, + "median_best_rank": 945.0 + }, + "jlens_minus_logit_auc": -0.006 + }, + "association": { + "logit_lens": { + "eval": "association", + "n_items": 102, + "n_items_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0, + "10": 0.0, + "25": 0.0109, + "100": 0.0326 + }, + "normalized_auc_over_logk": 0.0076, + "median_best_rank": 2671.0 + }, + "jlens": { + "eval": "association", + "n_items": 102, + "n_items_scored": 92, + "n_intermediates_unscorable_multitoken": 10, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0, + "2": 0.0, + "5": 0.0217, + "10": 0.0435, + "25": 0.0543, + "100": 0.1304 + }, + "normalized_auc_over_logk": 0.0446, + "median_best_rank": 1083.0 + }, + "jlens_minus_logit_auc": 0.037 + } + }, + "runtime_s": 15.6, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "centered": false +} \ No newline at end of file diff --git a/results/t7_smoke_410m.json b/results/t7_smoke_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..c92f0ae09144b0105a658269d81e092dacfa1778 --- /dev/null +++ b/results/t7_smoke_410m.json @@ -0,0 +1,149 @@ +{ + "experiment": "T7 lens comparison \u2014 anchor \u00a7A.6 pass@k, one metric, three transports", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prereg": null, + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "lens_n_prompts": 200, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "n_layers_used": 23, + "fold_norm_gain": false, + "eval_data_provenance": "anthropics/jacobian-lens @581d398, Apache-2.0, data/evaluations/*.json \u2014 vendored, not downloaded", + "metric": "pass@k = mean over ITEMS of the fraction of that item's intermediates whose min-over-layers rank <= k; AUC normalized over log k (anchor README)", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "results": { + "multihop": { + "logit_lens": { + "eval": "multihop", + "n_items": 30, + "n_items_scored": 30, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0333, + "2": 0.0333, + "5": 0.0333, + "10": 0.1333, + "25": 0.2, + "100": 0.4 + }, + "normalized_auc_over_logk": 0.1477, + "median_best_rank": 154.0 + }, + "jlens": { + "eval": "multihop", + "n_items": 30, + "n_items_scored": 30, + "n_intermediates_unscorable_multitoken": 0, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "ks": [ + 1, + 2, + 5, + 10, + 25, + 100 + ], + "pass_at_k": { + "1": 0.0333, + "2": 0.0333, + "5": 0.1667, + "10": 0.3, + "25": 0.4, + "100": 0.6 + }, + "normalized_auc_over_logk": 0.2802, + "median_best_rank": 31.0 + }, + "jlens_minus_logit_auc": 0.1325 + } + }, + "runtime_s": 10.1, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "centered": false +} \ No newline at end of file diff --git a/results/t8_probe_swap_410m.json b/results/t8_probe_swap_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..63f2334b46801e16bcfb66f152cc91d514234e01 --- /dev/null +++ b/results/t8_probe_swap_410m.json @@ -0,0 +1,1851 @@ +{ + "experiment": "T8 write battery on the anchor's probe-swap set", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "prereg": null, + "prereg_sha256": null, + "supersedes": "pythia/results/t5_write_410m.json (RETRACTED 2026-08-10)", + "data_provenance": "anthropics/jacobian-lens @581d398, Apache-2.0, data/experiments/probe-swap.json", + "seed": 1234, + "max_cells": 90, + "band_lo": 9, + "control_seed_policy": "sha256(seed|item|layer|arm) \u2014 independent per arm and per item", + "skip_positions": 1, + "skip_positions_note": "BOS/attention-sink position excluded; its residual norm is 7-9x every real token, so an all-positions write is dominated by a token the anchor's protocol does not include", + "argv": [ + "pythia/t8_probe_swap.py", + "--model", + "EleutherAI/pythia-410m-deduped", + "--lens", + "pythia/results/lens_410m_n200_db128.pt", + "--band-lo", + "9", + "--alphas", + "1", + "--out", + "pythia/results/t8_probe_swap_410m.json" + ], + "model": "EleutherAI/pythia-410m-deduped", + "lens": "pythia/results/lens_410m_n200_db128.pt", + "lens_n_prompts": 200, + "band": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "alphas": [ + 1.0 + ], + "positions": "all", + "centered": false, + "n_items_total": 90, + "n_dropped_tokenization": 17, + "n_dropped_baseline": 65, + "n_cells_run": 8, + "gate": "PER-DOSE: a cell counts at alpha only if the answer-token oracle flips at that alpha", + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "summary": { + "alpha=1": { + "n_admissible_this_dose": 1, + "median_clean_rank": 26.0, + "identity": { + "n": 1, + "flips": 0, + "median_rank_swap_answer": 26.0, + "median_kl_vs_clean": 0.0, + "median_delta_norm": null + }, + "swap@a1": { + "n": 1, + "flips": 0, + "median_rank_swap_answer": 22.0, + "median_kl_vs_clean": 0.0852, + "median_delta_norm": 202.467 + }, + "rand_target@a1": { + "n": 1, + "flips": 0, + "median_rank_swap_answer": 19.0, + "median_kl_vs_clean": 0.0455, + "median_delta_norm": 202.467 + }, + "orth@a1": { + "n": 1, + "flips": 0, + "median_rank_swap_answer": 45.0, + "median_kl_vs_clean": 0.0526, + "median_delta_norm": 202.467 + }, + "oracle@a1": { + "n": 1, + "flips": 1, + "median_rank_swap_answer": 0.0, + "median_kl_vs_clean": 0.8054, + "median_delta_norm": 202.467 + } + } + }, + "rows": [ + { + "name": "ex-city-language-Lyon-Naples", + "category": "city-language", + "prompt": "Fact: The language spoken in the country where Lyon is located is", + "intermediate": "France", + "swap_to": "Italy", + "answer": "French", + "swap_answer": "Italian", + "clean": { + "top1": " French", + "rank_swap_answer": 54, + "rank_answer": 0, + "logp_swap_answer": -5.8836, + "logp_answer": -2.4216, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " French", + "rank_swap_answer": 54, + "rank_answer": 0, + "logp_swap_answer": -5.8836, + "logp_answer": -2.4216, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " not", + "rank_swap_answer": 11, + "rank_answer": 4, + "logp_swap_answer": -4.4725, + "logp_answer": -3.1669, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0336, + "delta_norm_band_sum": 126.58, + "delta_norm_lastpos_sum": 11.8551, + "delta_norm_per_layer_sum": { + "9": 9.9617, + "10": 8.6928, + "11": 7.7968, + "12": 8.1571, + "13": 8.5623, + "14": 8.5382, + "15": 7.276, + "16": 7.0694, + "17": 9.1794, + "18": 10.0351, + "19": 8.8419, + "20": 7.8279, + "21": 9.6081, + "22": 15.0331 + }, + "delta_norm_per_layer_max": { + "9": 1.615, + "10": 1.8339, + "11": 1.5375, + "12": 1.4013, + "13": 1.8309, + "14": 1.8982, + "15": 1.6036, + "16": 1.4758, + "17": 1.6364, + "18": 1.715, + "19": 1.5119, + "20": 1.5975, + "21": 1.6914, + "22": 2.3589 + }, + "gram_cond_max": 9.817, + "n_degenerate_layers": 0, + "exchange_verified": { + "9": false, + "10": false, + "11": false, + "12": false, + "13": false, + "14": false, + "15": false, + "16": false, + "17": false, + "18": false, + "19": false, + "20": false, + "21": false, + "22": false + } + }, + "rand_target@a1": { + "top1": " not", + "rank_swap_answer": 61, + "rank_answer": 1, + "logp_swap_answer": -6.0271, + "logp_answer": -2.7981, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0222, + "delta_norm_band_sum": 126.58, + "delta_norm_lastpos_sum": 11.8551, + "delta_norm_per_layer_sum": { + "9": 9.9617, + "10": 8.6928, + "11": 7.7968, + "12": 8.1571, + "13": 8.5623, + "14": 8.5382, + "15": 7.276, + "16": 7.0694, + "17": 9.1794, + "18": 10.0351, + "19": 8.8419, + "20": 7.8279, + "21": 9.6081, + "22": 15.0331 + }, + "delta_norm_per_layer_max": { + "9": 1.615, + "10": 1.8339, + "11": 1.5375, + "12": 1.4013, + "13": 1.8309, + "14": 1.8982, + "15": 1.6036, + "16": 1.4758, + "17": 1.6364, + "18": 1.715, + "19": 1.5119, + "20": 1.5975, + "21": 1.6914, + "22": 2.3589 + }, + "gram_cond_max": 13.85, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " French", + "rank_swap_answer": 58, + "rank_answer": 0, + "logp_swap_answer": -5.8861, + "logp_answer": -2.3768, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0118, + "delta_norm_band_sum": 126.58, + "delta_norm_lastpos_sum": 11.8551, + "delta_norm_per_layer_sum": { + "9": 9.9617, + "10": 8.6928, + "11": 7.7968, + "12": 8.1571, + "13": 8.5623, + "14": 8.5382, + "15": 7.276, + "16": 7.0694, + "17": 9.1794, + "18": 10.0351, + "19": 8.8419, + "20": 7.8279, + "21": 9.6081, + "22": 15.0331 + }, + "delta_norm_per_layer_max": { + "9": 1.615, + "10": 1.8339, + "11": 1.5375, + "12": 1.4013, + "13": 1.8309, + "14": 1.8982, + "15": 1.6036, + "16": 1.4758, + "17": 1.6364, + "18": 1.715, + "19": 1.5119, + "20": 1.5975, + "21": 1.6914, + "22": 2.3589 + }, + "gram_cond_max": 13.607, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " not", + "rank_swap_answer": 8, + "rank_answer": 4, + "logp_swap_answer": -4.256, + "logp_answer": -3.2849, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0435, + "delta_norm_band_sum": 126.58, + "delta_norm_lastpos_sum": 11.8551, + "delta_norm_per_layer_sum": { + "9": 9.9617, + "10": 8.6928, + "11": 7.7968, + "12": 8.1571, + "13": 8.5623, + "14": 8.5382, + "15": 7.276, + "16": 7.0694, + "17": 9.1794, + "18": 10.0351, + "19": 8.8419, + "20": 7.8279, + "21": 9.6081, + "22": 15.0331 + }, + "delta_norm_per_layer_max": { + "9": 1.615, + "10": 1.8339, + "11": 1.5375, + "12": 1.4013, + "13": 1.8309, + "14": 1.8982, + "15": 1.6036, + "16": 1.4758, + "17": 1.6364, + "18": 1.715, + "19": 1.5119, + "20": 1.5975, + "21": 1.6914, + "22": 2.3589 + }, + "gram_cond_max": 10.096, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 219.454 + } + }, + "admissible": { + "a1": false + } + }, + { + "name": "ex2-city-language-Cairo", + "category": "city-language", + "prompt": "Fact: The language spoken in the country where Cairo is located is", + "intermediate": "Egypt", + "swap_to": "Russia", + "answer": "Arabic", + "swap_answer": "Russian", + "clean": { + "top1": " Arabic", + "rank_swap_answer": 532, + "rank_answer": 0, + "logp_swap_answer": -9.008, + "logp_answer": -2.7056, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " Arabic", + "rank_swap_answer": 532, + "rank_answer": 0, + "logp_swap_answer": -9.008, + "logp_answer": -2.7056, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Arabic", + "rank_swap_answer": 429, + "rank_answer": 0, + "logp_swap_answer": -8.6824, + "logp_answer": -2.244, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0732, + "delta_norm_band_sum": 230.8498, + "delta_norm_lastpos_sum": 39.2583, + "delta_norm_per_layer_sum": { + "9": 16.0163, + "10": 14.3409, + "11": 13.3185, + "12": 10.9932, + "13": 10.142, + "14": 6.888, + "15": 9.2492, + "16": 12.4591, + "17": 17.5867, + "18": 15.46, + "19": 20.1553, + "20": 26.131, + "21": 22.1274, + "22": 35.9824 + }, + "delta_norm_per_layer_max": { + "9": 5.7667, + "10": 5.6435, + "11": 4.6559, + "12": 3.213, + "13": 1.6538, + "14": 2.3204, + "15": 3.7106, + "16": 3.8067, + "17": 6.2322, + "18": 4.7377, + "19": 6.2483, + "20": 6.9857, + "21": 5.2262, + "22": 7.1201 + }, + "gram_cond_max": 4.358, + "n_degenerate_layers": 0, + "exchange_verified": { + "9": false, + "10": false, + "11": false, + "12": false, + "13": false, + "14": false, + "15": false, + "16": false, + "17": false, + "18": false, + "19": false, + "20": false, + "21": false, + "22": false + } + }, + "rand_target@a1": { + "top1": " not", + "rank_swap_answer": 1046, + "rank_answer": 2, + "logp_swap_answer": -9.8884, + "logp_answer": -2.7859, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.2419, + "delta_norm_band_sum": 230.8498, + "delta_norm_lastpos_sum": 39.2583, + "delta_norm_per_layer_sum": { + "9": 16.0163, + "10": 14.3409, + "11": 13.3185, + "12": 10.9932, + "13": 10.142, + "14": 6.888, + "15": 9.2492, + "16": 12.4591, + "17": 17.5867, + "18": 15.46, + "19": 20.1553, + "20": 26.131, + "21": 22.1274, + "22": 35.9824 + }, + "delta_norm_per_layer_max": { + "9": 5.7667, + "10": 5.6435, + "11": 4.6559, + "12": 3.213, + "13": 1.6538, + "14": 2.3204, + "15": 3.7106, + "16": 3.8067, + "17": 6.2322, + "18": 4.7377, + "19": 6.2483, + "20": 6.9857, + "21": 5.2262, + "22": 7.1201 + }, + "gram_cond_max": 7.759, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " the", + "rank_swap_answer": 196, + "rank_answer": 1, + "logp_swap_answer": -7.4737, + "logp_answer": -2.8958, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.2185, + "delta_norm_band_sum": 230.8498, + "delta_norm_lastpos_sum": 39.2583, + "delta_norm_per_layer_sum": { + "9": 16.0163, + "10": 14.3409, + "11": 13.3185, + "12": 10.9932, + "13": 10.142, + "14": 6.888, + "15": 9.2492, + "16": 12.4591, + "17": 17.5867, + "18": 15.46, + "19": 20.1553, + "20": 26.131, + "21": 22.1274, + "22": 35.9824 + }, + "delta_norm_per_layer_max": { + "9": 5.7667, + "10": 5.6435, + "11": 4.6559, + "12": 3.213, + "13": 1.6538, + "14": 2.3204, + "15": 3.7106, + "16": 3.8067, + "17": 6.2322, + "18": 4.7377, + "19": 6.2483, + "20": 6.9857, + "21": 5.2262, + "22": 7.1201 + }, + "gram_cond_max": 7.676, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " Arabic", + "rank_swap_answer": 229, + "rank_answer": 0, + "logp_swap_answer": -7.8635, + "logp_answer": -1.8575, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.114, + "delta_norm_band_sum": 230.8498, + "delta_norm_lastpos_sum": 39.2583, + "delta_norm_per_layer_sum": { + "9": 16.0163, + "10": 14.3409, + "11": 13.3185, + "12": 10.9932, + "13": 10.142, + "14": 6.888, + "15": 9.2492, + "16": 12.4591, + "17": 17.5867, + "18": 15.46, + "19": 20.1553, + "20": 26.131, + "21": 22.1274, + "22": 35.9824 + }, + "delta_norm_per_layer_max": { + "9": 5.7667, + "10": 5.6435, + "11": 4.6559, + "12": 3.213, + "13": 1.6538, + "14": 2.3204, + "15": 3.7106, + "16": 3.8067, + "17": 6.2322, + "18": 4.7377, + "19": 6.2483, + "20": 6.9857, + "21": 5.2262, + "22": 7.1201 + }, + "gram_cond_max": 3.628, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 196.701 + } + }, + "admissible": { + "a1": false + } + }, + { + "name": "ex2-city-language-Moscow", + "category": "city-language", + "prompt": "Fact: The language spoken in the country where Moscow is located is", + "intermediate": "Russia", + "swap_to": "Egypt", + "answer": "Russian", + "swap_answer": "Arabic", + "clean": { + "top1": " Russian", + "rank_swap_answer": 72, + "rank_answer": 0, + "logp_swap_answer": -6.2069, + "logp_answer": -2.4699, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " Russian", + "rank_swap_answer": 72, + "rank_answer": 0, + "logp_swap_answer": -6.2069, + "logp_answer": -2.4699, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " not", + "rank_swap_answer": 5, + "rank_answer": 17, + "logp_swap_answer": -3.8185, + "logp_answer": -4.6479, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.1314, + "delta_norm_band_sum": 198.0398, + "delta_norm_lastpos_sum": 10.0141, + "delta_norm_per_layer_sum": { + "9": 16.619, + "10": 14.7234, + "11": 15.5017, + "12": 15.0562, + "13": 16.7651, + "14": 11.7798, + "15": 11.6515, + "16": 10.4619, + "17": 11.6596, + "18": 11.4811, + "19": 10.6453, + "20": 14.1468, + "21": 14.675, + "22": 22.8735 + }, + "delta_norm_per_layer_max": { + "9": 6.9861, + "10": 6.5058, + "11": 6.4234, + "12": 5.9442, + "13": 6.8356, + "14": 5.8434, + "15": 4.8906, + "16": 3.2027, + "17": 2.8779, + "18": 3.3535, + "19": 2.5978, + "20": 2.1813, + "21": 2.3961, + "22": 3.4198 + }, + "gram_cond_max": 4.358, + "n_degenerate_layers": 0, + "exchange_verified": { + "9": false, + "10": false, + "11": false, + "12": false, + "13": false, + "14": false, + "15": false, + "16": false, + "17": false, + "18": false, + "19": false, + "20": false, + "21": false, + "22": false + } + }, + "rand_target@a1": { + "top1": " not", + "rank_swap_answer": 36, + "rank_answer": 4, + "logp_swap_answer": -5.4735, + "logp_answer": -3.5484, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0548, + "delta_norm_band_sum": 198.0398, + "delta_norm_lastpos_sum": 10.0141, + "delta_norm_per_layer_sum": { + "9": 16.619, + "10": 14.7234, + "11": 15.5017, + "12": 15.0562, + "13": 16.7651, + "14": 11.7798, + "15": 11.6515, + "16": 10.4619, + "17": 11.6596, + "18": 11.4811, + "19": 10.6453, + "20": 14.1468, + "21": 14.675, + "22": 22.8735 + }, + "delta_norm_per_layer_max": { + "9": 6.9861, + "10": 6.5058, + "11": 6.4234, + "12": 5.9442, + "13": 6.8356, + "14": 5.8434, + "15": 4.8906, + "16": 3.2027, + "17": 2.8779, + "18": 3.3535, + "19": 2.5978, + "20": 2.1813, + "21": 2.3961, + "22": 3.4198 + }, + "gram_cond_max": 8.629, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " Russian", + "rank_swap_answer": 105, + "rank_answer": 0, + "logp_swap_answer": -6.7349, + "logp_answer": -2.3173, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0279, + "delta_norm_band_sum": 198.0398, + "delta_norm_lastpos_sum": 10.0141, + "delta_norm_per_layer_sum": { + "9": 16.619, + "10": 14.7234, + "11": 15.5017, + "12": 15.0562, + "13": 16.7651, + "14": 11.7798, + "15": 11.6515, + "16": 10.4619, + "17": 11.6596, + "18": 11.4811, + "19": 10.6453, + "20": 14.1468, + "21": 14.675, + "22": 22.8735 + }, + "delta_norm_per_layer_max": { + "9": 6.9861, + "10": 6.5058, + "11": 6.4234, + "12": 5.9442, + "13": 6.8356, + "14": 5.8434, + "15": 4.8906, + "16": 3.2027, + "17": 2.8779, + "18": 3.3535, + "19": 2.5978, + "20": 2.1813, + "21": 2.3961, + "22": 3.4198 + }, + "gram_cond_max": 8.219, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " not", + "rank_swap_answer": 16, + "rank_answer": 4, + "logp_swap_answer": -4.5272, + "logp_answer": -3.5011, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0485, + "delta_norm_band_sum": 198.0398, + "delta_norm_lastpos_sum": 10.0141, + "delta_norm_per_layer_sum": { + "9": 16.619, + "10": 14.7234, + "11": 15.5017, + "12": 15.0562, + "13": 16.7651, + "14": 11.7798, + "15": 11.6515, + "16": 10.4619, + "17": 11.6596, + "18": 11.4811, + "19": 10.6453, + "20": 14.1468, + "21": 14.675, + "22": 22.8735 + }, + "delta_norm_per_layer_max": { + "9": 6.9861, + "10": 6.5058, + "11": 6.4234, + "12": 5.9442, + "13": 6.8356, + "14": 5.8434, + "15": 4.8906, + "16": 3.2027, + "17": 2.8779, + "18": 3.3535, + "19": 2.5978, + "20": 2.1813, + "21": 2.3961, + "22": 3.4198 + }, + "gram_cond_max": 3.628, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 249.814 + } + }, + "admissible": { + "a1": false + } + }, + { + "name": "ex2-language-capital-Hungarian", + "category": "language-capital", + "prompt": "Fact: The capital of the country where Hungarian is the primary language is", + "intermediate": "Hungary", + "swap_to": "Poland", + "answer": "Budapest", + "swap_answer": "Warsaw", + "clean": { + "top1": " Budapest", + "rank_swap_answer": 326, + "rank_answer": 0, + "logp_swap_answer": -8.1757, + "logp_answer": -1.9696, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " Budapest", + "rank_swap_answer": 326, + "rank_answer": 0, + "logp_swap_answer": -8.1757, + "logp_answer": -1.9696, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " called", + "rank_swap_answer": 35, + "rank_answer": 2, + "logp_swap_answer": -5.3013, + "logp_answer": -3.2375, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.1504, + "delta_norm_band_sum": 216.874, + "delta_norm_lastpos_sum": 21.6804, + "delta_norm_per_layer_sum": { + "9": 12.5534, + "10": 12.3845, + "11": 9.2405, + "12": 9.8595, + "13": 12.6758, + "14": 11.3205, + "15": 10.3734, + "16": 13.5071, + "17": 17.7207, + "18": 22.0412, + "19": 19.7383, + "20": 19.659, + "21": 21.6942, + "22": 24.1058 + }, + "delta_norm_per_layer_max": { + "9": 5.3138, + "10": 5.3962, + "11": 3.8862, + "12": 1.6832, + "13": 2.7313, + "14": 2.131, + "15": 2.0125, + "16": 1.782, + "17": 4.4155, + "18": 3.3095, + "19": 2.8224, + "20": 3.398, + "21": 2.4239, + "22": 2.9096 + }, + "gram_cond_max": 9.785, + "n_degenerate_layers": 0 + }, + "rand_target@a1": { + "top1": " called", + "rank_swap_answer": 250, + "rank_answer": 1, + "logp_swap_answer": -7.7335, + "logp_answer": -2.3631, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.1202, + "delta_norm_band_sum": 216.874, + "delta_norm_lastpos_sum": 21.6804, + "delta_norm_per_layer_sum": { + "9": 12.5534, + "10": 12.3845, + "11": 9.2405, + "12": 9.8595, + "13": 12.6758, + "14": 11.3205, + "15": 10.3734, + "16": 13.5071, + "17": 17.7207, + "18": 22.0412, + "19": 19.7383, + "20": 19.659, + "21": 21.6942, + "22": 24.1058 + }, + "delta_norm_per_layer_max": { + "9": 5.3138, + "10": 5.3962, + "11": 3.8862, + "12": 1.6832, + "13": 2.7313, + "14": 2.131, + "15": 2.0125, + "16": 1.782, + "17": 4.4155, + "18": 3.3095, + "19": 2.8224, + "20": 3.398, + "21": 2.4239, + "22": 2.9096 + }, + "gram_cond_max": 13.199, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " Budapest", + "rank_swap_answer": 347, + "rank_answer": 0, + "logp_swap_answer": -8.1772, + "logp_answer": -2.7399, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.1354, + "delta_norm_band_sum": 216.874, + "delta_norm_lastpos_sum": 21.6804, + "delta_norm_per_layer_sum": { + "9": 12.5534, + "10": 12.3845, + "11": 9.2405, + "12": 9.8595, + "13": 12.6758, + "14": 11.3205, + "15": 10.3734, + "16": 13.5071, + "17": 17.7207, + "18": 22.0412, + "19": 19.7383, + "20": 19.659, + "21": 21.6942, + "22": 24.1058 + }, + "delta_norm_per_layer_max": { + "9": 5.3138, + "10": 5.3962, + "11": 3.8862, + "12": 1.6832, + "13": 2.7313, + "14": 2.131, + "15": 2.0125, + "16": 1.782, + "17": 4.4155, + "18": 3.3095, + "19": 2.8224, + "20": 3.398, + "21": 2.4239, + "22": 2.9096 + }, + "gram_cond_max": 13.327, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " called", + "rank_swap_answer": 32, + "rank_answer": 10, + "logp_swap_answer": -5.2777, + "logp_answer": -4.1477, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.1853, + "delta_norm_band_sum": 216.874, + "delta_norm_lastpos_sum": 21.6804, + "delta_norm_per_layer_sum": { + "9": 12.5534, + "10": 12.3845, + "11": 9.2405, + "12": 9.8595, + "13": 12.6758, + "14": 11.3205, + "15": 10.3734, + "16": 13.5071, + "17": 17.7207, + "18": 22.0412, + "19": 19.7383, + "20": 19.659, + "21": 21.6942, + "22": 24.1058 + }, + "delta_norm_per_layer_max": { + "9": 5.3138, + "10": 5.3962, + "11": 3.8862, + "12": 1.6832, + "13": 2.7313, + "14": 2.131, + "15": 2.0125, + "16": 1.782, + "17": 4.4155, + "18": 3.3095, + "19": 2.8224, + "20": 3.398, + "21": 2.4239, + "22": 2.9096 + }, + "gram_cond_max": 6.16, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 184.477 + } + }, + "admissible": { + "a1": false + } + }, + { + "name": "ex2-language-capital-Swedish", + "category": "language-capital", + "prompt": "Fact: The capital of the country where Swedish is the primary language is", + "intermediate": "Sweden", + "swap_to": "Hungary", + "answer": "Stockholm", + "swap_answer": "Budapest", + "clean": { + "top1": " Stockholm", + "rank_swap_answer": 1013, + "rank_answer": 0, + "logp_swap_answer": -9.8655, + "logp_answer": -2.2639, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " Stockholm", + "rank_swap_answer": 1013, + "rank_answer": 0, + "logp_swap_answer": -9.8655, + "logp_answer": -2.2639, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " L", + "rank_swap_answer": 232, + "rank_answer": 2, + "logp_swap_answer": -7.5101, + "logp_answer": -3.2762, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.1808, + "delta_norm_band_sum": 373.8955, + "delta_norm_lastpos_sum": 54.7998, + "delta_norm_per_layer_sum": { + "9": 13.9527, + "10": 14.2858, + "11": 13.6326, + "12": 12.8837, + "13": 16.1069, + "14": 19.1788, + "15": 18.2092, + "16": 15.936, + "17": 35.8657, + "18": 34.0837, + "19": 44.503, + "20": 38.6218, + "21": 36.1637, + "22": 60.4718 + }, + "delta_norm_per_layer_max": { + "9": 6.0575, + "10": 6.2678, + "11": 5.3725, + "12": 3.7106, + "13": 4.0694, + "14": 4.4473, + "15": 5.044, + "16": 3.1114, + "17": 8.8078, + "18": 7.3631, + "19": 8.9388, + "20": 8.0965, + "21": 7.2885, + "22": 8.1958 + }, + "gram_cond_max": 7.058, + "n_degenerate_layers": 0 + }, + "rand_target@a1": { + "top1": " located", + "rank_swap_answer": 1937, + "rank_answer": 9, + "logp_swap_answer": -10.487, + "logp_answer": -4.2419, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.7562, + "delta_norm_band_sum": 373.8955, + "delta_norm_lastpos_sum": 54.7998, + "delta_norm_per_layer_sum": { + "9": 13.9527, + "10": 14.2858, + "11": 13.6326, + "12": 12.8837, + "13": 16.1069, + "14": 19.1788, + "15": 18.2092, + "16": 15.936, + "17": 35.8657, + "18": 34.0837, + "19": 44.503, + "20": 38.6218, + "21": 36.1637, + "22": 60.4718 + }, + "delta_norm_per_layer_max": { + "9": 6.0575, + "10": 6.2678, + "11": 5.3725, + "12": 3.7106, + "13": 4.0694, + "14": 4.4473, + "15": 5.044, + "16": 3.1114, + "17": 8.8078, + "18": 7.3631, + "19": 8.9388, + "20": 8.0965, + "21": 7.2885, + "22": 8.1958 + }, + "gram_cond_max": 10.121, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " Mal", + "rank_swap_answer": 2446, + "rank_answer": 21, + "logp_swap_answer": -10.7964, + "logp_answer": -5.0489, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.5486, + "delta_norm_band_sum": 373.8955, + "delta_norm_lastpos_sum": 54.7998, + "delta_norm_per_layer_sum": { + "9": 13.9527, + "10": 14.2858, + "11": 13.6326, + "12": 12.8837, + "13": 16.1069, + "14": 19.1788, + "15": 18.2092, + "16": 15.936, + "17": 35.8657, + "18": 34.0837, + "19": 44.503, + "20": 38.6218, + "21": 36.1637, + "22": 60.4718 + }, + "delta_norm_per_layer_max": { + "9": 6.0575, + "10": 6.2678, + "11": 5.3725, + "12": 3.7106, + "13": 4.0694, + "14": 4.4473, + "15": 5.044, + "16": 3.1114, + "17": 8.8078, + "18": 7.3631, + "19": 8.9388, + "20": 8.0965, + "21": 7.2885, + "22": 8.1958 + }, + "gram_cond_max": 10.526, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " Stockholm", + "rank_swap_answer": 487, + "rank_answer": 0, + "logp_swap_answer": -8.7078, + "logp_answer": -2.3923, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.1, + "delta_norm_band_sum": 373.8955, + "delta_norm_lastpos_sum": 54.7998, + "delta_norm_per_layer_sum": { + "9": 13.9527, + "10": 14.2858, + "11": 13.6326, + "12": 12.8837, + "13": 16.1069, + "14": 19.1788, + "15": 18.2092, + "16": 15.936, + "17": 35.8657, + "18": 34.0837, + "19": 44.503, + "20": 38.6218, + "21": 36.1637, + "22": 60.4718 + }, + "delta_norm_per_layer_max": { + "9": 6.0575, + "10": 6.2678, + "11": 5.3725, + "12": 3.7106, + "13": 4.0694, + "14": 4.4473, + "15": 5.044, + "16": 3.1114, + "17": 8.8078, + "18": 7.3631, + "19": 8.9388, + "20": 8.0965, + "21": 7.2885, + "22": 8.1958 + }, + "gram_cond_max": 6.883, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 330.1519 + } + }, + "admissible": { + "a1": false + } + }, + { + "name": "mars-color", + "category": "multihop", + "prompt": "Fact: The color of the planet fourth from the Sun is", + "intermediate": "Mars", + "swap_to": "Earth", + "answer": "red", + "swap_answer": "blue", + "clean": { + "top1": " red", + "rank_swap_answer": 6, + "rank_answer": 0, + "logp_swap_answer": -3.6707, + "logp_answer": -2.2565, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " red", + "rank_swap_answer": 6, + "rank_answer": 0, + "logp_swap_answer": -3.6707, + "logp_answer": -2.2565, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " red", + "rank_swap_answer": 6, + "rank_answer": 0, + "logp_swap_answer": -3.6381, + "logp_answer": -2.2399, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0236, + "delta_norm_band_sum": 157.9715, + "delta_norm_lastpos_sum": 11.9342, + "delta_norm_per_layer_sum": { + "9": 22.1818, + "10": 17.0011, + "11": 10.5778, + "12": 6.7399, + "13": 9.479, + "14": 14.8104, + "15": 11.3915, + "16": 7.3331, + "17": 8.1528, + "18": 9.6933, + "19": 8.1175, + "20": 10.3318, + "21": 9.3162, + "22": 12.8454 + }, + "delta_norm_per_layer_max": { + "9": 3.3495, + "10": 3.1516, + "11": 2.6539, + "12": 1.433, + "13": 2.4767, + "14": 2.9724, + "15": 2.7797, + "16": 2.377, + "17": 2.8613, + "18": 3.0177, + "19": 2.4322, + "20": 2.2767, + "21": 1.5396, + "22": 1.9869 + }, + "gram_cond_max": 5.788, + "n_degenerate_layers": 0 + }, + "rand_target@a1": { + "top1": " red", + "rank_swap_answer": 5, + "rank_answer": 0, + "logp_swap_answer": -3.6938, + "logp_answer": -2.3695, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.039, + "delta_norm_band_sum": 157.9715, + "delta_norm_lastpos_sum": 11.9342, + "delta_norm_per_layer_sum": { + "9": 22.1818, + "10": 17.0011, + "11": 10.5778, + "12": 6.7399, + "13": 9.479, + "14": 14.8104, + "15": 11.3915, + "16": 7.3331, + "17": 8.1528, + "18": 9.6933, + "19": 8.1175, + "20": 10.3318, + "21": 9.3162, + "22": 12.8454 + }, + "delta_norm_per_layer_max": { + "9": 3.3495, + "10": 3.1516, + "11": 2.6539, + "12": 1.433, + "13": 2.4767, + "14": 2.9724, + "15": 2.7797, + "16": 2.377, + "17": 2.8613, + "18": 3.0177, + "19": 2.4322, + "20": 2.2767, + "21": 1.5396, + "22": 1.9869 + }, + "gram_cond_max": 9.368, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " red", + "rank_swap_answer": 6, + "rank_answer": 0, + "logp_swap_answer": -3.6497, + "logp_answer": -2.1723, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0319, + "delta_norm_band_sum": 157.9715, + "delta_norm_lastpos_sum": 11.9342, + "delta_norm_per_layer_sum": { + "9": 22.1818, + "10": 17.0011, + "11": 10.5778, + "12": 6.7399, + "13": 9.479, + "14": 14.8104, + "15": 11.3915, + "16": 7.3331, + "17": 8.1528, + "18": 9.6933, + "19": 8.1175, + "20": 10.3318, + "21": 9.3162, + "22": 12.8454 + }, + "delta_norm_per_layer_max": { + "9": 3.3495, + "10": 3.1516, + "11": 2.6539, + "12": 1.433, + "13": 2.4767, + "14": 2.9724, + "15": 2.7797, + "16": 2.377, + "17": 2.8613, + "18": 3.0177, + "19": 2.4322, + "20": 2.2767, + "21": 1.5396, + "22": 1.9869 + }, + "gram_cond_max": 9.136, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " red", + "rank_swap_answer": 21, + "rank_answer": 0, + "logp_swap_answer": -5.3427, + "logp_answer": -1.0719, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.2241, + "delta_norm_band_sum": 157.9715, + "delta_norm_lastpos_sum": 11.9342, + "delta_norm_per_layer_sum": { + "9": 22.1818, + "10": 17.0011, + "11": 10.5778, + "12": 6.7399, + "13": 9.479, + "14": 14.8104, + "15": 11.3915, + "16": 7.3331, + "17": 8.1528, + "18": 9.6933, + "19": 8.1175, + "20": 10.3318, + "21": 9.3162, + "22": 12.8454 + }, + "delta_norm_per_layer_max": { + "9": 3.3495, + "10": 3.1516, + "11": 2.6539, + "12": 1.433, + "13": 2.4767, + "14": 2.9724, + "15": 2.7797, + "16": 2.377, + "17": 2.8613, + "18": 3.0177, + "19": 2.4322, + "20": 2.2767, + "21": 1.5396, + "22": 1.9869 + }, + "gram_cond_max": 12.599, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 107.4571 + } + }, + "admissible": { + "a1": false + } + }, + { + "name": "person-firstname-einstein", + "category": "person-firstname", + "prompt": "Fact: The first name of the physicist who developed the theory of general relativity was", + "intermediate": "Einstein", + "swap_to": "Newton", + "answer": "Albert", + "swap_answer": "Isaac", + "clean": { + "top1": " Albert", + "rank_swap_answer": 26, + "rank_answer": 0, + "logp_swap_answer": -5.1163, + "logp_answer": -2.273, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " Albert", + "rank_swap_answer": 26, + "rank_answer": 0, + "logp_swap_answer": -5.1163, + "logp_answer": -2.273, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " named", + "rank_swap_answer": 22, + "rank_answer": 1, + "logp_swap_answer": -4.9023, + "logp_answer": -3.1692, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0852, + "delta_norm_band_sum": 202.4671, + "delta_norm_lastpos_sum": 17.3784, + "delta_norm_per_layer_sum": { + "9": 15.0253, + "10": 14.242, + "11": 11.3418, + "12": 9.307, + "13": 10.0118, + "14": 10.2301, + "15": 13.5359, + "16": 9.9976, + "17": 10.7608, + "18": 14.5797, + "19": 13.384, + "20": 18.7932, + "21": 26.0096, + "22": 25.2484 + }, + "delta_norm_per_layer_max": { + "9": 2.4746, + "10": 2.229, + "11": 2.2946, + "12": 2.5323, + "13": 2.2768, + "14": 2.516, + "15": 1.5393, + "16": 1.3364, + "17": 1.6793, + "18": 2.1192, + "19": 2.5829, + "20": 2.9624, + "21": 3.2768, + "22": 3.3734 + }, + "gram_cond_max": 4.477, + "n_degenerate_layers": 0 + }, + "rand_target@a1": { + "top1": " Albert", + "rank_swap_answer": 19, + "rank_answer": 0, + "logp_swap_answer": -4.7872, + "logp_answer": -2.6311, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0455, + "delta_norm_band_sum": 202.4671, + "delta_norm_lastpos_sum": 17.3784, + "delta_norm_per_layer_sum": { + "9": 15.0253, + "10": 14.242, + "11": 11.3418, + "12": 9.307, + "13": 10.0118, + "14": 10.2301, + "15": 13.5359, + "16": 9.9976, + "17": 10.7608, + "18": 14.5797, + "19": 13.384, + "20": 18.7932, + "21": 26.0096, + "22": 25.2484 + }, + "delta_norm_per_layer_max": { + "9": 2.4746, + "10": 2.229, + "11": 2.2946, + "12": 2.5323, + "13": 2.2768, + "14": 2.516, + "15": 1.5393, + "16": 1.3364, + "17": 1.6793, + "18": 2.1192, + "19": 2.5829, + "20": 2.9624, + "21": 3.2768, + "22": 3.3734 + }, + "gram_cond_max": 8.756, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " Albert", + "rank_swap_answer": 45, + "rank_answer": 0, + "logp_swap_answer": -5.6974, + "logp_answer": -2.0197, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0526, + "delta_norm_band_sum": 202.4672, + "delta_norm_lastpos_sum": 17.3784, + "delta_norm_per_layer_sum": { + "9": 15.0253, + "10": 14.242, + "11": 11.3418, + "12": 9.307, + "13": 10.0118, + "14": 10.2301, + "15": 13.5359, + "16": 9.9976, + "17": 10.7608, + "18": 14.5797, + "19": 13.384, + "20": 18.7932, + "21": 26.0096, + "22": 25.2484 + }, + "delta_norm_per_layer_max": { + "9": 2.4746, + "10": 2.229, + "11": 2.2946, + "12": 2.5323, + "13": 2.2768, + "14": 2.516, + "15": 1.5393, + "16": 1.3364, + "17": 1.6793, + "18": 2.1192, + "19": 2.5829, + "20": 2.9624, + "21": 3.2768, + "22": 3.3734 + }, + "gram_cond_max": 8.483, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " Isaac", + "rank_swap_answer": 0, + "rank_answer": 42, + "logp_swap_answer": -1.406, + "logp_answer": -5.7077, + "flip_to_swap_answer": true, + "kl_vs_clean": 0.8054, + "delta_norm_band_sum": 202.4671, + "delta_norm_lastpos_sum": 17.3784, + "delta_norm_per_layer_sum": { + "9": 15.0253, + "10": 14.242, + "11": 11.3418, + "12": 9.307, + "13": 10.0118, + "14": 10.2301, + "15": 13.5359, + "16": 9.9976, + "17": 10.7608, + "18": 14.5797, + "19": 13.384, + "20": 18.7932, + "21": 26.0096, + "22": 25.2484 + }, + "delta_norm_per_layer_max": { + "9": 2.4746, + "10": 2.229, + "11": 2.2946, + "12": 2.5323, + "13": 2.2768, + "14": 2.516, + "15": 1.5393, + "16": 1.3364, + "17": 1.6793, + "18": 2.1192, + "19": 2.5829, + "20": 2.9624, + "21": 3.2768, + "22": 3.3734 + }, + "gram_cond_max": 4.304, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 176.7198 + } + }, + "admissible": { + "a1": true + } + }, + { + "name": "rhyme-spoon-orbit", + "category": "multihop", + "prompt": "Fact: The celestial body whose name rhymes with 'spoon' orbits the planet called", + "intermediate": "moon", + "swap_to": "Europa", + "answer": "Earth", + "swap_answer": "Jupiter", + "clean": { + "top1": " Earth", + "rank_swap_answer": 4, + "rank_answer": 0, + "logp_swap_answer": -2.9155, + "logp_answer": -2.0212, + "flip_to_swap_answer": false + }, + "arms": { + "identity": { + "top1": " Earth", + "rank_swap_answer": 4, + "rank_answer": 0, + "logp_swap_answer": -2.9155, + "logp_answer": -2.0212, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0 + }, + "swap@a1": { + "top1": " Earth", + "rank_swap_answer": 4, + "rank_answer": 0, + "logp_swap_answer": -2.8757, + "logp_answer": -1.9074, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0091, + "delta_norm_band_sum": 568.1537, + "delta_norm_lastpos_sum": 18.7854, + "delta_norm_per_layer_sum": { + "9": 43.6367, + "10": 50.494, + "11": 39.6333, + "12": 28.1908, + "13": 25.2312, + "14": 23.5359, + "15": 20.1228, + "16": 37.3531, + "17": 50.3055, + "18": 43.8018, + "19": 55.4261, + "20": 49.1226, + "21": 52.1762, + "22": 49.1237 + }, + "delta_norm_per_layer_max": { + "9": 5.4543, + "10": 6.3285, + "11": 5.2151, + "12": 4.7393, + "13": 4.3433, + "14": 4.0597, + "15": 3.5487, + "16": 3.8849, + "17": 4.6689, + "18": 4.1539, + "19": 5.3789, + "20": 4.766, + "21": 5.5939, + "22": 5.3463 + }, + "gram_cond_max": 2.438, + "n_degenerate_layers": 0 + }, + "rand_target@a1": { + "top1": " Earth", + "rank_swap_answer": 1, + "rank_answer": 0, + "logp_swap_answer": -2.4341, + "logp_answer": -1.8051, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0824, + "delta_norm_band_sum": 568.1537, + "delta_norm_lastpos_sum": 18.7854, + "delta_norm_per_layer_sum": { + "9": 43.6367, + "10": 50.494, + "11": 39.6333, + "12": 28.1908, + "13": 25.2312, + "14": 23.5358, + "15": 20.1228, + "16": 37.3531, + "17": 50.3055, + "18": 43.8018, + "19": 55.4261, + "20": 49.1226, + "21": 52.1762, + "22": 49.1237 + }, + "delta_norm_per_layer_max": { + "9": 5.4543, + "10": 6.3285, + "11": 5.2151, + "12": 4.7393, + "13": 4.3433, + "14": 4.0597, + "15": 3.5487, + "16": 3.8849, + "17": 4.6689, + "18": 4.1539, + "19": 5.3789, + "20": 4.766, + "21": 5.5939, + "22": 5.3463 + }, + "gram_cond_max": 6.859, + "n_degenerate_layers": 0 + }, + "orth@a1": { + "top1": " Earth", + "rank_swap_answer": 3, + "rank_answer": 0, + "logp_swap_answer": -2.778, + "logp_answer": -1.7552, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.0645, + "delta_norm_band_sum": 568.1537, + "delta_norm_lastpos_sum": 18.7854, + "delta_norm_per_layer_sum": { + "9": 43.6368, + "10": 50.494, + "11": 39.6333, + "12": 28.1908, + "13": 25.2312, + "14": 23.5359, + "15": 20.1228, + "16": 37.3531, + "17": 50.3055, + "18": 43.8018, + "19": 55.4261, + "20": 49.1226, + "21": 52.1762, + "22": 49.1237 + }, + "delta_norm_per_layer_max": { + "9": 5.4543, + "10": 6.3285, + "11": 5.2151, + "12": 4.7393, + "13": 4.3433, + "14": 4.0597, + "15": 3.5487, + "16": 3.8849, + "17": 4.6689, + "18": 4.1539, + "19": 5.3789, + "20": 4.766, + "21": 5.5939, + "22": 5.3463 + }, + "gram_cond_max": 6.534, + "n_degenerate_layers": 0 + }, + "oracle@a1": { + "top1": " '", + "rank_swap_answer": 6, + "rank_answer": 1, + "logp_swap_answer": -3.0102, + "logp_answer": -2.404, + "flip_to_swap_answer": false, + "kl_vs_clean": 0.019, + "delta_norm_band_sum": 568.1537, + "delta_norm_lastpos_sum": 18.7854, + "delta_norm_per_layer_sum": { + "9": 43.6368, + "10": 50.494, + "11": 39.6333, + "12": 28.1908, + "13": 25.2312, + "14": 23.5358, + "15": 20.1228, + "16": 37.3531, + "17": 50.3055, + "18": 43.8018, + "19": 55.4261, + "20": 49.1226, + "21": 52.1762, + "22": 49.1237 + }, + "delta_norm_per_layer_max": { + "9": 5.4543, + "10": 6.3285, + "11": 5.2151, + "12": 4.7393, + "13": 4.3433, + "14": 4.0597, + "15": 3.5487, + "16": 3.8849, + "17": 4.6689, + "18": 4.1539, + "19": 5.3789, + "20": 4.766, + "21": 5.5939, + "22": 5.3463 + }, + "gram_cond_max": 4.04, + "n_degenerate_layers": 0, + "native_dose_if_unmatched": 307.4968 + } + }, + "admissible": { + "a1": false + } + } + ], + "runtime_s": 24.6 +} \ No newline at end of file diff --git a/results/t9_n_convergence_160m.json b/results/t9_n_convergence_160m.json new file mode 100644 index 0000000000000000000000000000000000000000..d255a6ef92fb0c57f8a76a87518ea6fa4ee533d8 --- /dev/null +++ b/results/t9_n_convergence_160m.json @@ -0,0 +1,1076 @@ +{ + "experiment": "T9 n-convergence \u2014 operator vs task (THESIS.md C3)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "model": "EleutherAI/pythia-160m-deduped", + "ns": [ + 1, + 5, + 10, + 25, + 50, + 100, + 200 + ], + "reference_n": 200, + "top_r": 8, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "dim_batch": 128, + "centered": false, + "nested_prefixes": true, + "task_metric": { + "eval": "multihop", + "by_n": { + "1": { + "auc": 0.132, + "pass@10": 0.15 + }, + "5": { + "auc": 0.1257, + "pass@10": 0.15 + }, + "10": { + "auc": 0.1201, + "pass@10": 0.125 + }, + "25": { + "auc": 0.1114, + "pass@10": 0.1 + }, + "50": { + "auc": 0.1095, + "pass@10": 0.1 + }, + "100": { + "auc": 0.1138, + "pass@10": 0.125 + }, + "200": { + "auc": 0.1138, + "pass@10": 0.125 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124", + "device": "cuda", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "n": 1, + "layer": 0, + "stable_rank": 3.2067, + "effective_rank": 44.1386, + "top_sv": 22.9686, + "fro": 41.13, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.667178, + "rel_op_dist_to_ref": 2.110458, + "subspace_cos_top_r_vs_ref": 0.44557 + }, + { + "n": 1, + "layer": 1, + "stable_rank": 3.53, + "effective_rank": 52.4956, + "top_sv": 20.2576, + "fro": 38.0606, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.714725, + "rel_op_dist_to_ref": 1.839261, + "subspace_cos_top_r_vs_ref": 0.466817 + }, + { + "n": 1, + "layer": 2, + "stable_rank": 3.349, + "effective_rank": 56.9123, + "top_sv": 19.1074, + "fro": 34.967, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.74315, + "rel_op_dist_to_ref": 1.641927, + "subspace_cos_top_r_vs_ref": 0.532326 + }, + { + "n": 1, + "layer": 3, + "stable_rank": 4.2204, + "effective_rank": 82.6545, + "top_sv": 16.1229, + "fro": 33.1223, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.770739, + "rel_op_dist_to_ref": 1.341595, + "subspace_cos_top_r_vs_ref": 0.475861 + }, + { + "n": 1, + "layer": 4, + "stable_rank": 4.3374, + "effective_rank": 92.5019, + "top_sv": 17.1039, + "fro": 35.6212, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.79796, + "rel_op_dist_to_ref": 1.169548, + "subspace_cos_top_r_vs_ref": 0.52876 + }, + { + "n": 1, + "layer": 5, + "stable_rank": 3.9978, + "effective_rank": 98.4186, + "top_sv": 18.4952, + "fro": 36.98, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.822193, + "rel_op_dist_to_ref": 1.066404, + "subspace_cos_top_r_vs_ref": 0.517906 + }, + { + "n": 1, + "layer": 6, + "stable_rank": 5.6674, + "effective_rank": 152.2497, + "top_sv": 15.579, + "fro": 37.0877, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.861224, + "rel_op_dist_to_ref": 0.965718, + "subspace_cos_top_r_vs_ref": 0.51688 + }, + { + "n": 1, + "layer": 7, + "stable_rank": 7.6341, + "effective_rank": 234.5129, + "top_sv": 12.0398, + "fro": 33.2655, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.89844, + "rel_op_dist_to_ref": 0.692038, + "subspace_cos_top_r_vs_ref": 0.715488 + }, + { + "n": 1, + "layer": 8, + "stable_rank": 9.7216, + "effective_rank": 324.3381, + "top_sv": 10.9764, + "fro": 34.2235, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.935169, + "rel_op_dist_to_ref": 0.448103, + "subspace_cos_top_r_vs_ref": 0.657872 + }, + { + "n": 1, + "layer": 9, + "stable_rank": 9.7915, + "effective_rank": 393.4176, + "top_sv": 10.8056, + "fro": 33.8118, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.958693, + "rel_op_dist_to_ref": 0.295125, + "subspace_cos_top_r_vs_ref": 0.736729 + }, + { + "n": 1, + "layer": 10, + "stable_rank": 7.1178, + "effective_rank": 394.8333, + "top_sv": 12.2746, + "fro": 32.7476, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.981817, + "rel_op_dist_to_ref": 0.218186, + "subspace_cos_top_r_vs_ref": 0.814938 + }, + { + "n": 5, + "layer": 0, + "stable_rank": 5.9924, + "effective_rank": 108.7775, + "top_sv": 13.1052, + "fro": 32.0805, + "dispersion": 0.471434, + "heavy_tail_ratio": 1.1109, + "cos_to_ref": 0.872357, + "rel_op_dist_to_ref": 0.910656, + "subspace_cos_top_r_vs_ref": 0.61786 + }, + { + "n": 5, + "layer": 1, + "stable_rank": 6.4361, + "effective_rank": 115.4077, + "top_sv": 12.1547, + "fro": 30.8356, + "dispersion": 0.37579, + "heavy_tail_ratio": 1.1359, + "cos_to_ref": 0.893804, + "rel_op_dist_to_ref": 0.788877, + "subspace_cos_top_r_vs_ref": 0.63024 + }, + { + "n": 5, + "layer": 2, + "stable_rank": 6.2874, + "effective_rank": 120.0756, + "top_sv": 11.5239, + "fro": 28.8958, + "dispersion": 0.323088, + "heavy_tail_ratio": 1.1396, + "cos_to_ref": 0.910367, + "rel_op_dist_to_ref": 0.611396, + "subspace_cos_top_r_vs_ref": 0.762253 + }, + { + "n": 5, + "layer": 3, + "stable_rank": 6.7388, + "effective_rank": 137.1653, + "top_sv": 11.2011, + "fro": 29.0771, + "dispersion": 0.277086, + "heavy_tail_ratio": 1.1456, + "cos_to_ref": 0.919646, + "rel_op_dist_to_ref": 0.518572, + "subspace_cos_top_r_vs_ref": 0.715403 + }, + { + "n": 5, + "layer": 4, + "stable_rank": 6.4176, + "effective_rank": 144.7056, + "top_sv": 12.7097, + "fro": 32.1974, + "dispersion": 0.233123, + "heavy_tail_ratio": 1.1518, + "cos_to_ref": 0.928282, + "rel_op_dist_to_ref": 0.428584, + "subspace_cos_top_r_vs_ref": 0.696986 + }, + { + "n": 5, + "layer": 5, + "stable_rank": 5.8956, + "effective_rank": 154.5362, + "top_sv": 13.9654, + "fro": 33.9088, + "dispersion": 0.191985, + "heavy_tail_ratio": 1.1368, + "cos_to_ref": 0.937113, + "rel_op_dist_to_ref": 0.382277, + "subspace_cos_top_r_vs_ref": 0.72304 + }, + { + "n": 5, + "layer": 6, + "stable_rank": 7.5269, + "effective_rank": 198.3266, + "top_sv": 12.9688, + "fro": 35.5799, + "dispersion": 0.155882, + "heavy_tail_ratio": 1.1582, + "cos_to_ref": 0.942791, + "rel_op_dist_to_ref": 0.371495, + "subspace_cos_top_r_vs_ref": 0.633842 + }, + { + "n": 5, + "layer": 7, + "stable_rank": 7.104, + "effective_rank": 247.0668, + "top_sv": 12.1826, + "fro": 32.4705, + "dispersion": 0.105936, + "heavy_tail_ratio": 1.0956, + "cos_to_ref": 0.956975, + "rel_op_dist_to_ref": 0.268203, + "subspace_cos_top_r_vs_ref": 0.884751 + }, + { + "n": 5, + "layer": 8, + "stable_rank": 7.8218, + "effective_rank": 301.5766, + "top_sv": 12.389, + "fro": 34.6488, + "dispersion": 0.067588, + "heavy_tail_ratio": 1.0736, + "cos_to_ref": 0.966862, + "rel_op_dist_to_ref": 0.21631, + "subspace_cos_top_r_vs_ref": 0.777785 + }, + { + "n": 5, + "layer": 9, + "stable_rank": 7.9347, + "effective_rank": 358.53, + "top_sv": 12.2707, + "fro": 34.5644, + "dispersion": 0.039385, + "heavy_tail_ratio": 1.0512, + "cos_to_ref": 0.977083, + "rel_op_dist_to_ref": 0.165814, + "subspace_cos_top_r_vs_ref": 0.803761 + }, + { + "n": 5, + "layer": 10, + "stable_rank": 5.439, + "effective_rank": 319.3062, + "top_sv": 14.6354, + "fro": 34.1318, + "dispersion": 0.014961, + "heavy_tail_ratio": 1.0361, + "cos_to_ref": 0.991458, + "rel_op_dist_to_ref": 0.070818, + "subspace_cos_top_r_vs_ref": 0.865165 + }, + { + "n": 10, + "layer": 0, + "stable_rank": 7.4568, + "effective_rank": 126.3313, + "top_sv": 10.8889, + "fro": 29.7343, + "dispersion": 0.571935, + "heavy_tail_ratio": 1.1596, + "cos_to_ref": 0.932671, + "rel_op_dist_to_ref": 0.588178, + "subspace_cos_top_r_vs_ref": 0.780853 + }, + { + "n": 10, + "layer": 1, + "stable_rank": 7.5121, + "effective_rank": 129.5136, + "top_sv": 10.5948, + "fro": 29.0381, + "dispersion": 0.438509, + "heavy_tail_ratio": 1.1796, + "cos_to_ref": 0.945322, + "rel_op_dist_to_ref": 0.50941, + "subspace_cos_top_r_vs_ref": 0.773535 + }, + { + "n": 10, + "layer": 2, + "stable_rank": 6.7491, + "effective_rank": 129.4547, + "top_sv": 10.6494, + "fro": 27.6659, + "dispersion": 0.364307, + "heavy_tail_ratio": 1.1722, + "cos_to_ref": 0.953925, + "rel_op_dist_to_ref": 0.399208, + "subspace_cos_top_r_vs_ref": 0.835483 + }, + { + "n": 10, + "layer": 3, + "stable_rank": 6.4929, + "effective_rank": 139.5824, + "top_sv": 11.0445, + "fro": 28.1427, + "dispersion": 0.307737, + "heavy_tail_ratio": 1.1697, + "cos_to_ref": 0.958981, + "rel_op_dist_to_ref": 0.344328, + "subspace_cos_top_r_vs_ref": 0.808301 + }, + { + "n": 10, + "layer": 4, + "stable_rank": 5.9531, + "effective_rank": 144.014, + "top_sv": 12.8816, + "fro": 31.4294, + "dispersion": 0.254248, + "heavy_tail_ratio": 1.17, + "cos_to_ref": 0.962911, + "rel_op_dist_to_ref": 0.30161, + "subspace_cos_top_r_vs_ref": 0.756526 + }, + { + "n": 10, + "layer": 5, + "stable_rank": 5.6602, + "effective_rank": 157.1941, + "top_sv": 13.9542, + "fro": 33.1985, + "dispersion": 0.209761, + "heavy_tail_ratio": 1.1526, + "cos_to_ref": 0.967672, + "rel_op_dist_to_ref": 0.25947, + "subspace_cos_top_r_vs_ref": 0.8526 + }, + { + "n": 10, + "layer": 6, + "stable_rank": 7.7536, + "effective_rank": 208.6626, + "top_sv": 12.456, + "fro": 34.6839, + "dispersion": 0.174223, + "heavy_tail_ratio": 1.1788, + "cos_to_ref": 0.969599, + "rel_op_dist_to_ref": 0.279175, + "subspace_cos_top_r_vs_ref": 0.839817 + }, + { + "n": 10, + "layer": 7, + "stable_rank": 7.1353, + "effective_rank": 255.6203, + "top_sv": 11.8921, + "fro": 31.7659, + "dispersion": 0.121544, + "heavy_tail_ratio": 1.1121, + "cos_to_ref": 0.977608, + "rel_op_dist_to_ref": 0.182769, + "subspace_cos_top_r_vs_ref": 0.947055 + }, + { + "n": 10, + "layer": 8, + "stable_rank": 7.7011, + "effective_rank": 305.1233, + "top_sv": 12.3418, + "fro": 34.2492, + "dispersion": 0.07886, + "heavy_tail_ratio": 1.0805, + "cos_to_ref": 0.982679, + "rel_op_dist_to_ref": 0.156413, + "subspace_cos_top_r_vs_ref": 0.803199 + }, + { + "n": 10, + "layer": 9, + "stable_rank": 7.806, + "effective_rank": 359.9796, + "top_sv": 12.3007, + "fro": 34.367, + "dispersion": 0.047668, + "heavy_tail_ratio": 1.053, + "cos_to_ref": 0.98802, + "rel_op_dist_to_ref": 0.124267, + "subspace_cos_top_r_vs_ref": 0.945915 + }, + { + "n": 10, + "layer": 10, + "stable_rank": 5.2261, + "effective_rank": 309.3293, + "top_sv": 14.9849, + "fro": 34.2562, + "dispersion": 0.016628, + "heavy_tail_ratio": 1.0315, + "cos_to_ref": 0.99562, + "rel_op_dist_to_ref": 0.044313, + "subspace_cos_top_r_vs_ref": 0.973082 + }, + { + "n": 25, + "layer": 0, + "stable_rank": 6.4282, + "effective_rank": 118.7458, + "top_sv": 10.8166, + "fro": 27.4242, + "dispersion": 0.705901, + "heavy_tail_ratio": 1.3843, + "cos_to_ref": 0.952695, + "rel_op_dist_to_ref": 0.504107, + "subspace_cos_top_r_vs_ref": 0.824339 + }, + { + "n": 25, + "layer": 1, + "stable_rank": 6.6358, + "effective_rank": 123.614, + "top_sv": 10.4566, + "fro": 26.9361, + "dispersion": 0.553394, + "heavy_tail_ratio": 1.3585, + "cos_to_ref": 0.961016, + "rel_op_dist_to_ref": 0.431124, + "subspace_cos_top_r_vs_ref": 0.78933 + }, + { + "n": 25, + "layer": 2, + "stable_rank": 5.9887, + "effective_rank": 122.9825, + "top_sv": 10.5948, + "fro": 25.9274, + "dispersion": 0.4584, + "heavy_tail_ratio": 1.3146, + "cos_to_ref": 0.967351, + "rel_op_dist_to_ref": 0.340109, + "subspace_cos_top_r_vs_ref": 0.901698 + }, + { + "n": 25, + "layer": 3, + "stable_rank": 5.8601, + "effective_rank": 133.6615, + "top_sv": 10.9388, + "fro": 26.4799, + "dispersion": 0.379718, + "heavy_tail_ratio": 1.2102, + "cos_to_ref": 0.97116, + "rel_op_dist_to_ref": 0.290912, + "subspace_cos_top_r_vs_ref": 0.866996 + }, + { + "n": 25, + "layer": 4, + "stable_rank": 5.4879, + "effective_rank": 139.1677, + "top_sv": 12.6768, + "fro": 29.6968, + "dispersion": 0.313504, + "heavy_tail_ratio": 1.21, + "cos_to_ref": 0.974017, + "rel_op_dist_to_ref": 0.251136, + "subspace_cos_top_r_vs_ref": 0.814614 + }, + { + "n": 25, + "layer": 5, + "stable_rank": 5.2695, + "effective_rank": 151.3637, + "top_sv": 13.8295, + "fro": 31.7461, + "dispersion": 0.265596, + "heavy_tail_ratio": 1.2066, + "cos_to_ref": 0.97723, + "rel_op_dist_to_ref": 0.226536, + "subspace_cos_top_r_vs_ref": 0.8841 + }, + { + "n": 25, + "layer": 6, + "stable_rank": 7.2226, + "effective_rank": 205.7614, + "top_sv": 12.4107, + "fro": 33.3534, + "dispersion": 0.214991, + "heavy_tail_ratio": 1.2051, + "cos_to_ref": 0.979064, + "rel_op_dist_to_ref": 0.216204, + "subspace_cos_top_r_vs_ref": 0.861693 + }, + { + "n": 25, + "layer": 7, + "stable_rank": 6.638, + "effective_rank": 248.1137, + "top_sv": 12.0673, + "fro": 31.0904, + "dispersion": 0.150786, + "heavy_tail_ratio": 1.1217, + "cos_to_ref": 0.984436, + "rel_op_dist_to_ref": 0.155476, + "subspace_cos_top_r_vs_ref": 0.963569 + }, + { + "n": 25, + "layer": 8, + "stable_rank": 7.5087, + "effective_rank": 305.3083, + "top_sv": 12.251, + "fro": 33.57, + "dispersion": 0.101247, + "heavy_tail_ratio": 1.0911, + "cos_to_ref": 0.987767, + "rel_op_dist_to_ref": 0.132535, + "subspace_cos_top_r_vs_ref": 0.838095 + }, + { + "n": 25, + "layer": 9, + "stable_rank": 7.7155, + "effective_rank": 361.4228, + "top_sv": 12.2122, + "fro": 33.9211, + "dispersion": 0.059753, + "heavy_tail_ratio": 1.0607, + "cos_to_ref": 0.991545, + "rel_op_dist_to_ref": 0.098319, + "subspace_cos_top_r_vs_ref": 0.943804 + }, + { + "n": 25, + "layer": 10, + "stable_rank": 5.3117, + "effective_rank": 315.3587, + "top_sv": 14.7433, + "fro": 33.9788, + "dispersion": 0.022144, + "heavy_tail_ratio": 1.0371, + "cos_to_ref": 0.996782, + "rel_op_dist_to_ref": 0.049883, + "subspace_cos_top_r_vs_ref": 0.980356 + }, + { + "n": 50, + "layer": 0, + "stable_rank": 7.4984, + "effective_rank": 129.9975, + "top_sv": 9.632, + "fro": 26.3753, + "dispersion": 0.749762, + "heavy_tail_ratio": 1.4212, + "cos_to_ref": 0.979027, + "rel_op_dist_to_ref": 0.299976, + "subspace_cos_top_r_vs_ref": 0.88985 + }, + { + "n": 50, + "layer": 1, + "stable_rank": 7.4934, + "effective_rank": 133.3251, + "top_sv": 9.5221, + "fro": 26.0655, + "dispersion": 0.587571, + "heavy_tail_ratio": 1.3887, + "cos_to_ref": 0.982382, + "rel_op_dist_to_ref": 0.263245, + "subspace_cos_top_r_vs_ref": 0.865746 + }, + { + "n": 50, + "layer": 2, + "stable_rank": 6.3992, + "effective_rank": 130.3712, + "top_sv": 9.9734, + "fro": 25.2291, + "dispersion": 0.487417, + "heavy_tail_ratio": 1.3378, + "cos_to_ref": 0.984652, + "rel_op_dist_to_ref": 0.219636, + "subspace_cos_top_r_vs_ref": 0.937846 + }, + { + "n": 50, + "layer": 3, + "stable_rank": 5.8911, + "effective_rank": 135.9932, + "top_sv": 10.7352, + "fro": 26.0558, + "dispersion": 0.403426, + "heavy_tail_ratio": 1.2858, + "cos_to_ref": 0.98633, + "rel_op_dist_to_ref": 0.19222, + "subspace_cos_top_r_vs_ref": 0.856443 + }, + { + "n": 50, + "layer": 4, + "stable_rank": 5.4406, + "effective_rank": 140.8758, + "top_sv": 12.5351, + "fro": 29.2382, + "dispersion": 0.332938, + "heavy_tail_ratio": 1.2658, + "cos_to_ref": 0.987885, + "rel_op_dist_to_ref": 0.161882, + "subspace_cos_top_r_vs_ref": 0.834879 + }, + { + "n": 50, + "layer": 5, + "stable_rank": 5.2472, + "effective_rank": 155.0784, + "top_sv": 13.6408, + "fro": 31.2463, + "dispersion": 0.27927, + "heavy_tail_ratio": 1.2193, + "cos_to_ref": 0.989829, + "rel_op_dist_to_ref": 0.126734, + "subspace_cos_top_r_vs_ref": 0.954622 + }, + { + "n": 50, + "layer": 6, + "stable_rank": 7.328, + "effective_rank": 211.6983, + "top_sv": 12.1488, + "fro": 32.8869, + "dispersion": 0.231752, + "heavy_tail_ratio": 1.2139, + "cos_to_ref": 0.990235, + "rel_op_dist_to_ref": 0.123255, + "subspace_cos_top_r_vs_ref": 0.934372 + }, + { + "n": 50, + "layer": 7, + "stable_rank": 6.5091, + "effective_rank": 249.0737, + "top_sv": 12.1156, + "fro": 30.9104, + "dispersion": 0.162948, + "heavy_tail_ratio": 1.1224, + "cos_to_ref": 0.99259, + "rel_op_dist_to_ref": 0.093567, + "subspace_cos_top_r_vs_ref": 0.977624 + }, + { + "n": 50, + "layer": 8, + "stable_rank": 7.2737, + "effective_rank": 302.6149, + "top_sv": 12.4547, + "fro": 33.5898, + "dispersion": 0.110412, + "heavy_tail_ratio": 1.0859, + "cos_to_ref": 0.994178, + "rel_op_dist_to_ref": 0.081533, + "subspace_cos_top_r_vs_ref": 0.868376 + }, + { + "n": 50, + "layer": 9, + "stable_rank": 7.4391, + "effective_rank": 355.0888, + "top_sv": 12.4676, + "fro": 34.0047, + "dispersion": 0.065904, + "heavy_tail_ratio": 1.0551, + "cos_to_ref": 0.995916, + "rel_op_dist_to_ref": 0.062809, + "subspace_cos_top_r_vs_ref": 0.962355 + }, + { + "n": 50, + "layer": 10, + "stable_rank": 5.1282, + "effective_rank": 305.3875, + "top_sv": 15.0793, + "fro": 34.1473, + "dispersion": 0.023799, + "heavy_tail_ratio": 1.0311, + "cos_to_ref": 0.998474, + "rel_op_dist_to_ref": 0.027021, + "subspace_cos_top_r_vs_ref": 0.994677 + }, + { + "n": 100, + "layer": 0, + "stable_rank": 7.3165, + "effective_rank": 130.2373, + "top_sv": 9.4819, + "fro": 25.6474, + "dispersion": 0.840929, + "heavy_tail_ratio": 1.4249, + "cos_to_ref": 0.993206, + "rel_op_dist_to_ref": 0.142551, + "subspace_cos_top_r_vs_ref": 0.987916 + }, + { + "n": 100, + "layer": 1, + "stable_rank": 7.3466, + "effective_rank": 133.9076, + "top_sv": 9.3668, + "fro": 25.3881, + "dispersion": 0.662458, + "heavy_tail_ratio": 1.3932, + "cos_to_ref": 0.994192, + "rel_op_dist_to_ref": 0.133352, + "subspace_cos_top_r_vs_ref": 0.986723 + }, + { + "n": 100, + "layer": 2, + "stable_rank": 6.3326, + "effective_rank": 131.2959, + "top_sv": 9.7862, + "fro": 24.6264, + "dispersion": 0.552897, + "heavy_tail_ratio": 1.3413, + "cos_to_ref": 0.994872, + "rel_op_dist_to_ref": 0.113612, + "subspace_cos_top_r_vs_ref": 0.986932 + }, + { + "n": 100, + "layer": 3, + "stable_rank": 5.9368, + "effective_rank": 138.4671, + "top_sv": 10.4303, + "fro": 25.4136, + "dispersion": 0.463666, + "heavy_tail_ratio": 1.2909, + "cos_to_ref": 0.995382, + "rel_op_dist_to_ref": 0.101302, + "subspace_cos_top_r_vs_ref": 0.901942 + }, + { + "n": 100, + "layer": 4, + "stable_rank": 5.5036, + "effective_rank": 144.0574, + "top_sv": 12.1342, + "fro": 28.4665, + "dispersion": 0.385973, + "heavy_tail_ratio": 1.275, + "cos_to_ref": 0.995912, + "rel_op_dist_to_ref": 0.08302, + "subspace_cos_top_r_vs_ref": 0.984015 + }, + { + "n": 100, + "layer": 5, + "stable_rank": 5.2671, + "effective_rank": 157.647, + "top_sv": 13.2697, + "fro": 30.4541, + "dispersion": 0.32027, + "heavy_tail_ratio": 1.2314, + "cos_to_ref": 0.99632, + "rel_op_dist_to_ref": 0.074751, + "subspace_cos_top_r_vs_ref": 0.981976 + }, + { + "n": 100, + "layer": 6, + "stable_rank": 7.2409, + "effective_rank": 213.6203, + "top_sv": 11.9334, + "fro": 32.1114, + "dispersion": 0.274004, + "heavy_tail_ratio": 1.2224, + "cos_to_ref": 0.996472, + "rel_op_dist_to_ref": 0.079996, + "subspace_cos_top_r_vs_ref": 0.974155 + }, + { + "n": 100, + "layer": 7, + "stable_rank": 6.377, + "effective_rank": 247.4022, + "top_sv": 12.0676, + "fro": 30.4738, + "dispersion": 0.19803, + "heavy_tail_ratio": 1.1217, + "cos_to_ref": 0.997349, + "rel_op_dist_to_ref": 0.061463, + "subspace_cos_top_r_vs_ref": 0.992229 + }, + { + "n": 100, + "layer": 8, + "stable_rank": 7.0907, + "effective_rank": 299.5662, + "top_sv": 12.4886, + "fro": 33.2549, + "dispersion": 0.139622, + "heavy_tail_ratio": 1.0827, + "cos_to_ref": 0.997976, + "rel_op_dist_to_ref": 0.053131, + "subspace_cos_top_r_vs_ref": 0.940145 + }, + { + "n": 100, + "layer": 9, + "stable_rank": 7.3131, + "effective_rank": 352.133, + "top_sv": 12.4914, + "fro": 33.7799, + "dispersion": 0.08735, + "heavy_tail_ratio": 1.0516, + "cos_to_ref": 0.998661, + "rel_op_dist_to_ref": 0.038382, + "subspace_cos_top_r_vs_ref": 0.984076 + }, + { + "n": 100, + "layer": 10, + "stable_rank": 5.0518, + "effective_rank": 300.8528, + "top_sv": 15.1959, + "fro": 34.1542, + "dispersion": 0.030264, + "heavy_tail_ratio": 1.0331, + "cos_to_ref": 0.999529, + "rel_op_dist_to_ref": 0.015709, + "subspace_cos_top_r_vs_ref": 0.997254 + }, + { + "n": 200, + "layer": 0, + "stable_rank": 7.5313, + "effective_rank": 130.0835, + "top_sv": 9.5287, + "fro": 26.1498, + "dispersion": 0.809959, + "heavy_tail_ratio": 1.4094, + "cos_to_ref": 1.00001, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 1, + "stable_rank": 7.5814, + "effective_rank": 134.1696, + "top_sv": 9.3774, + "fro": 25.8198, + "dispersion": 0.641106, + "heavy_tail_ratio": 1.3877, + "cos_to_ref": 1.000012, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 2, + "stable_rank": 6.5058, + "effective_rank": 131.5279, + "top_sv": 9.7953, + "fro": 24.9842, + "dispersion": 0.536249, + "heavy_tail_ratio": 1.3646, + "cos_to_ref": 1.000012, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 3, + "stable_rank": 6.0896, + "effective_rank": 138.8573, + "top_sv": 10.4303, + "fro": 25.7388, + "dispersion": 0.453193, + "heavy_tail_ratio": 1.3543, + "cos_to_ref": 1.000015, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 4, + "stable_rank": 5.5862, + "effective_rank": 143.9211, + "top_sv": 12.1954, + "fro": 28.8239, + "dispersion": 0.379039, + "heavy_tail_ratio": 1.3327, + "cos_to_ref": 1.00001, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 5, + "stable_rank": 5.2699, + "effective_rank": 155.5186, + "top_sv": 13.4569, + "fro": 30.8919, + "dispersion": 0.314207, + "heavy_tail_ratio": 1.2542, + "cos_to_ref": 1.00001, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 6, + "stable_rank": 7.2423, + "effective_rank": 210.8612, + "top_sv": 12.0808, + "fro": 32.5111, + "dispersion": 0.272192, + "heavy_tail_ratio": 1.2206, + "cos_to_ref": 1.000011, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 7, + "stable_rank": 6.3295, + "effective_rank": 243.6525, + "top_sv": 12.2283, + "fro": 30.7644, + "dispersion": 0.198329, + "heavy_tail_ratio": 1.1305, + "cos_to_ref": 1.000011, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 8, + "stable_rank": 7.05, + "effective_rank": 296.5193, + "top_sv": 12.6184, + "fro": 33.5039, + "dispersion": 0.140055, + "heavy_tail_ratio": 1.095, + "cos_to_ref": 1.000013, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 9, + "stable_rank": 7.2653, + "effective_rank": 349.2328, + "top_sv": 12.5893, + "fro": 33.9332, + "dispersion": 0.087794, + "heavy_tail_ratio": 1.0558, + "cos_to_ref": 1.000019, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 10, + "stable_rank": 5.0004, + "effective_rank": 297.4048, + "top_sv": 15.3142, + "fro": 34.2446, + "dispersion": 0.030351, + "heavy_tail_ratio": 1.0338, + "cos_to_ref": 1.000018, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + } + ], + "runtime_s": 260.0 +} \ No newline at end of file diff --git a/results/t9_n_convergence_410m.json b/results/t9_n_convergence_410m.json new file mode 100644 index 0000000000000000000000000000000000000000..cc8f116714505f869d70880d6ce162136cc746e8 --- /dev/null +++ b/results/t9_n_convergence_410m.json @@ -0,0 +1,2180 @@ +{ + "experiment": "T9 n-convergence \u2014 operator vs task (THESIS.md C3)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "model": "EleutherAI/pythia-410m-deduped", + "ns": [ + 1, + 5, + 10, + 25, + 50, + 100, + 200 + ], + "reference_n": 200, + "top_r": 8, + "layers": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "dim_batch": 128, + "centered": false, + "nested_prefixes": true, + "task_metric": { + "eval": "multihop", + "by_n": { + "1": { + "auc": 0.1858, + "pass@10": 0.125 + }, + "5": { + "auc": 0.2195, + "pass@10": 0.225 + }, + "10": { + "auc": 0.2239, + "pass@10": 0.25 + }, + "25": { + "auc": 0.2189, + "pass@10": 0.2 + }, + "50": { + "auc": 0.2077, + "pass@10": 0.175 + }, + "100": { + "auc": 0.2139, + "pass@10": 0.225 + }, + "200": { + "auc": 0.2214, + "pass@10": 0.225 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.6.0+cu124", + "device": "cuda", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "n": 1, + "layer": 0, + "stable_rank": 4.9736, + "effective_rank": 92.1745, + "top_sv": 8.5361, + "fro": 19.0366, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.496828, + "rel_op_dist_to_ref": 3.580467, + "subspace_cos_top_r_vs_ref": 0.316047 + }, + { + "n": 1, + "layer": 1, + "stable_rank": 4.8271, + "effective_rank": 94.334, + "top_sv": 7.8867, + "fro": 17.3275, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.516042, + "rel_op_dist_to_ref": 3.417923, + "subspace_cos_top_r_vs_ref": 0.328107 + }, + { + "n": 1, + "layer": 2, + "stable_rank": 4.8402, + "effective_rank": 92.2543, + "top_sv": 7.905, + "fro": 17.3911, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.559876, + "rel_op_dist_to_ref": 3.098861, + "subspace_cos_top_r_vs_ref": 0.350483 + }, + { + "n": 1, + "layer": 3, + "stable_rank": 4.9198, + "effective_rank": 97.613, + "top_sv": 7.2409, + "fro": 16.0605, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.593584, + "rel_op_dist_to_ref": 2.74212, + "subspace_cos_top_r_vs_ref": 0.368387 + }, + { + "n": 1, + "layer": 4, + "stable_rank": 5.29, + "effective_rank": 106.4719, + "top_sv": 7.4145, + "fro": 17.053, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.634952, + "rel_op_dist_to_ref": 2.349295, + "subspace_cos_top_r_vs_ref": 0.387339 + }, + { + "n": 1, + "layer": 5, + "stable_rank": 5.3349, + "effective_rank": 116.1418, + "top_sv": 7.1819, + "fro": 16.588, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.654185, + "rel_op_dist_to_ref": 2.013701, + "subspace_cos_top_r_vs_ref": 0.418006 + }, + { + "n": 1, + "layer": 6, + "stable_rank": 7.4171, + "effective_rank": 157.774, + "top_sv": 5.8046, + "fro": 15.8084, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.696479, + "rel_op_dist_to_ref": 1.552857, + "subspace_cos_top_r_vs_ref": 0.444245 + }, + { + "n": 1, + "layer": 7, + "stable_rank": 9.154, + "effective_rank": 187.0869, + "top_sv": 5.4416, + "fro": 16.4638, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.731501, + "rel_op_dist_to_ref": 1.201429, + "subspace_cos_top_r_vs_ref": 0.463088 + }, + { + "n": 1, + "layer": 8, + "stable_rank": 9.6046, + "effective_rank": 202.3158, + "top_sv": 5.7641, + "fro": 17.8635, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.756754, + "rel_op_dist_to_ref": 0.906873, + "subspace_cos_top_r_vs_ref": 0.507241 + }, + { + "n": 1, + "layer": 9, + "stable_rank": 11.3669, + "effective_rank": 232.2674, + "top_sv": 5.6687, + "fro": 19.1115, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.768023, + "rel_op_dist_to_ref": 0.884338, + "subspace_cos_top_r_vs_ref": 0.496752 + }, + { + "n": 1, + "layer": 10, + "stable_rank": 12.6316, + "effective_rank": 251.4744, + "top_sv": 5.9113, + "fro": 21.0092, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.787836, + "rel_op_dist_to_ref": 0.710601, + "subspace_cos_top_r_vs_ref": 0.489498 + }, + { + "n": 1, + "layer": 11, + "stable_rank": 13.5228, + "effective_rank": 279.6542, + "top_sv": 6.4345, + "fro": 23.6614, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.800962, + "rel_op_dist_to_ref": 0.648603, + "subspace_cos_top_r_vs_ref": 0.50172 + }, + { + "n": 1, + "layer": 12, + "stable_rank": 22.3899, + "effective_rank": 341.4007, + "top_sv": 5.3179, + "fro": 25.1627, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.827988, + "rel_op_dist_to_ref": 0.714382, + "subspace_cos_top_r_vs_ref": 0.516978 + }, + { + "n": 1, + "layer": 13, + "stable_rank": 25.9353, + "effective_rank": 380.3207, + "top_sv": 5.2338, + "fro": 26.6536, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.843052, + "rel_op_dist_to_ref": 0.790749, + "subspace_cos_top_r_vs_ref": 0.527828 + }, + { + "n": 1, + "layer": 14, + "stable_rank": 28.396, + "effective_rank": 408.9458, + "top_sv": 5.6064, + "fro": 29.8747, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.859357, + "rel_op_dist_to_ref": 0.746117, + "subspace_cos_top_r_vs_ref": 0.526262 + }, + { + "n": 1, + "layer": 15, + "stable_rank": 30.5861, + "effective_rank": 437.0871, + "top_sv": 5.8954, + "fro": 32.6039, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.873932, + "rel_op_dist_to_ref": 0.704641, + "subspace_cos_top_r_vs_ref": 0.544145 + }, + { + "n": 1, + "layer": 16, + "stable_rank": 36.4191, + "effective_rank": 515.5894, + "top_sv": 5.8805, + "fro": 35.4873, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.89663, + "rel_op_dist_to_ref": 0.808011, + "subspace_cos_top_r_vs_ref": 0.680357 + }, + { + "n": 1, + "layer": 17, + "stable_rank": 44.4045, + "effective_rank": 597.4292, + "top_sv": 5.5651, + "fro": 37.0832, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.916949, + "rel_op_dist_to_ref": 0.606942, + "subspace_cos_top_r_vs_ref": 0.734358 + }, + { + "n": 1, + "layer": 18, + "stable_rank": 50.6399, + "effective_rank": 652.8115, + "top_sv": 5.3288, + "fro": 37.9199, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.935726, + "rel_op_dist_to_ref": 0.546048, + "subspace_cos_top_r_vs_ref": 0.754637 + }, + { + "n": 1, + "layer": 19, + "stable_rank": 61.6337, + "effective_rank": 720.0831, + "top_sv": 4.8097, + "fro": 37.7586, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.950945, + "rel_op_dist_to_ref": 0.490227, + "subspace_cos_top_r_vs_ref": 0.785635 + }, + { + "n": 1, + "layer": 20, + "stable_rank": 63.1174, + "effective_rank": 766.7431, + "top_sv": 4.7297, + "fro": 37.575, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.964541, + "rel_op_dist_to_ref": 0.470411, + "subspace_cos_top_r_vs_ref": 0.854453 + }, + { + "n": 1, + "layer": 21, + "stable_rank": 66.9309, + "effective_rank": 819.0578, + "top_sv": 4.5174, + "fro": 36.9565, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.978315, + "rel_op_dist_to_ref": 0.3506, + "subspace_cos_top_r_vs_ref": 0.87523 + }, + { + "n": 1, + "layer": 22, + "stable_rank": 43.0123, + "effective_rank": 868.4246, + "top_sv": 5.2812, + "fro": 34.6349, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.98914, + "rel_op_dist_to_ref": 0.259132, + "subspace_cos_top_r_vs_ref": 0.880662 + }, + { + "n": 5, + "layer": 0, + "stable_rank": 9.5331, + "effective_rank": 129.4117, + "top_sv": 4.3787, + "fro": 13.5195, + "dispersion": 0.909044, + "heavy_tail_ratio": 1.0497, + "cos_to_ref": 0.708136, + "rel_op_dist_to_ref": 1.658743, + "subspace_cos_top_r_vs_ref": 0.40121 + }, + { + "n": 5, + "layer": 1, + "stable_rank": 9.5511, + "effective_rank": 135.5241, + "top_sv": 4.0131, + "fro": 12.4024, + "dispersion": 0.838229, + "heavy_tail_ratio": 1.0519, + "cos_to_ref": 0.726312, + "rel_op_dist_to_ref": 1.560679, + "subspace_cos_top_r_vs_ref": 0.44291 + }, + { + "n": 5, + "layer": 2, + "stable_rank": 9.7873, + "effective_rank": 132.1683, + "top_sv": 4.1061, + "fro": 12.8457, + "dispersion": 0.700105, + "heavy_tail_ratio": 1.0385, + "cos_to_ref": 0.756173, + "rel_op_dist_to_ref": 1.422413, + "subspace_cos_top_r_vs_ref": 0.484004 + }, + { + "n": 5, + "layer": 3, + "stable_rank": 9.8164, + "effective_rank": 140.9046, + "top_sv": 3.8624, + "fro": 12.1013, + "dispersion": 0.611487, + "heavy_tail_ratio": 1.0513, + "cos_to_ref": 0.780825, + "rel_op_dist_to_ref": 1.255465, + "subspace_cos_top_r_vs_ref": 0.514076 + }, + { + "n": 5, + "layer": 4, + "stable_rank": 10.2944, + "effective_rank": 148.7775, + "top_sv": 4.16, + "fro": 13.347, + "dispersion": 0.519002, + "heavy_tail_ratio": 1.0663, + "cos_to_ref": 0.814089, + "rel_op_dist_to_ref": 1.033558, + "subspace_cos_top_r_vs_ref": 0.570152 + }, + { + "n": 5, + "layer": 5, + "stable_rank": 10.6434, + "effective_rank": 165.2954, + "top_sv": 4.0278, + "fro": 13.1403, + "dispersion": 0.486666, + "heavy_tail_ratio": 1.0696, + "cos_to_ref": 0.833414, + "rel_op_dist_to_ref": 0.801853, + "subspace_cos_top_r_vs_ref": 0.643859 + }, + { + "n": 5, + "layer": 6, + "stable_rank": 12.2324, + "effective_rank": 182.8216, + "top_sv": 3.8149, + "fro": 13.3424, + "dispersion": 0.423509, + "heavy_tail_ratio": 1.0979, + "cos_to_ref": 0.849306, + "rel_op_dist_to_ref": 0.707417, + "subspace_cos_top_r_vs_ref": 0.642434 + }, + { + "n": 5, + "layer": 7, + "stable_rank": 12.6222, + "effective_rank": 194.0955, + "top_sv": 4.0972, + "fro": 14.5563, + "dispersion": 0.367975, + "heavy_tail_ratio": 1.1137, + "cos_to_ref": 0.864606, + "rel_op_dist_to_ref": 0.613284, + "subspace_cos_top_r_vs_ref": 0.613764 + }, + { + "n": 5, + "layer": 8, + "stable_rank": 11.7971, + "effective_rank": 195.9143, + "top_sv": 4.8148, + "fro": 16.537, + "dispersion": 0.329308, + "heavy_tail_ratio": 1.1291, + "cos_to_ref": 0.872966, + "rel_op_dist_to_ref": 0.531669, + "subspace_cos_top_r_vs_ref": 0.671248 + }, + { + "n": 5, + "layer": 9, + "stable_rank": 13.5519, + "effective_rank": 223.4997, + "top_sv": 4.9021, + "fro": 18.0456, + "dispersion": 0.309444, + "heavy_tail_ratio": 1.1308, + "cos_to_ref": 0.877774, + "rel_op_dist_to_ref": 0.527366, + "subspace_cos_top_r_vs_ref": 0.638965 + }, + { + "n": 5, + "layer": 10, + "stable_rank": 13.3998, + "effective_rank": 238.2235, + "top_sv": 5.5099, + "fro": 20.1691, + "dispersion": 0.278883, + "heavy_tail_ratio": 1.1255, + "cos_to_ref": 0.884993, + "rel_op_dist_to_ref": 0.499363, + "subspace_cos_top_r_vs_ref": 0.642152 + }, + { + "n": 5, + "layer": 11, + "stable_rank": 14.2985, + "effective_rank": 266.2959, + "top_sv": 6.1785, + "fro": 23.3628, + "dispersion": 0.259398, + "heavy_tail_ratio": 1.1237, + "cos_to_ref": 0.888029, + "rel_op_dist_to_ref": 0.497767, + "subspace_cos_top_r_vs_ref": 0.642181 + }, + { + "n": 5, + "layer": 12, + "stable_rank": 24.9023, + "effective_rank": 341.4857, + "top_sv": 4.9076, + "fro": 24.4899, + "dispersion": 0.216918, + "heavy_tail_ratio": 1.1126, + "cos_to_ref": 0.90148, + "rel_op_dist_to_ref": 0.55493, + "subspace_cos_top_r_vs_ref": 0.600953 + }, + { + "n": 5, + "layer": 13, + "stable_rank": 28.8378, + "effective_rank": 384.9223, + "top_sv": 4.866, + "fro": 26.1307, + "dispersion": 0.185583, + "heavy_tail_ratio": 1.1158, + "cos_to_ref": 0.908642, + "rel_op_dist_to_ref": 0.578314, + "subspace_cos_top_r_vs_ref": 0.636359 + }, + { + "n": 5, + "layer": 14, + "stable_rank": 32.4463, + "effective_rank": 409.5457, + "top_sv": 5.2438, + "fro": 29.8691, + "dispersion": 0.15475, + "heavy_tail_ratio": 1.1169, + "cos_to_ref": 0.913718, + "rel_op_dist_to_ref": 0.633444, + "subspace_cos_top_r_vs_ref": 0.611424 + }, + { + "n": 5, + "layer": 15, + "stable_rank": 32.41, + "effective_rank": 431.1569, + "top_sv": 5.7889, + "fro": 32.9557, + "dispersion": 0.13101, + "heavy_tail_ratio": 1.1136, + "cos_to_ref": 0.92117, + "rel_op_dist_to_ref": 0.577833, + "subspace_cos_top_r_vs_ref": 0.62853 + }, + { + "n": 5, + "layer": 16, + "stable_rank": 41.0204, + "effective_rank": 533.5728, + "top_sv": 5.5468, + "fro": 35.525, + "dispersion": 0.097226, + "heavy_tail_ratio": 1.1018, + "cos_to_ref": 0.935888, + "rel_op_dist_to_ref": 0.519365, + "subspace_cos_top_r_vs_ref": 0.728989 + }, + { + "n": 5, + "layer": 17, + "stable_rank": 46.3566, + "effective_rank": 603.6808, + "top_sv": 5.5037, + "fro": 37.4717, + "dispersion": 0.074031, + "heavy_tail_ratio": 1.0884, + "cos_to_ref": 0.946311, + "rel_op_dist_to_ref": 0.507738, + "subspace_cos_top_r_vs_ref": 0.812992 + }, + { + "n": 5, + "layer": 18, + "stable_rank": 52.6296, + "effective_rank": 661.8532, + "top_sv": 5.2602, + "fro": 38.16, + "dispersion": 0.055841, + "heavy_tail_ratio": 1.0773, + "cos_to_ref": 0.960516, + "rel_op_dist_to_ref": 0.373966, + "subspace_cos_top_r_vs_ref": 0.885458 + }, + { + "n": 5, + "layer": 19, + "stable_rank": 64.4527, + "effective_rank": 734.4942, + "top_sv": 4.6919, + "fro": 37.6668, + "dispersion": 0.038156, + "heavy_tail_ratio": 1.0482, + "cos_to_ref": 0.970432, + "rel_op_dist_to_ref": 0.282191, + "subspace_cos_top_r_vs_ref": 0.851385 + }, + { + "n": 5, + "layer": 20, + "stable_rank": 63.2729, + "effective_rank": 779.0151, + "top_sv": 4.7112, + "fro": 37.4735, + "dispersion": 0.027569, + "heavy_tail_ratio": 1.041, + "cos_to_ref": 0.979222, + "rel_op_dist_to_ref": 0.22342, + "subspace_cos_top_r_vs_ref": 0.931826 + }, + { + "n": 5, + "layer": 21, + "stable_rank": 67.7718, + "effective_rank": 820.9597, + "top_sv": 4.5013, + "fro": 37.0551, + "dispersion": 0.017737, + "heavy_tail_ratio": 1.0309, + "cos_to_ref": 0.987495, + "rel_op_dist_to_ref": 0.199262, + "subspace_cos_top_r_vs_ref": 0.919554 + }, + { + "n": 5, + "layer": 22, + "stable_rank": 40.7747, + "effective_rank": 868.9216, + "top_sv": 5.4308, + "fro": 34.6771, + "dispersion": 0.008642, + "heavy_tail_ratio": 1.0145, + "cos_to_ref": 0.994498, + "rel_op_dist_to_ref": 0.152287, + "subspace_cos_top_r_vs_ref": 0.977684 + }, + { + "n": 10, + "layer": 0, + "stable_rank": 13.8505, + "effective_rank": 147.3172, + "top_sv": 3.1843, + "fro": 11.8505, + "dispersion": 1.313602, + "heavy_tail_ratio": 1.1601, + "cos_to_ref": 0.825006, + "rel_op_dist_to_ref": 1.071453, + "subspace_cos_top_r_vs_ref": 0.564309 + }, + { + "n": 10, + "layer": 1, + "stable_rank": 13.6073, + "effective_rank": 153.4812, + "top_sv": 2.9672, + "fro": 10.9451, + "dispersion": 1.182651, + "heavy_tail_ratio": 1.1448, + "cos_to_ref": 0.839137, + "rel_op_dist_to_ref": 0.999185, + "subspace_cos_top_r_vs_ref": 0.613551 + }, + { + "n": 10, + "layer": 2, + "stable_rank": 13.0677, + "effective_rank": 146.6855, + "top_sv": 3.2134, + "fro": 11.616, + "dispersion": 0.94342, + "heavy_tail_ratio": 1.1453, + "cos_to_ref": 0.858782, + "rel_op_dist_to_ref": 0.960514, + "subspace_cos_top_r_vs_ref": 0.661473 + }, + { + "n": 10, + "layer": 3, + "stable_rank": 13.5251, + "effective_rank": 158.6788, + "top_sv": 2.9968, + "fro": 11.0211, + "dispersion": 0.810344, + "heavy_tail_ratio": 1.1206, + "cos_to_ref": 0.874591, + "rel_op_dist_to_ref": 0.820964, + "subspace_cos_top_r_vs_ref": 0.650063 + }, + { + "n": 10, + "layer": 4, + "stable_rank": 13.3008, + "effective_rank": 163.2762, + "top_sv": 3.3938, + "fro": 12.3771, + "dispersion": 0.663656, + "heavy_tail_ratio": 1.1141, + "cos_to_ref": 0.894903, + "rel_op_dist_to_ref": 0.648659, + "subspace_cos_top_r_vs_ref": 0.691412 + }, + { + "n": 10, + "layer": 5, + "stable_rank": 12.3921, + "effective_rank": 175.7126, + "top_sv": 3.5153, + "fro": 12.3744, + "dispersion": 0.608813, + "heavy_tail_ratio": 1.1116, + "cos_to_ref": 0.90528, + "rel_op_dist_to_ref": 0.530455, + "subspace_cos_top_r_vs_ref": 0.721788 + }, + { + "n": 10, + "layer": 6, + "stable_rank": 13.639, + "effective_rank": 191.5958, + "top_sv": 3.4297, + "fro": 12.666, + "dispersion": 0.532355, + "heavy_tail_ratio": 1.1168, + "cos_to_ref": 0.914201, + "rel_op_dist_to_ref": 0.482731, + "subspace_cos_top_r_vs_ref": 0.742569 + }, + { + "n": 10, + "layer": 7, + "stable_rank": 13.3382, + "effective_rank": 201.4026, + "top_sv": 3.805, + "fro": 13.8963, + "dispersion": 0.462352, + "heavy_tail_ratio": 1.1284, + "cos_to_ref": 0.922326, + "rel_op_dist_to_ref": 0.445939, + "subspace_cos_top_r_vs_ref": 0.701332 + }, + { + "n": 10, + "layer": 8, + "stable_rank": 11.8348, + "effective_rank": 199.5033, + "top_sv": 4.6243, + "fro": 15.9082, + "dispersion": 0.41294, + "heavy_tail_ratio": 1.1384, + "cos_to_ref": 0.926032, + "rel_op_dist_to_ref": 0.420841, + "subspace_cos_top_r_vs_ref": 0.746685 + }, + { + "n": 10, + "layer": 9, + "stable_rank": 13.7352, + "effective_rank": 229.2669, + "top_sv": 4.6849, + "fro": 17.3625, + "dispersion": 0.386412, + "heavy_tail_ratio": 1.1422, + "cos_to_ref": 0.929064, + "rel_op_dist_to_ref": 0.421815, + "subspace_cos_top_r_vs_ref": 0.726368 + }, + { + "n": 10, + "layer": 10, + "stable_rank": 13.3965, + "effective_rank": 243.2679, + "top_sv": 5.3252, + "fro": 19.4907, + "dispersion": 0.349823, + "heavy_tail_ratio": 1.1336, + "cos_to_ref": 0.932923, + "rel_op_dist_to_ref": 0.403662, + "subspace_cos_top_r_vs_ref": 0.747013 + }, + { + "n": 10, + "layer": 11, + "stable_rank": 14.279, + "effective_rank": 273.7173, + "top_sv": 5.9902, + "fro": 22.6353, + "dispersion": 0.325727, + "heavy_tail_ratio": 1.1305, + "cos_to_ref": 0.934598, + "rel_op_dist_to_ref": 0.40061, + "subspace_cos_top_r_vs_ref": 0.773259 + }, + { + "n": 10, + "layer": 12, + "stable_rank": 25.199, + "effective_rank": 357.4459, + "top_sv": 4.7095, + "fro": 23.6407, + "dispersion": 0.269368, + "heavy_tail_ratio": 1.1285, + "cos_to_ref": 0.943738, + "rel_op_dist_to_ref": 0.417288, + "subspace_cos_top_r_vs_ref": 0.704064 + }, + { + "n": 10, + "layer": 13, + "stable_rank": 28.1083, + "effective_rank": 401.7177, + "top_sv": 4.7784, + "fro": 25.3332, + "dispersion": 0.228738, + "heavy_tail_ratio": 1.1305, + "cos_to_ref": 0.947693, + "rel_op_dist_to_ref": 0.435938, + "subspace_cos_top_r_vs_ref": 0.723627 + }, + { + "n": 10, + "layer": 14, + "stable_rank": 32.797, + "effective_rank": 430.5297, + "top_sv": 5.0492, + "fro": 28.9159, + "dispersion": 0.193913, + "heavy_tail_ratio": 1.1347, + "cos_to_ref": 0.95096, + "rel_op_dist_to_ref": 0.464951, + "subspace_cos_top_r_vs_ref": 0.673878 + }, + { + "n": 10, + "layer": 15, + "stable_rank": 32.0514, + "effective_rank": 452.1645, + "top_sv": 5.6469, + "fro": 31.969, + "dispersion": 0.165321, + "heavy_tail_ratio": 1.131, + "cos_to_ref": 0.955182, + "rel_op_dist_to_ref": 0.426364, + "subspace_cos_top_r_vs_ref": 0.715213 + }, + { + "n": 10, + "layer": 16, + "stable_rank": 41.6119, + "effective_rank": 555.1581, + "top_sv": 5.3797, + "fro": 34.7024, + "dispersion": 0.123602, + "heavy_tail_ratio": 1.1146, + "cos_to_ref": 0.96405, + "rel_op_dist_to_ref": 0.382941, + "subspace_cos_top_r_vs_ref": 0.828351 + }, + { + "n": 10, + "layer": 17, + "stable_rank": 45.9646, + "effective_rank": 626.9543, + "top_sv": 5.4024, + "fro": 36.626, + "dispersion": 0.095025, + "heavy_tail_ratio": 1.1028, + "cos_to_ref": 0.970002, + "rel_op_dist_to_ref": 0.383996, + "subspace_cos_top_r_vs_ref": 0.870964 + }, + { + "n": 10, + "layer": 18, + "stable_rank": 52.9461, + "effective_rank": 679.5734, + "top_sv": 5.1621, + "fro": 37.5602, + "dispersion": 0.071173, + "heavy_tail_ratio": 1.0867, + "cos_to_ref": 0.977737, + "rel_op_dist_to_ref": 0.282311, + "subspace_cos_top_r_vs_ref": 0.926886 + }, + { + "n": 10, + "layer": 19, + "stable_rank": 65.2893, + "effective_rank": 751.4191, + "top_sv": 4.6095, + "fro": 37.2448, + "dispersion": 0.049306, + "heavy_tail_ratio": 1.0545, + "cos_to_ref": 0.983471, + "rel_op_dist_to_ref": 0.204928, + "subspace_cos_top_r_vs_ref": 0.868352 + }, + { + "n": 10, + "layer": 20, + "stable_rank": 63.1787, + "effective_rank": 792.1557, + "top_sv": 4.6738, + "fro": 37.1486, + "dispersion": 0.034798, + "heavy_tail_ratio": 1.0464, + "cos_to_ref": 0.988396, + "rel_op_dist_to_ref": 0.160456, + "subspace_cos_top_r_vs_ref": 0.95784 + }, + { + "n": 10, + "layer": 21, + "stable_rank": 67.9842, + "effective_rank": 830.3894, + "top_sv": 4.4622, + "fro": 36.7909, + "dispersion": 0.021538, + "heavy_tail_ratio": 1.0364, + "cos_to_ref": 0.992847, + "rel_op_dist_to_ref": 0.14773, + "subspace_cos_top_r_vs_ref": 0.920609 + }, + { + "n": 10, + "layer": 22, + "stable_rank": 40.5282, + "effective_rank": 872.7852, + "top_sv": 5.4307, + "fro": 34.5713, + "dispersion": 0.010173, + "heavy_tail_ratio": 1.0168, + "cos_to_ref": 0.996891, + "rel_op_dist_to_ref": 0.113374, + "subspace_cos_top_r_vs_ref": 0.989768 + }, + { + "n": 25, + "layer": 0, + "stable_rank": 15.5571, + "effective_rank": 159.6087, + "top_sv": 2.5935, + "fro": 10.2292, + "dispersion": 1.672356, + "heavy_tail_ratio": 1.2505, + "cos_to_ref": 0.883505, + "rel_op_dist_to_ref": 0.742852, + "subspace_cos_top_r_vs_ref": 0.644814 + }, + { + "n": 25, + "layer": 1, + "stable_rank": 15.1232, + "effective_rank": 165.2229, + "top_sv": 2.4521, + "fro": 9.5356, + "dispersion": 1.481153, + "heavy_tail_ratio": 1.2324, + "cos_to_ref": 0.89281, + "rel_op_dist_to_ref": 0.698442, + "subspace_cos_top_r_vs_ref": 0.681693 + }, + { + "n": 25, + "layer": 2, + "stable_rank": 14.4438, + "effective_rank": 159.8291, + "top_sv": 2.6794, + "fro": 10.183, + "dispersion": 1.17437, + "heavy_tail_ratio": 1.2351, + "cos_to_ref": 0.906268, + "rel_op_dist_to_ref": 0.665472, + "subspace_cos_top_r_vs_ref": 0.717866 + }, + { + "n": 25, + "layer": 3, + "stable_rank": 14.1606, + "effective_rank": 170.7439, + "top_sv": 2.606, + "fro": 9.8063, + "dispersion": 1.003008, + "heavy_tail_ratio": 1.1974, + "cos_to_ref": 0.916491, + "rel_op_dist_to_ref": 0.576237, + "subspace_cos_top_r_vs_ref": 0.692099 + }, + { + "n": 25, + "layer": 4, + "stable_rank": 13.5832, + "effective_rank": 174.3389, + "top_sv": 3.0157, + "fro": 11.1145, + "dispersion": 0.824857, + "heavy_tail_ratio": 1.1846, + "cos_to_ref": 0.930004, + "rel_op_dist_to_ref": 0.477752, + "subspace_cos_top_r_vs_ref": 0.750022 + }, + { + "n": 25, + "layer": 5, + "stable_rank": 12.3507, + "effective_rank": 187.2862, + "top_sv": 3.1868, + "fro": 11.1995, + "dispersion": 0.759205, + "heavy_tail_ratio": 1.1746, + "cos_to_ref": 0.938423, + "rel_op_dist_to_ref": 0.377091, + "subspace_cos_top_r_vs_ref": 0.779736 + }, + { + "n": 25, + "layer": 6, + "stable_rank": 13.4542, + "effective_rank": 203.7316, + "top_sv": 3.1429, + "fro": 11.528, + "dispersion": 0.658594, + "heavy_tail_ratio": 1.1794, + "cos_to_ref": 0.943915, + "rel_op_dist_to_ref": 0.344609, + "subspace_cos_top_r_vs_ref": 0.821664 + }, + { + "n": 25, + "layer": 7, + "stable_rank": 13.1265, + "effective_rank": 213.7156, + "top_sv": 3.5087, + "fro": 12.712, + "dispersion": 0.570114, + "heavy_tail_ratio": 1.1904, + "cos_to_ref": 0.949452, + "rel_op_dist_to_ref": 0.311278, + "subspace_cos_top_r_vs_ref": 0.802409 + }, + { + "n": 25, + "layer": 8, + "stable_rank": 11.8782, + "effective_rank": 214.7997, + "top_sv": 4.226, + "fro": 14.5645, + "dispersion": 0.515475, + "heavy_tail_ratio": 1.2006, + "cos_to_ref": 0.951796, + "rel_op_dist_to_ref": 0.291815, + "subspace_cos_top_r_vs_ref": 0.791163 + }, + { + "n": 25, + "layer": 9, + "stable_rank": 13.5751, + "effective_rank": 245.3579, + "top_sv": 4.3474, + "fro": 16.0176, + "dispersion": 0.47726, + "heavy_tail_ratio": 1.1995, + "cos_to_ref": 0.953768, + "rel_op_dist_to_ref": 0.292157, + "subspace_cos_top_r_vs_ref": 0.776595 + }, + { + "n": 25, + "layer": 10, + "stable_rank": 13.1907, + "effective_rank": 259.6032, + "top_sv": 4.996, + "fro": 18.1445, + "dispersion": 0.428902, + "heavy_tail_ratio": 1.1836, + "cos_to_ref": 0.95639, + "rel_op_dist_to_ref": 0.280994, + "subspace_cos_top_r_vs_ref": 0.795815 + }, + { + "n": 25, + "layer": 11, + "stable_rank": 14.3098, + "effective_rank": 293.3303, + "top_sv": 5.5933, + "fro": 21.1582, + "dispersion": 0.395973, + "heavy_tail_ratio": 1.1786, + "cos_to_ref": 0.957594, + "rel_op_dist_to_ref": 0.28092, + "subspace_cos_top_r_vs_ref": 0.827176 + }, + { + "n": 25, + "layer": 12, + "stable_rank": 25.0965, + "effective_rank": 376.2212, + "top_sv": 4.4774, + "fro": 22.43, + "dispersion": 0.32603, + "heavy_tail_ratio": 1.1637, + "cos_to_ref": 0.963158, + "rel_op_dist_to_ref": 0.317584, + "subspace_cos_top_r_vs_ref": 0.820723 + }, + { + "n": 25, + "layer": 13, + "stable_rank": 27.6153, + "effective_rank": 420.9939, + "top_sv": 4.6046, + "fro": 24.1971, + "dispersion": 0.276157, + "heavy_tail_ratio": 1.1614, + "cos_to_ref": 0.965674, + "rel_op_dist_to_ref": 0.338988, + "subspace_cos_top_r_vs_ref": 0.807387 + }, + { + "n": 25, + "layer": 14, + "stable_rank": 32.2981, + "effective_rank": 451.986, + "top_sv": 4.8824, + "fro": 27.7467, + "dispersion": 0.233558, + "heavy_tail_ratio": 1.1633, + "cos_to_ref": 0.967399, + "rel_op_dist_to_ref": 0.365966, + "subspace_cos_top_r_vs_ref": 0.762879 + }, + { + "n": 25, + "layer": 15, + "stable_rank": 31.4883, + "effective_rank": 473.8998, + "top_sv": 5.4798, + "fro": 30.7489, + "dispersion": 0.19932, + "heavy_tail_ratio": 1.1591, + "cos_to_ref": 0.970029, + "rel_op_dist_to_ref": 0.346265, + "subspace_cos_top_r_vs_ref": 0.793908 + }, + { + "n": 25, + "layer": 16, + "stable_rank": 40.8832, + "effective_rank": 574.0251, + "top_sv": 5.2694, + "fro": 33.6917, + "dispersion": 0.149031, + "heavy_tail_ratio": 1.1353, + "cos_to_ref": 0.975716, + "rel_op_dist_to_ref": 0.327835, + "subspace_cos_top_r_vs_ref": 0.892307 + }, + { + "n": 25, + "layer": 17, + "stable_rank": 45.7033, + "effective_rank": 646.7239, + "top_sv": 5.2861, + "fro": 35.7355, + "dispersion": 0.113438, + "heavy_tail_ratio": 1.1209, + "cos_to_ref": 0.979766, + "rel_op_dist_to_ref": 0.32195, + "subspace_cos_top_r_vs_ref": 0.910943 + }, + { + "n": 25, + "layer": 18, + "stable_rank": 53.2751, + "effective_rank": 695.3282, + "top_sv": 5.055, + "fro": 36.8952, + "dispersion": 0.085038, + "heavy_tail_ratio": 1.0992, + "cos_to_ref": 0.98502, + "rel_op_dist_to_ref": 0.224208, + "subspace_cos_top_r_vs_ref": 0.950115 + }, + { + "n": 25, + "layer": 19, + "stable_rank": 64.7793, + "effective_rank": 763.7921, + "top_sv": 4.5639, + "fro": 36.7319, + "dispersion": 0.060642, + "heavy_tail_ratio": 1.0634, + "cos_to_ref": 0.98882, + "rel_op_dist_to_ref": 0.162286, + "subspace_cos_top_r_vs_ref": 0.922806 + }, + { + "n": 25, + "layer": 20, + "stable_rank": 63.3266, + "effective_rank": 803.0504, + "top_sv": 4.6117, + "fro": 36.6976, + "dispersion": 0.042873, + "heavy_tail_ratio": 1.0551, + "cos_to_ref": 0.992104, + "rel_op_dist_to_ref": 0.131034, + "subspace_cos_top_r_vs_ref": 0.964622 + }, + { + "n": 25, + "layer": 21, + "stable_rank": 68.6182, + "effective_rank": 838.9075, + "top_sv": 4.4069, + "fro": 36.5037, + "dispersion": 0.026426, + "heavy_tail_ratio": 1.0421, + "cos_to_ref": 0.995202, + "rel_op_dist_to_ref": 0.119098, + "subspace_cos_top_r_vs_ref": 0.949934 + }, + { + "n": 25, + "layer": 22, + "stable_rank": 41.1391, + "effective_rank": 878.2734, + "top_sv": 5.3731, + "fro": 34.4613, + "dispersion": 0.012428, + "heavy_tail_ratio": 1.0189, + "cos_to_ref": 0.997939, + "rel_op_dist_to_ref": 0.091891, + "subspace_cos_top_r_vs_ref": 0.9937 + }, + { + "n": 50, + "layer": 0, + "stable_rank": 18.2066, + "effective_rank": 160.2674, + "top_sv": 2.281, + "fro": 9.7327, + "dispersion": 1.926469, + "heavy_tail_ratio": 1.4607, + "cos_to_ref": 0.94772, + "rel_op_dist_to_ref": 0.460828, + "subspace_cos_top_r_vs_ref": 0.866851 + }, + { + "n": 50, + "layer": 1, + "stable_rank": 17.4592, + "effective_rank": 165.2247, + "top_sv": 2.1837, + "fro": 9.1242, + "dispersion": 1.684141, + "heavy_tail_ratio": 1.3714, + "cos_to_ref": 0.953051, + "rel_op_dist_to_ref": 0.411092, + "subspace_cos_top_r_vs_ref": 0.852254 + }, + { + "n": 50, + "layer": 2, + "stable_rank": 17.2417, + "effective_rank": 163.1551, + "top_sv": 2.3648, + "fro": 9.8191, + "dispersion": 1.33565, + "heavy_tail_ratio": 1.3483, + "cos_to_ref": 0.958686, + "rel_op_dist_to_ref": 0.373379, + "subspace_cos_top_r_vs_ref": 0.879221 + }, + { + "n": 50, + "layer": 3, + "stable_rank": 15.3133, + "effective_rank": 172.6692, + "top_sv": 2.4254, + "fro": 9.4909, + "dispersion": 1.136774, + "heavy_tail_ratio": 1.334, + "cos_to_ref": 0.962923, + "rel_op_dist_to_ref": 0.328032, + "subspace_cos_top_r_vs_ref": 0.832768 + }, + { + "n": 50, + "layer": 4, + "stable_rank": 14.9923, + "effective_rank": 175.5921, + "top_sv": 2.8027, + "fro": 10.8518, + "dispersion": 0.935079, + "heavy_tail_ratio": 1.3369, + "cos_to_ref": 0.967977, + "rel_op_dist_to_ref": 0.291577, + "subspace_cos_top_r_vs_ref": 0.847903 + }, + { + "n": 50, + "layer": 5, + "stable_rank": 12.847, + "effective_rank": 186.1368, + "top_sv": 3.0752, + "fro": 11.0222, + "dispersion": 0.852942, + "heavy_tail_ratio": 1.3151, + "cos_to_ref": 0.971113, + "rel_op_dist_to_ref": 0.253092, + "subspace_cos_top_r_vs_ref": 0.901395 + }, + { + "n": 50, + "layer": 6, + "stable_rank": 13.5529, + "effective_rank": 201.2551, + "top_sv": 3.0923, + "fro": 11.3839, + "dispersion": 0.744252, + "heavy_tail_ratio": 1.2995, + "cos_to_ref": 0.973708, + "rel_op_dist_to_ref": 0.237846, + "subspace_cos_top_r_vs_ref": 0.901352 + }, + { + "n": 50, + "layer": 7, + "stable_rank": 13.0346, + "effective_rank": 210.9129, + "top_sv": 3.4856, + "fro": 12.584, + "dispersion": 0.646671, + "heavy_tail_ratio": 1.2737, + "cos_to_ref": 0.976229, + "rel_op_dist_to_ref": 0.207798, + "subspace_cos_top_r_vs_ref": 0.845183 + }, + { + "n": 50, + "layer": 8, + "stable_rank": 11.7366, + "effective_rank": 212.7797, + "top_sv": 4.2119, + "fro": 14.4291, + "dispersion": 0.585852, + "heavy_tail_ratio": 1.2553, + "cos_to_ref": 0.977335, + "rel_op_dist_to_ref": 0.191462, + "subspace_cos_top_r_vs_ref": 0.86741 + }, + { + "n": 50, + "layer": 9, + "stable_rank": 13.7367, + "effective_rank": 246.2119, + "top_sv": 4.2734, + "fro": 15.8383, + "dispersion": 0.543481, + "heavy_tail_ratio": 1.2345, + "cos_to_ref": 0.97807, + "rel_op_dist_to_ref": 0.194002, + "subspace_cos_top_r_vs_ref": 0.854658 + }, + { + "n": 50, + "layer": 10, + "stable_rank": 13.3129, + "effective_rank": 260.8184, + "top_sv": 4.9216, + "fro": 17.9571, + "dispersion": 0.491369, + "heavy_tail_ratio": 1.2129, + "cos_to_ref": 0.979295, + "rel_op_dist_to_ref": 0.183926, + "subspace_cos_top_r_vs_ref": 0.877951 + }, + { + "n": 50, + "layer": 11, + "stable_rank": 14.7326, + "effective_rank": 296.9349, + "top_sv": 5.4604, + "fro": 20.9585, + "dispersion": 0.455349, + "heavy_tail_ratio": 1.2046, + "cos_to_ref": 0.979787, + "rel_op_dist_to_ref": 0.187578, + "subspace_cos_top_r_vs_ref": 0.914916 + }, + { + "n": 50, + "layer": 12, + "stable_rank": 24.8889, + "effective_rank": 376.2206, + "top_sv": 4.4584, + "fro": 22.2422, + "dispersion": 0.375917, + "heavy_tail_ratio": 1.1602, + "cos_to_ref": 0.981925, + "rel_op_dist_to_ref": 0.223535, + "subspace_cos_top_r_vs_ref": 0.848336 + }, + { + "n": 50, + "layer": 13, + "stable_rank": 27.3198, + "effective_rank": 424.01, + "top_sv": 4.5839, + "fro": 23.9588, + "dispersion": 0.319881, + "heavy_tail_ratio": 1.1533, + "cos_to_ref": 0.983406, + "rel_op_dist_to_ref": 0.206058, + "subspace_cos_top_r_vs_ref": 0.901221 + }, + { + "n": 50, + "layer": 14, + "stable_rank": 31.6785, + "effective_rank": 456.9492, + "top_sv": 4.8771, + "fro": 27.4499, + "dispersion": 0.272968, + "heavy_tail_ratio": 1.1576, + "cos_to_ref": 0.984579, + "rel_op_dist_to_ref": 0.212274, + "subspace_cos_top_r_vs_ref": 0.877636 + }, + { + "n": 50, + "layer": 15, + "stable_rank": 31.0905, + "effective_rank": 479.0886, + "top_sv": 5.4596, + "fro": 30.4416, + "dispersion": 0.234895, + "heavy_tail_ratio": 1.1538, + "cos_to_ref": 0.985963, + "rel_op_dist_to_ref": 0.204068, + "subspace_cos_top_r_vs_ref": 0.942111 + }, + { + "n": 50, + "layer": 16, + "stable_rank": 40.792, + "effective_rank": 578.3548, + "top_sv": 5.2403, + "fro": 33.4688, + "dispersion": 0.174794, + "heavy_tail_ratio": 1.1302, + "cos_to_ref": 0.98873, + "rel_op_dist_to_ref": 0.2019, + "subspace_cos_top_r_vs_ref": 0.977397 + }, + { + "n": 50, + "layer": 17, + "stable_rank": 45.1788, + "effective_rank": 652.0341, + "top_sv": 5.2924, + "fro": 35.5722, + "dispersion": 0.132354, + "heavy_tail_ratio": 1.1166, + "cos_to_ref": 0.990957, + "rel_op_dist_to_ref": 0.1883, + "subspace_cos_top_r_vs_ref": 0.970202 + }, + { + "n": 50, + "layer": 18, + "stable_rank": 52.7419, + "effective_rank": 696.2625, + "top_sv": 5.0896, + "fro": 36.9619, + "dispersion": 0.099624, + "heavy_tail_ratio": 1.0899, + "cos_to_ref": 0.992847, + "rel_op_dist_to_ref": 0.158262, + "subspace_cos_top_r_vs_ref": 0.980304 + }, + { + "n": 50, + "layer": 19, + "stable_rank": 64.4549, + "effective_rank": 765.2288, + "top_sv": 4.5795, + "fro": 36.7651, + "dispersion": 0.07096, + "heavy_tail_ratio": 1.0574, + "cos_to_ref": 0.994643, + "rel_op_dist_to_ref": 0.100531, + "subspace_cos_top_r_vs_ref": 0.966068 + }, + { + "n": 50, + "layer": 20, + "stable_rank": 62.8591, + "effective_rank": 803.392, + "top_sv": 4.64, + "fro": 36.7866, + "dispersion": 0.049808, + "heavy_tail_ratio": 1.0491, + "cos_to_ref": 0.996327, + "rel_op_dist_to_ref": 0.085213, + "subspace_cos_top_r_vs_ref": 0.989189 + }, + { + "n": 50, + "layer": 21, + "stable_rank": 68.4115, + "effective_rank": 839.0364, + "top_sv": 4.4196, + "fro": 36.5534, + "dispersion": 0.030835, + "heavy_tail_ratio": 1.0384, + "cos_to_ref": 0.997833, + "rel_op_dist_to_ref": 0.081434, + "subspace_cos_top_r_vs_ref": 0.98958 + }, + { + "n": 50, + "layer": 22, + "stable_rank": 40.604, + "effective_rank": 879.1193, + "top_sv": 5.4087, + "fro": 34.4637, + "dispersion": 0.014089, + "heavy_tail_ratio": 1.018, + "cos_to_ref": 0.999109, + "rel_op_dist_to_ref": 0.061414, + "subspace_cos_top_r_vs_ref": 0.995565 + }, + { + "n": 100, + "layer": 0, + "stable_rank": 16.7101, + "effective_rank": 164.3155, + "top_sv": 2.2364, + "fro": 9.1417, + "dispersion": 2.281211, + "heavy_tail_ratio": 1.4687, + "cos_to_ref": 0.982348, + "rel_op_dist_to_ref": 0.218983, + "subspace_cos_top_r_vs_ref": 0.963391 + }, + { + "n": 100, + "layer": 1, + "stable_rank": 16.1902, + "effective_rank": 167.8211, + "top_sv": 2.1404, + "fro": 8.6122, + "dispersion": 1.994275, + "heavy_tail_ratio": 1.3756, + "cos_to_ref": 0.984153, + "rel_op_dist_to_ref": 0.199026, + "subspace_cos_top_r_vs_ref": 0.974767 + }, + { + "n": 100, + "layer": 2, + "stable_rank": 15.8745, + "effective_rank": 167.0716, + "top_sv": 2.3273, + "fro": 9.2725, + "dispersion": 1.591408, + "heavy_tail_ratio": 1.3555, + "cos_to_ref": 0.985695, + "rel_op_dist_to_ref": 0.183567, + "subspace_cos_top_r_vs_ref": 0.952972 + }, + { + "n": 100, + "layer": 3, + "stable_rank": 14.5122, + "effective_rank": 174.8159, + "top_sv": 2.3712, + "fro": 9.0328, + "dispersion": 1.360327, + "heavy_tail_ratio": 1.3336, + "cos_to_ref": 0.987102, + "rel_op_dist_to_ref": 0.167789, + "subspace_cos_top_r_vs_ref": 0.969092 + }, + { + "n": 100, + "layer": 4, + "stable_rank": 14.2118, + "effective_rank": 179.3978, + "top_sv": 2.7349, + "fro": 10.3101, + "dispersion": 1.109917, + "heavy_tail_ratio": 1.3476, + "cos_to_ref": 0.988798, + "rel_op_dist_to_ref": 0.146257, + "subspace_cos_top_r_vs_ref": 0.884376 + }, + { + "n": 100, + "layer": 5, + "stable_rank": 12.0391, + "effective_rank": 187.9349, + "top_sv": 3.0343, + "fro": 10.5281, + "dispersion": 0.997664, + "heavy_tail_ratio": 1.326, + "cos_to_ref": 0.989776, + "rel_op_dist_to_ref": 0.128313, + "subspace_cos_top_r_vs_ref": 0.963147 + }, + { + "n": 100, + "layer": 6, + "stable_rank": 12.6675, + "effective_rank": 202.487, + "top_sv": 3.0633, + "fro": 10.9026, + "dispersion": 0.867809, + "heavy_tail_ratio": 1.3112, + "cos_to_ref": 0.990597, + "rel_op_dist_to_ref": 0.120127, + "subspace_cos_top_r_vs_ref": 0.961211 + }, + { + "n": 100, + "layer": 7, + "stable_rank": 12.3153, + "effective_rank": 212.0565, + "top_sv": 3.4415, + "fro": 12.077, + "dispersion": 0.757, + "heavy_tail_ratio": 1.2848, + "cos_to_ref": 0.991476, + "rel_op_dist_to_ref": 0.109567, + "subspace_cos_top_r_vs_ref": 0.972347 + }, + { + "n": 100, + "layer": 8, + "stable_rank": 10.9313, + "effective_rank": 214.1885, + "top_sv": 4.1837, + "fro": 13.8321, + "dispersion": 0.684466, + "heavy_tail_ratio": 1.2706, + "cos_to_ref": 0.991877, + "rel_op_dist_to_ref": 0.106481, + "subspace_cos_top_r_vs_ref": 0.945554 + }, + { + "n": 100, + "layer": 9, + "stable_rank": 12.754, + "effective_rank": 248.5368, + "top_sv": 4.2568, + "fro": 15.202, + "dispersion": 0.637772, + "heavy_tail_ratio": 1.2486, + "cos_to_ref": 0.992082, + "rel_op_dist_to_ref": 0.115178, + "subspace_cos_top_r_vs_ref": 0.958315 + }, + { + "n": 100, + "layer": 10, + "stable_rank": 12.4198, + "effective_rank": 263.0503, + "top_sv": 4.8959, + "fro": 17.2537, + "dispersion": 0.576056, + "heavy_tail_ratio": 1.2279, + "cos_to_ref": 0.992531, + "rel_op_dist_to_ref": 0.113702, + "subspace_cos_top_r_vs_ref": 0.975027 + }, + { + "n": 100, + "layer": 11, + "stable_rank": 13.658, + "effective_rank": 301.2931, + "top_sv": 5.4461, + "fro": 20.1268, + "dispersion": 0.537344, + "heavy_tail_ratio": 1.2205, + "cos_to_ref": 0.992668, + "rel_op_dist_to_ref": 0.117591, + "subspace_cos_top_r_vs_ref": 0.981467 + }, + { + "n": 100, + "layer": 12, + "stable_rank": 23.4339, + "effective_rank": 381.3989, + "top_sv": 4.4373, + "fro": 21.4798, + "dispersion": 0.451984, + "heavy_tail_ratio": 1.1695, + "cos_to_ref": 0.993725, + "rel_op_dist_to_ref": 0.136604, + "subspace_cos_top_r_vs_ref": 0.874372 + }, + { + "n": 100, + "layer": 13, + "stable_rank": 25.5574, + "effective_rank": 430.479, + "top_sv": 4.5817, + "fro": 23.1622, + "dispersion": 0.391331, + "heavy_tail_ratio": 1.162, + "cos_to_ref": 0.994224, + "rel_op_dist_to_ref": 0.132552, + "subspace_cos_top_r_vs_ref": 0.984607 + }, + { + "n": 100, + "layer": 14, + "stable_rank": 29.6936, + "effective_rank": 465.3944, + "top_sv": 4.8776, + "fro": 26.5783, + "dispersion": 0.34061, + "heavy_tail_ratio": 1.165, + "cos_to_ref": 0.994879, + "rel_op_dist_to_ref": 0.139868, + "subspace_cos_top_r_vs_ref": 0.984192 + }, + { + "n": 100, + "layer": 15, + "stable_rank": 29.6189, + "effective_rank": 487.4474, + "top_sv": 5.4303, + "fro": 29.5531, + "dispersion": 0.295992, + "heavy_tail_ratio": 1.1601, + "cos_to_ref": 0.995377, + "rel_op_dist_to_ref": 0.137729, + "subspace_cos_top_r_vs_ref": 0.983196 + }, + { + "n": 100, + "layer": 16, + "stable_rank": 39.1108, + "effective_rank": 584.5587, + "top_sv": 5.2214, + "fro": 32.6532, + "dispersion": 0.225188, + "heavy_tail_ratio": 1.1344, + "cos_to_ref": 0.996304, + "rel_op_dist_to_ref": 0.134205, + "subspace_cos_top_r_vs_ref": 0.995378 + }, + { + "n": 100, + "layer": 17, + "stable_rank": 44.1666, + "effective_rank": 659.4382, + "top_sv": 5.2357, + "fro": 34.7947, + "dispersion": 0.172382, + "heavy_tail_ratio": 1.1219, + "cos_to_ref": 0.997127, + "rel_op_dist_to_ref": 0.106497, + "subspace_cos_top_r_vs_ref": 0.994677 + }, + { + "n": 100, + "layer": 18, + "stable_rank": 51.6585, + "effective_rank": 701.8063, + "top_sv": 5.0628, + "fro": 36.3876, + "dispersion": 0.131643, + "heavy_tail_ratio": 1.0941, + "cos_to_ref": 0.997808, + "rel_op_dist_to_ref": 0.093891, + "subspace_cos_top_r_vs_ref": 0.996682 + }, + { + "n": 100, + "layer": 19, + "stable_rank": 62.7332, + "effective_rank": 768.1022, + "top_sv": 4.5962, + "fro": 36.4025, + "dispersion": 0.09372, + "heavy_tail_ratio": 1.0772, + "cos_to_ref": 0.998433, + "rel_op_dist_to_ref": 0.062242, + "subspace_cos_top_r_vs_ref": 0.994126 + }, + { + "n": 100, + "layer": 20, + "stable_rank": 61.8463, + "effective_rank": 805.5551, + "top_sv": 4.6517, + "fro": 36.5806, + "dispersion": 0.06326, + "heavy_tail_ratio": 1.0644, + "cos_to_ref": 0.998929, + "rel_op_dist_to_ref": 0.051734, + "subspace_cos_top_r_vs_ref": 0.997708 + }, + { + "n": 100, + "layer": 21, + "stable_rank": 67.4946, + "effective_rank": 839.0887, + "top_sv": 4.44, + "fro": 36.4758, + "dispersion": 0.038459, + "heavy_tail_ratio": 1.0491, + "cos_to_ref": 0.999365, + "rel_op_dist_to_ref": 0.043391, + "subspace_cos_top_r_vs_ref": 0.998177 + }, + { + "n": 100, + "layer": 22, + "stable_rank": 40.3813, + "effective_rank": 879.1093, + "top_sv": 5.4193, + "fro": 34.4361, + "dispersion": 0.017036, + "heavy_tail_ratio": 1.0199, + "cos_to_ref": 0.999778, + "rel_op_dist_to_ref": 0.034872, + "subspace_cos_top_r_vs_ref": 0.997533 + }, + { + "n": 200, + "layer": 0, + "stable_rank": 16.1584, + "effective_rank": 159.8266, + "top_sv": 2.2539, + "fro": 9.0599, + "dispersion": 2.281745, + "heavy_tail_ratio": 1.4818, + "cos_to_ref": 1.000026, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 1, + "stable_rank": 15.7467, + "effective_rank": 163.0765, + "top_sv": 2.1495, + "fro": 8.5297, + "dispersion": 2.00216, + "heavy_tail_ratio": 1.4799, + "cos_to_ref": 1.000022, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 2, + "stable_rank": 15.5701, + "effective_rank": 163.2328, + "top_sv": 2.3359, + "fro": 9.2171, + "dispersion": 1.588203, + "heavy_tail_ratio": 1.4084, + "cos_to_ref": 1.000027, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 3, + "stable_rank": 14.2893, + "effective_rank": 170.6733, + "top_sv": 2.376, + "fro": 8.9814, + "dispersion": 1.354001, + "heavy_tail_ratio": 1.3877, + "cos_to_ref": 1.000027, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 4, + "stable_rank": 14.0601, + "effective_rank": 175.9493, + "top_sv": 2.741, + "fro": 10.2779, + "dispersion": 1.10488, + "heavy_tail_ratio": 1.3534, + "cos_to_ref": 1.000026, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 5, + "stable_rank": 12.0389, + "effective_rank": 184.9036, + "top_sv": 3.0235, + "fro": 10.4907, + "dispersion": 0.994679, + "heavy_tail_ratio": 1.3318, + "cos_to_ref": 1.000025, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 6, + "stable_rank": 12.6213, + "effective_rank": 199.5394, + "top_sv": 3.0615, + "fro": 10.8763, + "dispersion": 0.870293, + "heavy_tail_ratio": 1.3135, + "cos_to_ref": 1.000024, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 7, + "stable_rank": 12.2791, + "effective_rank": 209.566, + "top_sv": 3.4406, + "fro": 12.0564, + "dispersion": 0.761686, + "heavy_tail_ratio": 1.2853, + "cos_to_ref": 1.000024, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 8, + "stable_rank": 10.8515, + "effective_rank": 211.3338, + "top_sv": 4.2026, + "fro": 13.844, + "dispersion": 0.686858, + "heavy_tail_ratio": 1.2765, + "cos_to_ref": 1.000024, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 9, + "stable_rank": 12.8053, + "effective_rank": 246.3744, + "top_sv": 4.2578, + "fro": 15.236, + "dispersion": 0.642592, + "heavy_tail_ratio": 1.2738, + "cos_to_ref": 1.000026, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 10, + "stable_rank": 12.4279, + "effective_rank": 260.8717, + "top_sv": 4.91, + "fro": 17.309, + "dispersion": 0.580269, + "heavy_tail_ratio": 1.2368, + "cos_to_ref": 1.000029, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 11, + "stable_rank": 13.6259, + "effective_rank": 298.9343, + "top_sv": 5.4765, + "fro": 20.2151, + "dispersion": 0.540947, + "heavy_tail_ratio": 1.2137, + "cos_to_ref": 1.000031, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 12, + "stable_rank": 23.34, + "effective_rank": 379.37, + "top_sv": 4.4609, + "fro": 21.5511, + "dispersion": 0.457011, + "heavy_tail_ratio": 1.1709, + "cos_to_ref": 1.000026, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 13, + "stable_rank": 25.4164, + "effective_rank": 427.1899, + "top_sv": 4.6153, + "fro": 23.2674, + "dispersion": 0.395684, + "heavy_tail_ratio": 1.1719, + "cos_to_ref": 1.000029, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 14, + "stable_rank": 29.533, + "effective_rank": 462.7915, + "top_sv": 4.9121, + "fro": 26.6938, + "dispersion": 0.34247, + "heavy_tail_ratio": 1.1591, + "cos_to_ref": 1.000034, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 15, + "stable_rank": 29.3374, + "effective_rank": 484.196, + "top_sv": 5.4843, + "fro": 29.7047, + "dispersion": 0.296722, + "heavy_tail_ratio": 1.1539, + "cos_to_ref": 1.000034, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 16, + "stable_rank": 39.0389, + "effective_rank": 583.4448, + "top_sv": 5.2476, + "fro": 32.7868, + "dispersion": 0.226701, + "heavy_tail_ratio": 1.1291, + "cos_to_ref": 1.000036, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 17, + "stable_rank": 44.0809, + "effective_rank": 658.7553, + "top_sv": 5.2543, + "fro": 34.884, + "dispersion": 0.17264, + "heavy_tail_ratio": 1.1189, + "cos_to_ref": 1.000045, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 18, + "stable_rank": 51.4726, + "effective_rank": 701.3905, + "top_sv": 5.082, + "fro": 36.4595, + "dispersion": 0.131642, + "heavy_tail_ratio": 1.092, + "cos_to_ref": 1.000051, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 19, + "stable_rank": 62.1278, + "effective_rank": 768.2172, + "top_sv": 4.6247, + "fro": 36.4515, + "dispersion": 0.093592, + "heavy_tail_ratio": 1.0758, + "cos_to_ref": 1.00006, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 20, + "stable_rank": 61.3346, + "effective_rank": 805.1055, + "top_sv": 4.6786, + "fro": 36.6396, + "dispersion": 0.063345, + "heavy_tail_ratio": 1.0627, + "cos_to_ref": 1.000065, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 21, + "stable_rank": 66.8667, + "effective_rank": 838.1411, + "top_sv": 4.4657, + "fro": 36.5158, + "dispersion": 0.038719, + "heavy_tail_ratio": 1.0478, + "cos_to_ref": 1.000077, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 200, + "layer": 22, + "stable_rank": 39.9709, + "effective_rank": 878.2449, + "top_sv": 5.4491, + "fro": 34.4489, + "dispersion": 0.017331, + "heavy_tail_ratio": 1.0204, + "cos_to_ref": 1.000092, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + } + ], + "runtime_s": 1082.9 +} \ No newline at end of file diff --git a/results/t9_n_convergence_70m.json b/results/t9_n_convergence_70m.json new file mode 100644 index 0000000000000000000000000000000000000000..0311f275b05665be825c5b50d29e5d86a336f4cf --- /dev/null +++ b/results/t9_n_convergence_70m.json @@ -0,0 +1,454 @@ +{ + "experiment": "T9 n-convergence \u2014 operator vs task (THESIS.md C3)", + "status": "EXPLORATORY / DIRECTIONAL \u2014 no pre-registration", + "model": "EleutherAI/pythia-70m-deduped", + "ns": [ + 1, + 5, + 10, + 25, + 50, + 100 + ], + "reference_n": 100, + "top_r": 8, + "layers": [ + 0, + 1, + 2, + 3, + 4 + ], + "dim_batch": 128, + "centered": false, + "nested_prefixes": true, + "task_metric": { + "eval": "multihop", + "by_n": { + "1": { + "auc": 0.0463, + "pass@10": 0.025 + }, + "5": { + "auc": 0.0444, + "pass@10": 0.025 + }, + "10": { + "auc": 0.0419, + "pass@10": 0.025 + }, + "25": { + "auc": 0.0463, + "pass@10": 0.025 + }, + "50": { + "auc": 0.0463, + "pass@10": 0.025 + }, + "100": { + "auc": 0.0526, + "pass@10": 0.025 + } + } + }, + "env": { + "transformers": "5.14.1", + "torch": "2.13.0", + "device": "cpu", + "dtype": "float32" + }, + "jlens_commit": "581d398613e5602a5af361e1c34d3a92ea82ba8e", + "rows": [ + { + "n": 1, + "layer": 0, + "stable_rank": 5.9015, + "effective_rank": 97.7253, + "top_sv": 22.5927, + "fro": 54.8841, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.907213, + "rel_op_dist_to_ref": 0.764033, + "subspace_cos_top_r_vs_ref": 0.570505 + }, + { + "n": 1, + "layer": 1, + "stable_rank": 4.3888, + "effective_rank": 88.2812, + "top_sv": 20.5001, + "fro": 42.9465, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.922288, + "rel_op_dist_to_ref": 0.510418, + "subspace_cos_top_r_vs_ref": 0.691863 + }, + { + "n": 1, + "layer": 2, + "stable_rank": 4.7694, + "effective_rank": 112.3876, + "top_sv": 17.6498, + "fro": 38.5452, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.931073, + "rel_op_dist_to_ref": 0.49399, + "subspace_cos_top_r_vs_ref": 0.632588 + }, + { + "n": 1, + "layer": 3, + "stable_rank": 4.8891, + "effective_rank": 139.0124, + "top_sv": 15.9057, + "fro": 35.1694, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.95554, + "rel_op_dist_to_ref": 0.340332, + "subspace_cos_top_r_vs_ref": 0.722113 + }, + { + "n": 1, + "layer": 4, + "stable_rank": 3.7601, + "effective_rank": 137.7389, + "top_sv": 17.3027, + "fro": 33.5512, + "dispersion": 0.0, + "heavy_tail_ratio": 1.0, + "cos_to_ref": 0.978501, + "rel_op_dist_to_ref": 0.173308, + "subspace_cos_top_r_vs_ref": 0.825213 + }, + { + "n": 5, + "layer": 0, + "stable_rank": 8.704, + "effective_rank": 112.1077, + "top_sv": 18.8236, + "fro": 55.5341, + "dispersion": 0.157984, + "heavy_tail_ratio": 1.1644, + "cos_to_ref": 0.956712, + "rel_op_dist_to_ref": 0.453191, + "subspace_cos_top_r_vs_ref": 0.740424 + }, + { + "n": 5, + "layer": 1, + "stable_rank": 5.3149, + "effective_rank": 99.5206, + "top_sv": 18.7943, + "fro": 43.3283, + "dispersion": 0.122763, + "heavy_tail_ratio": 1.1657, + "cos_to_ref": 0.965635, + "rel_op_dist_to_ref": 0.279743, + "subspace_cos_top_r_vs_ref": 0.786849 + }, + { + "n": 5, + "layer": 2, + "stable_rank": 5.6597, + "effective_rank": 122.5546, + "top_sv": 16.5379, + "fro": 39.3436, + "dispersion": 0.103435, + "heavy_tail_ratio": 1.1529, + "cos_to_ref": 0.97223, + "rel_op_dist_to_ref": 0.255922, + "subspace_cos_top_r_vs_ref": 0.881515 + }, + { + "n": 5, + "layer": 3, + "stable_rank": 5.0507, + "effective_rank": 140.5007, + "top_sv": 15.608, + "fro": 35.0768, + "dispersion": 0.060191, + "heavy_tail_ratio": 1.0812, + "cos_to_ref": 0.980132, + "rel_op_dist_to_ref": 0.209794, + "subspace_cos_top_r_vs_ref": 0.838559 + }, + { + "n": 5, + "layer": 4, + "stable_rank": 3.4324, + "effective_rank": 120.5683, + "top_sv": 18.6155, + "fro": 34.4881, + "dispersion": 0.025918, + "heavy_tail_ratio": 1.0474, + "cos_to_ref": 0.988411, + "rel_op_dist_to_ref": 0.113049, + "subspace_cos_top_r_vs_ref": 0.924091 + }, + { + "n": 10, + "layer": 0, + "stable_rank": 9.1709, + "effective_rank": 115.4639, + "top_sv": 17.5133, + "fro": 53.0362, + "dispersion": 0.172001, + "heavy_tail_ratio": 1.212, + "cos_to_ref": 0.976189, + "rel_op_dist_to_ref": 0.323026, + "subspace_cos_top_r_vs_ref": 0.760063 + }, + { + "n": 10, + "layer": 1, + "stable_rank": 5.5603, + "effective_rank": 103.7253, + "top_sv": 17.6857, + "fro": 41.7031, + "dispersion": 0.133624, + "heavy_tail_ratio": 1.2054, + "cos_to_ref": 0.980599, + "rel_op_dist_to_ref": 0.207508, + "subspace_cos_top_r_vs_ref": 0.864607 + }, + { + "n": 10, + "layer": 2, + "stable_rank": 5.844, + "effective_rank": 126.6813, + "top_sv": 15.8062, + "fro": 38.2104, + "dispersion": 0.109439, + "heavy_tail_ratio": 1.1839, + "cos_to_ref": 0.984101, + "rel_op_dist_to_ref": 0.196289, + "subspace_cos_top_r_vs_ref": 0.891197 + }, + { + "n": 10, + "layer": 3, + "stable_rank": 5.0126, + "effective_rank": 140.3686, + "top_sv": 15.5237, + "fro": 34.7556, + "dispersion": 0.06726, + "heavy_tail_ratio": 1.0875, + "cos_to_ref": 0.988703, + "rel_op_dist_to_ref": 0.164409, + "subspace_cos_top_r_vs_ref": 0.900576 + }, + { + "n": 10, + "layer": 4, + "stable_rank": 3.4371, + "effective_rank": 121.143, + "top_sv": 18.4993, + "fro": 34.2961, + "dispersion": 0.030725, + "heavy_tail_ratio": 1.0508, + "cos_to_ref": 0.993716, + "rel_op_dist_to_ref": 0.083528, + "subspace_cos_top_r_vs_ref": 0.949599 + }, + { + "n": 25, + "layer": 0, + "stable_rank": 8.8378, + "effective_rank": 115.576, + "top_sv": 16.9918, + "fro": 50.5138, + "dispersion": 0.203295, + "heavy_tail_ratio": 1.2558, + "cos_to_ref": 0.987298, + "rel_op_dist_to_ref": 0.209431, + "subspace_cos_top_r_vs_ref": 0.85367 + }, + { + "n": 25, + "layer": 1, + "stable_rank": 5.3723, + "effective_rank": 102.9877, + "top_sv": 17.3009, + "fro": 40.1002, + "dispersion": 0.16111, + "heavy_tail_ratio": 1.2386, + "cos_to_ref": 0.989298, + "rel_op_dist_to_ref": 0.140935, + "subspace_cos_top_r_vs_ref": 0.894818 + }, + { + "n": 25, + "layer": 2, + "stable_rank": 5.6526, + "effective_rank": 126.0299, + "top_sv": 15.5268, + "fro": 36.915, + "dispersion": 0.132796, + "heavy_tail_ratio": 1.2127, + "cos_to_ref": 0.991326, + "rel_op_dist_to_ref": 0.138925, + "subspace_cos_top_r_vs_ref": 0.957447 + }, + { + "n": 25, + "layer": 3, + "stable_rank": 4.9836, + "effective_rank": 141.3216, + "top_sv": 15.2956, + "fro": 34.1455, + "dispersion": 0.082699, + "heavy_tail_ratio": 1.0991, + "cos_to_ref": 0.993607, + "rel_op_dist_to_ref": 0.119826, + "subspace_cos_top_r_vs_ref": 0.932325 + }, + { + "n": 25, + "layer": 4, + "stable_rank": 3.4106, + "effective_rank": 120.4609, + "top_sv": 18.4106, + "fro": 34.0003, + "dispersion": 0.038169, + "heavy_tail_ratio": 1.0561, + "cos_to_ref": 0.996293, + "rel_op_dist_to_ref": 0.063802, + "subspace_cos_top_r_vs_ref": 0.971248 + }, + { + "n": 50, + "layer": 0, + "stable_rank": 8.4874, + "effective_rank": 113.6757, + "top_sv": 17.2924, + "fro": 50.378, + "dispersion": 0.220152, + "heavy_tail_ratio": 1.2505, + "cos_to_ref": 0.994333, + "rel_op_dist_to_ref": 0.140235, + "subspace_cos_top_r_vs_ref": 0.945167 + }, + { + "n": 50, + "layer": 1, + "stable_rank": 5.1293, + "effective_rank": 99.8406, + "top_sv": 17.6735, + "fro": 40.0266, + "dispersion": 0.175321, + "heavy_tail_ratio": 1.2334, + "cos_to_ref": 0.995396, + "rel_op_dist_to_ref": 0.091281, + "subspace_cos_top_r_vs_ref": 0.961325 + }, + { + "n": 50, + "layer": 2, + "stable_rank": 5.5037, + "effective_rank": 123.9425, + "top_sv": 15.709, + "fro": 36.853, + "dispersion": 0.140834, + "heavy_tail_ratio": 1.2105, + "cos_to_ref": 0.99632, + "rel_op_dist_to_ref": 0.084464, + "subspace_cos_top_r_vs_ref": 0.986531 + }, + { + "n": 50, + "layer": 3, + "stable_rank": 4.9149, + "effective_rank": 139.9073, + "top_sv": 15.4266, + "fro": 34.1999, + "dispersion": 0.087577, + "heavy_tail_ratio": 1.0948, + "cos_to_ref": 0.997322, + "rel_op_dist_to_ref": 0.070704, + "subspace_cos_top_r_vs_ref": 0.974727 + }, + { + "n": 50, + "layer": 4, + "stable_rank": 3.3568, + "effective_rank": 117.4976, + "top_sv": 18.6448, + "fro": 34.1601, + "dispersion": 0.040445, + "heavy_tail_ratio": 1.05, + "cos_to_ref": 0.998454, + "rel_op_dist_to_ref": 0.033533, + "subspace_cos_top_r_vs_ref": 0.991471 + }, + { + "n": 100, + "layer": 0, + "stable_rank": 8.5381, + "effective_rank": 115.53, + "top_sv": 16.9856, + "fro": 49.6318, + "dispersion": 0.263322, + "heavy_tail_ratio": 1.2474, + "cos_to_ref": 1.000009, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 100, + "layer": 1, + "stable_rank": 5.1284, + "effective_rank": 100.8964, + "top_sv": 17.4984, + "fro": 39.6268, + "dispersion": 0.207697, + "heavy_tail_ratio": 1.229, + "cos_to_ref": 1.000011, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 100, + "layer": 2, + "stable_rank": 5.4551, + "effective_rank": 123.7123, + "top_sv": 15.6818, + "fro": 36.6263, + "dispersion": 0.165311, + "heavy_tail_ratio": 1.2051, + "cos_to_ref": 1.000008, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 100, + "layer": 3, + "stable_rank": 4.8865, + "effective_rank": 139.4824, + "top_sv": 15.4074, + "fro": 34.0586, + "dispersion": 0.106262, + "heavy_tail_ratio": 1.0901, + "cos_to_ref": 1.00001, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + }, + { + "n": 100, + "layer": 4, + "stable_rank": 3.3581, + "effective_rank": 117.5836, + "top_sv": 18.5768, + "fro": 34.0421, + "dispersion": 0.052284, + "heavy_tail_ratio": 1.0477, + "cos_to_ref": 1.000012, + "rel_op_dist_to_ref": 0.0, + "subspace_cos_top_r_vs_ref": 1.0 + } + ], + "runtime_s": 836.8 +} \ No newline at end of file