smoke PASSED (4/4 sources per shard, 40/40 boundaries); add contamination audit; raw target 1.23B = 23% headroom

build/audit_contamination.py

@@ -0,0 +1,172 @@
+# Phase 2 contamination audit (§3.3): mechanical 13-token overlap between the mix and the eval tasks.
+#
+# What §3.3 permits, precisely: "benchmark validation/dev material may be used mechanically: a script
+# computes overlap counts and you never read or learn from the items themselves. Test splits stay
+# untouched until Phase 6." This script obeys that literally, and it is written so that violating it is
+# hard: the only values that ever leave the process are counts and rates per source. No item text, no
+# lengths, no samples, nothing a human could read as a question. The print surface is a dict of integers.
+#
+# Which splits are used as the reference, and why it is broader than the minimum:
+#   for most of these tasks the harness draws its few-shot exemplars from the TRAIN split (ARC, HellaSwag,
+#   PIQA, Winogrande) or from MMLU's `dev`, so overlap with those is contamination by construction -- a
+#   mix that memorises an exemplar is a mix that inflates the score. So the reference set is
+#   {train, validation, dev} where present, per task, and NEVER `test`. That is more conservative than the
+#   literal wording, and it is the set that actually matters.
+#
+# Method: for each reference document, tokenise with the frozen tokenizer and hash every 13-token window
+# into a set of uint64. For each mix document, do the same and count hits. 13-grams match the documented
+# practice of the decontaminated math source in the mix, so the threshold is comparable to published
+# hygiene rather than something invented here.
+#
+# Memory: the reference side is the small one -- a few hundred thousand short items x ~300 grams x 8 B,
+# held as a Python set of ints. The mix side streams document by document from the shard files, so its
+# size is irrelevant. Nothing here holds a billion of anything.
+#
+# Read-only over the mix. CPU session. No credentials.
+
+import argparse
+import hashlib
+import json
+import os
+import struct
+
+import array
+
+K = 13
+# task -> (repo, config(s), splits considered reference material; 'test' deliberately absent)
+REFS = [
+    ("arc_easy", "allenai/ai2_arc", ["ARC-Easy"], ["train", "validation"]),
+    ("arc_challenge", "allenai/ai2_arc", ["ARC-Challenge"], ["train", "validation"]),
+    ("hellaswag", "Rowan/hellaswag", [None], ["train", "validation"]),
+    ("piqa", "baber/piqa", [None], ["train", "validation"]),
+    ("winogrande", "allenai/winogrande", ["winogrande_xl"], ["train", "validation"]),
+    ("mmlu", "cais/mmlu", ["all"], ["dev", "validation"]),
+    ("truthfulqa", "sylinrl/TruthfulQA", [None], ["validation"]),
+    ("gsm8k", "openai/gsm8k", ["main"], ["train", "validation"]),
+]
+TEXT_KEYS = ("question", "text", "choices", "story", "sentence", "prompt", "ctx", "input", "query")
+
+
+def grams64(ids):
+    """Set of uint64 hashes over K-length token windows. Hashing ids rather than characters means the
+    comparison is in the same space as the mix, and no human-readable string is ever formed."""
+    if len(ids) < K:
+        return {struct.unpack("<Q", hashlib.sha256(bytes(array.array("H", ids))).digest()[:8])[0]} \
+            if ids else set()
+    out = set()
+    for i in range(len(ids) - K + 1):
+        w = ids[i:i + K]
+        out.add(struct.unpack("<Q", hashlib.sha256(
+            memoryview(bytes(array.array("H", w)))).digest()[:8])[0])
+    return out
+
+
+def flatten(v, acc):
+    if isinstance(v, str):
+        acc.append(v)
+    elif isinstance(v, dict):
+        for x in v.values():
+            flatten(x, acc)
+    elif isinstance(v, (list, tuple)):
+        for x in v:
+            flatten(x, acc)
+
+
+def build_reference(tk, max_items_per_split):
+    """Return {ref_key: gram_set}. Counts for the report are derived from these sets afterwards, so the
+    reference corpus is streamed exactly once."""
+    import datasets
+
+    sets = {}
+    for task, repo, configs, splits in REFS:
+        for cfg in configs:
+            for sp in splits:
+                key = f"{task}::{cfg or 'default'}::{sp}"
+                try:
+                    ds = (datasets.load_dataset(repo, cfg, split=sp, streaming=True) if cfg
+                          else datasets.load_dataset(repo, split=sp, streaming=True))
+                except Exception as e:
+                    sets[key] = {"error": f"{type(e).__name__}: {str(e)[:140]}"}
+                    print(f"  ref {key}: UNREADABLE {type(e).__name__}", flush=True)
+                    continue
+                g, n = set(), 0
+                for row in ds:
+                    if n >= max_items_per_split:
+                        break
+                    parts = []
+                    for k in TEXT_KEYS:
+                        if k in row:
+                            flatten(row[k], parts)
+                    txt = " ".join(parts)
+                    if not txt.strip():
+                        continue
+                    n += 1
+                    g |= grams64(tk.encode(txt, add_special_tokens=False).ids)
+                sets[key] = g
+                print(f"  ref {key}: {n} items -> {len(g):,} grams", flush=True)
+    return sets
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--root", default="/kaggle/working/mixroot")
+    ap.add_argument("--max-items-per-split", type=int, default=20000)
+    a = ap.parse_args()
+
+    from tokenizers import Tokenizer
+    from huggingface_hub import hf_hub_download
+    tk = Tokenizer.from_file(hf_hub_download("HuggingFaceTB/SmolLM2-135M", "tokenizer.json"))
+    tk.no_truncation()
+
+    print("building reference gram sets (only counts ever leave this process)", flush=True)
+    ref_sets = build_reference(tk, a.max_items_per_split)
+    errors = {k: v["error"] for k, v in ref_sets.items() if isinstance(v, dict)}
+    ref_sets = {k: v for k, v in ref_sets.items() if isinstance(v, set)}
+
+    # stream the mix's own shards; per-shard source lists let a hit be attributed to a source, which is
+    # the only actionable granularity ("drop or clean any source that fails")
+    man = json.load(open(os.path.join(a.root, "manifest.json")))
+    hits = {k: 0 for k in ref_sets}
+    hits_by_source = {}
+    docs = grams_seen = 0
+    import build_mix as BM
+    for rec in man["shards"]:
+        p = os.path.join(a.root, rec["file"])
+        if not os.path.exists(p):
+            continue
+        for d in BM.iter_shard_docs(p):
+            docs += 1
+            g = grams64(list(d))
+            if not g:
+                continue
+            grams_seen += len(g)
+            for k, s in ref_sets.items():
+                ov = len(g & s)
+                if ov:
+                    hits[k] += ov
+                    for srcname in rec["sources"]:
+                        hits_by_source.setdefault(srcname, {}).setdefault(k, 0)
+                        hits_by_source[srcname][k] += ov // max(1, len(rec["sources"]))
+        if docs % 200000 < 5000:
+            print(f"  audited {docs:,} docs, {grams_seen:,} mix grams", flush=True)
+
+    report = {
+        "k": K,
+        "reference": {k: {"distinct_grams": len(v)} for k, v in ref_sets.items()},
+        "reference_unreadable": errors,
+        "mix_documents_audited": docs,
+        "mix_grams_audited": grams_seen,
+        "overlap_grams_total_by_ref": hits,
+        "overlap_rate_by_ref": {k: round(v / max(1, grams_seen), 8) for k, v in hits.items()},
+        "overlap_by_source": hits_by_source,
+        "note": ("counts only; no item text was read or printed. Reference splits are train/validation/dev "
+                 "per task -- never test, per §3.3. Any source with a non-zero overlap here must be dropped "
+                 "or cleaned before Gate 2 is claimed."),
+    }
+    print("AUDIT_JSON_BEGIN")
+    print(json.dumps(report, indent=1, default=str))
+    print("AUDIT_JSON_END")
+
+
+if __name__ == "__main__":
+    main()

build/build_mix.py

@@ -54,9 +54,9 @@ STATE_NAME = "build_state.json"
 # Corrected config and column names, from the Phase 2 inventory (docs/02-mix-plan.md §9). The names in
 # §2 came from dataset cards, and five of them did not exist.
 SOURCES = [
-    dict(repo="HuggingFaceFW/fineweb-edu", config="sample-10BT", col="text", tokens=300_000_000,
+    dict(repo="HuggingFaceFW/fineweb-edu", config="sample-10BT", col="text", tokens=340_000_000,
          min_score=4, why="quality-scored general English"),
-    dict(repo="HuggingFaceFW/finepdfs-edu", config="eng_Latn", col="text", tokens=150_000_000,
+    dict(repo="HuggingFaceFW/finepdfs-edu", config="eng_Latn", col="text", tokens=180_000_000,
          en_filter=True, why="long-form educational PDFs; needs own en filter"),
     dict(repo="HuggingFaceTB/cosmopedia", config="stanford", col="text", tokens=60_000_000,
          why="textbook expository"),
@@ -72,7 +72,7 @@ SOURCES = [
          why="rewritten encyclopedic prose"),
     dict(repo="omarkamali/wikipedia-monthly", config="20260101.en", col="text", tokens=60_000_000,
          en_filter=True, why="current encyclopedic breadth"),
-    dict(repo="HuggingFaceTB/finemath", config="finemath-4plus", col="text", tokens=130_000_000,
+    dict(repo="HuggingFaceTB/finemath", config="finemath-4plus", col="text", tokens=160_000_000,
          why="math; upstream-decontaminated against GSM8K/MMLU/ARC"),
     dict(repo="open-web-math/open-web-math", config="default", col="text", tokens=70_000_000,
          en_filter=True, why="forum/webbook math prose"),