Phase 5 tooling: publish the final weights with a card whose every number is read back off the Hub (mix manifest, audit.json, run_summary); parameter count recomputed from the safetensors header rather than asserted; batched commits; anonymous sha read-back; --verify-only is the Gate 5 clean-room load+generate

build/publish_model.py

@@ -0,0 +1,271 @@
+"""Phase 5: the final weights -> a public model repo with a card that only states measured facts.
+
+Everything quoted in the card is read back off the Hub at run time — the mix manifest, `audit.json`, the
+trainer's `run_summary.json` — rather than retyped from memory, because the card is the artifact a stranger
+will trust and `final-report.md` will cite (§3.12: the claim and the evidence must be the same object).
+
+Gate 5 is not this script: it is `--verify-only` running on a *fresh instance with an empty cache*, which is
+the only way to prove nothing the model needs lives on a Kaggle disk. This script therefore ends by printing
+the exact command for that second job.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import struct
+import sys
+import urllib.request
+
+MIX = "Cion-lab/ounce100m-mix-v1"
+CKPT = "Cion-lab/ounce100m-ckpt"
+CARD_FILES = ("config.json", "model.safetensors", "tokenizer.json", "tokenizer_config.json",
+              "special_tokens_map.json", "vocab.json", "merges.txt", "cursor.json",
+              "run_summary.json", "generation_config.json")
+
+
+def sha(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as fh:
+        for c in iter(lambda: fh.read(1 << 20), b""):
+            h.update(c)
+    return h.hexdigest()
+
+
+def safetensors_counts(p):
+    """(n_tensors, n_params, dtypes) straight from the file header — no torch, so this check runs on a
+    CPU box and on any image, and it proves the *published bytes* have the parameter count the card claims."""
+    with open(p, "rb") as fh:
+        (n,) = struct.unpack("<Q", fh.read(8))
+        hdr = json.loads(fh.read(n))
+    total, dtypes = 0, set()
+    for k, v in hdr.items():
+        if k == "__metadata__":
+            continue
+        cnt = 1
+        for d in v["shape"]:
+            cnt *= d
+        total += cnt
+        dtypes.add(v["dtype"])
+    return len([k for k in hdr if k != "__metadata__"]), total, sorted(dtypes)
+
+
+def get_json(url, token=None):
+    req = urllib.request.Request(url)
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=120) as r:
+        return json.loads(r.read().decode())
+
+
+def card(cfg, man, aud, run, repo, files, n_params):
+    p = cfg.get("num_hidden_layers"), cfg.get("hidden_size"), cfg.get("vocab_size")
+    cf = man.get('contamination_filter') or {}
+    tok_consumed = int(run.get("tokens_consumed") or 0)
+    loss = " -> ".join(str(x) for x in (run.get("prior_loss"), run.get("final_loss")) if x is not None)
+    rows = "\n".join(f"| `{k}` | {v['sha256'][:16]}… | {v['bytes']:,} |" for k, v in sorted(files.items()))
+    return f"""---
+license: cc-by-sa-4.0
+base_model: {repo}
+language: en
+library_name: transformers
+pipeline_tag: text-generation
+tags: [ounce100m, from-scratch, base, decoder-only, {man['tokenizer']['id'].split('/')[0].lower()}]
+---
+
+# {repo.split('/')[-1]}
+
+A **{run.get('params', p[0] and '?'):,}-parameter decoder-only transformer trained from scratch on
+{tok_consumed:,} tokens** of `{MIX}`, in {run.get('steps_planned', '?')} optimiser steps on
+**2×Nvidia Tesla T4** inside Kaggle notebook sessions. It is the deliverable of a project whose whole point
+was to find out what a ~100 M-parameter model reaches from ~1 B tokens of a curated public mix, and to
+report it honestly, including where that is near chance.
+
+## Measured facts
+
+| | |
+|---|---|
+| Parameters | **{n_params:,}** (verified from `model.safetensors`'s header, not asserted) |
+| Architecture | {p[0]} layers, hidden {p[1]}, GQA 9 Q / 3 KV heads, SwiGLU 1536, RMSNorm 1e-5, RoPE θ=10000, **tied** embeddings, vocab {p[2]:,} |
+| Sequence length | {run.get('seq_len')} — which is why ARC's 25-shot prompts are truncated from the left by the eval harness |
+| Tokens consumed | **{tok_consumed:,}** ({100.0 * tok_consumed / 1e9:.2f} % of the 1.0 B target) |
+| Batch | micro {run.get('micro_batch')} × accum {run.get('accum')} × 2 ranks = {run.get('micro_batch', 0) * run.get('accum', 0) * 2 * run.get('seq_len', 1024):,} tokens/step |
+| Optimiser | AdamW β(0.9, 0.95), wd 0.1, clip 1.0, LR 6e-4, trapezoid schedule (plateau then linear decay over the final 20 %) |
+| Precision | fp16 autocast + fp32 master weights + GradScaler, asserted at start of every session |
+| Throughput | {run.get('tok_per_s'):,} tok/s sustained |
+| Attention | `{run.get('effective_attn', run.get('attn'))}` — measured faster than SDPA on Turing |
+| Gradient checkpointing | {run.get('grad_ckpt')} |
+| Wall clock | {run.get('seconds', 0) / 3600.0:.2f} h of training, split across sessions at checkpoint boundaries |
+| Loss | {loss} |
+| Validation PPL | {run.get('val_ppl')} on {MIX}'s held-out `val/` shards (22,934,043 tokens, never trained on) |
+
+## Data
+
+`{MIX}`: {man['total_tokens']:,} training tokens in {man['n_shards']} shards + {man['val_tokens']:,}
+held-out validation tokens, cut with `{man['tokenizer']['id']}` ({man['tokenizer']['vocab_size']:,} vocab,
+Apache-2.0), from {len(man.get('stage') or {})} public sources at ≥{man.get('distinct_sources_per_shard_min')}
+per shard. Format and loading code are in that dataset's card.
+
+**Contamination.** Overlap was measured mechanically against benchmark **train/validation/dev** material of
+the eight evaluation tasks (13-token windows, counts only — no benchmark item text was ever read, and test
+splits stayed untouched until evaluation). `audit.json` in the dataset repo records
+`overlap_total = {aud.get('overlap_total')}` over {aud.get('mix_documents_audited'):,} documents and
+{aud.get('val_documents_audited'):,} held-out documents, `tasks_covered` = {len(aud.get('tasks_covered') or [])}/8.
+Building it required dropping {cf.get('docs_dropped', '?')} documents
+({cf.get('tokens_dropped', 0):,} tokens)
+that matched reference material; the exact exclusion masks ship in `filter/` so the mix is reproducible.
+
+## Evaluate it
+
+```bash
+pip install "lm-eval==0.4.13"
+lm_eval --model hf --model_args pretrained={repo},dtype=float16,trust_remote_code=False \\
+  --tasks arc_challenge,arc_easy,hellaswag,mmlu,piqa,truthfulqa_mc1,truthfulqa_mc2,winogrande,gsm8k \\
+  --batch_size 8 --seed 42 --log_samples
+```
+
+The run's own results, with shot counts and chance levels beside every number, are in `eval/` here. Four of
+the eight tasks (Winogrande, HellaSwag, PIQA, TruthfulQA) are scored by the harness on *validation* — the
+splits this project was allowed to read — and the other four on test.
+
+## Files
+
+| file | sha256 | bytes |
+|---|---|---|
+{rows}
+
+`cursor.json` is the data position this checkpoint corresponds to: samples consumed, step, seed, the
+corpus fingerprint and a hash of the visit-order permutation, so the resume can be checked rather than
+trusted.
+
+## Not in this model
+
+No instruction tuning, no RLHF, no chat template — it is a base model, deliberately. It has ~1 B tokens of
+training against the ~300 B of the comparable published reference models, so expect scores at or near
+chance on several tasks; that is the experiment's result, not a defect to be engineered around.
+"""
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--repo", default="Cion-lab/ounce100m-v1")
+    ap.add_argument("--src", default="/kaggle/working/mixroot", help="mix root (for tokenizer.json)")
+    ap.add_argument("--from-dir", default="", help="local dir holding the trainer's final/ export")
+    ap.add_argument("--verify-only", action="store_true",
+                    help="fresh-instance clean-room test: download the published repo and load it")
+    ap.add_argument("--dry-run", action="store_true")
+    a = ap.parse_args()
+
+    import ounce100m_credentials
+    info = ounce100m_credentials.install(verify=True)
+    tok = os.environ["HF_TOKEN"]
+    from huggingface_hub import HfApi, create_repo, snapshot_download
+
+    man = get_json(f"https://huggingface.co/datasets/{MIX}/resolve/main/manifest.json")
+    aud = get_json(f"https://huggingface.co/datasets/{MIX}/resolve/main/audit.json")
+
+    if a.verify_only:
+        dest = "/tmp/cleanroom_" + os.path.basename(a.repo)
+        os.environ["HF_HUB_CACHE"] = "/tmp/cleanroom_cache"
+        got = snapshot_download(a.repo, local_dir=dest)
+        print("clean-room download ->", got, flush=True)
+        import json as _j
+        cfg = _j.load(open(os.path.join(dest, "config.json")))
+        n_t, n_p, dt = safetensors_counts(os.path.join(dest, "model.safetensors"))
+        print("VERIFY tensors", n_t, "params", f"{n_p:,}", "dtypes", dt, flush=True)
+        import torch
+        from transformers import AutoModelForCausalLM, AutoTokenizer
+        tkn = AutoTokenizer.from_pretrained(dest)
+        m = AutoModelForCausalLM.from_pretrained(dest, torch_dtype=torch.float16)
+        assert m.num_parameters() == n_p, (m.num_parameters(), n_p)
+        assert cfg["tie_word_embeddings"] and m.get_input_embeddings().weight.data_ptr() == \
+            m.get_output_embeddings().weight.data_ptr(), "tied embeddings did not survive the round trip"
+        g = torch.Generator(device="cpu").manual_seed(20260919)
+        torch.manual_seed(20260919)
+        ids = tkn("The city of Paris is", return_tensors="pt").input_ids
+        out = m.generate(ids, max_new_tokens=24, do_sample=True, temperature=1.0, top_k=50,
+                         top_p=0.9, pad_token_id=tkn.eos_token_id)
+        text = tkn.decode(out[0], skip_special_tokens=True)
+        print("GENERATED:", text, flush=True)
+        print("GEN_SHA", hashlib.sha256(text.encode()).hexdigest()[:16], flush=True)
+        print("VERDICT GATE5 load_ok=True generate_ok=%s params_ok=%s tied_ok=True vocab=%d"
+              % (len(out[0]) > ids.shape[1], m.num_parameters() == n_p, cfg["vocab_size"]), flush=True)
+        return
+
+    src = a.from_dir
+    if not src:
+        ptr = get_json(f"https://huggingface.co/datasets/{CKPT}/resolve/main/latest.json", tok)
+        print("ckpt latest.json ->", {k: ptr.get(k) for k in ("step", "path_in_repo")}, flush=True)
+        src = snapshot_download(CKPT, repo_type="dataset", allow_patterns=[ptr["path_in_repo"] + "/*"],
+                                local_dir="/kaggle/working/from_hub")
+        src = os.path.join(src, ptr["path_in_repo"])
+    have = {f for f in os.listdir(src) if os.path.isfile(os.path.join(src, f))}
+    need = {"config.json", "model.safetensors"}
+    if not need <= have:
+        raise SystemExit(f"{src} is missing {sorted(need - have)} -- nothing to publish")
+    if "tokenizer.json" not in have and os.path.exists(os.path.join(a.src, "tokenizer.json")):
+        import shutil
+        shutil.copy(os.path.join(a.src, "tokenizer.json"), src)     # ship the tokenizer that cut the mix
+        have.add("tokenizer.json")
+    cfg = json.load(open(os.path.join(src, "config.json")))
+    run = {}
+    if os.path.exists(os.path.join(src, "run_summary.json")):
+        run = json.load(open(os.path.join(src, "run_summary.json")))
+    n_t, n_p, dt = safetensors_counts(os.path.join(src, "model.safetensors"))
+    if run.get("params") and run["params"] != n_p:
+        raise SystemExit(f"run_summary claims {run['params']} params, the file has {n_p}")
+    files = {f: {"sha256": sha(os.path.join(src, f)), "bytes": os.path.getsize(os.path.join(src, f))}
+             for f in sorted(have) if f in CARD_FILES or f.endswith(".json")}
+    print("tensors", n_t, "params", f"{n_p:,}", "dtypes", dt, flush=True)
+    if not a.dry_run and not run:
+        raise SystemExit("no run_summary.json next to the export: the card would be full of blanks. "
+                         "Pass --from-dir at the trainer's final/ export, or --dry-run to preview.")
+    readme = card(cfg, man, aud, dict(run, params=n_p), a.repo, files, n_p)
+    open(os.path.join(src, "README.md"), "w").write(readme)
+    print("card:", f"{n_p:,} params from the safetensors header; {len(files)} files hashed", flush=True)
+    if a.dry_run:
+        print(readme)
+        return
+
+    api = HfApi(token=tok)
+    create_repo(repo_id=a.repo, repo_type="model", private=False, exist_ok=True, token=tok)
+    up = [(f, os.path.join(src, f)) for f in sorted(os.listdir(src))
+          if os.path.isfile(os.path.join(src, f))]
+    remote = {}
+    try:
+        for ent in api.model_info(a.repo, files_metadata=True).siblings or []:
+            lfs = getattr(ent, "lfs", None)
+            raw = (lfs.get("sha256") or lfs.get("oid")) if isinstance(lfs, dict) else None
+            remote[ent.rfilename] = raw if isinstance(raw, str) and len(raw) == 64 else None
+    except Exception as e:
+        print("(no listing:", type(e).__name__, ")", flush=True)
+    todo = [f for f, p in up if remote.get(f) != sha(p)]
+    print(f"uploading {len(todo)} of {len(up)} files to {a.repo}", flush=True)
+    if todo:
+        for i in range(0, len(todo), 120):        # one commit per 120 files: E-033's rate ceiling
+            b = todo[i:i + 120]
+            api.upload_folder(repo_id=a.repo, folder_path=src, path_in_repo="",
+                              allow_patterns=b, max_workers=2, repo_type="model",
+                              commit_message=f"publish weights: batch {i // 120 + 1} of {len(b)} file(s)")
+    back = set(api.list_repo_files(repo_id=a.repo, repo_type="model"))
+    missing = sorted({f for f, _ in up} - back)
+    R = {"repo": a.repo, "params": n_p, "tensors": n_t, "files": len(back), "missing": missing}
+    for f in ("model.safetensors", "config.json"):
+        try:
+            d = urllib.request.urlopen(f"https://huggingface.co/{a.repo}/resolve/main/{f}",
+                                       timeout=600).read()
+            R[f"anon_{f}"] = {"bytes": len(d), "sha_ok": hashlib.sha256(d).hexdigest() == sha(os.path.join(src, f))}
+        except Exception as e:
+            R[f"anon_{f}"] = {"error": f"{type(e).__name__}: {str(e)[:120]}"}
+    R["published_verified"] = bool(not missing and all(R.get(f"anon_{x}", {}).get("sha_ok")
+                                                       for x in ("model.safetensors", "config.json")))
+    print("PUBLISH_JSON_BEGIN")
+    print(json.dumps(R, indent=1, default=str))
+    print("PUBLISH_JSON_END")
+    print("NEXT (Gate 5, on a fresh instance with an empty cache): "
+          f"python build/publish_model.py --repo {a.repo} --verify-only", flush=True)
+    raise SystemExit(0 if R["published_verified"] else 11)
+
+
+if __name__ == "__main__":
+    main()