review round 3 (probe/model-card/eval): rank-visible eval failure + cross-rank peak memory + complete run_summary; card derived-and-asserted from config; harness install --no-deps and task ids confirmed; torchrun redirects 3 + tee 3

build/publish_model.py

@@ -13,6 +13,7 @@ import argparse
 import hashlib
 import json
 import os
+import shutil
 import struct
 import sys
 import urllib.request
@@ -60,6 +61,16 @@ def get_json(url, token=None):
 
 def card(cfg, man, aud, run, repo, files, n_params):
     p = cfg.get("num_hidden_layers"), cfg.get("hidden_size"), cfg.get("vocab_size")
+    # The card states the architecture; half of it was typed rather than read, so a variant export would
+    # have published a description of a model it is not. Derive it and refuse on drift.
+    frozen = {"num_hidden_layers": 22, "hidden_size": 576, "intermediate_size": 1536,
+              "num_attention_heads": 9, "num_key_value_heads": 3, "vocab_size": 49152,
+              "tie_word_embeddings": True, "rms_norm_eps": 1e-5}
+    drift = {k: (cfg.get(k), v) for k, v in frozen.items() if cfg.get(k) != v}
+    if drift:
+        raise SystemExit("config.json does not match the frozen architecture, so the card would be a false "
+                         "statement about this model: " + json.dumps(drift, default=str)[:600])
+    rope = cfg.get("rope_theta") or (cfg.get("rope_parameters") or {}).get("rope_theta")
     cf = man.get('contamination_filter') or {}
     tok_consumed = int(run.get("tokens_consumed") or 0)
     loss = " -> ".join(str(x) for x in (run.get("prior_loss"), run.get("final_loss")) if x is not None)
@@ -75,7 +86,7 @@ tags: [ounce100m, from-scratch, base, decoder-only, {man['tokenizer']['id'].spli
 
 # {repo.split('/')[-1]}
 
-A **{run.get('params', p[0] and '?'):,}-parameter decoder-only transformer trained from scratch on
+A **{n_params:,}-parameter decoder-only transformer trained from scratch on
 {tok_consumed:,} tokens** of `{MIX}`, in {run.get('steps_planned', '?')} optimiser steps on
 **2×Nvidia Tesla T4** inside Kaggle notebook sessions. It is the deliverable of a project whose whole point
 was to find out what a ~100 M-parameter model reaches from ~1 B tokens of a curated public mix, and to
@@ -85,8 +96,8 @@ report it honestly, including where that is near chance.
 
 | | |
 |---|---|
-| Parameters | **{n_params:,}** (verified from `model.safetensors`'s header, not asserted) |
-| Architecture | {p[0]} layers, hidden {p[1]}, GQA 9 Q / 3 KV heads, SwiGLU 1536, RMSNorm 1e-5, RoPE θ=10000, **tied** embeddings, vocab {p[2]:,} |
+| Parameters | **{n_params:,}** (recomputed from `model.safetensors`'s own header, not asserted) |
+| Architecture | {p[0]} layers, hidden {p[1]}, GQA {cfg["num_attention_heads"]} Q / {cfg["num_key_value_heads"]} KV heads, SwiGLU {cfg["intermediate_size"]}, RMSNorm {cfg["rms_norm_eps"]}, RoPE θ={rope}, **tied** embeddings, vocab {p[2]:,} |
 | Sequence length | {run.get('seq_len')} — which is why ARC's 25-shot prompts are truncated from the left by the eval harness |
 | Tokens consumed | **{tok_consumed:,}** ({100.0 * tok_consumed / 1e9:.2f} % of the 1.0 B target) |
 | Batch | micro {run.get('micro_batch')} × accum {run.get('accum')} × 2 ranks = {run.get('micro_batch', 0) * run.get('accum', 0) * 2 * run.get('seq_len', 1024):,} tokens/step |
@@ -95,9 +106,9 @@ report it honestly, including where that is near chance.
 | Throughput | {run.get('tok_per_s'):,} tok/s sustained |
 | Attention | `{run.get('effective_attn', run.get('attn'))}` — measured faster than SDPA on Turing |
 | Gradient checkpointing | {run.get('grad_ckpt')} |
-| Wall clock | {run.get('seconds', 0) / 3600.0:.2f} h of training, split across sessions at checkpoint boundaries |
+| Wall clock | {run.get('session_seconds', run.get('seconds', 0)) / 3600.0:.2f} h in the session that produced this export. The run is split across sessions at checkpoint boundaries, so this is **not** the total training time |
 | Loss | {loss} |
-| Validation PPL | {run.get('val_ppl')} on {MIX}'s held-out `val/` shards (22,934,043 tokens, never trained on) |
+| Validation PPL | {run.get('val_ppl') if run.get('val_ppl') is not None else 'NOT MEASURED (' + str(run.get('val_loss_error')) + ')'} on {MIX}'s held-out `val/` shards ({man['val_tokens']:,} tokens, never trained on) |
 
 ## Data
 
@@ -156,9 +167,15 @@ def main():
     ap.add_argument("--dry-run", action="store_true")
     a = ap.parse_args()
 
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))  # E-028
     import ounce100m_credentials
-    info = ounce100m_credentials.install(verify=True)
+    ounce100m_credentials.install(verify=True)
     tok = os.environ["HF_TOKEN"]
+    # Order matters: huggingface_hub resolves its cache locations at import time, so a "clean room" that
+    # sets HF_HOME afterwards is testing Kaggle's warm disk instead of the Hub (review finding A6).
+    if a.verify_only:
+        os.environ["HF_HOME"] = "/tmp/cleanroom_hf"
+        os.environ["HF_HUB_CACHE"] = "/tmp/cleanroom_hf/hub"
     from huggingface_hub import HfApi, create_repo, snapshot_download
 
     man = get_json(f"https://huggingface.co/datasets/{MIX}/resolve/main/manifest.json")
@@ -166,7 +183,8 @@ def main():
 
     if a.verify_only:
         dest = "/tmp/cleanroom_" + os.path.basename(a.repo)
-        os.environ["HF_HUB_CACHE"] = "/tmp/cleanroom_cache"
+        shutil.rmtree(dest, ignore_errors=True)
+        shutil.rmtree("/tmp/cleanroom_hf", ignore_errors=True)
         got = snapshot_download(a.repo, local_dir=dest)
         print("clean-room download ->", got, flush=True)
         import json as _j
@@ -176,7 +194,10 @@ def main():
         import torch
         from transformers import AutoModelForCausalLM, AutoTokenizer
         tkn = AutoTokenizer.from_pretrained(dest)
-        m = AutoModelForCausalLM.from_pretrained(dest, torch_dtype=torch.float16)
+        try:                          # transformers 5 renamed torch_dtype; Gate 5 is this very call
+            m = AutoModelForCausalLM.from_pretrained(dest, dtype=torch.float16)
+        except TypeError:
+            m = AutoModelForCausalLM.from_pretrained(dest, torch_dtype=torch.float16)
         assert m.num_parameters() == n_p, (m.num_parameters(), n_p)
         assert cfg["tie_word_embeddings"] and m.get_input_embeddings().weight.data_ptr() == \
             m.get_output_embeddings().weight.data_ptr(), "tied embeddings did not survive the round trip"
@@ -212,19 +233,24 @@ def main():
     if os.path.exists(os.path.join(src, "run_summary.json")):
         run = json.load(open(os.path.join(src, "run_summary.json")))
     n_t, n_p, dt = safetensors_counts(os.path.join(src, "model.safetensors"))
-    if run.get("params") and run["params"] != n_p:
-        raise SystemExit(f"run_summary claims {run['params']} params, the file has {n_p}")
+    # `params` in run_summary.json is a dict {sum_numel, unique, trainable}; comparing that to an integer
+    # is always unequal and would have failed every real export (review finding A1).
+    claim = run.get("params_total") or run.get("params")
+    if isinstance(claim, dict):
+        claim = claim.get("sum_numel")
+    if claim and int(claim) != n_p:
+        raise SystemExit(f"run_summary claims {claim} params, the published safetensors file has {n_p}")
     files = {f: {"sha256": sha(os.path.join(src, f)), "bytes": os.path.getsize(os.path.join(src, f))}
              for f in sorted(have) if f in CARD_FILES or f.endswith(".json")}
     print("tensors", n_t, "params", f"{n_p:,}", "dtypes", dt, flush=True)
     if not a.dry_run and not run:
         raise SystemExit("no run_summary.json next to the export: the card would be full of blanks. "
                          "Pass --from-dir at the trainer's final/ export, or --dry-run to preview.")
-    readme = card(cfg, man, aud, dict(run, params=n_p), a.repo, files, n_p)
-    open(os.path.join(src, "README.md"), "w").write(readme)
+    readme = card(cfg, man, aud, dict(run, params_total=n_p), a.repo, files, n_p)
+    open(os.path.join(src, "README.md"), "w", encoding="utf-8").write(readme)
     print("card:", f"{n_p:,} params from the safetensors header; {len(files)} files hashed", flush=True)
     if a.dry_run:
-        print(readme)
+        print(readme.encode("utf-8", "replace").decode("utf-8", "replace"))
         return
 
     api = HfApi(token=tok)
@@ -252,9 +278,13 @@ def main():
     R = {"repo": a.repo, "params": n_p, "tensors": n_t, "files": len(back), "missing": missing}
     for f in ("model.safetensors", "config.json"):
         try:
-            d = urllib.request.urlopen(f"https://huggingface.co/{a.repo}/resolve/main/{f}",
-                                       timeout=600).read()
-            R[f"anon_{f}"] = {"bytes": len(d), "sha_ok": hashlib.sha256(d).hexdigest() == sha(os.path.join(src, f))}
+            h, nbytes = hashlib.sha256(), 0      # stream it: 1.7 GB does not belong in RAM
+            with urllib.request.urlopen(f"https://huggingface.co/{a.repo}/resolve/main/{f}",
+                                        timeout=900) as r:
+                for chunk in iter(lambda: r.read(1 << 20), b""):
+                    nbytes += len(chunk)
+                    h.update(chunk)
+            R[f"anon_{f}"] = {"bytes": nbytes, "sha_ok": h.hexdigest() == sha(os.path.join(src, f))}
         except Exception as e:
             R[f"anon_{f}"] = {"error": f"{type(e).__name__}: {str(e)[:120]}"}
     R["published_verified"] = bool(not missing and all(R.get(f"anon_{x}", {}).get("sha_ok")