publish_model: anonymous clean room, every-file read-back, list-only-on-404 resume, card fields read from run_summary (review E-045)

build/publish_model.py

@@ -74,6 +74,30 @@ def card(cfg, man, aud, run, repo, files, n_params):
     cf = man.get('contamination_filter') or {}
     tok_consumed = int(run.get("tokens_consumed") or 0)
     loss = " -> ".join(str(x) for x in (run.get("prior_loss"), run.get("final_loss")) if x is not None)
+    # Same rule as the architecture: anything the card states about *this* model comes from its own
+    # `run_summary.json`, and a field that is not there stops the publish rather than rendering as None.
+    # The Optimiser/Precision/Batch rows used to be typed prose, which is how a card ends up describing the
+    # recipe somebody meant to run (review E-045/2).
+    need = ("optim", "lr", "warmup_frac", "decay_frac", "world", "tokens_per_step", "seq_len",
+            "micro_batch", "accum", "attn", "grad_ckpt", "tok_per_s", "seed", "data_seed")
+    absent = [k for k in need if run.get(k) is None]
+    if absent:
+        raise SystemExit(f"run_summary.json carries no {absent}, so the card cannot state them without "
+                         "guessing. Publish the summary the trainer wrote, or say in the card what is "
+                         "unknown -- never both.")
+    geom = run["micro_batch"] * run["accum"] * run["world"] * run["seq_len"]
+    if geom != run["tokens_per_step"]:
+        raise SystemExit(f"the batch row would be a false statement: {run['micro_batch']} x "
+                         f"{run['accum']} x {run['world']} x {run['seq_len']} = {geom} != "
+                         f"tokens_per_step {run['tokens_per_step']}")
+    if run.get("val_ppl") is None and run.get("val_skipped") and run.get("val_loss_error") is None:
+        val_row = ("not measured on this leg -- a session that stops at a checkpoint boundary skips the "
+                   "validation pass by design; the run's PPL comes from the horizon export")
+    elif run.get("val_ppl") is None:
+        val_row = f"NOT MEASURED ({run.get('val_loss_error')})"
+    else:
+        val_row = (f"**{run['val_ppl']:.2f}** on {MIX}'s held-out `val/` shards "
+                   f"({man['val_tokens']:,} tokens, never trained on)")
     rows = "\n".join(f"| `{k}` | {v['sha256'][:16]}… | {v['bytes']:,} |" for k, v in sorted(files.items()))
     return f"""---
 license: cc-by-sa-4.0
@@ -98,17 +122,18 @@ report it honestly, including where that is near chance.
 |---|---|
 | Parameters | **{n_params:,}** (recomputed from `model.safetensors`'s own header, not asserted) |
 | Architecture | {p[0]} layers, hidden {p[1]}, GQA {cfg["num_attention_heads"]} Q / {cfg["num_key_value_heads"]} KV heads, SwiGLU {cfg["intermediate_size"]}, RMSNorm {cfg["rms_norm_eps"]}, RoPE θ={rope}, **tied** embeddings, vocab {p[2]:,} |
-| Sequence length | {run.get('seq_len')} — which is why ARC's 25-shot prompts are truncated from the left by the eval harness |
+| Sequence length | {run['seq_len']} — which is why ARC's 25-shot prompts are truncated from the left by the eval harness |
 | Tokens consumed | **{tok_consumed:,}** ({100.0 * tok_consumed / 1e9:.2f} % of the 1.0 B target) |
-| Batch | micro {run.get('micro_batch')} × accum {run.get('accum')} × 2 ranks = {run.get('micro_batch', 0) * run.get('accum', 0) * 2 * run.get('seq_len', 1024):,} tokens/step |
-| Optimiser | AdamW β(0.9, 0.95), wd 0.1, clip 1.0, LR 6e-4, trapezoid schedule (plateau then linear decay over the final 20 %) |
-| Precision | fp16 autocast + fp32 master weights + GradScaler, asserted at start of every session |
-| Throughput | {run.get('tok_per_s'):,} tok/s sustained |
-| Attention | `{run.get('effective_attn', run.get('attn'))}` — measured faster than SDPA on Turing |
-| Gradient checkpointing | {run.get('grad_ckpt')} |
+| Batch | micro {run['micro_batch']} × accum {run['accum']} × {run['world']} ranks = {run['tokens_per_step']:,} tokens/step |
+| Optimiser | `{run['optim']}`, LR {run['lr']}, warmup fraction {run['warmup_frac']}, trapezoid decay over the final {100.0 * (1.0 - (run['decay_frac'] or 0)):.1f} % of the schedule |
+| Precision | fp16 autocast + fp32 master weights + GradScaler — asserted at the start of every session, and each session's log line `precision: fp16=True bf16=False scaler=GradScaler` is the evidence |
+| Throughput | {run['tok_per_s']:,} tok/s in the session that produced this export |
+| Attention | `{run.get('effective_attn') or run['attn']}` |
+| Gradient checkpointing | `{run['grad_ckpt']}` |
+| Randomness | model seed {run['seed']}, data seed {run['data_seed']} |
 | Wall clock | {run.get('session_seconds', run.get('seconds', 0)) / 3600.0:.2f} h in the session that produced this export. The run is split across sessions at checkpoint boundaries, so this is **not** the total training time |
 | Loss | {loss} |
-| Validation PPL | {run.get('val_ppl') if run.get('val_ppl') is not None else 'NOT MEASURED (' + str(run.get('val_loss_error')) + ')'} on {MIX}'s held-out `val/` shards ({man['val_tokens']:,} tokens, never trained on) |
+| Validation PPL | {val_row} |
 
 ## Data
 
@@ -185,8 +210,13 @@ def main():
         dest = "/tmp/cleanroom_" + os.path.basename(a.repo)
         shutil.rmtree(dest, ignore_errors=True)
         shutil.rmtree("/tmp/cleanroom_hf", ignore_errors=True)
-        got = snapshot_download(a.repo, local_dir=dest)
-        print("clean-room download ->", got, flush=True)
+        # Gate 5's whole claim is "a stranger can load this". `install()` above puts HF_TOKEN in the
+        # environment and `huggingface_hub` picks it up from there, so without this the clean room would
+        # be authenticated and a private or gated repo would pass (review E-045/1).
+        for k in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGING_FACE_TOKEN"):
+            os.environ.pop(k, None)
+        got = snapshot_download(a.repo, local_dir=dest, token=False)
+        print("clean-room download (anonymous) ->", got, flush=True)
         import json as _j
         cfg = _j.load(open(os.path.join(dest, "config.json")))
         n_t, n_p, dt = safetensors_counts(os.path.join(dest, "model.safetensors"))
@@ -247,25 +277,44 @@ def main():
         raise SystemExit("no run_summary.json next to the export: the card would be full of blanks. "
                          "Pass --from-dir at the trainer's final/ export, or --dry-run to preview.")
     readme = card(cfg, man, aud, dict(run, params_total=n_p), a.repo, files, n_p)
-    open(os.path.join(src, "README.md"), "w", encoding="utf-8").write(readme)
     print("card:", f"{n_p:,} params from the safetensors header; {len(files)} files hashed", flush=True)
     if a.dry_run:
+        # A preview that writes into the export is not a preview: it leaves a README behind that the next
+        # real run then treats as one of the files to publish (review E-045, minor).
         print(readme.encode("utf-8", "replace").decode("utf-8", "replace"))
         return
+    open(os.path.join(src, "README.md"), "w", encoding="utf-8").write(readme)
 
     api = HfApi(token=tok)
     create_repo(repo_id=a.repo, repo_type="model", private=False, exist_ok=True, token=tok)
     up = [(f, os.path.join(src, f)) for f in sorted(os.listdir(src))
           if os.path.isfile(os.path.join(src, f))]
+    local_sha = {f: sha(p) for f, p in up}
     remote = {}
     try:
-        for ent in api.model_info(a.repo, files_metadata=True).siblings or []:
-            lfs = getattr(ent, "lfs", None)
-            raw = (lfs.get("sha256") or lfs.get("oid")) if isinstance(lfs, dict) else None
-            remote[ent.rfilename] = raw if isinstance(raw, str) and len(raw) == 64 else None
+        for attempt in (1, 2, 3):
+            try:
+                for ent in api.model_info(a.repo, files_metadata=True).siblings or []:
+                    lfs = getattr(ent, "lfs", None)
+                    raw = (lfs.get("sha256") or lfs.get("oid")) if isinstance(lfs, dict) else None
+                    remote[ent.rfilename] = raw if isinstance(raw, str) and len(raw) == 64 else None
+                break
+            except Exception as e:
+                # E-034 again: "this repo has nothing in it yet" and "the Hub did not answer" are different
+                # facts. The first means upload everything; the second, swallowed into `remote = {}` here,
+                # means silently re-uploading 1.3 GB of bytes that are already on the Hub.
+                if type(e).__name__ in ("RepositoryNotFoundError", "GatedRepoError"):
+                    print("(remote listing empty -- first publish)", flush=True)
+                    break
+                if attempt == 3:
+                    raise
+                print(f"(listing failed {type(e).__name__}, retry {attempt}/3)", flush=True)
+                import time as _t
+                _t.sleep(10 * attempt)
     except Exception as e:
-        print("(no listing:", type(e).__name__, ")", flush=True)
-    todo = [f for f, p in up if remote.get(f) != sha(p)]
+        raise SystemExit(f"cannot read the remote listing after 3 tries ({type(e).__name__}: "
+                         f"{str(e)[:160]}) -- refusing to guess what is already uploaded")
+    todo = [f for f, _ in up if remote.get(f) != local_sha[f]]
     print(f"uploading {len(todo)} of {len(up)} files to {a.repo}", flush=True)
     if todo:
         for i in range(0, len(todo), 120):        # one commit per 120 files: E-033's rate ceiling
@@ -276,7 +325,11 @@ def main():
     back = set(api.list_repo_files(repo_id=a.repo, repo_type="model"))
     missing = sorted({f for f, _ in up} - back)
     R = {"repo": a.repo, "params": n_p, "tensors": n_t, "files": len(back), "missing": missing}
-    for f in ("model.safetensors", "config.json"):
+    # Every file, not just the weights and the config: `list_repo_files` proves a name exists, and a
+    # tokenizer whose bytes did not land loads as a *different* tokenizer, which is the failure that
+    # silently invalidates every number in Phase 6 (E-031's rule, applied to the model repo).
+    anon = {}
+    for f, _ in up:
         try:
             h, nbytes = hashlib.sha256(), 0      # stream it: 1.7 GB does not belong in RAM
             with urllib.request.urlopen(f"https://huggingface.co/{a.repo}/resolve/main/{f}",
@@ -284,11 +337,14 @@ def main():
                 for chunk in iter(lambda: r.read(1 << 20), b""):
                     nbytes += len(chunk)
                     h.update(chunk)
-            R[f"anon_{f}"] = {"bytes": nbytes, "sha_ok": h.hexdigest() == sha(os.path.join(src, f))}
+            anon[f] = {"bytes": nbytes, "sha_ok": h.hexdigest() == local_sha[f]}
         except Exception as e:
-            R[f"anon_{f}"] = {"error": f"{type(e).__name__}: {str(e)[:120]}"}
-    R["published_verified"] = bool(not missing and all(R.get(f"anon_{x}", {}).get("sha_ok")
-                                                       for x in ("model.safetensors", "config.json")))
+            anon[f] = {"error": f"{type(e).__name__}: {str(e)[:120]}"}
+    R["anon_readback"] = {k: (v.get("sha_ok") if "sha_ok" in v else v.get("error")) for k, v in anon.items()}
+    bad = sorted(k for k, v in anon.items() if not v.get("sha_ok"))
+    R["published_verified"] = bool(not missing and not bad)
+    if bad:
+        R["anon_failed"] = bad[:10]
     print("PUBLISH_JSON_BEGIN")
     print(json.dumps(R, indent=1, default=str))
     print("PUBLISH_JSON_END")