add publish_mix: LFS-first upload, generated card + attribution, Hub-side verification

build/publish_mix.py

@@ -0,0 +1,198 @@
+# Phase 2 publisher: mix + manifest + card -> public `ounce100m-mix-v1`, then verified from the Hub.
+
+import argparse
+import hashlib
+import json
+import os
+import struct
+import sys
+import urllib.request
+
+# licence per (repo, config), as read from the Hub cards on 2026-09-19. The mix contains CC-BY-SA-4.0
+# material, so the derived dataset inherits SA -- see docs/02-mix-plan.md §6.
+LICENCES = {
+    "HuggingFaceFW/fineweb-edu": "ODC-By-1.0",
+    "HuggingFaceFW/finepdfs-edu": "ODC-By-1.0 + Common Crawl Terms of Use",
+    "HuggingFaceTB/cosmopedia": "Apache-2.0",
+    "HuggingFaceFW/finewiki": "CC-BY-SA-4.0",
+    "omarkamali/wikipedia-monthly": "CC-BY-SA-4.0",
+    "HuggingFaceTB/finemath": "ODC-By-1.0",
+    "open-web-math/open-web-math": "ODC-By (card body; cardData.license empty)",
+    "HuggingFaceFW/finephrase": "ODC-By-1.0",
+    "SimpleStories/SimpleStories": "MIT",
+    "common-pile/arxiv_abstracts": "CC0 (arXiv metadata)",
+}
+
+
+def card(man, repo):
+    tot = man["total_tokens"]
+    vt = man.get("val_tokens", 0)
+    rows = []
+    for k, v in sorted(man.get("stage", {}).items()):
+        rows.append(f"| {k} | {v.get('tokens', 0):,} | {v.get('docs_seen', 0):,} | "
+                    f"{v.get('complete', False)} |")
+    src_table = "\n".join(rows) or "| (none) | | | |"
+    return f"""---
+license: cc-by-sa-4.0
+pretty_name: ounce100m pretraining mix v1
+tags: [pretraining, tokenized, english, ounce100m]
+---
+
+# {repo.split('/')[1]}
+
+Tokenised, deduplicated English pre-training mix built for project **ounce100m** — a 90–110M-parameter
+decoder-only base model trained from scratch on ~1B tokens on 2x Tesla T4. This dataset is the exact
+input to that run, published so the result can be reproduced.
+
+- **Training tokens:** {tot:,} across {len(man['shards'])} shards
+- **Held-out validation tokens:** {vt:,} in {len(man.get('val_shards', []))} shards (`val/`), sampled
+  every {man.get('val_holdout', {}).get('stride', '?')}th merged document — across sources, never whole
+  documents, and excluded from the training shards
+- **Tokenizer:** `{man['tokenizer']['id']}` — vocab {man['tokenizer']['vocab_size']:,},
+  sha256 `{man['tokenizer']['sha256'][:16]}…`
+- **Built:** {man.get('built_utc', 'see manifest')}
+
+## Format
+
+Raw binary, no wrapper library. Each shard:
+
+```
+header  : struct '<IIII'  = vocab, n_docs, n_tokens, flags      (16 bytes)
+offsets : uint32 x (n_docs + 1)   cumulative token positions, [0] = 0
+ids     : uint16 x n_tokens       SmolLM2 token ids
+```
+
+Document *i* of a shard is `ids[offsets[i]:offsets[i+1]]`. Concatenating the shards in filename order
+gives the training stream; a training position is exactly `(shard_index, token_offset)`, which is what
+lets the run resume at the identical data position after an interruption.
+
+```python
+import struct, array
+with open(path, "rb") as f:
+    vocab, n_docs, n_tokens, flags = struct.unpack("<IIII", f.read(16))
+    offs = array.array("I"); offs.fromfile(f, n_docs + 1)
+    ids  = array.array("H"); ids.fromfile(f, n_tokens)
+```
+
+## Composition
+
+| source | tokens staged | docs seen | complete |
+|---|---|---|---|
+{src_table}
+
+Per-shard contents, source mix and sha256 digests are in `manifest.json`. Selection rules and the
+reasoning behind each source are in the project's `docs/02-mix-plan.md`; licence attribution and the
+transformations applied to each source are in `ATTRIBUTION.md`.
+
+## Provenance and hygiene
+
+- Built by streaming public datasets on the Hugging Face Hub, quality/English-gated, deduplicated by
+  exact document hash and by source url/id.
+- **No benchmark content.** Overlap was computed mechanically against eval *train/validation/dev*
+  material only; test splits were never opened during construction. Counts are in `audit.json`.
+- No raw web crawl: web-derived sources here are classifier-scored or curated subsets.
+- Trained model: see the `ounce100m-*` model repos in the same namespace.
+"""
+
+
+ATTRIBUTION = """# Attribution
+
+This is a derived dataset. Each source keeps its own licence; the mix as a whole is offered under
+CC-BY-SA-4.0 because it contains CC-BY-SA-4.0 material (finewiki, wikipedia-monthly), whose share-alike
+clause applies to derivatives.
+
+| source | licence | transformation applied here |
+|---|---|---|
+"""
+
+
+def attribution(man):
+    lines = []
+    for k, v in sorted(man.get("stage", {}).items()):
+        repo = v.get("repo") or k.split("__")[0]
+        lic = LICENCES.get(repo, "see card")
+        lines.append(f"| `{k}` ({repo}) | {lic} | streamed, whitespace-normalised, "
+                     f"length/quality gated, exact- and url-deduplicated, tokenised to uint16 |")
+    return ATTRIBUTION + "\n".join(lines) + "\n"
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--root", default="/kaggle/working/mixroot")
+    ap.add_argument("--repo", default="Cion-lab/ounce100m-mix-v1")
+    ap.add_argument("--dry-run", action="store_true")
+    a = ap.parse_args()
+
+    import ounce100m_credentials
+    ounce100m_credentials.install(verify=True)
+    tok = os.environ["HF_TOKEN"]
+    from huggingface_hub import HfApi, create_repo
+
+    man = json.load(open(os.path.join(a.root, "manifest.json")))
+    if not man.get("shards"):
+        print("FATAL: refusing to publish a mix with zero shards", flush=True)
+        sys.exit(5)
+    man["built_utc"] = __import__("time").strftime("%Y-%m-%dT%H:%M:%SZ", __import__("time").gmtime())
+    with open(os.path.join(a.root, "manifest.json"), "w") as f:
+        json.dump(man, f, indent=1)
+    open(os.path.join(a.root, "README.md"), "w").write(card(man, a.repo))
+    open(os.path.join(a.root, "ATTRIBUTION.md"), "w").write(attribution(man))
+
+    api = HfApi(token=tok)
+    owner, name = a.repo.split("/", 1)
+    if not a.dry_run:
+        create_repo(repo_id=a.repo, repo_type="dataset", private=False, exist_ok=True, token=tok)
+        # .gitattributes must exist before the first .bin, or a >10MB shard is rejected as
+        # "should be tracked by LFS" rather than uploaded as LFS.
+        api.upload_file(path_or_fileobj=b"*.bin filter=lfs diff=lfs merge=lfs -text\n",
+                        path_in_repo=".gitattributes", repo_id=a.repo, repo_type="dataset",
+                        commit_message="track shard binaries with LFS")
+
+    to_upload = []
+    for grp in ("shards", "val_shards"):
+        for rec in man.get(grp, []):
+            to_upload.append((rec["file"], os.path.join(a.root, rec["file"])))
+    for extra in ("manifest.json", "README.md", "ATTRIBUTION.md", "audit.json",
+                  "build_state.json"):
+        p = os.path.join(a.root, extra)
+        if os.path.exists(p):
+            to_upload.append((extra, p))
+    print(f"uploading {len(to_upload)} files to {a.repo}", flush=True)
+    if a.dry_run:
+        for np, lp in to_upload:
+            print("  would upload", np, os.path.getsize(lp), flush=True)
+        return
+
+    for np, lp in to_upload:
+        with open(lp, "rb") as fh:
+            payload = fh.read()
+        api.upload_file(path_or_fileobj=payload, path_in_repo=np, repo_id=a.repo,
+                        repo_type="dataset", commit_message=f"add {np}")
+        print(f"  + {np} ({len(payload) / 1024**2:.1f} MB)", flush=True)
+
+    # Independent verification from the Hub, not from the client that just wrote it.
+    remote = set(api.list_repo_files(repo_id=a.repo, repo_type="dataset"))
+    want = {np for np, _ in to_upload} | {".gitattributes"}
+    R = {"repo": a.repo, "files_expected": len(want), "files_present": len(want & remote),
+         "missing": sorted(want - remote)}
+    first = man["shards"][0]["file"]
+    url = f"https://huggingface.co/datasets/{a.repo}/resolve/main/{first}"
+    try:
+        with urllib.request.urlopen(url, timeout=300) as r:
+            body = r.read()
+        hv = struct.unpack("<IIII", body[:16])
+        R["anonymous_shard_read"] = {"bytes": len(body), "header": list(hv),
+                                     "n_tokens_matches_manifest": hv[2] == man["shards"][0]["tokens"],
+                                     "sha256_matches_manifest": hashlib.sha256(body).hexdigest()
+                                     == man["shards"][0]["sha256"]}
+    except Exception as e:
+        R["anonymous_shard_read"] = {"error": f"{type(e).__name__}: {str(e)[:160]}"}
+    R["published_verified"] = bool(not R["missing"] and R["anonymous_shard_read"].get(
+        "sha256_matches_manifest"))
+    print("PUBLISH_JSON_BEGIN")
+    print(json.dumps(R, indent=1, default=str))
+    print("PUBLISH_JSON_END")
+
+
+if __name__ == "__main__":
+    main()