#!/usr/bin/env python3 """Generate weights/MANIFEST.md — every weight identified by origin/size/format/state. Reads weights/source_files.json (origin metadata + SHA-256 of each source file) and weights/weights_manifest.json (NOEON-produced parameters). Produces a single human-readable table that distinguishes *original source* weights from *NOEON produced* weights, and marks the latter's training state explicitly. Usage: python scripts/weights_report.py """ from __future__ import annotations import json from pathlib import Path ROOT = Path(__file__).resolve().parent.parent W = ROOT / "weights" def fmt_bytes(n: int) -> str: for unit in ("B", "KB", "MB", "GB", "TB"): if n < 1024: return f"{n:.1f} {unit}" n /= 1024 return f"{n:.1f} PB" def main() -> int: src = json.loads((W / "source_files.json").read_text()) lines = ["# NOEON weights manifest", "", "Every weight is identified by origin, size, format and state.", "`origin = original source` means the file belongs to a frozen", "upstream model and is byte-identical to it (SHA-256 from the Hub).", "`origin = NOEON` means NOEON produced it.", "", "## NOEON-produced parameters", ""] wm_path = W / "weights_manifest.json" if wm_path.exists(): wm = json.loads(wm_path.read_text()) lines += [f"State: **{wm['state']}**. {wm['note']}", "", f"- trainable parameters: **{wm['trainable_params']:,}**", f"- frozen source parameters: **{wm['frozen_source_params']:,}**", f"- tensors: {wm['n_tensors']}", "", "| file | size | sha256 | origin | state |", "|---|---|---|---|---|"] for f in wm["produced_files"]: lines.append(f"| `{f['path']}` | {fmt_bytes(f['size'])} | " f"`{f['sha256'][:16]}…` | NOEON | {f['state']} |") lines += ["", "## Original source weights (not downloaded here)", "", "These are the frozen upstream parameters. NOEON never modifies", "them. Sizes and SHA-256 come from the Hub tree API, so they can be", "verified after `huggingface-cli download`.", ""] for key, s in src["sources"].items(): weights = [f for f in s.get("files", []) if f.get("is_weight")] total = sum(f["size"] or 0 for f in weights) lines += [f"### {key} — `{s['repo']}` @ `{s.get('revision','?')}`", "", f"- license: {s.get('license')} · role: {s.get('role')}", f"- weight files: {len(weights)} · total: {fmt_bytes(total)}", "", "| file | size | format | sha256 | origin |", "|---|---|---|---|---|"] for f in weights: fmt = f["path"].rsplit(".", 1)[-1] sha = (f.get("lfs_sha256") or f.get("blob_sha1") or "—") lines.append(f"| `{f['path']}` | {fmt_bytes(f['size'] or 0)} | {fmt} | " f"`{sha[:16]}…` | original source |") lines.append("") (W / "MANIFEST.md").write_text("\n".join(lines) + "\n") print(f"wrote {W / 'MANIFEST.md'}") return 0 if __name__ == "__main__": raise SystemExit(main())