NOEON / scripts /weights_report.py
Frankenstein-Labs's picture
NOEON: full model, code, configs, tokenizers, manifests
419f250 verified
Raw History Blame Contribute Delete
3.27 kB
#!/usr/bin/env python3
"""Generate weights/MANIFEST.md — every weight identified by origin/size/format/state.
Reads weights/source_files.json (origin metadata + SHA-256 of each source file)
and weights/weights_manifest.json (NOEON-produced parameters). Produces a single
human-readable table that distinguishes *original source* weights from *NOEON
produced* weights, and marks the latter's training state explicitly.
Usage:
python scripts/weights_report.py
"""
from __future__ import annotations
import json
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
W = ROOT / "weights"
def fmt_bytes(n: int) -> str:
for unit in ("B", "KB", "MB", "GB", "TB"):
if n < 1024:
return f"{n:.1f} {unit}"
n /= 1024
return f"{n:.1f} PB"
def main() -> int:
src = json.loads((W / "source_files.json").read_text())
lines = ["# NOEON weights manifest", "",
"Every weight is identified by origin, size, format and state.",
"`origin = original source` means the file belongs to a frozen",
"upstream model and is byte-identical to it (SHA-256 from the Hub).",
"`origin = NOEON` means NOEON produced it.", "",
"## NOEON-produced parameters", ""]
wm_path = W / "weights_manifest.json"
if wm_path.exists():
wm = json.loads(wm_path.read_text())
lines += [f"State: **{wm['state']}**. {wm['note']}", "",
f"- trainable parameters: **{wm['trainable_params']:,}**",
f"- frozen source parameters: **{wm['frozen_source_params']:,}**",
f"- tensors: {wm['n_tensors']}", "",
"| file | size | sha256 | origin | state |",
"|---|---|---|---|---|"]
for f in wm["produced_files"]:
lines.append(f"| `{f['path']}` | {fmt_bytes(f['size'])} | "
f"`{f['sha256'][:16]}…` | NOEON | {f['state']} |")
lines += ["", "## Original source weights (not downloaded here)", "",
"These are the frozen upstream parameters. NOEON never modifies",
"them. Sizes and SHA-256 come from the Hub tree API, so they can be",
"verified after `huggingface-cli download`.", ""]
for key, s in src["sources"].items():
weights = [f for f in s.get("files", []) if f.get("is_weight")]
total = sum(f["size"] or 0 for f in weights)
lines += [f"### {key} — `{s['repo']}` @ `{s.get('revision','?')}`", "",
f"- license: {s.get('license')} · role: {s.get('role')}",
f"- weight files: {len(weights)} · total: {fmt_bytes(total)}",
"", "| file | size | format | sha256 | origin |",
"|---|---|---|---|---|"]
for f in weights:
fmt = f["path"].rsplit(".", 1)[-1]
sha = (f.get("lfs_sha256") or f.get("blob_sha1") or "—")
lines.append(f"| `{f['path']}` | {fmt_bytes(f['size'] or 0)} | {fmt} | "
f"`{sha[:16]}…` | original source |")
lines.append("")
(W / "MANIFEST.md").write_text("\n".join(lines) + "\n")
print(f"wrote {W / 'MANIFEST.md'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())