File size: 3,509 Bytes
12496fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """Create a fresh allowlisted release, never upload the whole working directory."""
from pathlib import Path
from hashlib import sha256
from datetime import datetime, timezone
import json
import shutil
import subprocess
import uuid
ROOT = Path(__file__).resolve().parents[1]
TOP_FILES = {"README.md", "LICENSE", "NOTICE", "pyproject.toml", "requirements-tested.txt", ".gitignore", ".gitattributes"}
SOURCE_DIRS = {"nexora", "scripts", "tests", "docs", "configs", "examples", "reports"}
ALLOWED_EXTENSIONS = {".py", ".md", ".json", ".jsonl", ".toml", ".txt", ".xml", ".png", ".svg", ".csv", ".ps1", ".sh"}
def main():
# Unique stage per invocation; no recursive cleanup can accidentally touch user files.
stage = ROOT / ".release" / ("stage-" + uuid.uuid4().hex[:12])
stage.mkdir(parents=True)
files = [ROOT / name for name in TOP_FILES if (ROOT / name).is_file()]
for name in SOURCE_DIRS:
files.extend(p for p in (ROOT / name).rglob("*") if p.is_file() and p.suffix in ALLOWED_EXTENSIONS and "__pycache__" not in p.parts and not p.is_symlink())
for folder in ["artifacts/data", "artifacts/tiny", "artifacts/posttraining-experiment"]:
files.extend(p for p in (ROOT / folder).glob("*") if p.is_file() and p.suffix in {".json", ".jsonl", ".npy", ".safetensors"})
checkpoint_dir = ROOT / "artifacts/tiny/checkpoints"
latest = json.loads((checkpoint_dir / "latest.json").read_text())
checkpoint = checkpoint_dir / latest["file"]
if checkpoint.resolve().parent != checkpoint_dir.resolve() or sha256(checkpoint.read_bytes()).hexdigest() != latest["sha256"]:
raise ValueError("Checkpoint manifest mismatch")
files.extend([checkpoint_dir / "latest.json", checkpoint])
inventory = []
for path in sorted(set(files)):
if path.is_symlink() or not path.resolve().is_relative_to(ROOT):
raise ValueError("Release path escapes project")
relative = path.relative_to(ROOT)
if relative.as_posix() in {"reports/release-manifest.json", "reports/upload-receipt.json"}:
continue
destination = stage / relative
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(path, destination)
inventory.append({"path": relative.as_posix(), "bytes": destination.stat().st_size,
"sha256": sha256(destination.read_bytes()).hexdigest()})
revision = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip()
dirty = subprocess.run(["git", "status", "--porcelain"], cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip()
if dirty:
raise RuntimeError("Commit release inputs before packaging; source revision must identify the staged files")
manifest = {"created_utc": datetime.now(timezone.utc).isoformat(), "source_commit": revision, "repository": "devildasdf/NEXORA",
"original_model_parameters": 820736, "status": "research prototype, not trained 120B", "files": inventory,
"manifest_self_hash_excluded": True, "total_bytes": sum(x["bytes"] for x in inventory)}
(stage / "reports/release-manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
(ROOT / ".release/current-stage.txt").write_text(str(stage), encoding="utf-8")
print(json.dumps({"stage": str(stage), "files": len(inventory), "bytes": manifest["total_bytes"], "source_commit": revision}))
if __name__ == "__main__":
main()
|