File size: 3,268 Bytes
3fd1a35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | #!/usr/bin/env python3
"""Finalize a separately built portable release; never overwrite source weights."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import tarfile
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--directory", type=Path, required=True)
parser.add_argument("--model-source", type=Path, required=True)
parser.add_argument("--validation", type=Path, help="Successful acceptance report for this executable")
parser.add_argument("--source-model", default="inclusionAI/Ling-3.0-tiny")
parser.add_argument("--source-revision", default="e3a47d5b986e7141b6efd62597d598ebb392060d")
args = parser.parse_args()
root = Path(__file__).resolve().parents[1]
target = args.directory.resolve()
model = args.model_source.resolve()
if not (target / "mindnano-infer").is_file() or not (target / "MANIFEST.json").is_file():
raise SystemExit("build_portable.py must create the new executable and manifest first")
if target == model.parent:
raise SystemExit("use a separate release directory")
weights = target / "ling3-tiny-w4.l3r"
if not weights.exists():
try:
os.link(model, weights)
except OSError:
shutil.copyfile(model, weights)
elif not os.path.samefile(model, weights):
raise SystemExit("release already contains different weights")
shutil.copyfile(root / "models" / "local-final" / "v6-board-test" / "README.md", target / "README.md")
with tarfile.open(target / "SOURCE.tar.gz", "w:gz") as archive:
for name in ("CMakeLists.txt", "LICENSE", "LICENSES.md", "README.md", "src", "include", "tests", "tools", "third_party", "deployment", "docs"):
def clean(info):
return None if "__pycache__" in info.name or info.name.endswith(".pyc") else info
archive.add(root / name, arcname="Ling3RKNN/" + name, filter=clean)
(target / "MODEL.json").write_text(json.dumps({
"model": "mindnano-ling3-tiny", "mindnano_integration_version": 6,
"weights": weights.name, "artifact": weights.name, "source_model": args.source_model,
"source_revision": args.source_revision,
"capabilities": ["flow_control_ack", "pause_resume", "request_cancel", "top_k", "repeat_penalty", "enable_thinking", "session_cache", "generated_state_reuse", "qa_cache_default", "resident_session_cache"],
}, ensure_ascii=False, indent=2) + "\n")
names = ["mindnano-infer", weights.name, "MANIFEST.json", "MODEL.json", "SOURCE.tar.gz", "README.md"]
if args.validation:
validation = json.loads(args.validation.read_text())
if validation.get("passed") is not True:
raise SystemExit("validation did not pass")
shutil.copyfile(args.validation, target / "VALIDATION.json")
names.append("VALIDATION.json")
lines = []
for name in names:
with (target / name).open("rb") as handle:
digest = hashlib.file_digest(handle, "sha256").hexdigest()
lines.append(f"{digest} {name}\n")
(target / "SHA256SUMS").write_text("".join(lines))
print(json.dumps({"directory": str(target), "files": list(names)}))
if __name__ == "__main__":
main()
|