"""Build an allowlisted, checksummed Hugging Face v0.1 release without caches.""" import argparse import gzip import hashlib import json import shutil import tarfile from pathlib import Path def sha256(path): digest = hashlib.sha256() with Path(path).open("rb") as stream: for block in iter(lambda: stream.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def copy_tree(source, target): for path in sorted(source.rglob("*")): if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc": if path.is_symlink(): raise ValueError(f"Release cannot include symlink: {path}") destination = target / path.relative_to(source) destination.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(path, destination) def archive(root, relative, output): with output.open("wb") as raw: with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=0) as zipped: with tarfile.open(fileobj=zipped, mode="w", format=tarfile.PAX_FORMAT) as tar: for path in sorted((root / relative).rglob("*")): if not path.is_file(): continue if path.is_symlink(): raise ValueError("Dataset symlinks cannot be packaged") info = tar.gettarinfo(str(path), arcname=str(path.relative_to(root))) info.uid = info.gid = info.mtime = 0 info.uname = info.gname = "" with path.open("rb") as stream: tar.addfile(info, stream) def build(root, output): root, output = Path(root).resolve(), Path(output).resolve() if output.exists() and any(output.iterdir()): raise ValueError("Use a fresh output folder; an existing release is never overwritten") output.mkdir(parents=True, exist_ok=True) for directory in ("tfm", "tests", "configs", "docs", "examples", "scripts"): copy_tree(root / directory, output / directory) for name in ("pyproject.toml", "uv.lock", "LICENSE", ".gitignore"): shutil.copyfile(root / name, output / name) (output / "docs/usage.md").write_text((root / "README.md").read_text().replace("(docs/", "(")) shutil.copyfile(root / "docs/huggingface-model-card.md", output / "README.md") artifacts = ( "transformer-v2", "transformer-h96-v2", "transformer-nohistory-v2", "mlp-v2", "gru-v2", "onnx-transformer-v2", "report-transformer-v2", "report-mlp-v2", "report-gru-v2", "report-ensemble-v2", "report-context-v2", ) for name in artifacts: relative = Path("artifacts/smoke") / name copy_tree(root / relative, output / relative) for name in ( "comparison-v2.json", "comparison-v2-config.json", "api-latency-cpu-v2.json", "api-latency-mps-v2.json", "api-latency-ensemble-cpu-v2.json", "http-prediction-v2.json", ): relative = Path("artifacts/smoke") / name shutil.copyfile(root / relative, output / relative) (output / "datasets").mkdir() datasets = [] for relative, name in ( (Path("artifacts/poc/data"), "synthetic-1200-v0.1.2.tar.gz"), (Path("artifacts/smoke/data-v2"), "smoke-90-v0.1.2.tar.gz"), ): manifest = json.loads((root / relative / "manifest.json").read_text()) if manifest["generator_version"] != "0.1.2": raise ValueError("Only corrected generator0.1.2 datasets can be released") archive(root, relative, output / "datasets" / name) datasets.append( { "archive": f"datasets/{name}", "extract_at_repository_root": True, "original_path": str(relative), "systems": manifest["systems"], "days": manifest["days"], "seed": manifest["seed"], "generator_version": manifest["generator_version"], "manifest_sha256": sha256(root / relative / "manifest.json"), "system_counts_by_split": { key: len(value) for key, value in manifest["split_system_ids"].items() }, "source": "entirely synthetic; no external observations", } ) (output / ".gitattributes").write_text( "*.pt filter=lfs diff=lfs merge=lfs -text\n" "*.onnx filter=lfs diff=lfs merge=lfs -text\n" "*.tar.gz filter=lfs diff=lfs merge=lfs -text\n" ) files = [path for path in sorted(output.rglob("*")) if path.is_file()] inventory = { str(path.relative_to(output)): {"bytes": path.stat().st_size, "sha256": sha256(path)} for path in files } release = { "version": "tfm-release-v1", "release": "0.1.0", "repository": "maxffarrell/transition-foundation-model", "status": "experimental_short_training_only", "license": "MIT", "source_provenance": "Original implementation created for this proof of concept; source included", "dataset_generator_version": "0.1.2", "datasets": datasets, "default_parameter_count": 1570434, "default_weights_training_systems": 63, "full_dataset_training_completed": False, "transformer_research_success_demonstrated": False, "files": inventory, } (output / "release.json").write_text(json.dumps(release, indent=2, sort_keys=True) + "\n") checksums = {name: item["sha256"] for name, item in inventory.items()} checksums["release.json"] = sha256(output / "release.json") (output / "SHA256SUMS").write_text( "".join(f"{digest} {name}\n" for name, digest in sorted(checksums.items())) ) print( json.dumps( { "output": str(output), "files": len(inventory) + 2, "bytes": sum(path.stat().st_size for path in output.rglob("*") if path.is_file()), "datasets": datasets, } ) ) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", default=".") parser.add_argument("--output", required=True) args = parser.parse_args() build(args.root, args.output)