Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download scripts/build_release.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 6.42 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/scripts/build_release.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/scripts/build_release.py
-
curl -L -o build_release.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/scripts/build_release.py
6.42 kB
| """Build an allowlisted, checksummed Hugging Face v0.1 release without caches.""" | |
| import argparse | |
| import gzip | |
| import hashlib | |
| import json | |
| import shutil | |
| import tarfile | |
| from pathlib import Path | |
| def sha256(path): | |
| digest = hashlib.sha256() | |
| with Path(path).open("rb") as stream: | |
| for block in iter(lambda: stream.read(1024 * 1024), b""): | |
| digest.update(block) | |
| return digest.hexdigest() | |
| def copy_tree(source, target): | |
| for path in sorted(source.rglob("*")): | |
| if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc": | |
| if path.is_symlink(): | |
| raise ValueError(f"Release cannot include symlink: {path}") | |
| destination = target / path.relative_to(source) | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.copyfile(path, destination) | |
| def archive(root, relative, output): | |
| with output.open("wb") as raw: | |
| with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=0) as zipped: | |
| with tarfile.open(fileobj=zipped, mode="w", format=tarfile.PAX_FORMAT) as tar: | |
| for path in sorted((root / relative).rglob("*")): | |
| if not path.is_file(): | |
| continue | |
| if path.is_symlink(): | |
| raise ValueError("Dataset symlinks cannot be packaged") | |
| info = tar.gettarinfo(str(path), arcname=str(path.relative_to(root))) | |
| info.uid = info.gid = info.mtime = 0 | |
| info.uname = info.gname = "" | |
| with path.open("rb") as stream: | |
| tar.addfile(info, stream) | |
| def build(root, output): | |
| root, output = Path(root).resolve(), Path(output).resolve() | |
| if output.exists() and any(output.iterdir()): | |
| raise ValueError("Use a fresh output folder; an existing release is never overwritten") | |
| output.mkdir(parents=True, exist_ok=True) | |
| for directory in ("tfm", "tests", "configs", "docs", "examples", "scripts"): | |
| copy_tree(root / directory, output / directory) | |
| for name in ("pyproject.toml", "uv.lock", "LICENSE", ".gitignore"): | |
| shutil.copyfile(root / name, output / name) | |
| (output / "docs/usage.md").write_text((root / "README.md").read_text().replace("(docs/", "(")) | |
| shutil.copyfile(root / "docs/huggingface-model-card.md", output / "README.md") | |
| artifacts = ( | |
| "transformer-v2", | |
| "transformer-h96-v2", | |
| "transformer-nohistory-v2", | |
| "mlp-v2", | |
| "gru-v2", | |
| "onnx-transformer-v2", | |
| "report-transformer-v2", | |
| "report-mlp-v2", | |
| "report-gru-v2", | |
| "report-ensemble-v2", | |
| "report-context-v2", | |
| ) | |
| for name in artifacts: | |
| relative = Path("artifacts/smoke") / name | |
| copy_tree(root / relative, output / relative) | |
| for name in ( | |
| "comparison-v2.json", | |
| "comparison-v2-config.json", | |
| "api-latency-cpu-v2.json", | |
| "api-latency-mps-v2.json", | |
| "api-latency-ensemble-cpu-v2.json", | |
| "http-prediction-v2.json", | |
| ): | |
| relative = Path("artifacts/smoke") / name | |
| shutil.copyfile(root / relative, output / relative) | |
| (output / "datasets").mkdir() | |
| datasets = [] | |
| for relative, name in ( | |
| (Path("artifacts/poc/data"), "synthetic-1200-v0.1.2.tar.gz"), | |
| (Path("artifacts/smoke/data-v2"), "smoke-90-v0.1.2.tar.gz"), | |
| ): | |
| manifest = json.loads((root / relative / "manifest.json").read_text()) | |
| if manifest["generator_version"] != "0.1.2": | |
| raise ValueError("Only corrected generator0.1.2 datasets can be released") | |
| archive(root, relative, output / "datasets" / name) | |
| datasets.append( | |
| { | |
| "archive": f"datasets/{name}", | |
| "extract_at_repository_root": True, | |
| "original_path": str(relative), | |
| "systems": manifest["systems"], | |
| "days": manifest["days"], | |
| "seed": manifest["seed"], | |
| "generator_version": manifest["generator_version"], | |
| "manifest_sha256": sha256(root / relative / "manifest.json"), | |
| "system_counts_by_split": { | |
| key: len(value) for key, value in manifest["split_system_ids"].items() | |
| }, | |
| "source": "entirely synthetic; no external observations", | |
| } | |
| ) | |
| (output / ".gitattributes").write_text( | |
| "*.pt filter=lfs diff=lfs merge=lfs -text\n" | |
| "*.onnx filter=lfs diff=lfs merge=lfs -text\n" | |
| "*.tar.gz filter=lfs diff=lfs merge=lfs -text\n" | |
| ) | |
| files = [path for path in sorted(output.rglob("*")) if path.is_file()] | |
| inventory = { | |
| str(path.relative_to(output)): {"bytes": path.stat().st_size, "sha256": sha256(path)} | |
| for path in files | |
| } | |
| release = { | |
| "version": "tfm-release-v1", | |
| "release": "0.1.0", | |
| "repository": "maxffarrell/transition-foundation-model", | |
| "status": "experimental_short_training_only", | |
| "license": "MIT", | |
| "source_provenance": "Original implementation created for this proof of concept; source included", | |
| "dataset_generator_version": "0.1.2", | |
| "datasets": datasets, | |
| "default_parameter_count": 1570434, | |
| "default_weights_training_systems": 63, | |
| "full_dataset_training_completed": False, | |
| "transformer_research_success_demonstrated": False, | |
| "files": inventory, | |
| } | |
| (output / "release.json").write_text(json.dumps(release, indent=2, sort_keys=True) + "\n") | |
| checksums = {name: item["sha256"] for name, item in inventory.items()} | |
| checksums["release.json"] = sha256(output / "release.json") | |
| (output / "SHA256SUMS").write_text( | |
| "".join(f"{digest} {name}\n" for name, digest in sorted(checksums.items())) | |
| ) | |
| print( | |
| json.dumps( | |
| { | |
| "output": str(output), | |
| "files": len(inventory) + 2, | |
| "bytes": sum(path.stat().st_size for path in output.rglob("*") if path.is_file()), | |
| "datasets": datasets, | |
| } | |
| ) | |
| ) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--root", default=".") | |
| parser.add_argument("--output", required=True) | |
| args = parser.parse_args() | |
| build(args.root, args.output) | |