maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
6.42 kB
"""Build an allowlisted, checksummed Hugging Face v0.1 release without caches."""
import argparse
import gzip
import hashlib
import json
import shutil
import tarfile
from pathlib import Path
def sha256(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def copy_tree(source, target):
for path in sorted(source.rglob("*")):
if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc":
if path.is_symlink():
raise ValueError(f"Release cannot include symlink: {path}")
destination = target / path.relative_to(source)
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(path, destination)
def archive(root, relative, output):
with output.open("wb") as raw:
with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=0) as zipped:
with tarfile.open(fileobj=zipped, mode="w", format=tarfile.PAX_FORMAT) as tar:
for path in sorted((root / relative).rglob("*")):
if not path.is_file():
continue
if path.is_symlink():
raise ValueError("Dataset symlinks cannot be packaged")
info = tar.gettarinfo(str(path), arcname=str(path.relative_to(root)))
info.uid = info.gid = info.mtime = 0
info.uname = info.gname = ""
with path.open("rb") as stream:
tar.addfile(info, stream)
def build(root, output):
root, output = Path(root).resolve(), Path(output).resolve()
if output.exists() and any(output.iterdir()):
raise ValueError("Use a fresh output folder; an existing release is never overwritten")
output.mkdir(parents=True, exist_ok=True)
for directory in ("tfm", "tests", "configs", "docs", "examples", "scripts"):
copy_tree(root / directory, output / directory)
for name in ("pyproject.toml", "uv.lock", "LICENSE", ".gitignore"):
shutil.copyfile(root / name, output / name)
(output / "docs/usage.md").write_text((root / "README.md").read_text().replace("(docs/", "("))
shutil.copyfile(root / "docs/huggingface-model-card.md", output / "README.md")
artifacts = (
"transformer-v2",
"transformer-h96-v2",
"transformer-nohistory-v2",
"mlp-v2",
"gru-v2",
"onnx-transformer-v2",
"report-transformer-v2",
"report-mlp-v2",
"report-gru-v2",
"report-ensemble-v2",
"report-context-v2",
)
for name in artifacts:
relative = Path("artifacts/smoke") / name
copy_tree(root / relative, output / relative)
for name in (
"comparison-v2.json",
"comparison-v2-config.json",
"api-latency-cpu-v2.json",
"api-latency-mps-v2.json",
"api-latency-ensemble-cpu-v2.json",
"http-prediction-v2.json",
):
relative = Path("artifacts/smoke") / name
shutil.copyfile(root / relative, output / relative)
(output / "datasets").mkdir()
datasets = []
for relative, name in (
(Path("artifacts/poc/data"), "synthetic-1200-v0.1.2.tar.gz"),
(Path("artifacts/smoke/data-v2"), "smoke-90-v0.1.2.tar.gz"),
):
manifest = json.loads((root / relative / "manifest.json").read_text())
if manifest["generator_version"] != "0.1.2":
raise ValueError("Only corrected generator0.1.2 datasets can be released")
archive(root, relative, output / "datasets" / name)
datasets.append(
{
"archive": f"datasets/{name}",
"extract_at_repository_root": True,
"original_path": str(relative),
"systems": manifest["systems"],
"days": manifest["days"],
"seed": manifest["seed"],
"generator_version": manifest["generator_version"],
"manifest_sha256": sha256(root / relative / "manifest.json"),
"system_counts_by_split": {
key: len(value) for key, value in manifest["split_system_ids"].items()
},
"source": "entirely synthetic; no external observations",
}
)
(output / ".gitattributes").write_text(
"*.pt filter=lfs diff=lfs merge=lfs -text\n"
"*.onnx filter=lfs diff=lfs merge=lfs -text\n"
"*.tar.gz filter=lfs diff=lfs merge=lfs -text\n"
)
files = [path for path in sorted(output.rglob("*")) if path.is_file()]
inventory = {
str(path.relative_to(output)): {"bytes": path.stat().st_size, "sha256": sha256(path)}
for path in files
}
release = {
"version": "tfm-release-v1",
"release": "0.1.0",
"repository": "maxffarrell/transition-foundation-model",
"status": "experimental_short_training_only",
"license": "MIT",
"source_provenance": "Original implementation created for this proof of concept; source included",
"dataset_generator_version": "0.1.2",
"datasets": datasets,
"default_parameter_count": 1570434,
"default_weights_training_systems": 63,
"full_dataset_training_completed": False,
"transformer_research_success_demonstrated": False,
"files": inventory,
}
(output / "release.json").write_text(json.dumps(release, indent=2, sort_keys=True) + "\n")
checksums = {name: item["sha256"] for name, item in inventory.items()}
checksums["release.json"] = sha256(output / "release.json")
(output / "SHA256SUMS").write_text(
"".join(f"{digest} {name}\n" for name, digest in sorted(checksums.items()))
)
print(
json.dumps(
{
"output": str(output),
"files": len(inventory) + 2,
"bytes": sum(path.stat().st_size for path in output.rglob("*") if path.is_file()),
"datasets": datasets,
}
)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", default=".")
parser.add_argument("--output", required=True)
args = parser.parse_args()
build(args.root, args.output)