File size: 6,416 Bytes
1baf215 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | """Build an allowlisted, checksummed Hugging Face v0.1 release without caches."""
import argparse
import gzip
import hashlib
import json
import shutil
import tarfile
from pathlib import Path
def sha256(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def copy_tree(source, target):
for path in sorted(source.rglob("*")):
if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc":
if path.is_symlink():
raise ValueError(f"Release cannot include symlink: {path}")
destination = target / path.relative_to(source)
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(path, destination)
def archive(root, relative, output):
with output.open("wb") as raw:
with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=0) as zipped:
with tarfile.open(fileobj=zipped, mode="w", format=tarfile.PAX_FORMAT) as tar:
for path in sorted((root / relative).rglob("*")):
if not path.is_file():
continue
if path.is_symlink():
raise ValueError("Dataset symlinks cannot be packaged")
info = tar.gettarinfo(str(path), arcname=str(path.relative_to(root)))
info.uid = info.gid = info.mtime = 0
info.uname = info.gname = ""
with path.open("rb") as stream:
tar.addfile(info, stream)
def build(root, output):
root, output = Path(root).resolve(), Path(output).resolve()
if output.exists() and any(output.iterdir()):
raise ValueError("Use a fresh output folder; an existing release is never overwritten")
output.mkdir(parents=True, exist_ok=True)
for directory in ("tfm", "tests", "configs", "docs", "examples", "scripts"):
copy_tree(root / directory, output / directory)
for name in ("pyproject.toml", "uv.lock", "LICENSE", ".gitignore"):
shutil.copyfile(root / name, output / name)
(output / "docs/usage.md").write_text((root / "README.md").read_text().replace("(docs/", "("))
shutil.copyfile(root / "docs/huggingface-model-card.md", output / "README.md")
artifacts = (
"transformer-v2",
"transformer-h96-v2",
"transformer-nohistory-v2",
"mlp-v2",
"gru-v2",
"onnx-transformer-v2",
"report-transformer-v2",
"report-mlp-v2",
"report-gru-v2",
"report-ensemble-v2",
"report-context-v2",
)
for name in artifacts:
relative = Path("artifacts/smoke") / name
copy_tree(root / relative, output / relative)
for name in (
"comparison-v2.json",
"comparison-v2-config.json",
"api-latency-cpu-v2.json",
"api-latency-mps-v2.json",
"api-latency-ensemble-cpu-v2.json",
"http-prediction-v2.json",
):
relative = Path("artifacts/smoke") / name
shutil.copyfile(root / relative, output / relative)
(output / "datasets").mkdir()
datasets = []
for relative, name in (
(Path("artifacts/poc/data"), "synthetic-1200-v0.1.2.tar.gz"),
(Path("artifacts/smoke/data-v2"), "smoke-90-v0.1.2.tar.gz"),
):
manifest = json.loads((root / relative / "manifest.json").read_text())
if manifest["generator_version"] != "0.1.2":
raise ValueError("Only corrected generator0.1.2 datasets can be released")
archive(root, relative, output / "datasets" / name)
datasets.append(
{
"archive": f"datasets/{name}",
"extract_at_repository_root": True,
"original_path": str(relative),
"systems": manifest["systems"],
"days": manifest["days"],
"seed": manifest["seed"],
"generator_version": manifest["generator_version"],
"manifest_sha256": sha256(root / relative / "manifest.json"),
"system_counts_by_split": {
key: len(value) for key, value in manifest["split_system_ids"].items()
},
"source": "entirely synthetic; no external observations",
}
)
(output / ".gitattributes").write_text(
"*.pt filter=lfs diff=lfs merge=lfs -text\n"
"*.onnx filter=lfs diff=lfs merge=lfs -text\n"
"*.tar.gz filter=lfs diff=lfs merge=lfs -text\n"
)
files = [path for path in sorted(output.rglob("*")) if path.is_file()]
inventory = {
str(path.relative_to(output)): {"bytes": path.stat().st_size, "sha256": sha256(path)}
for path in files
}
release = {
"version": "tfm-release-v1",
"release": "0.1.0",
"repository": "maxffarrell/transition-foundation-model",
"status": "experimental_short_training_only",
"license": "MIT",
"source_provenance": "Original implementation created for this proof of concept; source included",
"dataset_generator_version": "0.1.2",
"datasets": datasets,
"default_parameter_count": 1570434,
"default_weights_training_systems": 63,
"full_dataset_training_completed": False,
"transformer_research_success_demonstrated": False,
"files": inventory,
}
(output / "release.json").write_text(json.dumps(release, indent=2, sort_keys=True) + "\n")
checksums = {name: item["sha256"] for name, item in inventory.items()}
checksums["release.json"] = sha256(output / "release.json")
(output / "SHA256SUMS").write_text(
"".join(f"{digest} {name}\n" for name, digest in sorted(checksums.items()))
)
print(
json.dumps(
{
"output": str(output),
"files": len(inventory) + 2,
"bytes": sum(path.stat().st_size for path in output.rglob("*") if path.is_file()),
"datasets": datasets,
}
)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", default=".")
parser.add_argument("--output", required=True)
args = parser.parse_args()
build(args.root, args.output)
|