File size: 6,416 Bytes
1baf215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
"""Build an allowlisted, checksummed Hugging Face v0.1 release without caches."""

import argparse
import gzip
import hashlib
import json
import shutil
import tarfile
from pathlib import Path


def sha256(path):
    digest = hashlib.sha256()
    with Path(path).open("rb") as stream:
        for block in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def copy_tree(source, target):
    for path in sorted(source.rglob("*")):
        if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc":
            if path.is_symlink():
                raise ValueError(f"Release cannot include symlink: {path}")
            destination = target / path.relative_to(source)
            destination.parent.mkdir(parents=True, exist_ok=True)
            shutil.copyfile(path, destination)


def archive(root, relative, output):
    with output.open("wb") as raw:
        with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=0) as zipped:
            with tarfile.open(fileobj=zipped, mode="w", format=tarfile.PAX_FORMAT) as tar:
                for path in sorted((root / relative).rglob("*")):
                    if not path.is_file():
                        continue
                    if path.is_symlink():
                        raise ValueError("Dataset symlinks cannot be packaged")
                    info = tar.gettarinfo(str(path), arcname=str(path.relative_to(root)))
                    info.uid = info.gid = info.mtime = 0
                    info.uname = info.gname = ""
                    with path.open("rb") as stream:
                        tar.addfile(info, stream)


def build(root, output):
    root, output = Path(root).resolve(), Path(output).resolve()
    if output.exists() and any(output.iterdir()):
        raise ValueError("Use a fresh output folder; an existing release is never overwritten")
    output.mkdir(parents=True, exist_ok=True)
    for directory in ("tfm", "tests", "configs", "docs", "examples", "scripts"):
        copy_tree(root / directory, output / directory)
    for name in ("pyproject.toml", "uv.lock", "LICENSE", ".gitignore"):
        shutil.copyfile(root / name, output / name)
    (output / "docs/usage.md").write_text((root / "README.md").read_text().replace("(docs/", "("))
    shutil.copyfile(root / "docs/huggingface-model-card.md", output / "README.md")
    artifacts = (
        "transformer-v2",
        "transformer-h96-v2",
        "transformer-nohistory-v2",
        "mlp-v2",
        "gru-v2",
        "onnx-transformer-v2",
        "report-transformer-v2",
        "report-mlp-v2",
        "report-gru-v2",
        "report-ensemble-v2",
        "report-context-v2",
    )
    for name in artifacts:
        relative = Path("artifacts/smoke") / name
        copy_tree(root / relative, output / relative)
    for name in (
        "comparison-v2.json",
        "comparison-v2-config.json",
        "api-latency-cpu-v2.json",
        "api-latency-mps-v2.json",
        "api-latency-ensemble-cpu-v2.json",
        "http-prediction-v2.json",
    ):
        relative = Path("artifacts/smoke") / name
        shutil.copyfile(root / relative, output / relative)
    (output / "datasets").mkdir()
    datasets = []
    for relative, name in (
        (Path("artifacts/poc/data"), "synthetic-1200-v0.1.2.tar.gz"),
        (Path("artifacts/smoke/data-v2"), "smoke-90-v0.1.2.tar.gz"),
    ):
        manifest = json.loads((root / relative / "manifest.json").read_text())
        if manifest["generator_version"] != "0.1.2":
            raise ValueError("Only corrected generator0.1.2 datasets can be released")
        archive(root, relative, output / "datasets" / name)
        datasets.append(
            {
                "archive": f"datasets/{name}",
                "extract_at_repository_root": True,
                "original_path": str(relative),
                "systems": manifest["systems"],
                "days": manifest["days"],
                "seed": manifest["seed"],
                "generator_version": manifest["generator_version"],
                "manifest_sha256": sha256(root / relative / "manifest.json"),
                "system_counts_by_split": {
                    key: len(value) for key, value in manifest["split_system_ids"].items()
                },
                "source": "entirely synthetic; no external observations",
            }
        )
    (output / ".gitattributes").write_text(
        "*.pt filter=lfs diff=lfs merge=lfs -text\n"
        "*.onnx filter=lfs diff=lfs merge=lfs -text\n"
        "*.tar.gz filter=lfs diff=lfs merge=lfs -text\n"
    )
    files = [path for path in sorted(output.rglob("*")) if path.is_file()]
    inventory = {
        str(path.relative_to(output)): {"bytes": path.stat().st_size, "sha256": sha256(path)}
        for path in files
    }
    release = {
        "version": "tfm-release-v1",
        "release": "0.1.0",
        "repository": "maxffarrell/transition-foundation-model",
        "status": "experimental_short_training_only",
        "license": "MIT",
        "source_provenance": "Original implementation created for this proof of concept; source included",
        "dataset_generator_version": "0.1.2",
        "datasets": datasets,
        "default_parameter_count": 1570434,
        "default_weights_training_systems": 63,
        "full_dataset_training_completed": False,
        "transformer_research_success_demonstrated": False,
        "files": inventory,
    }
    (output / "release.json").write_text(json.dumps(release, indent=2, sort_keys=True) + "\n")
    checksums = {name: item["sha256"] for name, item in inventory.items()}
    checksums["release.json"] = sha256(output / "release.json")
    (output / "SHA256SUMS").write_text(
        "".join(f"{digest}  {name}\n" for name, digest in sorted(checksums.items()))
    )
    print(
        json.dumps(
            {
                "output": str(output),
                "files": len(inventory) + 2,
                "bytes": sum(path.stat().st_size for path in output.rglob("*") if path.is_file()),
                "datasets": datasets,
            }
        )
    )


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--root", default=".")
    parser.add_argument("--output", required=True)
    args = parser.parse_args()
    build(args.root, args.output)