File size: 5,419 Bytes
fd74a4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
"""Stage the scientific mirror and interactive browser explorer; publish with --publish."""
import argparse
import csv
import hashlib
import io
import json
from pathlib import Path
import subprocess
from urllib.request import Request, urlopen
from zipfile import ZipFile

ROOT = Path(__file__).resolve().parents[1]
NAME = "program-structure-certificates"
HUB = "DQSWORDMAN/" + NAME
GITHUB = "https://github.com/dqswordman/" + NAME
FROZEN = "8b0f2e0dd17ee36ce9233a9053dc55c22110dfce"
ASSET = NAME + "-v1.0.0.zip"


def git(*args):
    return subprocess.check_output(["git", *args], cwd=ROOT)


def digest(data):
    return hashlib.sha256(data).hexdigest()


def online(url, limit):
    with urlopen(Request(url,headers={"User-Agent":"program-structure-certificate-publisher"}),timeout=60) as response:
        data = response.read(limit+1)
    if len(data)>limit:
        raise ValueError("Remote file exceeds its allowed size")
    return data


def dump(path, obj):
    path.write_bytes((json.dumps(obj,ensure_ascii=False,indent=2)+"\n").encode())


def prepare(dest, local_zip):
    dest.mkdir(parents=True,exist_ok=True)
    if any(dest.iterdir()):
        raise ValueError("Choose an empty staging folder")
    head = git("rev-parse","HEAD").decode().strip()
    with ZipFile(io.BytesIO(git("archive","--format=zip",head))) as archive:
        for entry in archive.infolist():
            src = entry.filename
            if entry.is_dir() or src.startswith("."):
                continue
            name = src.removeprefix("space/") if src.startswith("space/") else "GITHUB_README.md" if src=="README.md" else src
            path = dest/name
            path.parent.mkdir(parents=True,exist_ok=True)
            path.write_bytes(archive.read(entry))
    if local_zip:
        if local_zip.stat().st_size>100_000_000:
            raise ValueError("ZIP exceeds publication limit")
        blob = local_zip.read_bytes()
    else:
        release = json.loads(online("https://api.github.com/repos/dqswordman/"+NAME+"/releases/tags/v1.0.0",1_000_000))
        asset = next(a for a in release["assets"] if a["name"]==ASSET)
        blob = online(asset["browser_download_url"],100_000_000)
        if len(blob)!=asset["size"] or "sha256:"+digest(blob)!=asset["digest"]:
            raise ValueError("Download differs from GitHub release asset")
    with ZipFile(io.BytesIO(blob)) as z:
        manifest = json.loads(z.read(ASSET[:-4]+"/PUBLICATION_MANIFEST.json"))
        if z.testzip() or manifest["scientific_commit"]!=FROZEN:
            raise ValueError("Unexpected release archive")
    (dest/ASSET).write_bytes(blob)
    fixture = json.loads((dest/"fixtures/end_to_end.json").read_text(encoding="utf-8"))
    with (dest/"results/PRIMARY_PANEL_READABLE.csv").open(encoding="utf-8",newline="") as file:
        panel = list(csv.DictReader(file))
    if len(panel)!=8 or fixture["expected"]["certificates"]["SEPARATION"]["minimum_length"]!=2:
        raise ValueError("Unexpected explorer source records")
    dump(dest/"explorer.json",{"fixture":fixture,"panel":panel})
    manifest_path = dest / "MANIFEST.json"
    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    manifest["distribution"] = "Hugging Face static Space"
    manifest["files"] = []
    files = {}
    for path in sorted(dest.rglob("*")):
        if not path.is_file() or path == manifest_path:
            continue
        name = path.relative_to(dest).as_posix()
        checksum = digest(path.read_bytes())
        manifest["files"].append({"path": name, "size_bytes": path.stat().st_size, "sha256": checksum})
        files[name] = checksum
    dump(manifest_path, manifest)
    files["MANIFEST.json"] = digest(manifest_path.read_bytes())
    dump(dest/"SNAPSHOT.json",{"github_commit":head,"github_repository":GITHUB,"frozen_release":{
        "name":ASSET,"bytes":len(blob),"sha256":digest(blob),"scientific_commit":FROZEN},"sha256":files})
    return head


def publish(dest, head):
    from huggingface_hub import HfApi
    api = HfApi()
    if git("ls-remote","--heads",GITHUB+".git","refs/heads/main").decode().split()[0]!=head:
        raise ValueError("GitHub main changed; reload before publishing")
    previous = api.space_info(HUB)
    if previous.private or previous.sdk!="static":
        raise ValueError("Expected the designated public static Space")
    commit = api.upload_folder(repo_type="space",repo_id=HUB,folder_path=dest,parent_commit=previous.sha,
                               commit_message=f"Mirror GitHub {head[:12]}",delete_patterns=["*","**/*"],
                               ignore_patterns=[".gitattributes"])
    base = f"https://huggingface.co/spaces/{HUB}/resolve/{commit.oid}/"
    for name in ("SNAPSHOT.json","explorer.json","index.html",ASSET):
        expected = (dest/name).read_bytes()
        if online(base+name,len(expected))!=expected:
            raise ValueError("Published file differs: "+name)
    print(commit.commit_url)


if __name__ == "__main__":
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument("destination",type=Path)
    parser.add_argument("--zip",type=Path,help="Use a locally verified release ZIP for staging")
    parser.add_argument("--publish",action="store_true")
    args=parser.parse_args()
    revision=prepare(args.destination.resolve(),args.zip)
    if args.publish:
        publish(args.destination.resolve(),revision)