File size: 5,419 Bytes
fd74a4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | """Stage the scientific mirror and interactive browser explorer; publish with --publish."""
import argparse
import csv
import hashlib
import io
import json
from pathlib import Path
import subprocess
from urllib.request import Request, urlopen
from zipfile import ZipFile
ROOT = Path(__file__).resolve().parents[1]
NAME = "program-structure-certificates"
HUB = "DQSWORDMAN/" + NAME
GITHUB = "https://github.com/dqswordman/" + NAME
FROZEN = "8b0f2e0dd17ee36ce9233a9053dc55c22110dfce"
ASSET = NAME + "-v1.0.0.zip"
def git(*args):
return subprocess.check_output(["git", *args], cwd=ROOT)
def digest(data):
return hashlib.sha256(data).hexdigest()
def online(url, limit):
with urlopen(Request(url,headers={"User-Agent":"program-structure-certificate-publisher"}),timeout=60) as response:
data = response.read(limit+1)
if len(data)>limit:
raise ValueError("Remote file exceeds its allowed size")
return data
def dump(path, obj):
path.write_bytes((json.dumps(obj,ensure_ascii=False,indent=2)+"\n").encode())
def prepare(dest, local_zip):
dest.mkdir(parents=True,exist_ok=True)
if any(dest.iterdir()):
raise ValueError("Choose an empty staging folder")
head = git("rev-parse","HEAD").decode().strip()
with ZipFile(io.BytesIO(git("archive","--format=zip",head))) as archive:
for entry in archive.infolist():
src = entry.filename
if entry.is_dir() or src.startswith("."):
continue
name = src.removeprefix("space/") if src.startswith("space/") else "GITHUB_README.md" if src=="README.md" else src
path = dest/name
path.parent.mkdir(parents=True,exist_ok=True)
path.write_bytes(archive.read(entry))
if local_zip:
if local_zip.stat().st_size>100_000_000:
raise ValueError("ZIP exceeds publication limit")
blob = local_zip.read_bytes()
else:
release = json.loads(online("https://api.github.com/repos/dqswordman/"+NAME+"/releases/tags/v1.0.0",1_000_000))
asset = next(a for a in release["assets"] if a["name"]==ASSET)
blob = online(asset["browser_download_url"],100_000_000)
if len(blob)!=asset["size"] or "sha256:"+digest(blob)!=asset["digest"]:
raise ValueError("Download differs from GitHub release asset")
with ZipFile(io.BytesIO(blob)) as z:
manifest = json.loads(z.read(ASSET[:-4]+"/PUBLICATION_MANIFEST.json"))
if z.testzip() or manifest["scientific_commit"]!=FROZEN:
raise ValueError("Unexpected release archive")
(dest/ASSET).write_bytes(blob)
fixture = json.loads((dest/"fixtures/end_to_end.json").read_text(encoding="utf-8"))
with (dest/"results/PRIMARY_PANEL_READABLE.csv").open(encoding="utf-8",newline="") as file:
panel = list(csv.DictReader(file))
if len(panel)!=8 or fixture["expected"]["certificates"]["SEPARATION"]["minimum_length"]!=2:
raise ValueError("Unexpected explorer source records")
dump(dest/"explorer.json",{"fixture":fixture,"panel":panel})
manifest_path = dest / "MANIFEST.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
manifest["distribution"] = "Hugging Face static Space"
manifest["files"] = []
files = {}
for path in sorted(dest.rglob("*")):
if not path.is_file() or path == manifest_path:
continue
name = path.relative_to(dest).as_posix()
checksum = digest(path.read_bytes())
manifest["files"].append({"path": name, "size_bytes": path.stat().st_size, "sha256": checksum})
files[name] = checksum
dump(manifest_path, manifest)
files["MANIFEST.json"] = digest(manifest_path.read_bytes())
dump(dest/"SNAPSHOT.json",{"github_commit":head,"github_repository":GITHUB,"frozen_release":{
"name":ASSET,"bytes":len(blob),"sha256":digest(blob),"scientific_commit":FROZEN},"sha256":files})
return head
def publish(dest, head):
from huggingface_hub import HfApi
api = HfApi()
if git("ls-remote","--heads",GITHUB+".git","refs/heads/main").decode().split()[0]!=head:
raise ValueError("GitHub main changed; reload before publishing")
previous = api.space_info(HUB)
if previous.private or previous.sdk!="static":
raise ValueError("Expected the designated public static Space")
commit = api.upload_folder(repo_type="space",repo_id=HUB,folder_path=dest,parent_commit=previous.sha,
commit_message=f"Mirror GitHub {head[:12]}",delete_patterns=["*","**/*"],
ignore_patterns=[".gitattributes"])
base = f"https://huggingface.co/spaces/{HUB}/resolve/{commit.oid}/"
for name in ("SNAPSHOT.json","explorer.json","index.html",ASSET):
expected = (dest/name).read_bytes()
if online(base+name,len(expected))!=expected:
raise ValueError("Published file differs: "+name)
print(commit.commit_url)
if __name__ == "__main__":
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument("destination",type=Path)
parser.add_argument("--zip",type=Path,help="Use a locally verified release ZIP for staging")
parser.add_argument("--publish",action="store_true")
args=parser.parse_args()
revision=prepare(args.destination.resolve(),args.zip)
if args.publish:
publish(args.destination.resolve(),revision)
|