Download scripts/publish_space.py from DQSWORDMAN/program-structure-certificates: direct link, hf CLI and curl.
- Browser
- Download file 5.42 kB
-
https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates/resolve/main/scripts/publish_space.py
- Command line
-
hf download hf://spaces/DQSWORDMAN/program-structure-certificates/scripts/publish_space.py
-
curl -L -o publish_space.py https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates/resolve/main/scripts/publish_space.py
5.42 kB
| """Stage the scientific mirror and interactive browser explorer; publish with --publish.""" | |
| import argparse | |
| import csv | |
| import hashlib | |
| import io | |
| import json | |
| from pathlib import Path | |
| import subprocess | |
| from urllib.request import Request, urlopen | |
| from zipfile import ZipFile | |
| ROOT = Path(__file__).resolve().parents[1] | |
| NAME = "program-structure-certificates" | |
| HUB = "DQSWORDMAN/" + NAME | |
| GITHUB = "https://github.com/dqswordman/" + NAME | |
| FROZEN = "8b0f2e0dd17ee36ce9233a9053dc55c22110dfce" | |
| ASSET = NAME + "-v1.0.0.zip" | |
| def git(*args): | |
| return subprocess.check_output(["git", *args], cwd=ROOT) | |
| def digest(data): | |
| return hashlib.sha256(data).hexdigest() | |
| def online(url, limit): | |
| with urlopen(Request(url,headers={"User-Agent":"program-structure-certificate-publisher"}),timeout=60) as response: | |
| data = response.read(limit+1) | |
| if len(data)>limit: | |
| raise ValueError("Remote file exceeds its allowed size") | |
| return data | |
| def dump(path, obj): | |
| path.write_bytes((json.dumps(obj,ensure_ascii=False,indent=2)+"\n").encode()) | |
| def prepare(dest, local_zip): | |
| dest.mkdir(parents=True,exist_ok=True) | |
| if any(dest.iterdir()): | |
| raise ValueError("Choose an empty staging folder") | |
| head = git("rev-parse","HEAD").decode().strip() | |
| with ZipFile(io.BytesIO(git("archive","--format=zip",head))) as archive: | |
| for entry in archive.infolist(): | |
| src = entry.filename | |
| if entry.is_dir() or src.startswith("."): | |
| continue | |
| name = src.removeprefix("space/") if src.startswith("space/") else "GITHUB_README.md" if src=="README.md" else src | |
| path = dest/name | |
| path.parent.mkdir(parents=True,exist_ok=True) | |
| path.write_bytes(archive.read(entry)) | |
| if local_zip: | |
| if local_zip.stat().st_size>100_000_000: | |
| raise ValueError("ZIP exceeds publication limit") | |
| blob = local_zip.read_bytes() | |
| else: | |
| release = json.loads(online("https://api.github.com/repos/dqswordman/"+NAME+"/releases/tags/v1.0.0",1_000_000)) | |
| asset = next(a for a in release["assets"] if a["name"]==ASSET) | |
| blob = online(asset["browser_download_url"],100_000_000) | |
| if len(blob)!=asset["size"] or "sha256:"+digest(blob)!=asset["digest"]: | |
| raise ValueError("Download differs from GitHub release asset") | |
| with ZipFile(io.BytesIO(blob)) as z: | |
| manifest = json.loads(z.read(ASSET[:-4]+"/PUBLICATION_MANIFEST.json")) | |
| if z.testzip() or manifest["scientific_commit"]!=FROZEN: | |
| raise ValueError("Unexpected release archive") | |
| (dest/ASSET).write_bytes(blob) | |
| fixture = json.loads((dest/"fixtures/end_to_end.json").read_text(encoding="utf-8")) | |
| with (dest/"results/PRIMARY_PANEL_READABLE.csv").open(encoding="utf-8",newline="") as file: | |
| panel = list(csv.DictReader(file)) | |
| if len(panel)!=8 or fixture["expected"]["certificates"]["SEPARATION"]["minimum_length"]!=2: | |
| raise ValueError("Unexpected explorer source records") | |
| dump(dest/"explorer.json",{"fixture":fixture,"panel":panel}) | |
| manifest_path = dest / "MANIFEST.json" | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| manifest["distribution"] = "Hugging Face static Space" | |
| manifest["files"] = [] | |
| files = {} | |
| for path in sorted(dest.rglob("*")): | |
| if not path.is_file() or path == manifest_path: | |
| continue | |
| name = path.relative_to(dest).as_posix() | |
| checksum = digest(path.read_bytes()) | |
| manifest["files"].append({"path": name, "size_bytes": path.stat().st_size, "sha256": checksum}) | |
| files[name] = checksum | |
| dump(manifest_path, manifest) | |
| files["MANIFEST.json"] = digest(manifest_path.read_bytes()) | |
| dump(dest/"SNAPSHOT.json",{"github_commit":head,"github_repository":GITHUB,"frozen_release":{ | |
| "name":ASSET,"bytes":len(blob),"sha256":digest(blob),"scientific_commit":FROZEN},"sha256":files}) | |
| return head | |
| def publish(dest, head): | |
| from huggingface_hub import HfApi | |
| api = HfApi() | |
| if git("ls-remote","--heads",GITHUB+".git","refs/heads/main").decode().split()[0]!=head: | |
| raise ValueError("GitHub main changed; reload before publishing") | |
| previous = api.space_info(HUB) | |
| if previous.private or previous.sdk!="static": | |
| raise ValueError("Expected the designated public static Space") | |
| commit = api.upload_folder(repo_type="space",repo_id=HUB,folder_path=dest,parent_commit=previous.sha, | |
| commit_message=f"Mirror GitHub {head[:12]}",delete_patterns=["*","**/*"], | |
| ignore_patterns=[".gitattributes"]) | |
| base = f"https://huggingface.co/spaces/{HUB}/resolve/{commit.oid}/" | |
| for name in ("SNAPSHOT.json","explorer.json","index.html",ASSET): | |
| expected = (dest/name).read_bytes() | |
| if online(base+name,len(expected))!=expected: | |
| raise ValueError("Published file differs: "+name) | |
| print(commit.commit_url) | |
| if __name__ == "__main__": | |
| parser=argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("destination",type=Path) | |
| parser.add_argument("--zip",type=Path,help="Use a locally verified release ZIP for staging") | |
| parser.add_argument("--publish",action="store_true") | |
| args=parser.parse_args() | |
| revision=prepare(args.destination.resolve(),args.zip) | |
| if args.publish: | |
| publish(args.destination.resolve(),revision) | |