Download code/scripts/download_file.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 2.83 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/download_file.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/download_file.py
-
curl -L -o download_file.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/download_file.py
2.83 kB
| """Resumable ranged public artifact download with a required SHA256.""" | |
| import argparse,concurrent.futures,hashlib,json,os,pathlib,time,urllib.request | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--url',required=True);p.add_argument('--size',type=int,required=True);p.add_argument('--sha256',required=True);p.add_argument('--output',required=True);p.add_argument('--workers',type=int,default=12) | |
| a=p.parse_args();out=pathlib.Path(a.output);out.parent.mkdir(parents=True,exist_ok=True) | |
| def digest(path): | |
| h=hashlib.sha256() | |
| with path.open('rb') as f: | |
| while chunk:=f.read(8*1024*1024):h.update(chunk) | |
| return h.hexdigest() | |
| if out.exists(): | |
| if out.stat().st_size!=a.size or digest(out)!=a.sha256:raise ValueError('Existing artifact differs') | |
| return | |
| tmp=out.with_name(out.name+'.partial');done=out.with_name(out.name+'.ranges');done.mkdir(exist_ok=True) | |
| identity=dict(url=a.url,size=a.size,sha256=a.sha256) | |
| ip=done/'identity.json' | |
| if ip.exists() and json.loads(ip.read_text())!=identity:raise ValueError('Download identity changed') | |
| ip.write_text(json.dumps(identity));fd=os.open(tmp,os.O_CREAT|os.O_RDWR,0o600);os.ftruncate(fd,a.size) | |
| spans=[(lo,min(lo+64*1024*1024,a.size)-1) for lo in range(0,a.size,64*1024*1024)] | |
| def fetch(span): | |
| lo,hi=span;receipt=done/str(lo) | |
| if receipt.exists():return | |
| for attempt in range(4): | |
| try: | |
| req=urllib.request.Request(a.url+('&' if '?' in a.url else '?')+f'range_start={lo}',headers={'Range':f'bytes={lo}-{hi}'}) | |
| offset=lo | |
| with urllib.request.urlopen(req,timeout=120) as src: | |
| if src.status!=206 or not src.headers.get('Content-Range','').startswith(f'bytes {lo}-{hi}/'):raise ValueError('Incorrect range response') | |
| while chunk:=src.read(1024*1024): | |
| if offset+len(chunk)>hi+1:raise ValueError('Range overflow') | |
| view=memoryview(chunk) | |
| while view: | |
| n=os.pwrite(fd,view,offset);offset+=n;view=view[n:] | |
| if offset!=hi+1:raise ValueError('Incomplete range') | |
| receipt.write_text('complete');print(json.dumps(dict(start=lo,end=hi)),flush=True);return | |
| except Exception: | |
| if attempt==3:raise | |
| time.sleep(1+attempt) | |
| try: | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:list(pool.map(fetch,spans)) | |
| os.fsync(fd) | |
| finally:os.close(fd) | |
| actual=digest(tmp) | |
| if actual!=a.sha256:raise ValueError('SHA256 mismatch: '+actual) | |
| tmp.rename(out);out.with_name(out.name+'.provenance.json').write_text(json.dumps(identity,indent=2)) | |
| if __name__=='__main__':main() | |