File size: 2,829 Bytes
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
"""Resumable ranged public artifact download with a required SHA256."""
import argparse,concurrent.futures,hashlib,json,os,pathlib,time,urllib.request

def main():
    p=argparse.ArgumentParser();p.add_argument('--url',required=True);p.add_argument('--size',type=int,required=True);p.add_argument('--sha256',required=True);p.add_argument('--output',required=True);p.add_argument('--workers',type=int,default=12)
    a=p.parse_args();out=pathlib.Path(a.output);out.parent.mkdir(parents=True,exist_ok=True)
    def digest(path):
        h=hashlib.sha256()
        with path.open('rb') as f:
            while chunk:=f.read(8*1024*1024):h.update(chunk)
        return h.hexdigest()
    if out.exists():
        if out.stat().st_size!=a.size or digest(out)!=a.sha256:raise ValueError('Existing artifact differs')
        return
    tmp=out.with_name(out.name+'.partial');done=out.with_name(out.name+'.ranges');done.mkdir(exist_ok=True)
    identity=dict(url=a.url,size=a.size,sha256=a.sha256)
    ip=done/'identity.json'
    if ip.exists() and json.loads(ip.read_text())!=identity:raise ValueError('Download identity changed')
    ip.write_text(json.dumps(identity));fd=os.open(tmp,os.O_CREAT|os.O_RDWR,0o600);os.ftruncate(fd,a.size)
    spans=[(lo,min(lo+64*1024*1024,a.size)-1) for lo in range(0,a.size,64*1024*1024)]
    def fetch(span):
        lo,hi=span;receipt=done/str(lo)
        if receipt.exists():return
        for attempt in range(4):
            try:
                req=urllib.request.Request(a.url+('&' if '?' in a.url else '?')+f'range_start={lo}',headers={'Range':f'bytes={lo}-{hi}'})
                offset=lo
                with urllib.request.urlopen(req,timeout=120) as src:
                    if src.status!=206 or not src.headers.get('Content-Range','').startswith(f'bytes {lo}-{hi}/'):raise ValueError('Incorrect range response')
                    while chunk:=src.read(1024*1024):
                        if offset+len(chunk)>hi+1:raise ValueError('Range overflow')
                        view=memoryview(chunk)
                        while view:
                            n=os.pwrite(fd,view,offset);offset+=n;view=view[n:]
                if offset!=hi+1:raise ValueError('Incomplete range')
                receipt.write_text('complete');print(json.dumps(dict(start=lo,end=hi)),flush=True);return
            except Exception:
                if attempt==3:raise
                time.sleep(1+attempt)
    try:
        with concurrent.futures.ThreadPoolExecutor(max_workers=a.workers) as pool:list(pool.map(fetch,spans))
        os.fsync(fd)
    finally:os.close(fd)
    actual=digest(tmp)
    if actual!=a.sha256:raise ValueError('SHA256 mismatch: '+actual)
    tmp.rename(out);out.with_name(out.name+'.provenance.json').write_text(json.dumps(identity,indent=2))

if __name__=='__main__':main()