Buckets:
| from pathlib import Path | |
| import urllib.request,subprocess,json,time,gzip,base64,hashlib | |
| r=Path('/data/xiaofan/kernel-source-graph-native-20261103-v1');openurl=urllib.request.build_opener(urllib.request.ProxyHandler({})).open | |
| asset=openurl('http://100.64.180.123:34777/assets',timeout=30).read();assert hashlib.sha256(asset).hexdigest()=='a3d15994d61ba57dd5a95732992c820e22b908e3781e866574686346620ce97b' | |
| for name,record in json.loads(gzip.decompress(asset)).items(): | |
| p=(r/name).resolve();assert p.is_relative_to(r);p.parent.mkdir(exist_ok=True,parents=True);b=base64.b64decode(record['b64']);assert hashlib.sha256(b).hexdigest()==record['sha256'];p.write_bytes(b) | |
| log=(r/'docker_load.stdout').open('w');err=(r/'docker_load.stderr').open('w');p=subprocess.Popen(['docker','load'],stdin=subprocess.PIPE,stdout=log,stderr=err);start=time.monotonic();count=0 | |
| with openurl('http://100.64.180.123:34777/image',timeout=120)as response: | |
| while True: | |
| b=response.read(1024*1024) | |
| if not b:break | |
| p.stdin.write(b);count+=len(b) | |
| (r/'runtime_transfer.json').write_text(json.dumps(dict(bytes=count,seconds=time.monotonic()-start,status='loading'))) | |
| p.stdin.close();code=p.wait(timeout=300);assert code==0,(r/'docker_load.stderr').read_text()[-1000:] | |
| image=subprocess.check_output(['docker','image','inspect','sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b','--format','{{.Id}}'],text=True).strip();assert image=='sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b';(r/'runtime_transfer.json').write_text(json.dumps(dict(bytes=count,seconds=time.monotonic()-start,status='complete',image=image)));print('RUNTIME_TRANSFER_COMPLETE',count,flush=True) | |
Xet Storage Details
- Size:
- 1.69 kB
- Xet hash:
- dd8414e94cf442da01f29aabd993694ec9bf6b0d891e36c47d2d5e9f49b72b36
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.