zefancainlp's picture
download
raw
1.8 kB
from pathlib import Path
import urllib.request,subprocess,json,time,gzip,base64,hashlib
r=Path('/data/xiaofan/kernel-source-graph-native-20261103-v1');openurl=urllib.request.build_opener(urllib.request.ProxyHandler({})).open
asset=openurl('http://100.64.180.123:34777/assets',timeout=30).read();assert hashlib.sha256(asset).hexdigest()=='a3d15994d61ba57dd5a95732992c820e22b908e3781e866574686346620ce97b'
for name,record in json.loads(gzip.decompress(asset)).items():
p=(r/name).resolve();assert p.is_relative_to(r);p.parent.mkdir(exist_ok=True,parents=True);b=base64.b64decode(record['b64']);assert hashlib.sha256(b).hexdigest()==record['sha256'];p.write_bytes(b)
log=(r/'docker_load.stdout').open('w');err=(r/'docker_load.stderr').open('w');p=subprocess.Popen(['docker','load'],stdin=subprocess.PIPE,stdout=log,stderr=err);start=time.monotonic();count=0
(r/'runtime_transfer.json').write_text(json.dumps(dict(status='waiting_for_docker_export',time=time.time())))
with openurl('http://100.64.180.123:34777/image',timeout=1800)as response:
while True:
b=response.read(1024*1024)
if not b:break
count+=len(b)
p.stdin.write(b)
(r/'runtime_transfer.json').write_text(json.dumps(dict(bytes=count,seconds=time.monotonic()-start,status='loading')))
p.stdin.close();code=p.wait(timeout=300);assert code==0,(r/'docker_load.stderr').read_text()[-1000:]
image=subprocess.check_output(['docker','image','inspect','sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b','--format','{{.Id}}'],text=True).strip();assert image=='sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b';(r/'runtime_transfer.json').write_text(json.dumps(dict(bytes=count,seconds=time.monotonic()-start,status='complete',image=image)));print('RUNTIME_TRANSFER_COMPLETE',count,flush=True)

Xet Storage Details

Size:
1.8 kB
·
Xet hash:
3ef3681d84a4f17ecc42b5a4bed786291fbc9505c89e3dece889ae8349656e20

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.