| from pathlib import Path |
| from huggingface_hub import hf_hub_download |
| import hashlib,subprocess,os,json |
| root=Path('/marimo/disef_cars');source=root/'data/source' |
| expected={'cars_train.tgz':'512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288','cars_test.tgz':'bffea656d6f425cba3c91c6d83336e4c5f86c6cffd8975b0f375d3a10da8e243'} |
| for name,want in expected.items(): |
| h=hashlib.sha256() |
| with (source/name).open('rb') as f: |
| for b in iter(lambda:f.read(8<<20),b''):h.update(b) |
| assert h.hexdigest()==want,(name,h.hexdigest()) |
| print('ARCHIVE_HASH_PASS',name,flush=True) |
| subprocess.run(['/tmp/uv-venv/bin/python',str(root/'prepare_cars.py'),'--root',str(root)],check=True) |
| subprocess.run(['/tmp/uv-venv/bin/python',str(root/'resolve_support_leakage.py')],check=True) |
| repo='DanielNguyen121/Fewshot-datasets';token=os.environ.get('HF_TOKEN') |
| for name in ['images.jsonl','fewshot_seed_1_leakfree.json']: |
| a=root/'manifests'/name;b=Path(hf_hub_download(repo,'Cars/metadata/'+name,repo_type='dataset',token=token)) |
| if name=='images.jsonl': |
| local={json.loads(x)['filename']:json.loads(x)['sha256'] for x in a.read_text().splitlines()} |
| remote={json.loads(x)['filename']:json.loads(x)['sha256'] for x in b.read_text().splitlines()} |
| assert local==remote,(len(local),len(remote)) |
| else: |
| local=json.loads(a.read_text());remote=json.loads(b.read_text()) |
| assert local['support_by_class']==remote['support_by_class'] |
| print('MANIFEST_MATCH',name,flush=True) |
| Path('/marimo/IsSynth-Cars/data/original_validation.json').write_text(json.dumps({'status':'PASS','images':16185,'classes':196,'official_train':8144,'official_test':8041,'source_archive_sha256':expected,'published_image_hashes_match':True,'published_seed1_leakfree_support_match':True},indent=2)) |
| print('ORIGINAL_VALIDATION_PASS',flush=True) |
|
|
