Download src/validate_synthetic.py from DanielNguyen121/IsSynth-Cars: direct link, hf CLI and curl.
- Browser
- Download file 2.24 kB
-
https://huggingface.co/DanielNguyen121/IsSynth-Cars/resolve/main/src/validate_synthetic.py
- Command line
-
hf download hf://DanielNguyen121/IsSynth-Cars/src/validate_synthetic.py
-
curl -L -o validate_synthetic.py https://huggingface.co/DanielNguyen121/IsSynth-Cars/resolve/main/src/validate_synthetic.py
2.24 kB
| from pathlib import Path,PurePosixPath | |
| from collections import Counter,defaultdict | |
| from PIL import Image | |
| import zipfile,hashlib,sys,json,csv | |
| sys.path.insert(0,'/marimo/SPC') | |
| from util_data import SUBSET_NAMES | |
| def sha256_file(path): | |
| h=hashlib.sha256() | |
| with path.open('rb') as f: | |
| for block in iter(lambda:f.read(8<<20),b''):h.update(block) | |
| return h.hexdigest() | |
| base=Path('/marimo/IsSynth-Cars/data');archive=base/'synthetic/cars_issynth_images.zip';dest=base/'synthetic_extracted' | |
| with zipfile.ZipFile(archive) as z: | |
| infos=[i for i in z.infolist() if not i.is_dir()] | |
| assert all(not PurePosixPath(i.filename).is_absolute() and '..' not in PurePosixPath(i.filename).parts for i in infos) | |
| names=[i.filename for i in infos if i.filename.lower().endswith(('.jpg','.jpeg','.png'))] | |
| assert len(names)==12544 and len(set(names))==12544 | |
| assert z.testzip() is None | |
| if not (dest/'.extracted').exists(): | |
| z.extractall(dest);(dest/'.extracted').write_text(sha256_file(archive)) | |
| root=dest/'cars_issynth_images';classes=SUBSET_NAMES['cars'];counts=Counter();digests=defaultdict(list);rows=[] | |
| for cls_id,name in enumerate(classes): | |
| dirname=name.replace('/','_');folder=root/dirname | |
| assert folder.is_dir(),(cls_id,dirname) | |
| paths=sorted(p for p in folder.iterdir() if p.suffix.lower() in ['.jpg','.jpeg','.png']) | |
| assert len(paths)==64,(cls_id,dirname,len(paths)) | |
| for p in paths: | |
| with Image.open(p) as im:im.verify() | |
| h=hashlib.sha256(p.read_bytes()).hexdigest();digests[h].append(str(p.relative_to(root)));rows.append((cls_id,name,str(p.relative_to(root)),h)) | |
| counts[cls_id]=len(paths) | |
| with (base/'synthetic_manifest.csv').open('w',newline='') as f: | |
| w=csv.writer(f);w.writerow(['class_id','class_name','relative_path','sha256']);w.writerows(rows) | |
| report={'status':'PASS','archive_sha256':sha256_file(archive),'images':len(rows),'classes':len(counts),'samples_per_class':64,'duplicate_byte_groups':sum(len(v)>1 for v in digests.values()),'unexpected_directories':sorted(set(x.name for x in root.iterdir() if x.is_dir())-{n.replace('/','_') for n in classes})} | |
| assert not report['unexpected_directories'] | |
| (base/'synthetic_validation.json').write_text(json.dumps(report,indent=2)) | |
| print('SYNTHETIC_VALIDATION',json.dumps(report),flush=True) | |