IsSynth-Cars / src /validate_synthetic.py
DanielNguyen121's picture
Publish idempotent Cars data recovery scripts and dashboard
24b1962 verified
Raw History Blame Contribute Delete
2.24 kB
from pathlib import Path,PurePosixPath
from collections import Counter,defaultdict
from PIL import Image
import zipfile,hashlib,sys,json,csv
sys.path.insert(0,'/marimo/SPC')
from util_data import SUBSET_NAMES
def sha256_file(path):
h=hashlib.sha256()
with path.open('rb') as f:
for block in iter(lambda:f.read(8<<20),b''):h.update(block)
return h.hexdigest()
base=Path('/marimo/IsSynth-Cars/data');archive=base/'synthetic/cars_issynth_images.zip';dest=base/'synthetic_extracted'
with zipfile.ZipFile(archive) as z:
infos=[i for i in z.infolist() if not i.is_dir()]
assert all(not PurePosixPath(i.filename).is_absolute() and '..' not in PurePosixPath(i.filename).parts for i in infos)
names=[i.filename for i in infos if i.filename.lower().endswith(('.jpg','.jpeg','.png'))]
assert len(names)==12544 and len(set(names))==12544
assert z.testzip() is None
if not (dest/'.extracted').exists():
z.extractall(dest);(dest/'.extracted').write_text(sha256_file(archive))
root=dest/'cars_issynth_images';classes=SUBSET_NAMES['cars'];counts=Counter();digests=defaultdict(list);rows=[]
for cls_id,name in enumerate(classes):
dirname=name.replace('/','_');folder=root/dirname
assert folder.is_dir(),(cls_id,dirname)
paths=sorted(p for p in folder.iterdir() if p.suffix.lower() in ['.jpg','.jpeg','.png'])
assert len(paths)==64,(cls_id,dirname,len(paths))
for p in paths:
with Image.open(p) as im:im.verify()
h=hashlib.sha256(p.read_bytes()).hexdigest();digests[h].append(str(p.relative_to(root)));rows.append((cls_id,name,str(p.relative_to(root)),h))
counts[cls_id]=len(paths)
with (base/'synthetic_manifest.csv').open('w',newline='') as f:
w=csv.writer(f);w.writerow(['class_id','class_name','relative_path','sha256']);w.writerows(rows)
report={'status':'PASS','archive_sha256':sha256_file(archive),'images':len(rows),'classes':len(counts),'samples_per_class':64,'duplicate_byte_groups':sum(len(v)>1 for v in digests.values()),'unexpected_directories':sorted(set(x.name for x in root.iterdir() if x.is_dir())-{n.replace('/','_') for n in classes})}
assert not report['unexpected_directories']
(base/'synthetic_validation.json').write_text(json.dumps(report,indent=2))
print('SYNTHETIC_VALIDATION',json.dumps(report),flush=True)