from pathlib import Path,PurePosixPath from collections import Counter,defaultdict from PIL import Image import zipfile,hashlib,sys,json,csv sys.path.insert(0,'/marimo/SPC') from util_data import SUBSET_NAMES def sha256_file(path): h=hashlib.sha256() with path.open('rb') as f: for block in iter(lambda:f.read(8<<20),b''):h.update(block) return h.hexdigest() base=Path('/marimo/IsSynth-Cars/data');archive=base/'synthetic/cars_issynth_images.zip';dest=base/'synthetic_extracted' with zipfile.ZipFile(archive) as z: infos=[i for i in z.infolist() if not i.is_dir()] assert all(not PurePosixPath(i.filename).is_absolute() and '..' not in PurePosixPath(i.filename).parts for i in infos) names=[i.filename for i in infos if i.filename.lower().endswith(('.jpg','.jpeg','.png'))] assert len(names)==12544 and len(set(names))==12544 assert z.testzip() is None if not (dest/'.extracted').exists(): z.extractall(dest);(dest/'.extracted').write_text(sha256_file(archive)) root=dest/'cars_issynth_images';classes=SUBSET_NAMES['cars'];counts=Counter();digests=defaultdict(list);rows=[] for cls_id,name in enumerate(classes): dirname=name.replace('/','_');folder=root/dirname assert folder.is_dir(),(cls_id,dirname) paths=sorted(p for p in folder.iterdir() if p.suffix.lower() in ['.jpg','.jpeg','.png']) assert len(paths)==64,(cls_id,dirname,len(paths)) for p in paths: with Image.open(p) as im:im.verify() h=hashlib.sha256(p.read_bytes()).hexdigest();digests[h].append(str(p.relative_to(root)));rows.append((cls_id,name,str(p.relative_to(root)),h)) counts[cls_id]=len(paths) with (base/'synthetic_manifest.csv').open('w',newline='') as f: w=csv.writer(f);w.writerow(['class_id','class_name','relative_path','sha256']);w.writerows(rows) report={'status':'PASS','archive_sha256':sha256_file(archive),'images':len(rows),'classes':len(counts),'samples_per_class':64,'duplicate_byte_groups':sum(len(v)>1 for v in digests.values()),'unexpected_directories':sorted(set(x.name for x in root.iterdir() if x.is_dir())-{n.replace('/','_') for n in classes})} assert not report['unexpected_directories'] (base/'synthetic_validation.json').write_text(json.dumps(report,indent=2)) print('SYNTHETIC_VALIDATION',json.dumps(report),flush=True)