Spaces:
Running on Zero
Running on Zero
Download scripts/verify_dataset_manifest.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 5.18 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/verify_dataset_manifest.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/verify_dataset_manifest.py
-
curl -L -o verify_dataset_manifest.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/verify_dataset_manifest.py
5.18 kB
| """ | |
| Verification script for the benchmark evaluation manifest. | |
| Validates file existence, file sizes, SHA-256 hashes, image formats, | |
| resolutions, ground-truth label consistency, and split partitioning. | |
| """ | |
| import csv | |
| import hashlib | |
| import os | |
| import sys | |
| from pathlib import Path | |
| from PIL import Image | |
| PROJECT_ROOT = Path(__file__).resolve().parent.parent | |
| MANIFEST_PATH = PROJECT_ROOT / "benchmark" / "evaluation_manifest.csv" | |
| def compute_sha256(path: Path) -> str: | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| while chunk := f.read(65536): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def verify_manifest() -> bool: | |
| print("=" * 80) | |
| print("DATASET MANIFEST INTEGRITY VERIFICATION") | |
| print("=" * 80) | |
| print(f"Manifest Path: {MANIFEST_PATH}") | |
| if not MANIFEST_PATH.exists(): | |
| print(f"[FAIL] Manifest file does not exist: {MANIFEST_PATH}") | |
| return False | |
| with open(MANIFEST_PATH, "r", encoding="utf-8") as f: | |
| reader = list(csv.DictReader(f)) | |
| total_entries = len(reader) | |
| print(f"Total manifest entries: {total_entries}") | |
| expected_total = 159 | |
| if total_entries != expected_total: | |
| print(f"[FAIL] Expected {expected_total} rows, found {total_entries}") | |
| return False | |
| seen_ids = set() | |
| seen_hashes = set() | |
| split_counts = {} | |
| gt_counts = {} | |
| domain_counts = {} | |
| errors = [] | |
| for idx, row in enumerate(reader): | |
| sample_id = row.get("sample_id", f"row_{idx}") | |
| if sample_id in seen_ids: | |
| errors.append(f"Duplicate sample_id: {sample_id}") | |
| seen_ids.add(sample_id) | |
| file_path_str = row.get("file_path", "") | |
| file_path = Path(file_path_str) | |
| if not file_path.exists(): | |
| errors.append(f"File missing on disk: {file_path} (ID: {sample_id})") | |
| continue | |
| actual_size = os.path.getsize(file_path) | |
| expected_size = int(row.get("file_size_bytes", 0)) | |
| if actual_size != expected_size: | |
| errors.append(f"Size mismatch for {sample_id}: actual {actual_size} != recorded {expected_size}") | |
| actual_hash = compute_sha256(file_path) | |
| expected_hash = row.get("sha256", "") | |
| if actual_hash != expected_hash: | |
| errors.append(f"SHA-256 mismatch for {sample_id}: actual {actual_hash} != recorded {expected_hash}") | |
| if actual_hash in seen_hashes: | |
| errors.append(f"Duplicate hash across dataset for {sample_id}: {actual_hash}") | |
| seen_hashes.add(actual_hash) | |
| fmt = row.get("file_format", "") | |
| if fmt in ["JPEG", "PNG"]: | |
| try: | |
| with Image.open(file_path) as img: | |
| w, h = img.size | |
| exp_w = int(row.get("width", 0)) | |
| exp_h = int(row.get("height", 0)) | |
| if w != exp_w or h != exp_h: | |
| errors.append(f"Resolution mismatch for {sample_id}: actual {w}x{h} != recorded {exp_w}x{exp_h}") | |
| except Exception as e: | |
| errors.append(f"Failed to open image {file_path} for {sample_id}: {e}") | |
| elif row.get("category") not in ["corrupted_file", "unsupported_format"]: | |
| errors.append(f"Unrecognized file format without control category: {sample_id}") | |
| split = row.get("split", "unknown") | |
| gt = row.get("ground_truth", "unknown") | |
| dom = row.get("domain", "unknown") | |
| split_counts[split] = split_counts.get(split, 0) + 1 | |
| gt_counts[gt] = gt_counts.get(gt, 0) + 1 | |
| domain_counts[dom] = domain_counts.get(dom, 0) + 1 | |
| if errors: | |
| print(f"\n[FAIL] Found {len(errors)} integrity errors:") | |
| for err in errors[:10]: | |
| print(f" - {err}") | |
| return False | |
| print("\n[Split Breakdown]") | |
| for s, c in sorted(split_counts.items()): | |
| print(f" - {s}: {c} samples") | |
| print("\n[Ground Truth Breakdown]") | |
| for g, c in sorted(gt_counts.items()): | |
| print(f" - {g}: {c} samples") | |
| print("\n[Domain Breakdown]") | |
| for d, c in sorted(domain_counts.items()): | |
| print(f" - {d}: {c} samples") | |
| # Check expected partition sizes | |
| assert split_counts.get("held_out_test") == 105, "held_out_test count mismatch" | |
| assert split_counts.get("development_val") == 50, "development_val count mismatch" | |
| assert split_counts.get("operational_smoke") == 4, "operational_smoke count mismatch" | |
| assert gt_counts.get("REAL") == 77, "REAL count mismatch" | |
| assert gt_counts.get("FAKE") == 79, "FAKE count mismatch" | |
| assert gt_counts.get("REJECTION_CONTROL") == 3, "REJECTION_CONTROL count mismatch" | |
| assert domain_counts.get("facial") == 150, "facial domain count mismatch" | |
| assert domain_counts.get("general_scene") == 6, "general_scene domain count mismatch" | |
| assert domain_counts.get("control") == 3, "control domain count mismatch" | |
| print("\n[SUCCESS] Manifest verification passed 100% with zero errors!") | |
| return True | |
| if __name__ == "__main__": | |
| success = verify_manifest() | |
| sys.exit(0 if success else 1) | |