Multimodel_setup / scripts /verify_dataset_manifest.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
5.18 kB
"""
Verification script for the benchmark evaluation manifest.
Validates file existence, file sizes, SHA-256 hashes, image formats,
resolutions, ground-truth label consistency, and split partitioning.
"""
import csv
import hashlib
import os
import sys
from pathlib import Path
from PIL import Image
PROJECT_ROOT = Path(__file__).resolve().parent.parent
MANIFEST_PATH = PROJECT_ROOT / "benchmark" / "evaluation_manifest.csv"
def compute_sha256(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
while chunk := f.read(65536):
h.update(chunk)
return h.hexdigest()
def verify_manifest() -> bool:
print("=" * 80)
print("DATASET MANIFEST INTEGRITY VERIFICATION")
print("=" * 80)
print(f"Manifest Path: {MANIFEST_PATH}")
if not MANIFEST_PATH.exists():
print(f"[FAIL] Manifest file does not exist: {MANIFEST_PATH}")
return False
with open(MANIFEST_PATH, "r", encoding="utf-8") as f:
reader = list(csv.DictReader(f))
total_entries = len(reader)
print(f"Total manifest entries: {total_entries}")
expected_total = 159
if total_entries != expected_total:
print(f"[FAIL] Expected {expected_total} rows, found {total_entries}")
return False
seen_ids = set()
seen_hashes = set()
split_counts = {}
gt_counts = {}
domain_counts = {}
errors = []
for idx, row in enumerate(reader):
sample_id = row.get("sample_id", f"row_{idx}")
if sample_id in seen_ids:
errors.append(f"Duplicate sample_id: {sample_id}")
seen_ids.add(sample_id)
file_path_str = row.get("file_path", "")
file_path = Path(file_path_str)
if not file_path.exists():
errors.append(f"File missing on disk: {file_path} (ID: {sample_id})")
continue
actual_size = os.path.getsize(file_path)
expected_size = int(row.get("file_size_bytes", 0))
if actual_size != expected_size:
errors.append(f"Size mismatch for {sample_id}: actual {actual_size} != recorded {expected_size}")
actual_hash = compute_sha256(file_path)
expected_hash = row.get("sha256", "")
if actual_hash != expected_hash:
errors.append(f"SHA-256 mismatch for {sample_id}: actual {actual_hash} != recorded {expected_hash}")
if actual_hash in seen_hashes:
errors.append(f"Duplicate hash across dataset for {sample_id}: {actual_hash}")
seen_hashes.add(actual_hash)
fmt = row.get("file_format", "")
if fmt in ["JPEG", "PNG"]:
try:
with Image.open(file_path) as img:
w, h = img.size
exp_w = int(row.get("width", 0))
exp_h = int(row.get("height", 0))
if w != exp_w or h != exp_h:
errors.append(f"Resolution mismatch for {sample_id}: actual {w}x{h} != recorded {exp_w}x{exp_h}")
except Exception as e:
errors.append(f"Failed to open image {file_path} for {sample_id}: {e}")
elif row.get("category") not in ["corrupted_file", "unsupported_format"]:
errors.append(f"Unrecognized file format without control category: {sample_id}")
split = row.get("split", "unknown")
gt = row.get("ground_truth", "unknown")
dom = row.get("domain", "unknown")
split_counts[split] = split_counts.get(split, 0) + 1
gt_counts[gt] = gt_counts.get(gt, 0) + 1
domain_counts[dom] = domain_counts.get(dom, 0) + 1
if errors:
print(f"\n[FAIL] Found {len(errors)} integrity errors:")
for err in errors[:10]:
print(f" - {err}")
return False
print("\n[Split Breakdown]")
for s, c in sorted(split_counts.items()):
print(f" - {s}: {c} samples")
print("\n[Ground Truth Breakdown]")
for g, c in sorted(gt_counts.items()):
print(f" - {g}: {c} samples")
print("\n[Domain Breakdown]")
for d, c in sorted(domain_counts.items()):
print(f" - {d}: {c} samples")
# Check expected partition sizes
assert split_counts.get("held_out_test") == 105, "held_out_test count mismatch"
assert split_counts.get("development_val") == 50, "development_val count mismatch"
assert split_counts.get("operational_smoke") == 4, "operational_smoke count mismatch"
assert gt_counts.get("REAL") == 77, "REAL count mismatch"
assert gt_counts.get("FAKE") == 79, "FAKE count mismatch"
assert gt_counts.get("REJECTION_CONTROL") == 3, "REJECTION_CONTROL count mismatch"
assert domain_counts.get("facial") == 150, "facial domain count mismatch"
assert domain_counts.get("general_scene") == 6, "general_scene domain count mismatch"
assert domain_counts.get("control") == 3, "control domain count mismatch"
print("\n[SUCCESS] Manifest verification passed 100% with zero errors!")
return True
if __name__ == "__main__":
success = verify_manifest()
sys.exit(0 if success else 1)