Multimodel_setup / scripts /parity_audit /audit_dataset_extraction.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
8.56 kB
"""
Audit Phase 3: Dataset Extraction and Semantic Composition Audit
Cross-references the Parquet source shard with the extracted Phase 7 image files and manifest.
"""
import os
import sys
import io
import csv
import hashlib
import json
from collections import Counter
import pyarrow.parquet as pq
from PIL import Image
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv")
PARQUET_PATH = r"C:\Users\user\.cache\huggingface\hub\datasets--TheKernel01--Tiny-GenImage\snapshots\89c4fe9efd0ebc7ce5c7641ef57d578ccd639c69\data\validation-00000-of-00004.parquet"
def compute_sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def main():
print("=" * 80)
print("AUDITING PHASE 7 DATASET EXTRACTION AND COMPOSITION")
print("=" * 80)
# Check parquet existence
if not os.path.exists(PARQUET_PATH):
print(f"ERROR: Parquet file not found at {PARQUET_PATH}")
sys.exit(1)
print(f"Reading Parquet file: {PARQUET_PATH}")
table = pq.read_table(PARQUET_PATH)
print(f"Parquet schema:\n{table.schema}")
print(f"Total rows in shard 0: {table.num_rows}")
p_image = table.column("image")
p_label = table.column("label").to_pylist()
p_gen = table.column("generator").to_pylist()
# Check distinct generators and labels in shard 0
gen_counter = Counter(p_gen)
lbl_counter = Counter(p_label)
print(f"Generator distribution in shard 0: {gen_counter}")
print(f"Label distribution in shard 0: {lbl_counter}")
# Replicate selection
selected_real = []
selected_midjourney = []
selected_adm = []
for idx in range(table.num_rows):
gen_id = p_gen[idx]
if gen_id == 0 and len(selected_real) < 100:
selected_real.append(idx)
elif gen_id == 4 and len(selected_midjourney) < 50:
selected_midjourney.append(idx)
elif gen_id == 1 and len(selected_adm) < 50:
selected_adm.append(idx)
if len(selected_real) == 100 and len(selected_midjourney) == 50 and len(selected_adm) == 50:
break
print(f"Selection counts: REAL={len(selected_real)}, Midjourney={len(selected_midjourney)}, ADM={len(selected_adm)}")
# Load manifest
manifest_rows = []
with open(MANIFEST_PATH, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
for r in reader:
manifest_rows.append(r)
print(f"Manifest rows loaded: {len(manifest_rows)}")
# Map by sample_id
manifest_by_id = {r["sample_id"]: r for r in manifest_rows}
# Perform 1-to-1 byte and metadata comparison
targets = [
("real", selected_real, "REAL", "none"),
("midjourney", selected_midjourney, "FAKE", "midjourney"),
("adm", selected_adm, "FAKE", "adm"),
]
byte_mismatches = 0
size_mismatches = 0
hash_mismatches = 0
total_audited = 0
extraction_details = []
counter = {"real": 0, "midjourney": 0, "adm": 0}
for prefix, indices, expected_label, expected_gen in targets:
for idx in indices:
counter[prefix] += 1
if prefix == "real":
sample_id = f"phase7_ext_real_{counter[prefix]:03d}"
elif prefix == "midjourney":
sample_id = f"phase7_ext_fake_midjourney_{counter[prefix]:03d}"
else:
sample_id = f"phase7_ext_fake_adm_{counter[prefix]:03d}"
m_row = manifest_by_id[sample_id]
file_on_disk = os.path.join(PROJECT_ROOT, m_row["relative_path"])
# Read parquet record
p_rec = p_image[idx].as_py()
p_bytes = p_rec["bytes"]
p_path = p_rec.get("path", "")
p_sha = compute_sha256(p_bytes).lower()
p_size = len(p_bytes)
# Read disk file
with open(file_on_disk, "rb") as f_disk:
disk_bytes = f_disk.read()
disk_sha = compute_sha256(disk_bytes).lower()
disk_size = len(disk_bytes)
# Compare bytes
if p_bytes != disk_bytes:
byte_mismatches += 1
if p_size != disk_size:
size_mismatches += 1
if p_sha != disk_sha:
hash_mismatches += 1
# Decode images to inspect format, mode, channels, color profiles
with Image.open(io.BytesIO(p_bytes)) as pil_p:
p_format = pil_p.format
p_mode = pil_p.mode
p_w, p_h = pil_p.size
p_icc = pil_p.info.get("icc_profile") is not None
with Image.open(file_on_disk) as pil_d:
d_format = pil_d.format
d_mode = pil_d.mode
d_w, d_h = pil_d.size
d_icc = pil_d.info.get("icc_profile") is not None
extraction_details.append({
"sample_id": sample_id,
"prefix": prefix,
"label": expected_label,
"gen": expected_gen,
"parquet_orig_filename": os.path.basename(p_path),
"disk_filename": os.path.basename(file_on_disk),
"parquet_size": p_size,
"disk_size": disk_size,
"parquet_sha": p_sha,
"disk_sha": disk_sha,
"format": p_format,
"disk_format": d_format,
"mode": p_mode,
"width": p_w,
"height": p_h,
"has_icc": p_icc,
"exact_byte_match": p_bytes == disk_bytes,
})
total_audited += 1
print(f"\nAUDIT RESULTS ({total_audited} samples):")
print(f" Exact byte matches: {total_audited - byte_mismatches} / {total_audited}")
print(f" Byte mismatches: {byte_mismatches}")
print(f" Size mismatches: {size_mismatches}")
print(f" SHA-256 mismatches: {hash_mismatches}")
# Save extraction audit json
audit_out_path = os.path.join(PROJECT_ROOT, "benchmark", "results", "parity_audit_dataset_extraction.json")
with open(audit_out_path, "w", encoding="utf-8") as f:
json.dump({
"total_audited": total_audited,
"byte_mismatches": byte_mismatches,
"size_mismatches": size_mismatches,
"hash_mismatches": hash_mismatches,
"details": extraction_details,
}, f, indent=2)
print(f"Saved extraction details to: {audit_out_path}")
# Composition and dimension analysis
print("\n" + "=" * 80)
print("DATASET COMPOSITION & DIMENSION ANALYSIS")
print("=" * 80)
for group_name in ["real", "midjourney", "adm"]:
group_items = [d for d in extraction_details if d["prefix"] == group_name]
formats = Counter(d["format"] for d in group_items)
disk_formats = Counter(d["disk_format"] for d in group_items)
modes = Counter(d["mode"] for d in group_items)
has_icc = sum(1 for d in group_items if d["has_icc"])
widths = [d["width"] for d in group_items]
heights = [d["height"] for d in group_items]
aspect_ratios = [round(d["width"] / d["height"], 3) for d in group_items]
file_sizes_kb = [round(d["disk_size"] / 1024, 1) for d in group_items]
print(f"\n--- Group: {group_name.upper()} (N = {len(group_items)}) ---")
print(f" Source Formats in Parquet: {dict(formats)}")
print(f" Decoded Formats on Disk: {dict(disk_formats)}")
print(f" Color Modes: {dict(modes)}")
print(f" Has ICC profile: {has_icc} / {len(group_items)}")
print(f" Widths: min={min(widths)}, max={max(widths)}, mean={sum(widths)/len(widths):.1f}")
print(f" Heights: min={min(heights)}, max={max(heights)}, mean={sum(heights)/len(heights):.1f}")
print(f" Aspect Ratios: min={min(aspect_ratios)}, max={max(aspect_ratios)}, square_count={sum(1 for r in aspect_ratios if r == 1.0)}")
print(f" File Size (KB): min={min(file_sizes_kb)}, max={max(file_sizes_kb)}, mean={sum(file_sizes_kb)/len(file_sizes_kb):.1f}")
sample_filenames = [d["parquet_orig_filename"] for d in group_items[:5]]
print(f" Sample Source Filenames: {sample_filenames}")
if __name__ == "__main__":
main()