Multimodel_setup / scripts /prepare_phase7_dataset.py
Thiru1122's picture
feat(deepfake-detection): comprehensive cross-model evaluation, parity audits, and Phase 7 v2 reference preprocessing
151f388
Raw History Blame Contribute Delete
8.55 kB
"""
Phase 7 Dataset Preparation Script
Downloads GenImage validation shard 0 (TheKernel01/Tiny-GenImage),
extracts 200 balanced, pristine images:
- 100 REAL photographs (ImageNet-1k val/nature)
- 50 FAKE Midjourney images
- 50 FAKE ADM images
Screens against historical benchmark hashes, validates RGB decodability,
and generates benchmark/external_generalization_manifest.csv.
"""
import os
import sys
import io
import csv
import hashlib
from typing import Dict, Any, List
import pyarrow.parquet as pq
from PIL import Image
from huggingface_hub import hf_hub_download
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATASET_DIR = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization")
IMAGES_DIR = os.path.join(DATASET_DIR, "images")
REAL_DIR = os.path.join(IMAGES_DIR, "real")
MIDJOURNEY_DIR = os.path.join(IMAGES_DIR, "fake_midjourney")
ADM_DIR = os.path.join(IMAGES_DIR, "fake_adm")
MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv")
REPO_ID = "TheKernel01/Tiny-GenImage"
SHARD_FILENAME = "data/validation-00000-of-00004.parquet"
def compute_sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def load_historical_hashes() -> set:
"""Collects all SHA-256 hashes from prior project manifests and logs to guarantee zero leakage."""
historical_hashes = set()
# Phase 6 test manifest
p6_manifest = os.path.join(PROJECT_ROOT, "benchmark", "phase6_test_manifest.csv")
if os.path.exists(p6_manifest):
with open(p6_manifest, "r", encoding="utf-8") as f:
for row in csv.DictReader(f):
h = row.get("sha256")
if h:
historical_hashes.add(h.lower())
# Phase 4/5 evaluation manifest
eval_manifest = os.path.join(PROJECT_ROOT, "benchmark", "evaluation_manifest.csv")
if os.path.exists(eval_manifest):
with open(eval_manifest, "r", encoding="utf-8") as f:
for row in csv.DictReader(f):
h = row.get("sha256")
if h:
historical_hashes.add(h.lower())
print(f"Loaded {len(historical_hashes)} historical hashes for anti-leakage screening.")
return historical_hashes
def main():
print("=" * 80)
print("PHASE 7: EXTRACTING & FREEZING EXTERNAL GENERALIZATION DATASET (GENIMAGE)")
print("=" * 80)
for d in [REAL_DIR, MIDJOURNEY_DIR, ADM_DIR]:
os.makedirs(d, exist_ok=True)
historical_hashes = load_historical_hashes()
# Step 1: Download Shard 0 Parquet file
print(f"\n[Step 1] Downloading GenImage validation shard 0 from Hugging Face Hub ({REPO_ID})...")
parquet_path = hf_hub_download(
repo_id=REPO_ID,
filename=SHARD_FILENAME,
repo_type="dataset",
)
print(f" Downloaded to local cache: {parquet_path}")
# Step 2: Read Parquet table
print("\n[Step 2] Reading Parquet records from shard 0...")
table = pq.read_table(parquet_path)
print(f" Table loaded: {table.num_rows} rows, {table.num_columns} columns.")
p_image = table.column("image")
p_label = table.column("label").to_pylist()
p_gen = table.column("generator").to_pylist()
# Step 3: Extract balanced subsets
print("\n[Step 3] Selecting 100 Real, 50 Midjourney, 50 ADM samples...")
selected_real = []
selected_midjourney = []
selected_adm = []
for idx in range(table.num_rows):
gen_id = p_gen[idx]
lbl = p_label[idx]
if gen_id == 0 and len(selected_real) < 100:
selected_real.append(idx)
elif gen_id == 4 and len(selected_midjourney) < 50:
selected_midjourney.append(idx)
elif gen_id == 1 and len(selected_adm) < 50:
selected_adm.append(idx)
if len(selected_real) == 100 and len(selected_midjourney) == 50 and len(selected_adm) == 50:
break
print(f" Selected counts: REAL={len(selected_real)}, Midjourney={len(selected_midjourney)}, ADM={len(selected_adm)}")
assert len(selected_real) == 100, f"Expected 100 Real, got {len(selected_real)}"
assert len(selected_midjourney) == 50, f"Expected 50 Midjourney, got {len(selected_midjourney)}"
assert len(selected_adm) == 50, f"Expected 50 ADM, got {len(selected_adm)}"
# Step 4: Extract and save image files to disk with integrity validation
print("\n[Step 4] Writing pristine images to disk and validating integrity...")
manifest_rows = []
seen_hashes = set()
extraction_targets = [
("real", selected_real, REAL_DIR, "REAL", "none"),
("midjourney", selected_midjourney, MIDJOURNEY_DIR, "FAKE", "midjourney"),
("adm", selected_adm, ADM_DIR, "FAKE", "adm"),
]
sample_counter = 1
for name_prefix, indices, target_dir, ground_truth, gen_fam in extraction_targets:
for idx in indices:
row_dict = p_image[idx].as_py()
img_bytes = row_dict["bytes"]
orig_path = row_dict.get("path") or f"{name_prefix}_{idx}.jpg"
orig_filename = os.path.basename(orig_path)
sha256 = compute_sha256(img_bytes).lower()
file_size = len(img_bytes)
# Assert no duplicates
if sha256 in seen_hashes:
raise ValueError(f"Duplicate hash within selection: {sha256}")
seen_hashes.add(sha256)
if sha256 in historical_hashes:
raise ValueError(f"Hash collision with historical benchmark dataset: {sha256}")
# Verify image decode
with Image.open(io.BytesIO(img_bytes)) as pil_img:
width, height = pil_img.size
img_format = pil_img.format or "JPEG"
# Ensure RGB decodable
_ = pil_img.convert("RGB")
# Determine clean filename and relative/absolute paths
if name_prefix == "real":
sample_id = f"phase7_ext_real_{len([r for r in manifest_rows if r['label'] == 'REAL']) + 1:03d}"
out_filename = f"genimage_real_{len([r for r in manifest_rows if r['label'] == 'REAL']) + 1:03d}.jpg"
elif name_prefix == "midjourney":
sample_id = f"phase7_ext_fake_midjourney_{len([r for r in manifest_rows if r['generator_family'] == 'midjourney']) + 1:03d}"
out_filename = f"genimage_midjourney_{len([r for r in manifest_rows if r['generator_family'] == 'midjourney']) + 1:03d}.jpg"
else:
sample_id = f"phase7_ext_fake_adm_{len([r for r in manifest_rows if r['generator_family'] == 'adm']) + 1:03d}"
out_filename = f"genimage_adm_{len([r for r in manifest_rows if r['generator_family'] == 'adm']) + 1:03d}.jpg"
out_path = os.path.join(target_dir, out_filename)
with open(out_path, "wb") as f_out:
f_out.write(img_bytes)
rel_path = os.path.relpath(out_path, PROJECT_ROOT).replace("\\", "/")
abs_path = os.path.abspath(out_path).replace("\\", "/")
manifest_rows.append({
"sample_id": sample_id,
"relative_path": rel_path,
"absolute_path": abs_path,
"label": ground_truth,
"generator_family": gen_fam,
"source_dataset": "GenImage (TheKernel01/Tiny-GenImage)",
"source_split": "validation",
"original_filename": orig_filename,
"file_size_bytes": file_size,
"sha256": sha256,
"width": width,
"height": height,
"format": img_format,
"category": "ImageNet-1k object/scene",
"license_or_usage_reference": "CC BY-NC-SA 4.0 / ImageNet research non-commercial",
})
print(f" Successfully extracted and verified {len(manifest_rows)} pristine images.")
# Step 5: Save manifest CSV
print(f"\n[Step 5] Writing frozen manifest to {MANIFEST_PATH}...")
fieldnames = list(manifest_rows[0].keys())
with open(MANIFEST_PATH, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(manifest_rows)
print(f" Frozen manifest generated successfully: {MANIFEST_PATH}")
print(f" Rows: {len(manifest_rows)} (100 REAL, 50 Midjourney, 50 ADM).")
print("=" * 80)
print("STAGE 2 DATASET PREPARATION COMPLETED SUCCESSFULLY!")
print("=" * 80)
if __name__ == "__main__":
main()