Spaces:
Running on Zero
Running on Zero
File size: 8,550 Bytes
151f388 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | """
Phase 7 Dataset Preparation Script
Downloads GenImage validation shard 0 (TheKernel01/Tiny-GenImage),
extracts 200 balanced, pristine images:
- 100 REAL photographs (ImageNet-1k val/nature)
- 50 FAKE Midjourney images
- 50 FAKE ADM images
Screens against historical benchmark hashes, validates RGB decodability,
and generates benchmark/external_generalization_manifest.csv.
"""
import os
import sys
import io
import csv
import hashlib
from typing import Dict, Any, List
import pyarrow.parquet as pq
from PIL import Image
from huggingface_hub import hf_hub_download
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATASET_DIR = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization")
IMAGES_DIR = os.path.join(DATASET_DIR, "images")
REAL_DIR = os.path.join(IMAGES_DIR, "real")
MIDJOURNEY_DIR = os.path.join(IMAGES_DIR, "fake_midjourney")
ADM_DIR = os.path.join(IMAGES_DIR, "fake_adm")
MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv")
REPO_ID = "TheKernel01/Tiny-GenImage"
SHARD_FILENAME = "data/validation-00000-of-00004.parquet"
def compute_sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def load_historical_hashes() -> set:
"""Collects all SHA-256 hashes from prior project manifests and logs to guarantee zero leakage."""
historical_hashes = set()
# Phase 6 test manifest
p6_manifest = os.path.join(PROJECT_ROOT, "benchmark", "phase6_test_manifest.csv")
if os.path.exists(p6_manifest):
with open(p6_manifest, "r", encoding="utf-8") as f:
for row in csv.DictReader(f):
h = row.get("sha256")
if h:
historical_hashes.add(h.lower())
# Phase 4/5 evaluation manifest
eval_manifest = os.path.join(PROJECT_ROOT, "benchmark", "evaluation_manifest.csv")
if os.path.exists(eval_manifest):
with open(eval_manifest, "r", encoding="utf-8") as f:
for row in csv.DictReader(f):
h = row.get("sha256")
if h:
historical_hashes.add(h.lower())
print(f"Loaded {len(historical_hashes)} historical hashes for anti-leakage screening.")
return historical_hashes
def main():
print("=" * 80)
print("PHASE 7: EXTRACTING & FREEZING EXTERNAL GENERALIZATION DATASET (GENIMAGE)")
print("=" * 80)
for d in [REAL_DIR, MIDJOURNEY_DIR, ADM_DIR]:
os.makedirs(d, exist_ok=True)
historical_hashes = load_historical_hashes()
# Step 1: Download Shard 0 Parquet file
print(f"\n[Step 1] Downloading GenImage validation shard 0 from Hugging Face Hub ({REPO_ID})...")
parquet_path = hf_hub_download(
repo_id=REPO_ID,
filename=SHARD_FILENAME,
repo_type="dataset",
)
print(f" Downloaded to local cache: {parquet_path}")
# Step 2: Read Parquet table
print("\n[Step 2] Reading Parquet records from shard 0...")
table = pq.read_table(parquet_path)
print(f" Table loaded: {table.num_rows} rows, {table.num_columns} columns.")
p_image = table.column("image")
p_label = table.column("label").to_pylist()
p_gen = table.column("generator").to_pylist()
# Step 3: Extract balanced subsets
print("\n[Step 3] Selecting 100 Real, 50 Midjourney, 50 ADM samples...")
selected_real = []
selected_midjourney = []
selected_adm = []
for idx in range(table.num_rows):
gen_id = p_gen[idx]
lbl = p_label[idx]
if gen_id == 0 and len(selected_real) < 100:
selected_real.append(idx)
elif gen_id == 4 and len(selected_midjourney) < 50:
selected_midjourney.append(idx)
elif gen_id == 1 and len(selected_adm) < 50:
selected_adm.append(idx)
if len(selected_real) == 100 and len(selected_midjourney) == 50 and len(selected_adm) == 50:
break
print(f" Selected counts: REAL={len(selected_real)}, Midjourney={len(selected_midjourney)}, ADM={len(selected_adm)}")
assert len(selected_real) == 100, f"Expected 100 Real, got {len(selected_real)}"
assert len(selected_midjourney) == 50, f"Expected 50 Midjourney, got {len(selected_midjourney)}"
assert len(selected_adm) == 50, f"Expected 50 ADM, got {len(selected_adm)}"
# Step 4: Extract and save image files to disk with integrity validation
print("\n[Step 4] Writing pristine images to disk and validating integrity...")
manifest_rows = []
seen_hashes = set()
extraction_targets = [
("real", selected_real, REAL_DIR, "REAL", "none"),
("midjourney", selected_midjourney, MIDJOURNEY_DIR, "FAKE", "midjourney"),
("adm", selected_adm, ADM_DIR, "FAKE", "adm"),
]
sample_counter = 1
for name_prefix, indices, target_dir, ground_truth, gen_fam in extraction_targets:
for idx in indices:
row_dict = p_image[idx].as_py()
img_bytes = row_dict["bytes"]
orig_path = row_dict.get("path") or f"{name_prefix}_{idx}.jpg"
orig_filename = os.path.basename(orig_path)
sha256 = compute_sha256(img_bytes).lower()
file_size = len(img_bytes)
# Assert no duplicates
if sha256 in seen_hashes:
raise ValueError(f"Duplicate hash within selection: {sha256}")
seen_hashes.add(sha256)
if sha256 in historical_hashes:
raise ValueError(f"Hash collision with historical benchmark dataset: {sha256}")
# Verify image decode
with Image.open(io.BytesIO(img_bytes)) as pil_img:
width, height = pil_img.size
img_format = pil_img.format or "JPEG"
# Ensure RGB decodable
_ = pil_img.convert("RGB")
# Determine clean filename and relative/absolute paths
if name_prefix == "real":
sample_id = f"phase7_ext_real_{len([r for r in manifest_rows if r['label'] == 'REAL']) + 1:03d}"
out_filename = f"genimage_real_{len([r for r in manifest_rows if r['label'] == 'REAL']) + 1:03d}.jpg"
elif name_prefix == "midjourney":
sample_id = f"phase7_ext_fake_midjourney_{len([r for r in manifest_rows if r['generator_family'] == 'midjourney']) + 1:03d}"
out_filename = f"genimage_midjourney_{len([r for r in manifest_rows if r['generator_family'] == 'midjourney']) + 1:03d}.jpg"
else:
sample_id = f"phase7_ext_fake_adm_{len([r for r in manifest_rows if r['generator_family'] == 'adm']) + 1:03d}"
out_filename = f"genimage_adm_{len([r for r in manifest_rows if r['generator_family'] == 'adm']) + 1:03d}.jpg"
out_path = os.path.join(target_dir, out_filename)
with open(out_path, "wb") as f_out:
f_out.write(img_bytes)
rel_path = os.path.relpath(out_path, PROJECT_ROOT).replace("\\", "/")
abs_path = os.path.abspath(out_path).replace("\\", "/")
manifest_rows.append({
"sample_id": sample_id,
"relative_path": rel_path,
"absolute_path": abs_path,
"label": ground_truth,
"generator_family": gen_fam,
"source_dataset": "GenImage (TheKernel01/Tiny-GenImage)",
"source_split": "validation",
"original_filename": orig_filename,
"file_size_bytes": file_size,
"sha256": sha256,
"width": width,
"height": height,
"format": img_format,
"category": "ImageNet-1k object/scene",
"license_or_usage_reference": "CC BY-NC-SA 4.0 / ImageNet research non-commercial",
})
print(f" Successfully extracted and verified {len(manifest_rows)} pristine images.")
# Step 5: Save manifest CSV
print(f"\n[Step 5] Writing frozen manifest to {MANIFEST_PATH}...")
fieldnames = list(manifest_rows[0].keys())
with open(MANIFEST_PATH, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(manifest_rows)
print(f" Frozen manifest generated successfully: {MANIFEST_PATH}")
print(f" Rows: {len(manifest_rows)} (100 REAL, 50 Midjourney, 50 ADM).")
print("=" * 80)
print("STAGE 2 DATASET PREPARATION COMPLETED SUCCESSFULLY!")
print("=" * 80)
if __name__ == "__main__":
main()
|