Spaces:
Running on Zero
Running on Zero
Download scripts/prepare_phase7_dataset.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 8.55 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/prepare_phase7_dataset.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/prepare_phase7_dataset.py
-
curl -L -o prepare_phase7_dataset.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/prepare_phase7_dataset.py
8.55 kB
| """ | |
| Phase 7 Dataset Preparation Script | |
| Downloads GenImage validation shard 0 (TheKernel01/Tiny-GenImage), | |
| extracts 200 balanced, pristine images: | |
| - 100 REAL photographs (ImageNet-1k val/nature) | |
| - 50 FAKE Midjourney images | |
| - 50 FAKE ADM images | |
| Screens against historical benchmark hashes, validates RGB decodability, | |
| and generates benchmark/external_generalization_manifest.csv. | |
| """ | |
| import os | |
| import sys | |
| import io | |
| import csv | |
| import hashlib | |
| from typing import Dict, Any, List | |
| import pyarrow.parquet as pq | |
| from PIL import Image | |
| from huggingface_hub import hf_hub_download | |
| PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| DATASET_DIR = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization") | |
| IMAGES_DIR = os.path.join(DATASET_DIR, "images") | |
| REAL_DIR = os.path.join(IMAGES_DIR, "real") | |
| MIDJOURNEY_DIR = os.path.join(IMAGES_DIR, "fake_midjourney") | |
| ADM_DIR = os.path.join(IMAGES_DIR, "fake_adm") | |
| MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv") | |
| REPO_ID = "TheKernel01/Tiny-GenImage" | |
| SHARD_FILENAME = "data/validation-00000-of-00004.parquet" | |
| def compute_sha256(data: bytes) -> str: | |
| return hashlib.sha256(data).hexdigest() | |
| def load_historical_hashes() -> set: | |
| """Collects all SHA-256 hashes from prior project manifests and logs to guarantee zero leakage.""" | |
| historical_hashes = set() | |
| # Phase 6 test manifest | |
| p6_manifest = os.path.join(PROJECT_ROOT, "benchmark", "phase6_test_manifest.csv") | |
| if os.path.exists(p6_manifest): | |
| with open(p6_manifest, "r", encoding="utf-8") as f: | |
| for row in csv.DictReader(f): | |
| h = row.get("sha256") | |
| if h: | |
| historical_hashes.add(h.lower()) | |
| # Phase 4/5 evaluation manifest | |
| eval_manifest = os.path.join(PROJECT_ROOT, "benchmark", "evaluation_manifest.csv") | |
| if os.path.exists(eval_manifest): | |
| with open(eval_manifest, "r", encoding="utf-8") as f: | |
| for row in csv.DictReader(f): | |
| h = row.get("sha256") | |
| if h: | |
| historical_hashes.add(h.lower()) | |
| print(f"Loaded {len(historical_hashes)} historical hashes for anti-leakage screening.") | |
| return historical_hashes | |
| def main(): | |
| print("=" * 80) | |
| print("PHASE 7: EXTRACTING & FREEZING EXTERNAL GENERALIZATION DATASET (GENIMAGE)") | |
| print("=" * 80) | |
| for d in [REAL_DIR, MIDJOURNEY_DIR, ADM_DIR]: | |
| os.makedirs(d, exist_ok=True) | |
| historical_hashes = load_historical_hashes() | |
| # Step 1: Download Shard 0 Parquet file | |
| print(f"\n[Step 1] Downloading GenImage validation shard 0 from Hugging Face Hub ({REPO_ID})...") | |
| parquet_path = hf_hub_download( | |
| repo_id=REPO_ID, | |
| filename=SHARD_FILENAME, | |
| repo_type="dataset", | |
| ) | |
| print(f" Downloaded to local cache: {parquet_path}") | |
| # Step 2: Read Parquet table | |
| print("\n[Step 2] Reading Parquet records from shard 0...") | |
| table = pq.read_table(parquet_path) | |
| print(f" Table loaded: {table.num_rows} rows, {table.num_columns} columns.") | |
| p_image = table.column("image") | |
| p_label = table.column("label").to_pylist() | |
| p_gen = table.column("generator").to_pylist() | |
| # Step 3: Extract balanced subsets | |
| print("\n[Step 3] Selecting 100 Real, 50 Midjourney, 50 ADM samples...") | |
| selected_real = [] | |
| selected_midjourney = [] | |
| selected_adm = [] | |
| for idx in range(table.num_rows): | |
| gen_id = p_gen[idx] | |
| lbl = p_label[idx] | |
| if gen_id == 0 and len(selected_real) < 100: | |
| selected_real.append(idx) | |
| elif gen_id == 4 and len(selected_midjourney) < 50: | |
| selected_midjourney.append(idx) | |
| elif gen_id == 1 and len(selected_adm) < 50: | |
| selected_adm.append(idx) | |
| if len(selected_real) == 100 and len(selected_midjourney) == 50 and len(selected_adm) == 50: | |
| break | |
| print(f" Selected counts: REAL={len(selected_real)}, Midjourney={len(selected_midjourney)}, ADM={len(selected_adm)}") | |
| assert len(selected_real) == 100, f"Expected 100 Real, got {len(selected_real)}" | |
| assert len(selected_midjourney) == 50, f"Expected 50 Midjourney, got {len(selected_midjourney)}" | |
| assert len(selected_adm) == 50, f"Expected 50 ADM, got {len(selected_adm)}" | |
| # Step 4: Extract and save image files to disk with integrity validation | |
| print("\n[Step 4] Writing pristine images to disk and validating integrity...") | |
| manifest_rows = [] | |
| seen_hashes = set() | |
| extraction_targets = [ | |
| ("real", selected_real, REAL_DIR, "REAL", "none"), | |
| ("midjourney", selected_midjourney, MIDJOURNEY_DIR, "FAKE", "midjourney"), | |
| ("adm", selected_adm, ADM_DIR, "FAKE", "adm"), | |
| ] | |
| sample_counter = 1 | |
| for name_prefix, indices, target_dir, ground_truth, gen_fam in extraction_targets: | |
| for idx in indices: | |
| row_dict = p_image[idx].as_py() | |
| img_bytes = row_dict["bytes"] | |
| orig_path = row_dict.get("path") or f"{name_prefix}_{idx}.jpg" | |
| orig_filename = os.path.basename(orig_path) | |
| sha256 = compute_sha256(img_bytes).lower() | |
| file_size = len(img_bytes) | |
| # Assert no duplicates | |
| if sha256 in seen_hashes: | |
| raise ValueError(f"Duplicate hash within selection: {sha256}") | |
| seen_hashes.add(sha256) | |
| if sha256 in historical_hashes: | |
| raise ValueError(f"Hash collision with historical benchmark dataset: {sha256}") | |
| # Verify image decode | |
| with Image.open(io.BytesIO(img_bytes)) as pil_img: | |
| width, height = pil_img.size | |
| img_format = pil_img.format or "JPEG" | |
| # Ensure RGB decodable | |
| _ = pil_img.convert("RGB") | |
| # Determine clean filename and relative/absolute paths | |
| if name_prefix == "real": | |
| sample_id = f"phase7_ext_real_{len([r for r in manifest_rows if r['label'] == 'REAL']) + 1:03d}" | |
| out_filename = f"genimage_real_{len([r for r in manifest_rows if r['label'] == 'REAL']) + 1:03d}.jpg" | |
| elif name_prefix == "midjourney": | |
| sample_id = f"phase7_ext_fake_midjourney_{len([r for r in manifest_rows if r['generator_family'] == 'midjourney']) + 1:03d}" | |
| out_filename = f"genimage_midjourney_{len([r for r in manifest_rows if r['generator_family'] == 'midjourney']) + 1:03d}.jpg" | |
| else: | |
| sample_id = f"phase7_ext_fake_adm_{len([r for r in manifest_rows if r['generator_family'] == 'adm']) + 1:03d}" | |
| out_filename = f"genimage_adm_{len([r for r in manifest_rows if r['generator_family'] == 'adm']) + 1:03d}.jpg" | |
| out_path = os.path.join(target_dir, out_filename) | |
| with open(out_path, "wb") as f_out: | |
| f_out.write(img_bytes) | |
| rel_path = os.path.relpath(out_path, PROJECT_ROOT).replace("\\", "/") | |
| abs_path = os.path.abspath(out_path).replace("\\", "/") | |
| manifest_rows.append({ | |
| "sample_id": sample_id, | |
| "relative_path": rel_path, | |
| "absolute_path": abs_path, | |
| "label": ground_truth, | |
| "generator_family": gen_fam, | |
| "source_dataset": "GenImage (TheKernel01/Tiny-GenImage)", | |
| "source_split": "validation", | |
| "original_filename": orig_filename, | |
| "file_size_bytes": file_size, | |
| "sha256": sha256, | |
| "width": width, | |
| "height": height, | |
| "format": img_format, | |
| "category": "ImageNet-1k object/scene", | |
| "license_or_usage_reference": "CC BY-NC-SA 4.0 / ImageNet research non-commercial", | |
| }) | |
| print(f" Successfully extracted and verified {len(manifest_rows)} pristine images.") | |
| # Step 5: Save manifest CSV | |
| print(f"\n[Step 5] Writing frozen manifest to {MANIFEST_PATH}...") | |
| fieldnames = list(manifest_rows[0].keys()) | |
| with open(MANIFEST_PATH, "w", encoding="utf-8", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=fieldnames) | |
| writer.writeheader() | |
| writer.writerows(manifest_rows) | |
| print(f" Frozen manifest generated successfully: {MANIFEST_PATH}") | |
| print(f" Rows: {len(manifest_rows)} (100 REAL, 50 Midjourney, 50 ADM).") | |
| print("=" * 80) | |
| print("STAGE 2 DATASET PREPARATION COMPLETED SUCCESSFULLY!") | |
| print("=" * 80) | |
| if __name__ == "__main__": | |
| main() | |