Spaces:
Running on Zero
Running on Zero
Download scripts/parity_audit/audit_dataset_extraction.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 8.56 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/parity_audit/audit_dataset_extraction.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/parity_audit/audit_dataset_extraction.py
-
curl -L -o audit_dataset_extraction.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/parity_audit/audit_dataset_extraction.py
8.56 kB
| """ | |
| Audit Phase 3: Dataset Extraction and Semantic Composition Audit | |
| Cross-references the Parquet source shard with the extracted Phase 7 image files and manifest. | |
| """ | |
| import os | |
| import sys | |
| import io | |
| import csv | |
| import hashlib | |
| import json | |
| from collections import Counter | |
| import pyarrow.parquet as pq | |
| from PIL import Image | |
| PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| MANIFEST_PATH = os.path.join(PROJECT_ROOT, "benchmark", "external_generalization_manifest.csv") | |
| PARQUET_PATH = r"C:\Users\user\.cache\huggingface\hub\datasets--TheKernel01--Tiny-GenImage\snapshots\89c4fe9efd0ebc7ce5c7641ef57d578ccd639c69\data\validation-00000-of-00004.parquet" | |
| def compute_sha256(data: bytes) -> str: | |
| return hashlib.sha256(data).hexdigest() | |
| def main(): | |
| print("=" * 80) | |
| print("AUDITING PHASE 7 DATASET EXTRACTION AND COMPOSITION") | |
| print("=" * 80) | |
| # Check parquet existence | |
| if not os.path.exists(PARQUET_PATH): | |
| print(f"ERROR: Parquet file not found at {PARQUET_PATH}") | |
| sys.exit(1) | |
| print(f"Reading Parquet file: {PARQUET_PATH}") | |
| table = pq.read_table(PARQUET_PATH) | |
| print(f"Parquet schema:\n{table.schema}") | |
| print(f"Total rows in shard 0: {table.num_rows}") | |
| p_image = table.column("image") | |
| p_label = table.column("label").to_pylist() | |
| p_gen = table.column("generator").to_pylist() | |
| # Check distinct generators and labels in shard 0 | |
| gen_counter = Counter(p_gen) | |
| lbl_counter = Counter(p_label) | |
| print(f"Generator distribution in shard 0: {gen_counter}") | |
| print(f"Label distribution in shard 0: {lbl_counter}") | |
| # Replicate selection | |
| selected_real = [] | |
| selected_midjourney = [] | |
| selected_adm = [] | |
| for idx in range(table.num_rows): | |
| gen_id = p_gen[idx] | |
| if gen_id == 0 and len(selected_real) < 100: | |
| selected_real.append(idx) | |
| elif gen_id == 4 and len(selected_midjourney) < 50: | |
| selected_midjourney.append(idx) | |
| elif gen_id == 1 and len(selected_adm) < 50: | |
| selected_adm.append(idx) | |
| if len(selected_real) == 100 and len(selected_midjourney) == 50 and len(selected_adm) == 50: | |
| break | |
| print(f"Selection counts: REAL={len(selected_real)}, Midjourney={len(selected_midjourney)}, ADM={len(selected_adm)}") | |
| # Load manifest | |
| manifest_rows = [] | |
| with open(MANIFEST_PATH, "r", encoding="utf-8") as f: | |
| reader = csv.DictReader(f) | |
| for r in reader: | |
| manifest_rows.append(r) | |
| print(f"Manifest rows loaded: {len(manifest_rows)}") | |
| # Map by sample_id | |
| manifest_by_id = {r["sample_id"]: r for r in manifest_rows} | |
| # Perform 1-to-1 byte and metadata comparison | |
| targets = [ | |
| ("real", selected_real, "REAL", "none"), | |
| ("midjourney", selected_midjourney, "FAKE", "midjourney"), | |
| ("adm", selected_adm, "FAKE", "adm"), | |
| ] | |
| byte_mismatches = 0 | |
| size_mismatches = 0 | |
| hash_mismatches = 0 | |
| total_audited = 0 | |
| extraction_details = [] | |
| counter = {"real": 0, "midjourney": 0, "adm": 0} | |
| for prefix, indices, expected_label, expected_gen in targets: | |
| for idx in indices: | |
| counter[prefix] += 1 | |
| if prefix == "real": | |
| sample_id = f"phase7_ext_real_{counter[prefix]:03d}" | |
| elif prefix == "midjourney": | |
| sample_id = f"phase7_ext_fake_midjourney_{counter[prefix]:03d}" | |
| else: | |
| sample_id = f"phase7_ext_fake_adm_{counter[prefix]:03d}" | |
| m_row = manifest_by_id[sample_id] | |
| file_on_disk = os.path.join(PROJECT_ROOT, m_row["relative_path"]) | |
| # Read parquet record | |
| p_rec = p_image[idx].as_py() | |
| p_bytes = p_rec["bytes"] | |
| p_path = p_rec.get("path", "") | |
| p_sha = compute_sha256(p_bytes).lower() | |
| p_size = len(p_bytes) | |
| # Read disk file | |
| with open(file_on_disk, "rb") as f_disk: | |
| disk_bytes = f_disk.read() | |
| disk_sha = compute_sha256(disk_bytes).lower() | |
| disk_size = len(disk_bytes) | |
| # Compare bytes | |
| if p_bytes != disk_bytes: | |
| byte_mismatches += 1 | |
| if p_size != disk_size: | |
| size_mismatches += 1 | |
| if p_sha != disk_sha: | |
| hash_mismatches += 1 | |
| # Decode images to inspect format, mode, channels, color profiles | |
| with Image.open(io.BytesIO(p_bytes)) as pil_p: | |
| p_format = pil_p.format | |
| p_mode = pil_p.mode | |
| p_w, p_h = pil_p.size | |
| p_icc = pil_p.info.get("icc_profile") is not None | |
| with Image.open(file_on_disk) as pil_d: | |
| d_format = pil_d.format | |
| d_mode = pil_d.mode | |
| d_w, d_h = pil_d.size | |
| d_icc = pil_d.info.get("icc_profile") is not None | |
| extraction_details.append({ | |
| "sample_id": sample_id, | |
| "prefix": prefix, | |
| "label": expected_label, | |
| "gen": expected_gen, | |
| "parquet_orig_filename": os.path.basename(p_path), | |
| "disk_filename": os.path.basename(file_on_disk), | |
| "parquet_size": p_size, | |
| "disk_size": disk_size, | |
| "parquet_sha": p_sha, | |
| "disk_sha": disk_sha, | |
| "format": p_format, | |
| "disk_format": d_format, | |
| "mode": p_mode, | |
| "width": p_w, | |
| "height": p_h, | |
| "has_icc": p_icc, | |
| "exact_byte_match": p_bytes == disk_bytes, | |
| }) | |
| total_audited += 1 | |
| print(f"\nAUDIT RESULTS ({total_audited} samples):") | |
| print(f" Exact byte matches: {total_audited - byte_mismatches} / {total_audited}") | |
| print(f" Byte mismatches: {byte_mismatches}") | |
| print(f" Size mismatches: {size_mismatches}") | |
| print(f" SHA-256 mismatches: {hash_mismatches}") | |
| # Save extraction audit json | |
| audit_out_path = os.path.join(PROJECT_ROOT, "benchmark", "results", "parity_audit_dataset_extraction.json") | |
| with open(audit_out_path, "w", encoding="utf-8") as f: | |
| json.dump({ | |
| "total_audited": total_audited, | |
| "byte_mismatches": byte_mismatches, | |
| "size_mismatches": size_mismatches, | |
| "hash_mismatches": hash_mismatches, | |
| "details": extraction_details, | |
| }, f, indent=2) | |
| print(f"Saved extraction details to: {audit_out_path}") | |
| # Composition and dimension analysis | |
| print("\n" + "=" * 80) | |
| print("DATASET COMPOSITION & DIMENSION ANALYSIS") | |
| print("=" * 80) | |
| for group_name in ["real", "midjourney", "adm"]: | |
| group_items = [d for d in extraction_details if d["prefix"] == group_name] | |
| formats = Counter(d["format"] for d in group_items) | |
| disk_formats = Counter(d["disk_format"] for d in group_items) | |
| modes = Counter(d["mode"] for d in group_items) | |
| has_icc = sum(1 for d in group_items if d["has_icc"]) | |
| widths = [d["width"] for d in group_items] | |
| heights = [d["height"] for d in group_items] | |
| aspect_ratios = [round(d["width"] / d["height"], 3) for d in group_items] | |
| file_sizes_kb = [round(d["disk_size"] / 1024, 1) for d in group_items] | |
| print(f"\n--- Group: {group_name.upper()} (N = {len(group_items)}) ---") | |
| print(f" Source Formats in Parquet: {dict(formats)}") | |
| print(f" Decoded Formats on Disk: {dict(disk_formats)}") | |
| print(f" Color Modes: {dict(modes)}") | |
| print(f" Has ICC profile: {has_icc} / {len(group_items)}") | |
| print(f" Widths: min={min(widths)}, max={max(widths)}, mean={sum(widths)/len(widths):.1f}") | |
| print(f" Heights: min={min(heights)}, max={max(heights)}, mean={sum(heights)/len(heights):.1f}") | |
| print(f" Aspect Ratios: min={min(aspect_ratios)}, max={max(aspect_ratios)}, square_count={sum(1 for r in aspect_ratios if r == 1.0)}") | |
| print(f" File Size (KB): min={min(file_sizes_kb)}, max={max(file_sizes_kb)}, mean={sum(file_sizes_kb)/len(file_sizes_kb):.1f}") | |
| sample_filenames = [d["parquet_orig_filename"] for d in group_items[:5]] | |
| print(f" Sample Source Filenames: {sample_filenames}") | |
| if __name__ == "__main__": | |
| main() | |