Deploy from GitHub 1af737237f5d10dfa9cf12849d98f47c3b30ce37 (filtered: code + Dockerfile + README + NOTICES only)

Dockerfile

@@ -0,0 +1,78 @@
+# DeepFakeScanner — single-image deployment for HF Spaces, Cloud Run, Modal, etc.
+#
+# Build:
+#   docker build -t deepfake-scanner .
+# Run:
+#   docker run -p 7860:7860 deepfake-scanner
+#
+# Lives at the repo root because HF Spaces' Docker SDK builds from ./Dockerfile.
+# Port 7860 is the HF Spaces default. Override with the PORT env var on
+# other platforms (Cloud Run injects PORT automatically).
+
+FROM python:3.11-slim AS builder
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+# Build deps (some Python wheels still compile on slim).
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        libgl1 \
+        libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install CPU-only PyTorch first to keep the image lean (~700 MB instead of ~3 GB).
+COPY requirements.txt .
+RUN pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision \
+    && pip install -r requirements.txt
+
+# Copy source.
+COPY src/ ./src/
+COPY scripts/ ./scripts/
+COPY pyproject.toml .
+
+# Install the package.
+RUN pip install -e .
+
+# Pre-download CLIP backbone weights so first request isn't slow. This is a
+# public download — no auth needed.
+RUN python scripts/download_weights.py
+
+# NOTE: the Stage 2 head weights (Veridicate/scanner-head-v1) are NOT
+# pre-downloaded here. That repo is private, and HF Spaces does not expose
+# secrets to a plain `RUN` at build time (it would need a BuildKit
+# --mount=type=secret). Instead, ClipClassifier downloads the head at
+# container startup using the HF_TOKEN secret, which IS available at
+# runtime. The head is only ~530 KB so the startup cost is negligible.
+# See config.py:head_checkpoint_hf_repo and clip_classifier.py.
+
+# ---------------------------------------------------------------------------
+# Final image
+# ---------------------------------------------------------------------------
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PORT=7860
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libgl1 \
+        libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy installed packages and HF cache from builder.
+COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
+COPY --from=builder /root/.cache/huggingface /root/.cache/huggingface
+COPY --from=builder /app /app
+
+WORKDIR /app
+
+EXPOSE 7860
+
+# Use a wrapper so the PORT env var (Cloud Run) is honoured.
+CMD ["sh", "-c", "uvicorn deepfake_scanner.api.v1:app --host 0.0.0.0 --port ${PORT:-7860}"]

NOTICES.md

@@ -0,0 +1,99 @@
+# Third-Party Notices
+
+This project is built on commercially-licensed open-source components only.
+Every dependency below has been verified against its license terms.
+
+## Runtime dependencies
+
+| Component                  | Version (≥) | License        | Use                                        |
+|----------------------------|-------------|----------------|--------------------------------------------|
+| Python                     | 3.11        | PSF            | Runtime                                    |
+| PyTorch (CPU build)        | 2.3         | BSD-3-Clause   | Tensor / NN runtime                        |
+| torchvision                | 0.18        | BSD-3-Clause   | Image transforms                           |
+| HuggingFace `transformers` | 4.41        | Apache-2.0     | CLIP loading                               |
+| HuggingFace `accelerate`   | 0.30        | Apache-2.0     | Inference helpers                          |
+| OpenAI CLIP ViT-B/32       | n/a         | MIT            | Frozen image-encoder weights               |
+| Pillow (PIL fork)          | 10.3        | HPND           | Image decoding                             |
+| NumPy                      | 1.26        | BSD-3-Clause   | Tensor ops                                 |
+| FastAPI                    | 0.111       | MIT            | Web framework                              |
+| Starlette                  | (transitive)| BSD-3-Clause   | ASGI core                                  |
+| Uvicorn                    | 0.29        | BSD-3-Clause   | ASGI server                                |
+| python-multipart           | 0.0.9       | Apache-2.0     | File-upload parsing                        |
+| Pydantic                   | 2.7         | MIT            | Schema validation                          |
+| `c2pa-python`              | 0.5         | Apache-2.0     | C2PA / Content-Credentials verification    |
+
+## Development / test dependencies
+
+| Component | License    | Use            |
+|-----------|------------|----------------|
+| pytest    | MIT        | Test runner    |
+| httpx     | BSD-3-Clause | Test client  |
+
+## Stage 2 dataset-curation dependencies (NOT shipped with the inference image)
+
+These are only installed on the GPU machine that builds the dataset.
+
+| Component                           | License      | Use                                     |
+|-------------------------------------|--------------|-----------------------------------------|
+| FiftyOne                            | Apache-2.0   | Open Images V7 download / sampling      |
+| `diffusers` (HuggingFace)           | Apache-2.0   | Flux.1-schnell pipeline                 |
+| `black-forest-labs/FLUX.1-schnell`  | Apache-2.0   | Synthetic-image generator weights       |
+
+## Stage 3A candidate generator license review
+
+Review date: 2026-05-15. No Stage 3A images have been generated yet; this table
+records which generators are approved for the initial multi-generator dataset.
+
+| Component | License / terms | Status | Use |
+|---|---|---|---|
+| `black-forest-labs/FLUX.1-schnell` | Apache-2.0 ([HF model card](https://huggingface.co/black-forest-labs/FLUX.1-schnell)) | Approved | Already-used rectified-flow baseline. |
+| `stabilityai/stable-diffusion-xl-base-1.0` | CreativeML Open RAIL++-M ([HF license](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/LICENSE.md)) | Approved with use-policy restrictions | Stage 3A diffusion U-Net training source. Outputs may be used for detector training, subject to OpenRAIL use restrictions; do not use to train a competing generative model. |
+| `stabilityai/stable-diffusion-3.5-medium` | Stability AI Community License ([HF model card](https://huggingface.co/stabilityai/stable-diffusion-3.5-medium), [Stability license](https://stability.ai/license)) | Conditionally approved | Stage 3A MMDiT / diffusion-transformer training source while Veridicate remains under the Community License revenue threshold and registers as required. Enterprise license required before use if annual revenue exceeds USD $1M. Outputs must not be used to create or improve a foundational generative AI model. |
+| `fal/AuraFlow-v0.3` | Apache-2.0 ([HF model card](https://huggingface.co/fal/AuraFlow-v0.3)) | Approved | Stage 3A independent rectified-flow training source. |
+| `PixArt-alpha/PixArt-Sigma-XL-2-1024-MS` | HF model card labels weights as CreativeML Open RAIL++-M; project code is Apache-2.0, but the model card's direct-use section says research purposes only ([HF model card](https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS), [GitHub repo](https://github.com/PixArt-alpha/PixArt-sigma)) | Deferred / blocked pending clarification | Do not use for Stage 3A training until the model-weight license and model-card intended-use language are clarified. |
+
+## Datasets used at training time
+
+| Dataset                    | License      | Notes                                   |
+|----------------------------|--------------|-----------------------------------------|
+| Open Images V7             | CC BY 2.0    | Real-image class. 50,000 images sampled. Attribution preserved in `data/manifest.csv`. |
+| Self-generated (Flux schnell) | Apache-2.0 (model) | AI-image class. 50,000 images generated. We own the outputs; recorded in manifest with prompt + seed. |
+
+## Distributed artifacts (downstream of training)
+
+| Artifact | License | Notes |
+|---|---|---|
+| `Veridicate/scanner-head-v1` (private HF Hub model repo) | Apache-2.0 | Trained CLIP classifier head weights (~530 KB). Downloaded at container startup by `ClipClassifier` using the `HF_TOKEN` Space secret. Trained on the Open Images + Flux-schnell dataset above, so the output weights are unencumbered. Repo is private (commercial-IP reasons) but the license on the weights themselves is Apache-2.0. |
+
+## Explicitly NOT used (research-only / non-commercial / unclear license)
+
+These are deliberately excluded to preserve commercial usability:
+
+- **FaceForensics++** — research/educational use only.
+- **DFDC (Deepfake Detection Challenge)** — research only.
+- **Celeb-DF** — research only.
+- **`flux.1-dev`** — non-commercial license. Use `flux.1-schnell` instead.
+- **Outputs from closed-API image generators** — including Google
+  Gemini / Imagen 3 / "Nano Banana", OpenAI DALL-E, Midjourney, xAI
+  Grok Imagine / Aurora, Adobe Firefly. Their terms of service either
+  restrict using outputs as training data for downstream models or
+  leave it ambiguous enough that, for a commercial detector, the safe
+  reading is "not allowed without an explicit license." Stage 3 of the
+  roadmap (see [`docs/plan.md`](docs/plan.md)) explicitly *catches*
+  these generators by training on the *open-source* equivalents of
+  their underlying architectures (diffusion U-Net, DiT, rectified flow,
+  GAN) and learning the family fingerprint — without ever using their
+  outputs as training data. Stage 5 may revisit the closed-API path
+  with negotiated enterprise license agreements, but only once
+  revenue justifies the legal and per-image API spend.
+- Any HuggingFace model whose license tag is `cc-by-nc-*`,
+  `non-commercial`, or unspecified.
+
+## Updating this document
+
+Add a row whenever a new dependency is introduced. If a license is unclear,
+flag it and resolve before merging — never assume.
+
+For **forward-looking** license concerns (revenue thresholds, deferred
+items, in-flight investigations), see [`docs/licensing.md`](docs/licensing.md).
+That file is the live tracker; this file is the approved-and-shipped record.

README.md

@@ -0,0 +1,150 @@
+---
+title: DeepFakeScanner
+sdk: docker
+app_port: 7860
+pinned: false
+license: apache-2.0
+short_description: AI-generated & manipulated image detection (v0.4.0)
+---
+
+# DeepFakeScanner
+
+A commercial deepfake / AI-generated image detection service.
+
+## What it is
+
+A FastAPI web service that scans an uploaded image and returns a structured
+verdict: `authentic`, `ai_generated`, `deepfake`, `edited`, or `uncertain` —
+with per-class probabilities, per-detector signals, and a C2PA provenance
+check.
+
+## Live
+
+| | URL |
+|---|---|
+| Frontend | https://veridicate.com |
+| API | https://api.veridicate.com |
+| Source (HF mirror) | https://huggingface.co/spaces/veridicate/scanner |
+
+## Status
+
+**Stage 2 deployed (`v0.3.0-stage2`).** The CLIP classifier head was
+fine-tuned on a 100k commercially-licensed dataset (50k Open Images +
+50k Flux.1-schnell) and is currently serving real predictions in
+production.
+
+In-distribution test-set metrics (10k held-out images):
+- Accuracy: **98.44%**
+- Precision (AI class): 98.25%, recall: 98.64%, F1: 98.44%
+
+**Stage 3A wired + verified 2026-05-30 (`v0.4.0-stage3a`); ships to
+production on merge of `feat/akila-20260515 → main`.** Multi-generator
+dataset built: 50k Flux + 20k SDXL + 20k SD 3.5 Medium + 10k AuraFlow
+on the AI side, matched authentic from Open Images V7. SDXL held out of
+training so the heldout split is a true generalisation test. The trained
+head (`head_v3a.pt`) is published to the private HF Hub repo, the runtime
+config now defaults to it (`config.py`), and a filtered, test-gated
+GitHub Action deploys the inference service on merge. Headline verified
+numbers vs the Stage 2 baseline:
+
+| Split | Baseline | Candidate | Δ |
+|---|---|---|---|
+| heldout SDXL (UNSEEN in training) | 83.93% | **89.48%** | **+5.55 pp** |
+| test (in-distribution) | 96.47% | 98.43% | +1.96 pp |
+| test_augmented (robustness) | 93.96% | 98.41% | +4.46 pp |
+
+The +5.55 pp on the SDXL holdout is the load-bearing number — SDXL was
+held entirely out of training, so it's the closest available proxy for
+how the model will behave on generators it never saw. The
+family-fingerprint approach (train on a diverse mix of open
+generators, inherit coverage of closed generators) is validated.
+Detailed audit trail in
+[`docs/stage3a-implementation.md`](docs/stage3a-implementation.md).
+
+**Known limitation (until the Stage 3A merge deploys):** the live
+model is still the Stage 2 head, trained on Flux.1-schnell only. Other
+generators (Gemini/Imagen 3, DALL-E 3, Midjourney, Grok, Stable
+Diffusion) are out-of-distribution for the *currently-live* model and
+it often returns `uncertain` verdicts. The Stage 3A head (queued to
+ship) closes most of this gap. Rollback is a one-line env override
+(`MODEL_VERSION` + `HEAD_CHECKPOINT_HF_FILENAME`) — both heads live in
+the same private HF Hub repo.
+
+## Roadmap at a glance
+
+| Stage | What it delivers | Status |
+|---|---|---|
+| **1** | Working website, API, deploy pipeline. Detector returns random guesses. | ✅ Done |
+| **2** | A trained classifier — 98% accurate on Flux-family AI images. | ✅ Done, live (`v0.3.0-stage2`) |
+| **3A** | Broad coverage across the AI image-generation landscape — CLIP head retrained on Flux + SDXL + SD 3.5 + AuraFlow. | ✅ Wired + verified 2026-05-30 (`v0.4.0-stage3a`); ships on merge to `main` |
+| **3B** | Frequency-artifact detector (FFT/DCT, generator-agnostic) brought online. | 🔭 Queued after Stage 3A ships |
+| **4** | Production scale: faster hosting, paid tier, user accounts. | ⏸️ After Stage 3 |
+| **5** | Enterprise capability: licensed paid-API training data, face-swap detection, adversarial robustness. | 🔭 Future |
+
+Full roadmap with per-stage strengths, weaknesses, and how each
+weakness gets fixed: [`docs/plan.md`](docs/plan.md). Plain-English
+summary up front; technical detail below; glossary at the end for
+non-technical readers.
+
+## Quick start
+
+```bash
+# CPU PyTorch first (lean install)
+pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+pip install -r requirements.txt
+pip install -e .
+
+# Pre-download CLIP weights
+python scripts/download_weights.py
+
+# Run the server
+uvicorn deepfake_scanner.api.v1:app --reload --port 7860
+```
+
+Then:
+
+```bash
+curl -F "file=@some_image.jpg" http://localhost:7860/v1/scan/image | jq
+```
+
+Or against the live API:
+
+```bash
+curl -F "file=@some_image.jpg" https://api.veridicate.com/v1/scan/image | jq
+```
+
+Dataset-generation dependencies are documented in the optional GPU section of
+[`requirements.txt`](requirements.txt). Keep those packages out of the
+production inference image.
+
+## API
+
+- `GET  /health` — liveness probe
+- `GET  /v1/info` — model + config metadata
+- `POST /v1/scan/image` — scan an image (multipart/form-data, max 10 MB,
+  JPEG/PNG/WebP)
+
+See [`ARCHITECTURE.md`](ARCHITECTURE.md) for the full response schema.
+
+## Privacy
+
+Visitor uploads are processed in-memory and **never persisted**. Only scan
+metadata (verdict, confidence, latency, model version) is recorded.
+
+## Documentation
+
+- [`docs/plan.md`](docs/plan.md) — **product roadmap** with per-stage
+  strengths, weaknesses, and fix paths. Written so a non-technical
+  reader can follow the strategy, with deeper technical detail and a
+  glossary inline.
+- [`ARCHITECTURE.md`](ARCHITECTURE.md) — technical design of the
+  detection pipeline + API contract
+- [`docs/decisions.md`](docs/decisions.md) — running decision log
+  (good context if picking up this project later)
+- [`NOTICES.md`](NOTICES.md) — third-party licensing record
+- [`CLAUDE.md`](CLAUDE.md) — project context (auto-loaded by Claude Code)
+- [`scripts/dataset/README.md`](scripts/dataset/README.md) — dataset curation pipeline
+
+## License
+
+Apache 2.0.

pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "deepfake-scanner"
+version = "0.2.0"
+description = "Detect AI-generated and manipulated images via a versioned FastAPI inference service."
+requires-python = ">=3.11"
+license = { text = "Apache-2.0" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"

requirements.txt

@@ -0,0 +1,67 @@
+# ---------------------------------------------------------------------------
+# IMPORTANT — PyTorch CPU-only install (Stage 1 deployment target is CPU)
+#
+# Do NOT run plain `pip install torch` — that pulls the CUDA build (~2.5 GB).
+# Use the CPU wheel index instead:
+#
+#   pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+#
+# Then install the rest:
+#   pip install -r requirements.txt
+#
+# All packages below are commercially licensed (MIT / Apache-2.0 / BSD-3 / HPND).
+# ---------------------------------------------------------------------------
+
+# Deep-learning runtime
+torch>=2.3.0
+torchvision>=0.18.0
+
+# CLIP backbone & general HF model loading
+transformers>=4.41.0,<5    # Apache-2.0 — pinned to 4.x: 5.x changed CLIPModel.get_image_features to return a wrapper object
+accelerate>=0.30.0         # Apache-2.0 — recommended companion for transformers
+
+# Image I/O & preprocessing
+Pillow>=10.3.0
+numpy>=1.26.4
+
+# C2PA / Content Credentials verification
+# Apache-2.0; native deps may not build on every platform — provenance/c2pa.py
+# degrades gracefully if the import fails.
+# Upper bound is deliberate: the Reader API changed between 0.2x and 0.3x
+# (`Reader.from_stream` was removed) and the blanket except in
+# provenance/c2pa.py turned that into a SILENT three-month outage —
+# see docs/decisions.md 2026-08-09. Verify provenance still works before
+# widening this, and run tests/test_provenance_c2pa.py on a machine that
+# has a C2PA-signed sample.
+c2pa-python>=0.32.0,<0.33
+
+# API server
+fastapi>=0.111.0
+uvicorn[standard]>=0.29.0
+python-multipart>=0.0.9    # required for FastAPI file uploads
+pydantic>=2.7.0
+
+# Testing
+pytest>=8.2.0
+httpx>=0.27.0              # required by FastAPI TestClient
+
+# ---------------------------------------------------------------------------
+# Optional dataset-curation deps (NOT installed by this file).
+#
+# This file is the consolidated Python dependency ledger, but the production
+# Dockerfile installs it directly. Keep GPU/dataset-only packages commented so
+# the inference image stays lean. On a rented GPU box, install the CUDA torch
+# wheel first, then install the dataset packages listed below explicitly:
+#
+#   pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121
+#   pip install 'fiftyone>=0.24.0' 'diffusers>=0.30.0' 'sentencepiece>=0.2.0' 'protobuf>=4.25.0'
+#
+# Dataset-only packages, licenses, and purpose:
+#   fiftyone>=0.24.0          # Apache-2.0 — Open Images V7 sampling
+#   diffusers>=0.30.0         # Apache-2.0 — Flux/SDXL/SD3/AuraFlow pipelines
+#   sentencepiece>=0.2.0      # Apache-2.0 — required by Flux's T5 tokenizer
+#   protobuf>=4.25.0          # BSD-3 — required by sentencepiece
+#
+# Stage 2/3 head training itself runs on cached embeddings and uses only the
+# installed runtime/test dependencies above.
+# ---------------------------------------------------------------------------

scripts/dataset/README.md

@@ -0,0 +1,159 @@
+# Dataset curation
+
+This folder contains the scripts that produced the **commercially-licensed
+training dataset** for Stage 2 fine-tuning (DONE, 2026-05).
+
+Stage 3A extends this folder with one new generation script per
+approved open-source generator (initially AuraFlow, SDXL, and SD 3.5
+Medium; PixArt-Σ is deferred pending license/intended-use clarification).
+The rest of the pipeline
+(`build_manifest.py`, `split.py`, `../precompute_embeddings.py`,
+`../train_head.py`) is generator-agnostic and will be reused as-is —
+the manifest format already supports multiple AI source labels.
+See [`docs/plan.md`](../../docs/plan.md) Stage 3 for the full plan.
+
+## What we're building
+
+A dataset of roughly **100k images** total:
+
+| Split      | Source                                | Class           | License            |
+|------------|---------------------------------------|-----------------|--------------------|
+| ~50k real  | Open Images V7 (sampled)              | `authentic`     | CC BY 2.0          |
+| ~50k AI    | Self-generated with Flux.1-schnell    | `ai_generated`  | Apache 2.0 weights → outputs are unrestricted |
+
+Stage 2 trains the CLIP classifier head on this 2-class data. The `deepfake`
+and `edited` classes will be added later via separate scripts.
+
+## Workflow
+
+1. **`fetch_open_images.py`** — downloads a stratified sample of Open Images
+   into `data/raw/real/`. Run once. Idempotent.
+
+2. **`generate_flux_synthetic.py`** — generates AI images with Flux.1-schnell
+   into `data/raw/ai_generated/`. Run on a rented cloud GPU
+   (~$30 total at RunPod / Lambda Labs spot rates).
+
+   **`generate_sdxl_synthetic.py`** — generates AI images with SDXL into
+   `data/raw/ai_generated/sdxl/`. Use `--dry-run` locally before running on a
+   GPU box.
+
+   **`generate_sd35_synthetic.py`** — generates AI images with SD 3.5 Medium
+   into `data/raw/ai_generated/sd35-medium/`. This source is conditionally
+   approved under the Stability AI Community License; review `NOTICES.md`
+   before running it.
+
+   **`generate_auraflow_synthetic.py`** — generates AI images with AuraFlow
+   v0.3 into `data/raw/ai_generated/auraflow-v0.3/`. Use `--dry-run` locally
+   before running on a GPU box.
+
+   Stage 3A generator scripts should reuse `generation_utils.py` for prompt
+   loading, deterministic seeds, stable output keys, file hashes, image
+   dimensions, approved generator metadata, and manifest-row construction.
+
+3. **`build_manifest.py`** — produces `data/manifest.csv` with one row per
+   image. Required columns are
+   `path,class,source,license,license_url,sha256`. Stage 3A adds optional
+   generator metadata columns:
+   `generator,model_family,model_id,prompt,seed,width,height,generation_params_json`.
+   Missing optional values are written as empty strings, and approved
+   AI-generator metadata is backfilled from known `source` values when older
+   fragments do not include the optional columns. You can pass fragments
+   explicitly with `--inputs`, or point `--input-dir` at `data/raw` to discover
+   all `*_manifest.csv` fragments recursively. This is the
+   **legal record** that protects you when you commercialise and the
+   source-of-truth for per-generator evaluation.
+
+4. **`split.py`** — produces train/val/test splits stratified by class. Stage
+   3A adds optional generator-aware splitting with
+   `--stratify-by class-generator`, plus `--holdout-generator <name>` to write
+   a `heldout.csv` evaluation split while excluding that generator from
+   train/val/test.
+
+After splitting, training itself lives outside this directory:
+
+5. **`augment_images.py`** — creates deterministic Stage 3A robustness
+   augmentations from any manifest or split CSV. It writes new JPEG images plus
+   a manifest-compatible CSV with `original_path`, `augmentation`,
+   `augmentation_seed`, and `augmentation_params_json`. Run it on `train.csv`
+   for augmented training rows, or on `val.csv` / `test.csv` into separate
+   manifests such as `test_augmented.csv` for robustness evaluation while
+   keeping the clean splits unchanged.
+
+6. **`../precompute_embeddings.py`** — encodes every image through frozen
+   CLIP once and caches the 512-d feature vectors. Runs on CPU (1-4 hours
+   on the dev laptop). One-shot per dataset version. By default it encodes
+   clean `train.csv`, `val.csv`, and `test.csv`. Stage 3A adds
+   `--train-augment-manifest train_augmented.csv` to append augmented rows to
+   `train.npz`, and `--extra-split test_augmented=test_augmented.csv` for
+   explicit augmented robustness-eval embeddings.
+
+7. **`../train_head.py`** — trains the small classifier head on the cached
+   features. Seconds per epoch on CPU; iterate hyperparameters freely. Stage
+   3A reports overall metrics, per-source/per-generator/per-family metrics when
+   embedding metadata is available, and augmentation robustness metrics for
+   explicitly encoded augmented eval splits. Use `--report-out` to save the
+   JSON report.
+
+8. **`../evaluate_head.py`** — evaluates a candidate checkpoint, optionally
+   against a Stage 2 baseline checkpoint, on any cached embedding split
+   (`test`, `heldout`, `test_augmented`, etc.). It writes a reproducible JSON
+   report with overall, per-generator, held-out-generator, uncertainty, and
+   augmentation robustness metrics.
+
+9. **`run_stage3a_smoke.py`** — runs a tiny local fixture pipeline before
+   expensive GPU work. It builds fixture images/manifests, splits with an SDXL
+   holdout, creates train/test augmentations, writes deterministic mocked
+   512-d embeddings, trains a smoke head, and evaluates it against a constant
+   baseline:
+
+   ```bash
+   python scripts/dataset/run_stage3a_smoke.py \
+       --work-dir /tmp/deepfakescanner-stage3a-smoke
+   ```
+
+   This intentionally bypasses real CLIP precompute so the smoke test stays
+   fast and offline; use `../precompute_embeddings.py` for real dataset runs.
+
+10. **`../run_stage3a_pipeline.py`** — orchestrates the full Stage 3A sequence
+    after the smoke test passes. It can plan the run, execute GPU generation,
+    build the manifest, split, augment, precompute embeddings, train the
+    candidate head, evaluate against the Stage 2 baseline, write a JSON
+    ship/no-ship recommendation, and optionally upload an accepted checkpoint
+    to the private HF Hub repo.
+
+    ```bash
+    python scripts/run_stage3a_pipeline.py --dry-run --stop-after evaluate
+    python scripts/run_stage3a_pipeline.py --stop-after generate
+    python scripts/run_stage3a_pipeline.py \
+        --skip-generation \
+        --baseline data/checkpoints/head_v1.pt
+    ```
+
+    Publishing is intentionally opt-in via `--publish-if-accepted` and requires
+    `HF_TOKEN`.
+
+For a collaborator-facing GPU handoff, use
+[`docs/stage3a-gpu-collaborator-guide.md`](../../docs/stage3a-gpu-collaborator-guide.md).
+
+## Why each piece is licensed for commercial use
+
+- **Open Images V7** — Google's dataset, all images are CC BY 2.0 (commercial
+  use allowed with attribution). Attribution lives in `manifest.csv`.
+- **Flux.1-schnell** — released by Black Forest Labs under Apache 2.0. Outputs
+  are not restricted; you own them. (Note: `flux.1-dev` is non-commercial —
+  do NOT use it.)
+- **Stage 3A approved additions** — SDXL, SD 3.5 Medium, and AuraFlow were
+  reviewed on 2026-05-15. See [`NOTICES.md`](../../NOTICES.md) for the exact
+  license status and constraints before generating any images.
+- **NOT used here**: FaceForensics++, DFDC, Celeb-DF (research-only),
+  outputs from closed-API generators (Gemini, Midjourney, DALL-E, Grok,
+  Firefly — ToS restrictions or ambiguous terms), `flux.1-dev`
+  (non-commercial), and PixArt-Σ until its model-card intended-use ambiguity
+  is clarified. See [`NOTICES.md`](../../NOTICES.md) for the full list and
+  [`docs/decisions.md`](../../docs/decisions.md) 2026-05-14 and 2026-05-15
+  entries for the reasoning.
+
+## Running it
+
+The scripts are designed to be run independently. See each script's docstring
+for prerequisites and command-line flags.

scripts/dataset/augment_images.py

@@ -0,0 +1,269 @@
+"""
+Create deterministic Stage 3A image augmentations from a manifest or split CSV.
+
+The script writes new image files plus a manifest-compatible CSV fragment. It
+does not modify the source images or source CSV, so validation/test splits stay
+clean unless you explicitly run this script for a separate augmented eval set.
+
+Augmentations
+-------------
+- JPEG recompression
+- Downscale then upscale
+- Deterministic crop
+- Mild blur or sharpen
+- Mild color/contrast/brightness jitter
+
+Usage
+-----
+    python scripts/dataset/augment_images.py \\
+        --manifest data/train.csv \\
+        --data-root data \\
+        --out-dir data/augmented/train \\
+        --out-manifest data/train_augmented.csv \\
+        --copies 1 \\
+        --seed 0
+
+For a separate robustness eval set, point --manifest at val.csv or test.csv and
+write to a distinct output manifest such as data/test_augmented.csv.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import random
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+
+from PIL import Image, ImageEnhance, ImageFilter
+
+
+AUGMENTATION_FIELDS = [
+    "width",
+    "height",
+    "original_path",
+    "augmentation",
+    "augmentation_seed",
+    "augmentation_params_json",
+]
+
+
+def _stable_seed(global_seed: int, path: str, copy_index: int) -> int:
+    payload = f"{global_seed}|{path}|{copy_index}".encode("utf-8")
+    return int(hashlib.sha256(payload).hexdigest()[:16], 16) % (2**31)
+
+
+def _sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _params_json(params: dict[str, Any]) -> str:
+    return json.dumps(params, sort_keys=True, separators=(",", ":"))
+
+
+def _relative_to_data_root(path: Path, data_root: Path) -> str:
+    try:
+        return path.relative_to(data_root).as_posix()
+    except ValueError as exc:
+        raise ValueError(
+            f"Output path {path} is not under data root {data_root}; "
+            "choose an --out-dir inside --data-root"
+        ) from exc
+
+
+def _resize_roundtrip(
+    image: Image.Image,
+    rng: random.Random,
+) -> tuple[Image.Image, dict]:
+    width, height = image.size
+    scale = rng.uniform(0.65, 0.95)
+    small_size = (
+        max(1, int(round(width * scale))),
+        max(1, int(round(height * scale))),
+    )
+    resample_down = Image.Resampling.BICUBIC
+    resample_up = Image.Resampling.BILINEAR
+    resized = image.resize(small_size, resample_down).resize(
+        (width, height),
+        resample_up,
+    )
+    return resized, {"resize_scale": round(scale, 4)}
+
+
+def _crop_and_restore(
+    image: Image.Image,
+    rng: random.Random,
+) -> tuple[Image.Image, dict]:
+    width, height = image.size
+    crop_scale = rng.uniform(0.9, 1.0)
+    crop_w = max(1, int(round(width * crop_scale)))
+    crop_h = max(1, int(round(height * crop_scale)))
+    max_left = max(0, width - crop_w)
+    max_top = max(0, height - crop_h)
+    left = rng.randint(0, max_left) if max_left else 0
+    top = rng.randint(0, max_top) if max_top else 0
+    cropped = image.crop((left, top, left + crop_w, top + crop_h))
+    restored = cropped.resize((width, height), Image.Resampling.BICUBIC)
+    return restored, {
+        "crop_scale": round(crop_scale, 4),
+        "crop_left": left,
+        "crop_top": top,
+    }
+
+
+def _filter(image: Image.Image, rng: random.Random) -> tuple[Image.Image, dict]:
+    mode = rng.choice(["none", "blur", "sharpen"])
+    if mode == "blur":
+        radius = rng.uniform(0.15, 0.45)
+        return image.filter(ImageFilter.GaussianBlur(radius=radius)), {
+            "filter": mode,
+            "blur_radius": round(radius, 4),
+        }
+    if mode == "sharpen":
+        return image.filter(ImageFilter.SHARPEN), {"filter": mode}
+    return image, {"filter": mode}
+
+
+def _enhance(image: Image.Image, rng: random.Random) -> tuple[Image.Image, dict]:
+    brightness = rng.uniform(0.92, 1.08)
+    contrast = rng.uniform(0.9, 1.1)
+    color = rng.uniform(0.9, 1.1)
+    image = ImageEnhance.Brightness(image).enhance(brightness)
+    image = ImageEnhance.Contrast(image).enhance(contrast)
+    image = ImageEnhance.Color(image).enhance(color)
+    return image, {
+        "brightness": round(brightness, 4),
+        "contrast": round(contrast, 4),
+        "color": round(color, 4),
+    }
+
+
+def _jpeg_roundtrip(image: Image.Image, rng: random.Random) -> tuple[Image.Image, dict]:
+    quality = rng.randint(55, 95)
+    buffer = BytesIO()
+    image.save(buffer, format="JPEG", quality=quality, optimize=False)
+    buffer.seek(0)
+    with Image.open(buffer) as jpeg:
+        jpeg.load()
+        result = jpeg.convert("RGB")
+    return result, {"jpeg_quality": quality}
+
+
+def augment_image(image: Image.Image, seed: int) -> tuple[Image.Image, dict[str, Any]]:
+    """Return a deterministic augmented image and its parameter record."""
+    rng = random.Random(seed)
+    augmented = image.convert("RGB")
+    params: dict[str, Any] = {"version": "stage3a-v1"}
+
+    augmented, resize_params = _resize_roundtrip(augmented, rng)
+    params.update(resize_params)
+    augmented, crop_params = _crop_and_restore(augmented, rng)
+    params.update(crop_params)
+    augmented, filter_params = _filter(augmented, rng)
+    params.update(filter_params)
+    augmented, enhance_params = _enhance(augmented, rng)
+    params.update(enhance_params)
+    augmented, jpeg_params = _jpeg_roundtrip(augmented, rng)
+    params.update(jpeg_params)
+
+    return augmented, params
+
+
+def _output_path(out_dir: Path, seed: int, original_path: str, copy_index: int) -> Path:
+    key = hashlib.sha256(f"{original_path}|{seed}|{copy_index}".encode()).hexdigest()
+    return out_dir / f"{key[:24]}.jpg"
+
+
+def _augmented_row(
+    row: dict[str, str],
+    *,
+    original_path: str,
+    output_path: Path,
+    data_root: Path,
+    seed: int,
+    params: dict[str, Any],
+) -> dict[str, str]:
+    updated = dict(row)
+    updated["path"] = _relative_to_data_root(output_path, data_root)
+    updated["sha256"] = _sha256_file(output_path)
+    updated["width"] = str(params["width"])
+    updated["height"] = str(params["height"])
+    updated["original_path"] = original_path
+    updated["augmentation"] = params["version"]
+    updated["augmentation_seed"] = str(seed)
+    updated["augmentation_params_json"] = _params_json(params)
+    return updated
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--data-root", type=Path, required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--out-manifest", type=Path, required=True)
+    parser.add_argument("--copies", type=int, default=1)
+    parser.add_argument("--seed", type=int, default=0)
+    args = parser.parse_args()
+
+    if args.copies < 1:
+        raise ValueError("--copies must be >= 1")
+
+    data_root = args.data_root.resolve()
+    out_dir = args.out_dir.resolve()
+    out_manifest = args.out_manifest.resolve()
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_manifest.parent.mkdir(parents=True, exist_ok=True)
+
+    with args.manifest.open() as fh:
+        reader = csv.DictReader(fh)
+        source_rows = list(reader)
+        source_fieldnames = reader.fieldnames or []
+
+    rows: list[dict[str, str]] = []
+    for row in source_rows:
+        original_path = row["path"]
+        src = data_root / original_path
+        for copy_index in range(args.copies):
+            seed = _stable_seed(args.seed, original_path, copy_index)
+            dst = _output_path(out_dir, seed, original_path, copy_index)
+            with Image.open(src) as image:
+                augmented, params = augment_image(image, seed)
+                params["width"], params["height"] = augmented.size
+                augmented.save(dst, format="JPEG", quality=params["jpeg_quality"])
+
+            rows.append(
+                _augmented_row(
+                    row,
+                    original_path=original_path,
+                    output_path=dst,
+                    data_root=data_root,
+                    seed=seed,
+                    params=params,
+                )
+            )
+
+    fieldnames = [
+        *source_fieldnames,
+        *[field for field in AUGMENTATION_FIELDS if field not in source_fieldnames],
+    ]
+    with out_manifest.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"Augmented rows: {len(rows)}")
+    print(f"Images: {out_dir}")
+    print(f"Manifest: {out_manifest}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/build_manifest.py

@@ -0,0 +1,193 @@
+"""
+Combine per-source manifests into a single master manifest.
+
+The master manifest is the authoritative legal record for the dataset. It is:
+  • Committed alongside the trained model (so the lineage is auditable).
+  • Used by the training script (no image is loaded that isn't in the manifest).
+  • The first thing a lawyer or auditor will ask to see.
+
+Stage 3A manifest schema
+------------------------
+Required legal/training fields:
+  path, class, source, license, license_url, sha256
+
+Optional generator metadata fields:
+  generator, model_family, model_id, prompt, seed, width, height,
+  generation_params_json
+
+Missing optional values are written as empty strings. This keeps Stage 2
+manifest fragments readable while giving Stage 3A scripts stable columns for
+per-generator evaluation.
+
+Usage
+-----
+    python scripts/dataset/build_manifest.py \
+        --inputs data/raw/real_manifest.csv data/raw/ai_generated_manifest.csv \
+        --out data/manifest.csv
+
+    python scripts/dataset/build_manifest.py \
+        --input-dir data/raw \
+        --out data/manifest.csv
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+from pathlib import Path
+
+from generation_utils import APPROVED_GENERATORS
+
+REQUIRED_FIELDS = ["path", "class", "source", "license", "license_url", "sha256"]
+OPTIONAL_STAGE3A_FIELDS = [
+    "generator",
+    "model_family",
+    "model_id",
+    "prompt",
+    "seed",
+    "width",
+    "height",
+    "generation_params_json",
+]
+MANIFEST_FIELDS = REQUIRED_FIELDS + OPTIONAL_STAGE3A_FIELDS
+MANIFEST_FRAGMENT_GLOB = "*_manifest.csv"
+GENERATOR_SPECS_BY_SOURCE = {
+    spec.source: spec for spec in APPROVED_GENERATORS.values()
+}
+
+
+def _normalise_row(row: dict[str, str]) -> dict[str, str]:
+    """Return a manifest row with all Stage 3A optional fields present."""
+    normalised = dict(row)
+    source = normalised.get("source", "")
+    spec = GENERATOR_SPECS_BY_SOURCE.get(source)
+    if normalised.get("class") == "ai_generated" and spec is not None:
+        if not normalised.get("generator"):
+            normalised["generator"] = spec.generator
+        if not normalised.get("model_family"):
+            normalised["model_family"] = spec.model_family
+        if not normalised.get("model_id"):
+            normalised["model_id"] = spec.model_id
+
+    for field in OPTIONAL_STAGE3A_FIELDS:
+        if normalised.get(field) is None:
+            normalised[field] = ""
+        else:
+            normalised[field] = str(normalised.get(field, ""))
+    return normalised
+
+
+def _discover_manifest_fragments(input_dirs: list[Path], out: Path) -> list[Path]:
+    """Find manifest fragments below each input directory in stable order."""
+    discovered: list[Path] = []
+    out_resolved = out.resolve()
+    for input_dir in input_dirs:
+        if not input_dir.exists():
+            raise FileNotFoundError(f"Input directory does not exist: {input_dir}")
+        if not input_dir.is_dir():
+            raise NotADirectoryError(f"Input path is not a directory: {input_dir}")
+        for path in sorted(input_dir.rglob(MANIFEST_FRAGMENT_GLOB)):
+            if path.resolve() == out_resolved:
+                continue
+            discovered.append(path)
+    return discovered
+
+
+def _unique_paths(paths: list[Path]) -> list[Path]:
+    """Deduplicate paths while preserving caller/discovery order."""
+    unique: list[Path] = []
+    seen: set[Path] = set()
+    for path in paths:
+        resolved = path.resolve()
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+        unique.append(path)
+    return unique
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--inputs",
+        type=Path,
+        nargs="+",
+        default=[],
+        help="Explicit manifest fragments to merge",
+    )
+    parser.add_argument(
+        "--input-dir",
+        type=Path,
+        action="append",
+        default=[],
+        help=(
+            f"Directory to scan recursively for {MANIFEST_FRAGMENT_GLOB}; "
+            "can be passed multiple times"
+        ),
+    )
+    parser.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args()
+
+    input_paths = _unique_paths(
+        [*args.inputs, *_discover_manifest_fragments(args.input_dir, args.out)]
+    )
+    if not input_paths:
+        raise ValueError("Provide at least one --inputs file or --input-dir")
+
+    rows: list[dict] = []
+    for src in input_paths:
+        if not src.exists():
+            raise FileNotFoundError(f"Input manifest does not exist: {src}")
+        with src.open() as fh:
+            reader = csv.DictReader(fh)
+            for r in reader:
+                missing = [f for f in REQUIRED_FIELDS if not r.get(f)]
+                if missing:
+                    raise ValueError(
+                        f"{src}: row missing required fields {missing}: {r}"
+                    )
+                rows.append(_normalise_row(r))
+
+    # Detect duplicates by sha256 — important for license cleanliness AND
+    # to avoid train/test leakage.
+    seen: dict[str, str] = {}
+    deduped: list[dict] = []
+    for r in rows:
+        sha = r["sha256"]
+        if sha in seen:
+            print(f"  dropping duplicate {r['path']} (matches {seen[sha]})")
+            continue
+        seen[sha] = r["path"]
+        deduped.append(r)
+
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    # Write a stable schema first, then any extra legacy/source-specific fields.
+    fieldnames = sorted({k for r in deduped for k in r.keys()})
+    ordered = MANIFEST_FIELDS + [f for f in fieldnames if f not in MANIFEST_FIELDS]
+
+    with args.out.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=ordered)
+        writer.writeheader()
+        writer.writerows(deduped)
+
+    classes: dict[str, int] = {}
+    for r in deduped:
+        classes[r["class"]] = classes.get(r["class"], 0) + 1
+
+    print(f"\nMaster manifest: {args.out}")
+    print(f"  fragments: {len(input_paths)}")
+    print(f"  total rows: {len(deduped)}")
+    for cls, n in sorted(classes.items()):
+        print(f"  {cls}: {n}")
+
+    generators: dict[str, int] = {}
+    for r in deduped:
+        if r["class"] != "ai_generated":
+            continue
+        generator = r.get("generator") or r["source"]
+        generators[generator] = generators.get(generator, 0) + 1
+    for generator, n in sorted(generators.items()):
+        print(f"  generator {generator}: {n}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/closed_eval_prompts.txt

@@ -0,0 +1,104 @@
+# Closed-generator evaluation prompt set (P0 benchmark)
+#
+# PURPOSE: produce a like-for-like AI-image eval set from closed consumer
+# tools (ChatGPT/DALL-E, Gemini, Midjourney, Firefly). EVALUATION ONLY —
+# these images are never added to a training manifest or data/splits*/.
+#
+# WHY THESE PROMPTS: our detector's weakness is photorealism, not art.
+# Stylised output is already easy to catch. So every prompt below asks for
+# a photograph, and ~40% deliberately ask for MEDIOCRE photographs —
+# harsh flash, motion blur, clutter, bad white balance. Polished "AI
+# beauty shots" flatter a detector; ordinary snapshots are what people
+# actually share and what defeats detectors in the wild. A benchmark built
+# only from beautiful images would overstate our accuracy.
+#
+# REUSE THE SAME LIST FOR EVERY TOOL. Identical prompts across ChatGPT,
+# Gemini and Midjourney make tool-to-tool comparison apples-to-apples.
+# (Shared prompts would be leakage in a *training* set — see the Stage 3A
+# content-leak finding — but this set is never trained on, so it is safe
+# and desirable here.)
+#
+# AR = aspect ratio to request. Mixing them matters: a detector can learn
+# "1024x1024 square = AI" instead of learning anything about pixels.
+#
+# ---------------------------------------------------------------------
+# PEOPLE / PORTRAITS (12) - highest-stakes category for a deepfake product
+# ---------------------------------------------------------------------
+1  | AR 4:5  | A candid photo of an elderly man with deep wrinkles laughing in a bright kitchen, shot on a phone, slight overexposure near the window.
+2  | AR 4:5  | A woman in her thirties in a navy raincoat waiting at a bus stop in light drizzle, overcast grey light, unposed, looking off-camera.
+3  | AR 1:1  | A tired nurse in blue scrubs sitting on a plastic chair in a hospital corridor at the end of a shift, harsh fluorescent overhead lighting.
+4  | AR 3:2  | Two teenage boys on a skate ramp mid-conversation, late afternoon sun, one blurred by motion, chain-link fence behind.
+5  | AR 4:5  | A bearded man in his forties taking a mirror selfie in a cluttered bathroom, phone visible, direct flash blowing out the mirror.
+6  | AR 1:1  | A grandmother and a toddler on a sofa looking at a picture book, warm lamp light, slightly out of focus, ordinary living room.
+7  | AR 3:2  | A construction worker in a hi-vis vest drinking from a thermos, sitting on a kerb, dusty site behind, midday sun overhead.
+8  | AR 4:5  | A young woman with curly hair reading on a train, window reflections across her face, natural light, shot from the seat opposite.
+9  | AR 1:1  | A chef with a burn scar on his forearm plating food in a cramped restaurant kitchen, steam, motion blur on his hands.
+10 | AR 3:2  | A crowd of commuters on an escalator seen from below, mixed ages, nobody looking at the camera, station lighting.
+11 | AR 4:5  | A man in his sixties asleep in an armchair with a newspaper on his lap, television glow, dim room, grainy low-light phone photo.
+12 | AR 3:2  | A woman jogging on a coastal path at dawn, seen from behind at distance, slight camera shake, thin morning fog.
+#
+# ---------------------------------------------------------------------
+# ANIMALS (6)
+# ---------------------------------------------------------------------
+13 | AR 1:1  | A wet golden retriever shaking off water in a back garden, droplets frozen mid-air, bright overcast light.
+14 | AR 3:2  | A tabby cat asleep on a radiator shelf beside a dying houseplant, ordinary flat, afternoon light through net curtains.
+15 | AR 3:2  | Three cows standing in a muddy field beside a wire fence, flat grey sky, unremarkable rural scene.
+16 | AR 4:5  | A pigeon with a deformed foot standing on wet pavement outside a shop, litter nearby, dull daylight.
+17 | AR 3:2  | A horse mid-canter in a paddock, slight motion blur on the legs, dust kicked up, low evening sun behind.
+18 | AR 1:1  | A goldfish in a small glass bowl on a kitchen counter, refraction distorting its body, kitchen clutter behind.
+#
+# ---------------------------------------------------------------------
+# FOOD (5) - AI food is often too perfect; ask for the opposite
+# ---------------------------------------------------------------------
+19 | AR 1:1  | A half-eaten plate of curry and rice with a bent fork, on a scratched dining table, direct phone flash.
+20 | AR 4:5  | A cappuccino with slightly broken foam art on a cafe table, phone and keys beside it, window light.
+21 | AR 1:1  | A greasy takeaway burger in its wrapper on a car dashboard, at night, lit by streetlight through the windscreen.
+22 | AR 3:2  | A birthday cake with uneven icing and candles blown out, smoke still rising, dim kitchen, people's arms in frame.
+23 | AR 1:1  | A supermarket cheese counter with condensation on the glass, price labels visible but unreadable, fluorescent light.
+#
+# ---------------------------------------------------------------------
+# LANDSCAPE / NATURE (6)
+# ---------------------------------------------------------------------
+24 | AR 3:2  | A flat farmland horizon under heavy grey cloud, a single pylon, no dramatic light, ordinary and dull.
+25 | AR 3:2  | A rocky beach at low tide with seaweed and plastic debris, overcast, taken standing at eye level.
+26 | AR 4:5  | A narrow forest path after rain, mud and exposed roots, flat diffuse light under the canopy.
+27 | AR 3:2  | A mountain valley in clear afternoon light with a road switchbacking through it, haze in the distance.
+28 | AR 3:2  | A frozen pond edge with reeds and thin cracked ice, close to the ground, winter, weak sun.
+29 | AR 4:5  | A eucalyptus trunk in close detail, peeling bark, harsh dappled midday sunlight, shallow depth of field.
+#
+# ---------------------------------------------------------------------
+# STREET / URBAN (6)
+# ---------------------------------------------------------------------
+30 | AR 3:2  | A wet city street at night, taxi headlights, reflections on tarmac, handheld, slight motion blur.
+31 | AR 4:5  | A row of terraced houses with wheelie bins out, parked cars, grey daylight, nothing happening.
+32 | AR 3:2  | Scaffolding on a building facade with a torn safety net, seen from across the road, flat light.
+33 | AR 1:1  | A launderette interior at night seen through the window from outside, one person inside, fluorescent glare.
+34 | AR 3:2  | A pedestrian crossing seen from a car windscreen in rain, wipers mid-sweep, blurred figures crossing.
+35 | AR 4:5  | Graffiti on a shutter beside a closed shop, cigarette ends on the pavement, early morning, empty street.
+#
+# ---------------------------------------------------------------------
+# INTERIORS / DOMESTIC (5)
+# ---------------------------------------------------------------------
+36 | AR 3:2  | An unmade bed with creased sheets in morning light, phone charger cable trailing off the side table.
+37 | AR 4:5  | A cluttered home office desk with two mugs, tangled cables and sticky notes, mixed daylight and lamp light.
+38 | AR 3:2  | A kitchen sink piled with washing-up, window above it, dull afternoon light, ordinary and unstyled.
+39 | AR 1:1  | A child's bedroom floor covered in toys, ceiling light on, slightly overexposed, taken from the doorway.
+40 | AR 3:2  | A stairwell in an apartment block with scuffed walls and a bicycle chained to the railing, motion-sensor lighting.
+#
+# ---------------------------------------------------------------------
+# OBJECTS / PRODUCT (4)
+# ---------------------------------------------------------------------
+41 | AR 1:1  | A worn leather wallet with cards half out, on a wooden table, single window light, visible scratches.
+42 | AR 1:1  | A cracked phone screen still displaying a home screen, held in a hand, reflections across the glass.
+43 | AR 4:5  | A pair of muddy walking boots on a doorstep, drying, late afternoon shadow across them.
+44 | AR 1:1  | A houseplant with several browning leaves in a chipped ceramic pot on a windowsill, backlit.
+#
+# ---------------------------------------------------------------------
+# EVENTS / GROUPS (6)
+# ---------------------------------------------------------------------
+45 | AR 3:2  | A school sports day, parents watching from the sideline, children mid-race, bright hazy sun, handheld.
+46 | AR 3:2  | A crowded pub interior during a match, faces lit by screens, motion blur, noisy low-light photo.
+47 | AR 4:5  | A market stall holder handing change to a customer, produce in the foreground, busy background.
+48 | AR 3:2  | A small wedding party posing awkwardly outside a registry office, overcast, one person blinking.
+49 | AR 3:2  | A queue outside a bakery on a cold morning, breath visible, people on phones, low winter sun.
+50 | AR 1:1  | A family barbecue in a small garden, smoke from the grill, folding chairs, harsh midday light.

scripts/dataset/fetch_open_images.py

@@ -0,0 +1,138 @@
+"""
+Download a stratified sample of Open Images V7.
+
+Open Images is published by Google under CC BY 2.0 (commercial use OK).
+Each downloaded image is recorded in the manifest with its source URL and
+license string for audit trail.
+
+Usage
+-----
+    python scripts/dataset/fetch_open_images.py \
+        --out data/raw/real \
+        --count 50000
+
+Notes
+-----
+We use the FiftyOne library because it has first-class Open Images support
+and handles the (large, non-trivial) metadata files for us. FiftyOne is
+Apache 2.0 licensed.
+
+This script is idempotent: re-running with the same --out resumes where it
+left off and skips already-downloaded files.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import time
+from pathlib import Path
+
+
+def _load_with_retry(foz, split: str, count: int, max_attempts: int = 20):
+    """Call foz.load_zoo_dataset with retry on transient network errors.
+
+    Open Images is hosted on S3; a single dropped TCP connection bubbles up
+    as botocore.EndpointConnectionError and kills the whole download even
+    when 99% of the work is done. Each successful image is cached locally,
+    so retrying the call resumes from where it left off — we just need to
+    keep retrying until the network cooperates for one full pass.
+
+    Caught broadly to also handle ConnectionError, OSError, and the
+    botocore wrappers without a hard import on botocore.
+    """
+    for attempt in range(1, max_attempts + 1):
+        try:
+            return foz.load_zoo_dataset(
+                "open-images-v7",
+                split=split,
+                max_samples=count,
+                shuffle=True,
+                # We only need image data, not labels — bbox/segmentation/etc.
+                # add tens of GB of metadata we don't use.
+                label_types=[],
+            )
+        except Exception as exc:  # noqa: BLE001 — intentional broad catch
+            cls = type(exc).__name__
+            transient = any(
+                token in cls
+                for token in (
+                    "Connection", "Endpoint", "Timeout", "SSL", "Proxy",
+                )
+            ) or isinstance(exc, OSError)
+            if not transient or attempt >= max_attempts:
+                raise
+            wait = min(60, 5 * attempt)
+            print(
+                f"\n[retry {attempt}/{max_attempts}] {cls}: {exc}\n"
+                f"  Sleeping {wait}s before retrying — already-downloaded "
+                "images are cached, so the next attempt resumes."
+            )
+            time.sleep(wait)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--out", type=Path, required=True, help="Output directory")
+    parser.add_argument("--count", type=int, default=50_000, help="Number of images")
+    parser.add_argument(
+        "--split",
+        choices=["train", "validation", "test"],
+        default="train",
+        help="Open Images split to draw from",
+    )
+    parser.add_argument(
+        "--max-attempts",
+        type=int,
+        default=20,
+        help="Max retry attempts on network errors before giving up",
+    )
+    args = parser.parse_args()
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    manifest_path = args.out.parent / "real_manifest.csv"
+
+    print(f"Downloading {args.count} images from Open Images {args.split} split...")
+    print("Importing FiftyOne (this is slow the first time)...")
+
+    # Lazy import — fiftyone has heavy native deps.
+    import fiftyone.zoo as foz
+
+    dataset = _load_with_retry(foz, args.split, args.count, args.max_attempts)
+
+    rows: list[dict] = []
+    for sample in dataset:
+        src = Path(sample.filepath)
+        dst = args.out / src.name
+        if not dst.exists():
+            dst.write_bytes(src.read_bytes())
+
+        with dst.open("rb") as fh:
+            sha = hashlib.sha256(fh.read()).hexdigest()
+
+        rows.append({
+            # as_posix() so the manifest is portable to Linux GPU boxes —
+            # str() on Windows produces backslashes that break Path parsing
+            # on POSIX.
+            "path": dst.relative_to(args.out.parent.parent).as_posix(),
+            "class": "authentic",
+            "source": "open_images_v7",
+            "license": "CC-BY-2.0",
+            "license_url": "https://creativecommons.org/licenses/by/2.0/",
+            "sha256": sha,
+        })
+
+    # Write manifest fragment for the real half.
+    with manifest_path.open("w", newline="") as fh:
+        writer = csv.DictWriter(
+            fh,
+            fieldnames=["path", "class", "source", "license", "license_url", "sha256"],
+        )
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"Done. {len(rows)} images. Manifest fragment: {manifest_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/generate_auraflow_synthetic.py

@@ -0,0 +1,217 @@
+"""
+Generate synthetic images with AuraFlow v0.3.
+
+License
+-------
+`fal/AuraFlow-v0.3` is released under Apache-2.0. Stage 3A approved it as an
+independent rectified-flow training source. See `NOTICES.md` before running a
+real generation job.
+
+Usage
+-----
+    python scripts/dataset/generate_auraflow_synthetic.py \\
+        --out data/raw/ai_generated/auraflow-v0.3 \\
+        --count 20000 \\
+        --prompts scripts/dataset/prompts.txt
+
+Dry run, no model import/download:
+
+    python scripts/dataset/generate_auraflow_synthetic.py \\
+        --out data/raw/ai_generated/auraflow-v0.3 \\
+        --count 3 \\
+        --dry-run
+
+Hardware
+--------
+AuraFlow 1024x1024 generation should be run on a CUDA GPU with enough VRAM for
+half-precision inference. Start with a small smoke run on the target GPU before
+renting a long job.
+
+Idempotency
+-----------
+Each generated file is named by a hash of (prompt, seed), so re-runs skip
+already-generated images. Crash-resume works automatically.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import random
+from pathlib import Path
+
+from generation_utils import (
+    APPROVED_GENERATORS,
+    STAGE3A_MANIFEST_FIELDS,
+    choose_prompt,
+    image_dimensions,
+    infer_data_root,
+    load_prompts,
+    manifest_row,
+    next_seed,
+    sha256_file,
+    stable_image_key,
+)
+
+
+FALLBACK_PROMPTS = [
+    "a realistic phone photo of a ceramic mug beside a laptop",
+    "a casual snapshot of a parking lot after a summer storm",
+    "a natural light photo of houseplants on a crowded windowsill",
+    "a slightly blurry photo of a folded jacket on a cafe chair",
+]
+
+
+AURAFLOW_SPEC = APPROVED_GENERATORS["auraflow-v0.3"]
+
+
+def _manifest_path(out_dir: Path) -> Path:
+    return out_dir.parent / "auraflow_manifest.csv"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--out", type=Path, required=True, help="Output directory")
+    parser.add_argument(
+        "--data-root",
+        type=Path,
+        default=None,
+        help=(
+            "Dataset root for manifest paths. Defaults to the parent of the "
+            "'raw' path segment in --out."
+        ),
+    )
+    parser.add_argument("--count", type=int, default=20_000)
+    parser.add_argument(
+        "--prompts",
+        type=Path,
+        default=None,
+        help="Optional file with one prompt per line",
+    )
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--steps", type=int, default=30)
+    parser.add_argument("--guidance-scale", type=float, default=3.5)
+    parser.add_argument("--width", type=int, default=1024)
+    parser.add_argument("--height", type=int, default=1024)
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Validate prompt/seed/output planning without loading AuraFlow or writing files",
+    )
+    args = parser.parse_args()
+
+    out_dir = args.out.resolve()
+    out_dir.mkdir(parents=True, exist_ok=True)
+    data_root = (
+        args.data_root.resolve()
+        if args.data_root is not None
+        else infer_data_root(out_dir)
+    )
+    manifest_path = _manifest_path(out_dir)
+
+    prompts = load_prompts(args.prompts, fallback_prompts=FALLBACK_PROMPTS)
+    if args.prompts is not None:
+        print(f"Loaded {len(prompts)} prompts from {args.prompts}")
+    else:
+        print(
+            f"WARNING: --prompts not given; using {len(prompts)} built-in "
+            "fallback prompts (smoke-test only, not enough diversity for "
+            "a real training run)"
+        )
+
+    rng = random.Random(args.seed)
+
+    if args.dry_run:
+        print(
+            "Dry run: AuraFlow pipeline will not be loaded and no images "
+            "will be written."
+        )
+        for i in range(args.count):
+            prompt = choose_prompt(rng, prompts)
+            seed = next_seed(rng)
+            key = stable_image_key(prompt, seed)
+            dst = out_dir / f"{key}.png"
+            print(f"  {i + 1:04d}: seed={seed} path={dst} prompt={prompt!r}")
+        print(f"Dry run complete. Planned manifest fragment: {manifest_path}")
+        return
+
+    print("Loading AuraFlow pipeline (large download on first run)...")
+    import torch
+    from diffusers import AuraFlowPipeline
+
+    pipe = AuraFlowPipeline.from_pretrained(
+        AURAFLOW_SPEC.model_id,
+        torch_dtype=torch.float16,
+    )
+    pipe.to("cuda")
+    # AuraFlow VAE dtype-mismatch fix.
+    # Why: AuraFlow's VAE has biases that don't survive `torch_dtype=torch.float16`
+    # cleanly. Diffusers' internal `upcast_vae()` path (now deprecated for AuraFlow)
+    # only partially upcasts, leaving conv biases stranded in fp32 while inputs are
+    # fp16 -> RuntimeError "Input type (c10::Half) and bias type (float) should be
+    # the same" during `vae.decode`.
+    # Fix: cast the whole VAE to fp32, AND monkey-patch `vae.decode` to cast the
+    # latent input to match. fp32 VAE adds ~500 MB memory and ~10-20% time to the
+    # decode step (negligible on 48 GB cards).
+    pipe.vae = pipe.vae.to(dtype=torch.float32)
+    _orig_vae_decode = pipe.vae.decode
+    def _decode_with_dtype_cast(z, *args, **kwargs):
+        z = z.to(pipe.vae.dtype)
+        return _orig_vae_decode(z, *args, **kwargs)
+    pipe.vae.decode = _decode_with_dtype_cast
+
+    rows: list[dict] = []
+    for i in range(args.count):
+        prompt = choose_prompt(rng, prompts)
+        seed = next_seed(rng)
+        key = stable_image_key(prompt, seed)
+        dst = out_dir / f"{key}.png"
+
+        if not dst.exists():
+            generator = torch.Generator("cuda").manual_seed(seed)
+            image = pipe(
+                prompt=prompt,
+                num_inference_steps=args.steps,
+                guidance_scale=args.guidance_scale,
+                width=args.width,
+                height=args.height,
+                generator=generator,
+            ).images[0]
+            image.save(dst, format="PNG")
+
+        width, height = image_dimensions(dst)
+        rows.append(
+            manifest_row(
+                path=dst.relative_to(data_root).as_posix(),
+                cls="ai_generated",
+                spec=AURAFLOW_SPEC,
+                sha256=sha256_file(dst),
+                prompt=prompt,
+                seed=seed,
+                width=width,
+                height=height,
+                generation_params={
+                    "steps": args.steps,
+                    "guidance_scale": args.guidance_scale,
+                    "width": args.width,
+                    "height": args.height,
+                    "pipeline": AURAFLOW_SPEC.pipeline,
+                },
+            )
+        )
+
+        if (i + 1) % 100 == 0:
+            print(f"  generated {i + 1}/{args.count}")
+
+    with manifest_path.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=STAGE3A_MANIFEST_FIELDS)
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"Done. {len(rows)} images. Manifest fragment: {manifest_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/generate_flux_synthetic.py

@@ -0,0 +1,168 @@
+"""
+Generate synthetic images with Flux.1-schnell.
+
+Flux.1-schnell is released by Black Forest Labs under Apache 2.0. Outputs are
+not restricted, so we can use them as commercial training data.
+
+Usage
+-----
+    python scripts/dataset/generate_flux_synthetic.py \
+        --out data/raw/ai_generated \
+        --count 50000 \
+        --prompts scripts/dataset/prompts.txt
+
+Prompts file format
+-------------------
+One complete prompt per line. Blank lines and lines starting with `#` are
+ignored. Diversity is critical: a model trained on monotonous prompts learns
+to detect prompt style, not AI-generation artefacts. Aim for the prompts
+file to span the full distribution of subjects, settings, lighting, and
+camera styles a regular person might upload — including mundane and
+imperfect ones (snapshots, blurry shots, boring objects), not just
+gallery-worthy compositions.
+
+If `--prompts` is omitted, a tiny built-in fallback set is used. That set
+is only sufficient for smoke tests, not for a real training run.
+
+Hardware
+--------
+Flux.1-schnell needs a GPU with ~12-16 GB VRAM. Runs comfortably on:
+  • RunPod / Lambda Labs / Vast.ai spot instances (RTX 3090 / 4090 / A10G)
+  • ~3-5 seconds per image at 4 inference steps on an A10G
+  • 50k images ≈ 60-80 GPU-hours ≈ $25-50 at typical spot rates
+
+Idempotency
+-----------
+Each generated file is named by a hash of (prompt, seed) so re-runs skip
+already-generated images. Crash-resume works automatically.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import random
+from pathlib import Path
+
+from generation_utils import (
+    APPROVED_GENERATORS,
+    choose_prompt,
+    image_dimensions,
+    load_prompts,
+    manifest_row,
+    next_seed,
+    sha256_file,
+    stable_image_key,
+)
+
+
+# Fallback used only when --prompts is not provided. Intentionally tiny —
+# enough for a smoke test, not for a real training run.
+FALLBACK_PROMPTS = [
+    "a photo of a dog in a park, golden hour",
+    "candid photograph of a person walking on a busy street",
+    "studio lighting portrait of two friends talking",
+    "a wide-angle shot of a kitchen with a child playing",
+]
+
+
+FLUX_SPEC = APPROVED_GENERATORS["flux.1-schnell"]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--out", type=Path, required=True, help="Output directory")
+    parser.add_argument("--count", type=int, default=50_000)
+    parser.add_argument(
+        "--prompts",
+        type=Path,
+        default=None,
+        help="Optional file with one prompt template per line",
+    )
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--steps", type=int, default=4, help="Flux schnell uses 4 steps")
+    args = parser.parse_args()
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    manifest_path = args.out.parent / "ai_generated_manifest.csv"
+
+    if args.prompts is not None:
+        prompts = load_prompts(args.prompts)
+        print(f"Loaded {len(prompts)} prompts from {args.prompts}")
+    else:
+        prompts = load_prompts(None, fallback_prompts=FALLBACK_PROMPTS)
+        print(
+            f"WARNING: --prompts not given; using {len(prompts)} built-in "
+            "fallback prompts (smoke-test only, not enough diversity for "
+            "a real training run)"
+        )
+
+    rng = random.Random(args.seed)
+
+    # Lazy import — diffusers + torch + accelerate are heavy.
+    print("Loading Flux.1-schnell pipeline (~24 GB download on first run)...")
+    import torch
+    from diffusers import FluxPipeline
+
+    pipe = FluxPipeline.from_pretrained(
+        "black-forest-labs/FLUX.1-schnell",
+        torch_dtype=torch.bfloat16,
+    )
+    pipe.to("cuda")
+
+    rows: list[dict] = []
+    for i in range(args.count):
+        prompt = choose_prompt(rng, prompts)
+        seed = next_seed(rng)
+        # Filename is content-addressable so re-runs are idempotent.
+        key = stable_image_key(prompt, seed)
+        dst = args.out / f"{key}.png"
+
+        if not dst.exists():
+            generator = torch.Generator("cuda").manual_seed(seed)
+            image = pipe(
+                prompt,
+                guidance_scale=0.0,  # Flux schnell ignores guidance
+                num_inference_steps=args.steps,
+                generator=generator,
+            ).images[0]
+            image.save(dst, format="PNG")
+
+        width, height = image_dimensions(dst)
+
+        rows.append(manifest_row(
+            # as_posix() — keep manifests portable across OS boundaries.
+            path=dst.relative_to(args.out.parent.parent).as_posix(),
+            cls="ai_generated",
+            spec=FLUX_SPEC,
+            sha256=sha256_file(dst),
+            prompt=prompt,
+            seed=seed,
+            width=width,
+            height=height,
+            generation_params={
+                "steps": args.steps,
+                "guidance_scale": 0.0,
+                "pipeline": FLUX_SPEC.pipeline,
+            },
+        ))
+
+        if (i + 1) % 100 == 0:
+            print(f"  generated {i + 1}/{args.count}")
+
+    with manifest_path.open("w", newline="") as fh:
+        writer = csv.DictWriter(
+            fh,
+            fieldnames=[
+                "path", "class", "source", "license", "license_url",
+                "sha256", "generator", "model_family", "model_id", "prompt",
+                "seed", "width", "height", "generation_params_json",
+            ],
+        )
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"Done. {len(rows)} images. Manifest fragment: {manifest_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/generate_sd35_synthetic.py

@@ -0,0 +1,205 @@
+"""
+Generate synthetic images with Stable Diffusion 3.5 Medium.
+
+License
+-------
+`stabilityai/stable-diffusion-3.5-medium` is available under the Stability AI
+Community License. Stage 3A conditionally approved it while Veridicate remains
+under the license revenue threshold and satisfies registration requirements.
+Enterprise licensing is required before use above that threshold. Do not use
+outputs to create or improve a foundational generative AI model. See
+`NOTICES.md` before running a real generation job.
+
+Usage
+-----
+    python scripts/dataset/generate_sd35_synthetic.py \\
+        --out data/raw/ai_generated/sd35-medium \\
+        --count 20000 \\
+        --prompts scripts/dataset/prompts.txt
+
+Dry run, no model import/download:
+
+    python scripts/dataset/generate_sd35_synthetic.py \\
+        --out data/raw/ai_generated/sd35-medium \\
+        --count 3 \\
+        --dry-run
+
+Hardware
+--------
+SD 3.5 Medium generation should be run on a CUDA GPU with enough VRAM for
+1024x1024 half-precision inference. Start with a small smoke run on the target
+GPU before renting a long job.
+
+Idempotency
+-----------
+Each generated file is named by a hash of (prompt, seed), so re-runs skip
+already-generated images. Crash-resume works automatically.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import random
+from pathlib import Path
+
+from generation_utils import (
+    APPROVED_GENERATORS,
+    STAGE3A_MANIFEST_FIELDS,
+    choose_prompt,
+    image_dimensions,
+    infer_data_root,
+    load_prompts,
+    manifest_row,
+    next_seed,
+    sha256_file,
+    stable_image_key,
+)
+
+
+FALLBACK_PROMPTS = [
+    "a realistic phone photo of a grocery receipt on a kitchen table",
+    "a handheld snapshot of a small dog looking out a car window",
+    "a natural photo of a rainy suburban street at dusk",
+    "a casual indoor photo of laundry folded on a couch",
+]
+
+
+SD35_SPEC = APPROVED_GENERATORS["sd3.5-medium"]
+
+
+def _manifest_path(out_dir: Path) -> Path:
+    return out_dir.parent / "sd35_medium_manifest.csv"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--out", type=Path, required=True, help="Output directory")
+    parser.add_argument(
+        "--data-root",
+        type=Path,
+        default=None,
+        help=(
+            "Dataset root for manifest paths. Defaults to the parent of the "
+            "'raw' path segment in --out."
+        ),
+    )
+    parser.add_argument("--count", type=int, default=20_000)
+    parser.add_argument(
+        "--prompts",
+        type=Path,
+        default=None,
+        help="Optional file with one prompt per line",
+    )
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--steps", type=int, default=28)
+    parser.add_argument("--guidance-scale", type=float, default=4.5)
+    parser.add_argument("--width", type=int, default=1024)
+    parser.add_argument("--height", type=int, default=1024)
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Validate prompt/seed/output planning without loading SD 3.5 or writing files",
+    )
+    args = parser.parse_args()
+
+    out_dir = args.out.resolve()
+    out_dir.mkdir(parents=True, exist_ok=True)
+    data_root = (
+        args.data_root.resolve()
+        if args.data_root is not None
+        else infer_data_root(out_dir)
+    )
+    manifest_path = _manifest_path(out_dir)
+
+    prompts = load_prompts(args.prompts, fallback_prompts=FALLBACK_PROMPTS)
+    if args.prompts is not None:
+        print(f"Loaded {len(prompts)} prompts from {args.prompts}")
+    else:
+        print(
+            f"WARNING: --prompts not given; using {len(prompts)} built-in "
+            "fallback prompts (smoke-test only, not enough diversity for "
+            "a real training run)"
+        )
+
+    rng = random.Random(args.seed)
+
+    if args.dry_run:
+        print(
+            "Dry run: SD 3.5 Medium pipeline will not be loaded and no images "
+            "will be written."
+        )
+        for i in range(args.count):
+            prompt = choose_prompt(rng, prompts)
+            seed = next_seed(rng)
+            key = stable_image_key(prompt, seed)
+            dst = out_dir / f"{key}.png"
+            print(f"  {i + 1:04d}: seed={seed} path={dst} prompt={prompt!r}")
+        print(f"Dry run complete. Planned manifest fragment: {manifest_path}")
+        return
+
+    print("Loading SD 3.5 Medium pipeline (large download on first run)...")
+    import torch
+    from diffusers import StableDiffusion3Pipeline
+
+    pipe = StableDiffusion3Pipeline.from_pretrained(
+        SD35_SPEC.model_id,
+        torch_dtype=torch.float16,
+    )
+    pipe.to("cuda")
+
+    rows: list[dict] = []
+    for i in range(args.count):
+        prompt = choose_prompt(rng, prompts)
+        seed = next_seed(rng)
+        key = stable_image_key(prompt, seed)
+        dst = out_dir / f"{key}.png"
+
+        if not dst.exists():
+            generator = torch.Generator("cuda").manual_seed(seed)
+            image = pipe(
+                prompt=prompt,
+                num_inference_steps=args.steps,
+                guidance_scale=args.guidance_scale,
+                width=args.width,
+                height=args.height,
+                generator=generator,
+            ).images[0]
+            image.save(dst, format="PNG")
+
+        width, height = image_dimensions(dst)
+        rows.append(
+            manifest_row(
+                path=dst.relative_to(data_root).as_posix(),
+                cls="ai_generated",
+                spec=SD35_SPEC,
+                sha256=sha256_file(dst),
+                prompt=prompt,
+                seed=seed,
+                width=width,
+                height=height,
+                generation_params={
+                    "steps": args.steps,
+                    "guidance_scale": args.guidance_scale,
+                    "width": args.width,
+                    "height": args.height,
+                    "pipeline": SD35_SPEC.pipeline,
+                },
+            )
+        )
+
+        if (i + 1) % 100 == 0:
+            print(f"  generated {i + 1}/{args.count}")
+
+    with manifest_path.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=STAGE3A_MANIFEST_FIELDS)
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"Done. {len(rows)} images. Manifest fragment: {manifest_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/generate_sdxl_synthetic.py

@@ -0,0 +1,202 @@
+"""
+Generate synthetic images with Stable Diffusion XL base 1.0.
+
+License
+-------
+`stabilityai/stable-diffusion-xl-base-1.0` is released under CreativeML Open
+RAIL++-M. Stage 3A approved it for detector-training data with use-policy
+restrictions. Do not use SDXL outputs to train or improve a competing
+generative model. See `NOTICES.md` before running a real generation job.
+
+Usage
+-----
+    python scripts/dataset/generate_sdxl_synthetic.py \\
+        --out data/raw/ai_generated/sdxl \\
+        --count 20000 \\
+        --prompts scripts/dataset/prompts.txt
+
+Dry run, no model import/download:
+
+    python scripts/dataset/generate_sdxl_synthetic.py \\
+        --out data/raw/ai_generated/sdxl \\
+        --count 3 \\
+        --dry-run
+
+Hardware
+--------
+SDXL 1024x1024 generation generally needs a CUDA GPU with roughly 12-16 GB VRAM
+when using half precision and attention optimizations. Runtime varies strongly
+by GPU and step count; start with a small smoke run before renting a long job.
+
+Idempotency
+-----------
+Each generated file is named by a hash of (prompt, seed), so re-runs skip
+already-generated images. Crash-resume works automatically.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import random
+from pathlib import Path
+
+from generation_utils import (
+    APPROVED_GENERATORS,
+    STAGE3A_MANIFEST_FIELDS,
+    choose_prompt,
+    image_dimensions,
+    infer_data_root,
+    load_prompts,
+    manifest_row,
+    next_seed,
+    sha256_file,
+    stable_image_key,
+)
+
+
+FALLBACK_PROMPTS = [
+    "a casual phone photo of a bicycle leaning against a brick wall",
+    "a realistic photo of a bowl of soup on a kitchen counter",
+    "a slightly blurry snapshot of commuters waiting at a train platform",
+    "a natural light photo of a cluttered home office desk",
+]
+
+
+SDXL_SPEC = APPROVED_GENERATORS["sdxl"]
+
+
+def _manifest_path(out_dir: Path) -> Path:
+    return out_dir.parent / "sdxl_manifest.csv"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--out", type=Path, required=True, help="Output directory")
+    parser.add_argument(
+        "--data-root",
+        type=Path,
+        default=None,
+        help=(
+            "Dataset root for manifest paths. Defaults to the parent of the "
+            "'raw' path segment in --out."
+        ),
+    )
+    parser.add_argument("--count", type=int, default=20_000)
+    parser.add_argument(
+        "--prompts",
+        type=Path,
+        default=None,
+        help="Optional file with one prompt per line",
+    )
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--steps", type=int, default=30)
+    parser.add_argument("--guidance-scale", type=float, default=7.0)
+    parser.add_argument("--width", type=int, default=1024)
+    parser.add_argument("--height", type=int, default=1024)
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Validate prompt/seed/output planning without loading SDXL or writing files",
+    )
+    args = parser.parse_args()
+
+    out_dir = args.out.resolve()
+    out_dir.mkdir(parents=True, exist_ok=True)
+    data_root = (
+        args.data_root.resolve()
+        if args.data_root is not None
+        else infer_data_root(out_dir)
+    )
+    manifest_path = _manifest_path(out_dir)
+
+    prompts = load_prompts(args.prompts, fallback_prompts=FALLBACK_PROMPTS)
+    if args.prompts is not None:
+        print(f"Loaded {len(prompts)} prompts from {args.prompts}")
+    else:
+        print(
+            f"WARNING: --prompts not given; using {len(prompts)} built-in "
+            "fallback prompts (smoke-test only, not enough diversity for "
+            "a real training run)"
+        )
+
+    rng = random.Random(args.seed)
+
+    if args.dry_run:
+        print("Dry run: SDXL pipeline will not be loaded and no images will be written.")
+        for i in range(args.count):
+            prompt = choose_prompt(rng, prompts)
+            seed = next_seed(rng)
+            key = stable_image_key(prompt, seed)
+            dst = out_dir / f"{key}.png"
+            print(f"  {i + 1:04d}: seed={seed} path={dst} prompt={prompt!r}")
+        print(f"Dry run complete. Planned manifest fragment: {manifest_path}")
+        return
+
+    print("Loading SDXL pipeline (~7 GB download on first run)...")
+    import torch
+    from diffusers import StableDiffusionXLPipeline
+
+    pipe = StableDiffusionXLPipeline.from_pretrained(
+        SDXL_SPEC.model_id,
+        torch_dtype=torch.float16,
+        variant="fp16",
+        use_safetensors=True,
+    )
+    pipe.to("cuda")
+
+    rows: list[dict] = []
+    for i in range(args.count):
+        prompt = choose_prompt(rng, prompts)
+        seed = next_seed(rng)
+        key = stable_image_key(prompt, seed)
+        dst = out_dir / f"{key}.png"
+
+        if not dst.exists():
+            generator = torch.Generator("cuda").manual_seed(seed)
+            image = pipe(
+                prompt=prompt,
+                num_inference_steps=args.steps,
+                guidance_scale=args.guidance_scale,
+                width=args.width,
+                height=args.height,
+                generator=generator,
+            ).images[0]
+            image.save(dst, format="PNG")
+
+        width, height = image_dimensions(dst)
+        rows.append(
+            manifest_row(
+                path=dst.relative_to(data_root).as_posix(),
+                cls="ai_generated",
+                spec=SDXL_SPEC,
+                sha256=sha256_file(dst),
+                prompt=prompt,
+                seed=seed,
+                width=width,
+                height=height,
+                generation_params={
+                    "steps": args.steps,
+                    "guidance_scale": args.guidance_scale,
+                    "width": args.width,
+                    "height": args.height,
+                    "pipeline": SDXL_SPEC.pipeline,
+                },
+            )
+        )
+
+        if (i + 1) % 100 == 0:
+            print(f"  generated {i + 1}/{args.count}")
+
+    with manifest_path.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=STAGE3A_MANIFEST_FIELDS)
+        writer.writeheader()
+        writer.writerows(rows)
+
+    print(f"Done. {len(rows)} images. Manifest fragment: {manifest_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/generation_utils.py

@@ -0,0 +1,205 @@
+"""
+Shared helpers for Stage 3A dataset generation scripts.
+
+Keep this module dependency-light: it is used on GPU dataset boxes, not in the
+inference container. Generator-specific scripts should own their pipeline setup;
+this module only centralizes the boring parts that must stay consistent across
+sources.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import random
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Sequence
+
+
+@dataclass(frozen=True)
+class GeneratorSpec:
+    """License and metadata constants for one approved generator source."""
+
+    source: str
+    generator: str
+    model_family: str
+    model_id: str
+    license: str
+    license_url: str
+    pipeline: str
+
+
+APPROVED_GENERATORS: dict[str, GeneratorSpec] = {
+    "flux.1-schnell": GeneratorSpec(
+        source="flux.1-schnell",
+        generator="flux.1-schnell",
+        model_family="rectified_flow",
+        model_id="black-forest-labs/FLUX.1-schnell",
+        license="Apache-2.0",
+        license_url="https://www.apache.org/licenses/LICENSE-2.0",
+        pipeline="FluxPipeline",
+    ),
+    "sdxl": GeneratorSpec(
+        source="sdxl",
+        generator="sdxl",
+        model_family="diffusion_unet",
+        model_id="stabilityai/stable-diffusion-xl-base-1.0",
+        license="CreativeML Open RAIL++-M",
+        license_url=(
+            "https://huggingface.co/stabilityai/"
+            "stable-diffusion-xl-base-1.0/blob/main/LICENSE.md"
+        ),
+        pipeline="StableDiffusionXLPipeline",
+    ),
+    "sd3.5-medium": GeneratorSpec(
+        source="sd3.5-medium",
+        generator="sd3.5-medium",
+        model_family="diffusion_transformer",
+        model_id="stabilityai/stable-diffusion-3.5-medium",
+        license="Stability AI Community License",
+        license_url="https://stability.ai/license",
+        pipeline="StableDiffusion3Pipeline",
+    ),
+    "auraflow-v0.3": GeneratorSpec(
+        source="auraflow-v0.3",
+        generator="auraflow-v0.3",
+        model_family="rectified_flow",
+        model_id="fal/AuraFlow-v0.3",
+        license="Apache-2.0",
+        license_url="https://www.apache.org/licenses/LICENSE-2.0",
+        pipeline="AuraFlowPipeline",
+    ),
+}
+
+STAGE3A_MANIFEST_FIELDS = [
+    "path",
+    "class",
+    "source",
+    "license",
+    "license_url",
+    "sha256",
+    "generator",
+    "model_family",
+    "model_id",
+    "prompt",
+    "seed",
+    "width",
+    "height",
+    "generation_params_json",
+]
+
+
+def load_prompts(
+    path: Path | None,
+    *,
+    fallback_prompts: Sequence[str] | None = None,
+) -> list[str]:
+    """Read non-blank, non-comment prompts or return a fallback prompt list."""
+    if path is None:
+        if fallback_prompts is None:
+            raise ValueError("No prompts path provided and no fallback prompts given")
+        prompts = [p.strip() for p in fallback_prompts if p.strip()]
+        if not prompts:
+            raise ValueError("Fallback prompts list is empty")
+        return prompts
+
+    prompts: list[str] = []
+    with path.open(encoding="utf-8") as fh:
+        for line in fh:
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+            prompts.append(stripped)
+    if not prompts:
+        raise ValueError(f"No prompts found in {path}")
+    return prompts
+
+
+def choose_prompt(rng: random.Random, prompts: Sequence[str]) -> str:
+    """Pick one prompt with the caller's deterministic RNG."""
+    if not prompts:
+        raise ValueError("prompts must not be empty")
+    return rng.choice(list(prompts))
+
+
+def next_seed(rng: random.Random) -> int:
+    """Return a deterministic positive seed compatible with torch generators."""
+    return rng.randint(0, 2**31 - 1)
+
+
+def stable_image_key(prompt: str, seed: int) -> str:
+    """Return the legacy content-addressed key for generated image files."""
+    return hashlib.sha256(f"{prompt}|{seed}".encode()).hexdigest()[:24]
+
+
+def infer_data_root(out_dir: Path) -> Path:
+    """Infer the dataset root from an output path containing a `raw` segment."""
+    resolved = out_dir.resolve()
+    parts = resolved.parts
+    if "raw" not in parts:
+        raise ValueError(
+            f"Cannot infer data root from {out_dir}; pass --data-root explicitly"
+        )
+    raw_index = parts.index("raw")
+    if raw_index == 0:
+        raise ValueError(
+            f"Cannot infer data root from {out_dir}; pass --data-root explicitly"
+        )
+    return Path(*parts[:raw_index])
+
+
+def sha256_file(path: Path) -> str:
+    """Return the SHA-256 digest for a file."""
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def image_dimensions(path: Path) -> tuple[int, int]:
+    """Return image width and height without keeping the image open."""
+    from PIL import Image
+
+    with Image.open(path) as image:
+        return image.size
+
+
+def params_json(params: dict[str, Any]) -> str:
+    """Serialize generation parameters consistently for manifest rows."""
+    return json.dumps(params, sort_keys=True, separators=(",", ":"))
+
+
+def manifest_row(
+    *,
+    path: str,
+    cls: str,
+    spec: GeneratorSpec,
+    sha256: str,
+    prompt: str = "",
+    seed: int | str = "",
+    width: int | str = "",
+    height: int | str = "",
+    generation_params: dict[str, Any] | None = None,
+    extra: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build a Stage 3A-compatible manifest row for one generated image."""
+    row: dict[str, Any] = {
+        "path": path,
+        "class": cls,
+        "source": spec.source,
+        "license": spec.license,
+        "license_url": spec.license_url,
+        "sha256": sha256,
+        "generator": spec.generator,
+        "model_family": spec.model_family,
+        "model_id": spec.model_id,
+        "prompt": prompt,
+        "seed": seed,
+        "width": width,
+        "height": height,
+        "generation_params_json": params_json(generation_params or {}),
+    }
+    if extra:
+        row.update(extra)
+    return row

scripts/dataset/prompts.txt

@@ -0,0 +1,1711 @@
+# Prompts for Flux.1-schnell synthetic generation (Stage 2 dataset).
+#
+# One complete prompt per line. Blank lines and lines starting with `#`
+# are ignored. Lines should describe what a regular person might upload —
+# include mundane subjects, casual snapshots, varied camera styles and
+# imperfect compositions, not just gallery-worthy shots.
+#
+# Avoided on purpose: "cinematic", "8k", "highly detailed", "trending on
+# artstation", "masterpiece", "award winning". These bias Flux toward the
+# polished AI-art aesthetic that is trivially easy to detect, leaving the
+# model blind to AI images that *try* to look casual.
+#
+# Structure: roughly grouped by category. Order does not matter at runtime
+# (lines are sampled uniformly at random).
+
+# ---------------------------------------------------------------------------
+# People — portraits, candid, group, activities (~250)
+# ---------------------------------------------------------------------------
+a portrait of an elderly woman with grey hair smiling at the camera
+a teenage boy in a blue hoodie skateboarding down a quiet street
+a chef in a white apron tasting soup from a wooden spoon
+two children laughing on a swing set in a backyard
+an office worker on her phone walking through a glass lobby
+a man with a long beard reading a newspaper on a park bench
+a dancer mid-leap on a wooden stage during rehearsal
+a barista pulling a shot of espresso behind a marble counter
+a construction worker in a hard hat eating lunch on a steel beam
+a nurse in scrubs adjusting an IV drip beside a hospital bed
+a young woman in a yellow raincoat waiting at a bus stop
+a grandfather teaching his grandson how to tie a fishing knot
+a cyclist taking a water break beside a country road
+a librarian shelving books in a tall wooden bookcase
+a mechanic in greasy overalls leaning under an open car bonnet
+a guitar player busking on a cobblestone street corner
+a bride and groom posing in front of a chapel
+two coworkers laughing over coffee at an open-plan desk
+a man in a suit running to catch a train at a station platform
+a child blowing out candles on a birthday cake
+an elderly couple holding hands on a park bench in autumn
+a young father carrying his toddler on his shoulders
+a woman jogging along a foggy waterfront at dawn
+a teenager playing video games on a couch in a dim living room
+a baker dusting flour onto a kneaded dough on a wooden table
+a stylist trimming a customer's hair in a small salon
+a yoga teacher demonstrating a pose to a small class
+a security guard standing at the entrance to a museum
+a postal worker delivering parcels from a small van
+a fisherman repairing a net on a wooden dock
+a farmer feeding chickens in a wire-mesh coop
+a tattoo artist concentrating on a half-finished sleeve
+a soldier reading a letter in a barracks
+a dental hygienist explaining x-rays to a patient
+a group of friends taking a selfie in front of a brick wall
+a pregnant woman touching her stomach in front of a window
+a child holding a balloon at a fairground
+a man in a wheelchair stretching before a marathon
+a girl playing the violin at a school concert
+a businesswoman pacing while on a phone call near a glass window
+a homeless man sharing his sandwich with a stray dog
+a teacher writing on a whiteboard in a classroom
+a man pumping gas at a small rural petrol station
+a woman watering plants on a small apartment balcony
+two old men playing chess in a sunny park
+a girl reading a book under a tree on a summer afternoon
+a chef plating a dish in a busy restaurant kitchen
+a cleaner mopping a hospital corridor at night
+a delivery cyclist with a thermal bag waiting at a traffic light
+a surfer waxing his board on the sand
+a beekeeper inspecting a frame from a hive
+a vet examining a small dog on a metal table
+a couple sharing an umbrella in a heavy downpour
+a runner stretching against a wall in a stairwell
+a man fixing a leaking pipe under a bathroom sink
+a child painting a watercolor at a small wooden desk
+a butcher arranging cuts of meat behind a glass display
+a plumber writing an invoice on a clipboard
+a delivery driver scanning a parcel in a hallway
+a mother feeding a baby in a high chair
+a hairdresser blowing out a customer's hair
+two friends carrying boxes into a new apartment
+an artist working on a canvas in a cluttered studio
+a beekeeper holding a smoker beside hives in a field
+a knitter working on a half-finished scarf in lamplight
+a man washing his car in a suburban driveway
+a couple cooking pasta together in a small kitchen
+a violinist tuning her instrument backstage
+a young man trying on glasses in an optician's mirror
+a ranger pointing at a map on a forest trail
+a woman doing pull-ups at an outdoor calisthenics park
+a child eating an ice cream that is dripping down her hand
+a man in a parka shoveling snow from his front path
+a librarian helping a child find a book on a low shelf
+a barber giving a fade haircut with electric clippers
+two friends roasting marshmallows over a campfire
+a forensic technician dusting a doorframe for fingerprints
+a chef sharpening a knife on a steel rod
+a bicycle courier stopped at a kerb checking his phone
+a surfer paddling out through breaking waves at sunrise
+a ballerina lacing her pointe shoes in a dressing room
+a janitor emptying a bin in a quiet office
+a model walking a runway under harsh stage lights
+a piano teacher correcting a student's hand position
+a window cleaner suspended from a high-rise on ropes
+a courier signing for a package in a building lobby
+a businessman tying his tie in a hotel mirror
+a child throwing breadcrumbs to ducks at a pond
+a man stretching his back in a small home gym
+a woman applying lipstick in a car rear-view mirror
+a homeless man asleep on a cardboard mat under an overpass
+a child blowing bubbles in a sunlit garden
+a jeweler examining a ring under a loupe
+a father teaching his daughter to ride a bike on a quiet street
+a paramedic loading a stretcher into an ambulance
+a nun walking down a stone corridor of a convent
+a grandmother braiding her granddaughter's hair on a porch
+a woman sneezing into a tissue at a desk
+a craftsman shaping a clay pot on a spinning wheel
+a beekeeper extracting honey from a frame in a workshop
+two girls jumping rope on a chalk-marked sidewalk
+a sushi chef rolling a maki behind a counter
+a man tying his running shoes on a park bench
+a couple cuddling under a blanket on a sofa
+a child crying in a supermarket aisle
+a soldier polishing boots on a wooden chair
+a violin maker varnishing an unfinished instrument
+a chef weighing flour on a small kitchen scale
+a teenager sitting on a curb eating fries from a paper bag
+a man trying on a leather jacket in a thrift shop
+a barista latte-arting a heart on a flat white
+a runner crossing a finish line with arms raised
+a man asleep on a long-haul train with a book on his lap
+a graduation photo of a young woman in a black gown
+a girl reading a comic book in a treehouse
+a chef checking a roast in a steaming oven
+a man patching a bicycle tube on a kitchen table
+a paramedic taking notes on a clipboard beside a patient
+a woman braiding garlic into a long string
+a baker pulling sourdough loaves from a stone oven
+a teenager eating cereal in pyjamas in a bright kitchen
+a couple unloading groceries from a car boot
+a child catching a butterfly in a glass jar
+a runner stopping to drink water at a wooden trail marker
+a drummer setting up a kit on a small stage
+a craftsman carving wood with a chisel in a sunlit workshop
+a woman applying mascara in a small bathroom mirror
+a man writing in a notebook at a noisy diner
+a fisherman gutting a fish on a wooden dock
+a barista counting out change at a small bakery
+a janitor mopping a school cafeteria after lunch
+a man selling fruit from the back of a pickup truck
+a woman scrolling on her phone in a packed subway car
+a child building a sandcastle at low tide
+a young man playing a harmonica on a porch
+a baker piping cream onto a tray of eclairs
+a warehouse worker scanning a barcode on a tall shelf
+a schoolteacher lining up students in a hallway
+a mother and daughter trying on hats in a department store
+a barista wiping down the counter at closing time
+a girl carrying a stack of books taller than her face
+a man trying to fold a fitted sheet in a laundry room
+a couple arguing in front of a parked car at night
+a chef scoring a baguette with a razor blade
+a teenager learning to drive with her father in the passenger seat
+a man watching a football match alone in a small pub
+a child napping with a stuffed rabbit on a sofa
+a doctor washing his hands at a stainless-steel sink
+a ranger pointing out a bird through binoculars to tourists
+a busker drawing a small crowd in a metro station
+a couple bringing flowers to a grave
+a runner tying a shoelace at a starting line
+a child trying to ride a unicycle on a driveway
+a barista training a new hire on the espresso machine
+a writer typing on a laptop at a window seat in a cafe
+a ferryman pushing a small boat away from a wooden jetty
+a chef yelling orders across a hot kitchen pass
+a girl learning to use chopsticks at a dinner table
+a couple painting the walls of a half-empty apartment
+a maintenance worker changing a fluorescent tube on a tall ladder
+a surveyor looking through a tripod-mounted instrument on a roadside
+a man practicing tai chi alone in a misty park
+a boy showing his grandfather a frog he caught in a jar
+a paramedic checking a patient's pulse in the back of an ambulance
+a janitor sweeping leaves from the steps of a courthouse
+a mother spoon-feeding mashed banana to a baby
+a tour guide pointing up at a cathedral ceiling
+a couple looking at a map on a phone at a train platform
+a barber sweeping hair clippings from a tiled floor
+a woman drinking soup from a thermos on a park bench
+a child trying to skate on a backyard pond in winter
+a worker pouring fresh concrete into a wooden form
+a security guard watching cctv monitors in a cramped booth
+a teacher pinning student artwork to a corkboard
+a baker proofing dough in cloth-lined baskets
+a young woman crying at a bus station
+a couple kissing under streetlights in a foggy alley
+a man feeding pigeons from a paper bag on a stone bench
+a woman braiding wet hair in a steamy bathroom mirror
+a child running through a sprinkler in a backyard
+a fishmonger arranging ice and fish in a market display
+a runner pacing back and forth before a starting gun
+a girl trying on her mother's high heels in a hallway
+a mechanic showing a worn brake pad to a customer
+a young woman lighting a cigarette in a doorway at night
+a child climbing on a metal jungle gym at recess
+a man stacking logs neatly against a shed wall
+a couple posing awkwardly for a passport photo
+a teenager listening to headphones on a school bus
+a paramedic comforting a frightened child in the back of an ambulance
+a janitor changing the bin liner in an office kitchen
+a chef tasting a sauce from a wooden spoon and frowning
+a mother chasing a toddler around a coffee table
+a delivery driver looking lost on a residential street
+a fisherman casting a line off a stone breakwater at dusk
+a runner glancing at a smartwatch mid-stride
+a baker brushing egg wash onto a tray of pastries
+a woman trying to start a car on a cold morning
+a man smoking a cigar on a wraparound porch
+a couple flipping through a photo album on a couch
+a barber lathering a customer for a straight-razor shave
+a child holding a sparkler at a backyard party
+a writer crossing out lines in a notebook with a red pen
+a roofer hammering shingles under a hot midday sun
+a child learning to tie shoelaces with their tongue out
+a man hanging laundry on a line between two flats
+a couple cooking eggs in a small studio apartment
+a librarian stamping a date into a returned book
+a baker mixing dough by hand in a stainless steel bowl
+
+# ---------------------------------------------------------------------------
+# Animals — pets, wildlife, farm, insects (~150)
+# ---------------------------------------------------------------------------
+a tabby cat sleeping on a sunlit windowsill
+a golden retriever shaking water off after a bath
+a small terrier digging a hole in a flower bed
+a black cat staring intently at something offscreen
+a parrot perched on a curtain rod inside a living room
+a goldfish swimming in a small round bowl on a desk
+a hamster running on a wheel in a glass cage
+a beagle sniffing the base of a lamppost
+a horse grazing in a green pasture under a cloudy sky
+a Holstein cow chewing cud in a muddy field
+a flock of sheep crossing a single-lane country road
+a piglet nosing around in a straw-lined pen
+a goat standing on a fallen log staring at the camera
+a flock of chickens pecking around a wooden coop
+a duck leading ducklings across a path beside a pond
+a swan gliding across a glassy lake at sunrise
+a fox cautiously emerging from underbrush at twilight
+a deer grazing at the edge of a forest at dawn
+a squirrel holding an acorn on a low branch
+a raccoon raiding an overturned trash can
+a hedgehog curled up among autumn leaves
+a rabbit nibbling clover in a sunlit meadow
+a bald eagle perched on a dead tree against a grey sky
+a barn owl staring straight at the camera in dim light
+a hummingbird hovering at a red feeder
+a robin tugging a worm from wet grass
+a flock of starlings swirling across a winter sky
+a heron standing motionless in shallow water
+a kingfisher diving toward the surface of a stream
+a butterfly resting on a purple flower
+a bumblebee crawling across the inside of a foxglove
+a praying mantis on a green leaf at close range
+a dragonfly perched on the tip of a reed
+a snail leaving a trail across a rain-slicked stone
+a spider in the centre of a dew-covered web at dawn
+a tortoise eating a strawberry on a patio
+a leopard gecko on a warm rock under a heat lamp
+a python coiled on a branch inside a glass terrarium
+a tropical fish swimming through coral in an aquarium
+a pod of dolphins surfacing alongside a small boat
+a humpback whale's tail flicking above the ocean surface
+a seal lounging on a rocky outcrop in the sun
+a sea otter floating on its back holding a clam
+a starfish clinging to a tide-pool rock at low tide
+a crab scuttling sideways across wet sand
+a flock of penguins waddling across an icy beach
+a polar bear walking across pack ice
+a brown bear catching a leaping salmon at a waterfall
+a moose wading across a shallow river at sunset
+a herd of zebras drinking at a watering hole
+a giraffe stretching its neck to reach high acacia leaves
+a lion yawning under the shade of a tree
+a cheetah resting on a low termite mound
+a baby elephant nudging its mother's leg
+a meerkat standing on its hind legs scanning the horizon
+a wolf pack moving through deep snow at dusk
+a bald eagle's nest with two chicks high in a pine tree
+a hen roosting on a clutch of eggs in a wooden nest box
+a flock of geese flying in a v formation against a sunset
+a tabby kitten batting at a dangling string
+a black labrador catching a tennis ball mid-air
+a border collie herding sheep down a green hillside
+a great dane lying across a small couch with legs hanging off
+a poodle wearing a colorful scarf at a dog park
+a bulldog asleep with its tongue hanging out
+a husky howling on a snowy ridge
+a chihuahua peeking out of a tote bag on a bus
+a kitten asleep inside a coffee mug
+a one-eyed cat lounging on a sunny porch
+a stray dog drinking from a puddle in an alley
+a parakeet preening another in a wire cage
+a stick insect almost invisible against a green leaf
+a chameleon clinging to a branch with curled tail
+a baby giraffe peeking out from behind its mother
+a calf nuzzling its mother in a barn
+a rooster crowing on a wooden fence at sunrise
+a turkey strutting across a farmyard
+a llama spitting at a curious tourist
+a donkey resting its head over a paddock fence
+a peacock fanning its tail in a stately garden
+a swan hissing at a dog approaching its nest
+a magpie carrying a shiny piece of foil in its beak
+a flock of pigeons rising from a city square
+a sparrow bathing in a puddle on a patio
+a koala asleep in the fork of a eucalyptus tree
+a kangaroo standing alert in tall dry grass
+a wombat shuffling across a dirt track
+a possum staring from a tree branch at night
+an octopus changing color in a shallow tide pool
+a jellyfish drifting through deep blue water
+a flock of flamingos wading in a pink-tinted lake
+a school of small silver fish darting around coral
+a hermit crab carrying a whelk shell across a beach
+a butterfly chrysalis hanging from a leaf
+a cocoon hatching with a damp moth emerging
+a worm half-buried in dark garden soil
+a colony of ants carrying a leaf fragment
+a beetle climbing a long blade of grass
+a moth pressed against a lit window at night
+a guinea pig eating a slice of cucumber
+a ferret peeking out of a sweatshirt sleeve
+a rat sniffing a piece of cheese on a kitchen counter
+a chipmunk with cheeks full of seeds on a stone wall
+a baby seal alone on an icy beach looking around
+an arctic fox curled up in a snowdrift
+a herd of bison crossing a yellow plain
+a wild turkey foraging at the edge of a forest
+a black bear cub climbing a tree trunk
+a bobcat slinking through tall grass at dusk
+a porcupine ambling along a dirt road
+a skunk lifting its tail at a curious dog
+a cat napping curled around a houseplant
+a dog standing in the bed of a pickup truck with the wind in its fur
+a goldfish staring through the glass of an aquarium
+a parakeet sitting on a child's shoulder
+a tortoise eating dandelions in a backyard
+a dog asleep on a doormat with muddy paws
+a cat watching birds from inside a window
+a horse drinking from a metal trough at a stable
+a lamb wobbling on new legs in a barn
+a piglet sleeping on a pile of others
+a mouse crouched at the back of a kitchen cupboard
+a swan family swimming across a misty lake
+a pelican with a fish in its pouch standing on a dock
+a flock of seagulls fighting over chips on a pier
+a dog covered in mud after running through a field
+a cat with one ear flicked back staring at a vacuum cleaner
+a goat standing on the roof of a parked car
+a ferret tumbling out of a cardboard box
+a flock of starlings filling a winter tree like leaves
+a herd of cattle blocking a country road
+a black swan among white swans on a city pond
+a parrot tilting its head at a mirror
+a dog wearing a knitted sweater on a snowy walk
+a cat refusing to come out from under a bed
+a goldfish leaping briefly out of a pond
+a hedgehog drinking from a shallow saucer in a garden
+a stray dog asleep under a parked car at noon
+a dragonfly skimming the surface of a pond
+a pair of swallows building a nest under an eave
+a fox crossing a snowy front lawn at dusk
+
+# ---------------------------------------------------------------------------
+# Landscapes — nature, weather, seasons, time of day (~200)
+# ---------------------------------------------------------------------------
+a misty forest path at sunrise
+a snow-covered mountain pass under a clear blue sky
+a calm lake reflecting autumn trees on a still morning
+a rocky coastline with waves crashing against cliffs
+a desert landscape with red dunes under a low sun
+a wheat field rippling in the wind under a bright sky
+a vineyard on a rolling hillside in late summer
+a single oak tree standing alone in an empty field
+a forest of birch trees with white trunks and yellow leaves
+a path through a bamboo grove with sunlight filtering through
+a meadow full of wildflowers in early summer
+a dry riverbed cracked by the sun in late summer
+a lighthouse on a rocky headland during a storm
+a thunderstorm rolling across an open prairie
+a tornado funnel touching the ground in distant farmland
+a tropical beach at midday with white sand and turquoise water
+a black-sand beach in iceland with crashing waves
+a frozen lake with cracks visible beneath a thin layer of snow
+a pine forest after a fresh snowfall
+a glacier flowing between two rocky peaks
+a salt flat reflecting clouds like a mirror at sunset
+an active volcano with a glowing red lava flow at night
+a rainbow over a green valley after a rainstorm
+a foggy bay with boats barely visible at dawn
+a marsh with tall reeds at sunset
+a single cottonwood tree on a vast plain
+a winding mountain road climbing through pine forest
+a country road with stone walls on both sides under a grey sky
+a gravel path through a heather moor
+a single canoe pulled up on the shore of a quiet lake
+a footbridge crossing a stream in a fern-filled forest
+a wooden boardwalk through a coastal dune
+a cliff edge with a railing overlooking distant mountains
+a single farmhouse surrounded by autumn corn
+a herd of cows grazing in a green field at golden hour
+a freshly plowed field with tractor tracks under a low sky
+a pumpkin patch ready for harvest in october
+a sunflower field at peak bloom
+a lavender field in summer with rows stretching to the horizon
+a tea plantation on stepped hillsides at dawn
+a rice paddy reflecting clouds in a tropical valley
+a coffee plantation under a high canopy in misty mountains
+a banana plantation with broad green leaves
+an olive grove with twisted trunks under a hot sky
+a citrus orchard heavy with oranges
+a cherry orchard in full pink bloom
+an apple orchard heavy with red fruit before harvest
+a mossy stream running through a temperate rainforest
+a waterfall plunging into a clear pool
+a sea cave with light coming through a small opening
+a tidal flat at low tide with seabirds feeding
+a coral reef visible just below clear shallow water
+a kelp forest swaying in green underwater light
+a deep canyon with switchbacks descending to a river
+a sandstone arch glowing red in evening light
+a slot canyon with narrow walls of striped sandstone
+a hot spring steaming against snow-covered ground
+a geyser erupting in front of a wooded ridge
+a frozen waterfall in a rocky canyon
+a winter trail with footprints leading away into pines
+a deer trail through tall grass at dawn
+a hiking trail with switchbacks visible on a mountain face
+a dirt road disappearing into the desert horizon
+a country lane lined with stone walls under autumn trees
+a cobblestone path through a ruined castle garden
+a bridge of weathered planks over a small stream
+a lookout point with a chain-link safety fence over a deep valley
+a remote cabin in a clearing with smoke rising from the chimney
+a yurt at dusk on a windswept steppe
+a tent pitched beside a small alpine lake
+a campsite with a smoldering fire ring at first light
+a hammock strung between two trees overlooking a valley
+a kayak on the shore of a fjord at low tide
+a sailboat anchored in a quiet bay at sunset
+an old wooden rowboat tied to a weathered jetty
+a narrow stone bridge over a moss-banked stream
+an autumn forest carpeted in fallen leaves
+a winter morning with frost outlining every blade of grass
+a spring meadow with bluebells and sunlight through fresh leaves
+a summer afternoon with cumulus clouds piled high
+an evening sky with horizontal pink and orange clouds
+a starry night with the milky way over a desert horizon
+a moonlit field with a single tree casting a long shadow
+the aurora borealis over a snowy spruce forest
+a rare double rainbow over a wet country road
+a heavy fog rolling over a reservoir at dawn
+heavy rain blurring the windows of a country cottage
+a lightning strike forking against dark thunderclouds
+a hailstorm pelting a parked car
+a windswept dune with sand grains visible in motion
+a salt marsh at low tide with channels of water
+a beach with bleached driftwood scattered on the sand
+a tide pool with anemones and small fish at low tide
+a shipwreck partly buried in a beach
+a riverbank with willows leaning toward the water
+a meadow stream meandering past wildflowers
+a freshwater spring bubbling up through clear sand
+an estuary at sunset with reflections of an old jetty
+a ridge line at sunrise with mist filling the valleys below
+a basalt cliff with hexagonal columns
+a pebble beach at low tide with a stone breakwater behind
+a moor with heather in late summer
+a reindeer crossing a snowy plateau in winter
+a herd of bison crossing a yellowstone field
+a herd of wild horses kicking up dust on a high plain
+a mountain peak emerging above a sea of clouds
+a forest of giant redwoods with sun shafts through the canopy
+a ancient yew tree in the corner of a country churchyard
+a willow tree leaning over a slow river
+a baobab tree silhouetted at sunset on a savanna
+a frozen stream with bubbles trapped in clear ice
+a desert valley dotted with saguaro cacti at dusk
+a narrow alpine lake reflecting a snowy peak
+a mossy boulder beside a forest stream
+a forest floor covered in ferns and dappled light
+a marsh boardwalk leading to a wooden observation hide
+a hiker's view from a high pass on a clear afternoon
+a mountain hut perched on the edge of a steep ridge
+a windmill on a hillside in northern europe
+an old stone barn at the edge of a snowy field
+a reservoir at low water with mud cracks
+a wooden fishing pier extending into a calm sea
+a row of beach huts in pastel colors on a british beach
+a mangrove forest at low tide with exposed roots
+a rice field at sunset with farmers walking home
+a mountain village clinging to a steep slope
+a desert oasis with palm trees and a small pool
+a windswept island with a single white lighthouse
+a cliff with seabirds nesting on every ledge
+a coastline at low tide with a long line of seaweed on the sand
+an inland salt lake with a pink hue from algae
+a swamp with cypress trees and hanging spanish moss
+a frozen waterfall illuminated by morning light
+a small footbridge crossing a creek in a botanical garden
+a hayfield freshly cut into long parallel rows
+a grain silo standing alone on the edge of a field
+a wind farm on a green hillside with slowly turning blades
+a solar farm in a desert under a clear sky
+a hydroelectric dam with water spilling over the top
+a prairie under a heavy storm cloud
+a grove of aspens with white bark and yellow leaves
+a quiet beach at dusk with footprints leading to the water
+a frozen pond with skate marks etched into the surface
+a country fence with barbed wire and morning frost
+a country road with a covered bridge in the distance
+a single red barn against a green hillside
+a row of haystacks at sunset
+a field of fireflies at twilight in a wooded area
+a hiking trail descending through wildflowers in midsummer
+a forest after rain with droplets on every leaf
+a beach at first light with a low pink sun on the horizon
+a desert at midnight with a clear sky full of stars
+a tropical jungle from above with a meandering river
+a snowy clearing with deer tracks crossing it
+a remote chapel on a cliff with the sea behind
+an oak avenue leading to a country house
+a country garden in summer with hollyhocks and bees
+a frozen reservoir with ice fishing huts dotted across it
+a salt-glazed coastal field at sunset
+a pebble cove at low tide with sea cliffs behind
+a lake at dawn with mist rising off the surface
+a forest stream with a small wooden footbridge
+a meadow at twilight with fireflies just starting to glow
+a marsh at sunrise with mist between the reeds
+a frozen river snaking through a snowy forest
+a desert canyon at midday with sharp shadows
+a tropical lagoon with a single sailboat anchored
+a tundra in summer with low wildflowers
+a bog with cotton grass swaying in the breeze
+a high desert at dawn with sage and distant peaks
+a coastal cliff at sunset with a path along the edge
+a forest of dead trees standing in a flooded valley
+a windswept moorland with stone cairns marking a path
+a temperate rainforest with ferns and dripping moss
+a bay at low tide with a wooden wreck visible
+a quiet bay with seabirds wheeling over fishing boats
+a steppe at dusk with a single rider on horseback
+a lakeshore in autumn with red and yellow trees reflected
+a tundra in winter with the sun barely above the horizon
+a mountain ridge in fog with a single hiker silhouetted
+a savanna with a watering hole at golden hour
+a freshly fallen snow on a bench in a city park
+a country lane after rain with puddles reflecting the sky
+a grass airfield with a small white plane parked at the side
+a sand bar exposed at low tide with seabirds gathering
+a marsh on a calm morning with mist in the reeds
+a frozen lake reflecting alpenglow on a high peak
+a mossy log over a mountain stream
+a beach at dawn with sand still cool and untouched
+a grassy hillside in summer with grazing sheep
+a coastline with a string of small islands offshore
+
+# ---------------------------------------------------------------------------
+# Urban — streets, architecture, vehicles, signage (~200)
+# ---------------------------------------------------------------------------
+a quiet residential street at dawn
+a busy crosswalk in a city center at noon
+a narrow alley with bins and a fire escape
+a downtown intersection at night with light trails from cars
+an old brick building with ivy growing up the side
+a glass office tower reflecting a sunset sky
+a row of victorian terraces in a london street
+a small bakery with fogged windows on a winter morning
+a corner cafe with patrons at an outdoor table
+a hardware store with sandwich-board signs on the sidewalk
+a butcher shop with cuts of meat in the window
+a flower stand on a busy street corner
+a kiosk selling magazines and lottery tickets
+an old wooden phone booth on a quiet corner
+a mailbox at the edge of a leafy suburban street
+a fire hydrant leaking into a curbside puddle
+a stop sign half-covered in stickers
+a yellow taxi waiting at a red light
+a city bus pulling up to a covered stop
+a bicycle locked to a metal rack
+a row of parked scooters on a paris street
+a delivery van blocking half a one-way street
+a tow truck loading an illegally parked car
+a garbage truck working its way down a residential block
+a milk truck making early-morning deliveries
+a postal van with the back door open
+a food truck with a queue at a lunch hour
+an ice cream van with kids gathered at the window
+a hot dog cart with steam rising from the kettle
+a row of bicycles for hire chained to a docking station
+a tram crossing a square with cobblestone tracks
+a metro entrance with a green art-nouveau sign
+a subway platform with passengers waiting at a yellow line
+an empty subway car at midnight
+a busy commuter train pulling into a suburban station
+an overpass with traffic streaming below
+a highway interchange seen from above
+a toll booth on a country highway
+a roundabout with a fountain in the center
+a pedestrian crossing with worn paint
+a one-lane bridge with a give-way sign
+a multistory parking garage at dusk
+a downtown crosswalk at rush hour
+a hipster coffee shop with exposed brick walls
+a vintage cinema with neon marquee letters
+a corner bookstore with stacks visible through the window
+a record shop with crates of vinyl in the doorway
+a barbershop with a striped pole outside
+a tattoo parlor with neon signage in the front window
+a tailor shop with a sewing machine visible through the window
+a thrift store with hanging racks visible from the street
+a corner pub with hanging baskets at the entrance
+a small synagogue between two larger buildings
+a domed church at the end of a narrow street
+a mosque with a minaret over a rooftop view
+a hindu temple with painted figures over the doorway
+a rural chapel with a small graveyard
+a glass-and-steel art museum on a city square
+a brutalist concrete building with weather-stained walls
+an industrial warehouse converted into apartments
+an old mill on a river with a working waterwheel
+a row of victorian factory chimneys against a grey sky
+a boarded-up storefront with peeling paint
+a mural covering the side of an old brick building
+a graffiti tag on a metal roller shutter
+a stencil street-art piece on a concrete wall
+a public sculpture in a small park
+a bronze statue of a politician with pigeons on the head
+an art-deco theater facade at night
+a victorian train station with iron-and-glass roof
+an underground passage with tiled walls
+an underpass with a busker playing acoustic guitar
+a cobblestone street in a rainy european old town
+a street market with fruit and vegetable stalls
+a fish market with crushed ice and styrofoam crates
+a flea market with old radios and tools laid out on a blanket
+a christmas market with wooden stalls and string lights
+a night market with food stalls in southeast asia
+a souk with hanging lamps and spice piles
+a bazaar with woven baskets stacked high
+an outdoor flea market in the rain with vendors covered in tarps
+a high street with closed shops on a sunday afternoon
+a residential street the morning after a snowfall
+a city street flooded after a thunderstorm
+a city park with cherry blossoms and benches
+a small playground with a single child on the swings
+a basketball court with chain nets in a public park
+a tennis court at dusk with floodlights coming on
+a city pool with empty deck chairs and a lifeguard reading
+a community garden behind a chain-link fence
+a small farmers market in a parking lot on a saturday morning
+a row of allotments with sheds and beanpoles
+an old gas station converted into a coffee shop
+a drive-through bank with a single car at the window
+a 24-hour laundromat with one customer reading
+a phone repair shop with handsets in a glass case
+a watch repair shop with clocks visible through the window
+a key cutter's stall in a covered market
+an alley behind a restaurant with stacked crates and a chef on a smoke break
+a fire escape with potted plants on every landing
+a rooftop with laundry hanging on a line
+a rooftop terrace with strung lights and a small grill
+a high-rise window cleaner mid-task on a glass facade
+a construction site with cranes against a city skyline
+a half-built skyscraper with exposed steel framing
+a new apartment block with scaffolding still up
+a freshly paved road with workers spreading hot tar
+roadworks with cones and a flagger directing traffic
+a power line technician at the top of a wooden pole
+a streetlight just turning on at twilight
+a row of bus stop benches with peeling paint
+a public phone with a cracked screen and graffiti
+a public bench with a coffee cup left behind
+a city bench with a newspaper folded on the seat
+a wooden boardwalk along a city waterfront
+a footbridge across a busy highway
+a highway shoulder with a hitchhiker holding a sign
+a long-haul trucker filling up at a midnight gas station
+a gas station at sunset with a single car parked
+a warehouse loading dock with a truck reversing in
+an industrial yard with stacked shipping containers
+a ferry terminal at dawn with foot passengers waiting
+an airport terminal at night with cleaners pushing carts
+a small airfield with a single propeller plane
+a bus depot at dawn with rows of parked buses
+a tram depot with engineers at work on a tram
+a railway crossing with the barrier coming down
+a level crossing in a small town with a freight train passing
+a metro station entrance covered in posters
+a sleek high-speed train pulling into a modern station
+a steam train at a heritage railway station
+a single old wooden caboose abandoned on a siding
+a train carriage being scrapped in a rail yard
+a city skyline at night with a full moon rising
+a city skyline at sunrise from a far-off ridge
+a port with cranes loading a container ship
+a marina with sailboats moored in neat rows
+a small fishing harbor with crab pots stacked on the quayside
+a beachfront promenade with a victorian pier
+a seaside boardwalk with arcade games and a ferris wheel
+a riverside walk with joggers and dog walkers
+a canal with houseboats moored along one side
+a lock-keeper's cottage beside a working canal lock
+a quiet canal with reflections of buildings on the water
+a city square with a fountain and pigeons
+a town square with a clock tower and a war memorial
+a piazza in italy with cafes and a renaissance fountain
+a plaza in mexico with a bandstand and string lights
+a courtyard between three apartment buildings with a single tree
+a hidden garden behind an iron gate in a busy city
+a residential street in summer with kids running in a sprinkler
+a snowy residential street with cars half-buried in drifts
+a rainy bus stop with people sharing umbrellas
+a winter street with shops still lit at dusk
+a busy market street in a southeast asian capital
+a quiet residential street in tokyo at night
+a wide boulevard in paris during morning rush hour
+a narrow street in venice with washing strung overhead
+a steep street in lisbon with a yellow tram
+a stepped alley in a moroccan medina
+a colorful street in havana with classic cars
+a tree-lined avenue in autumn with leaves on the ground
+a riverside walk with weeping willows
+a city park in winter with a frozen pond and skaters
+a rooftop bar at sunset with a city skyline behind
+a hotel lobby with a doorman and a brass luggage cart
+a corner store at night with the lights still on
+a 24-hour diner with a single customer at the counter
+a roadside motel with a flickering vacancy sign
+a quiet train carriage at dawn
+a busy train carriage at rush hour
+a tram passenger looking out at a city street
+a bus passenger leaning against a window with headphones on
+a passenger plane interior with the cabin lights dimmed
+a domestic airport gate area with people sleeping on chairs
+a regional airport at sunrise with one plane on the tarmac
+a small ferry terminal in the rain
+a tug boat pushing a barge upstream
+a freight train crossing a long steel bridge
+a single car driving down a long straight road through cornfields
+a rural intersection with a stop sign and a single farmhouse
+a rural mailbox row at the end of a long driveway
+a rural church with a graveyard at the edge of a field
+a small town main street on a quiet sunday morning
+a small town hardware store with old signage
+a small town diner with a row of stools at the counter
+a small town gas station with a hand-painted sign
+a small town barber shop with a striped pole
+a small town post office with a flag pole
+a small town fire station with a vintage truck visible
+a small town library in an old victorian house
+a county fair with a ferris wheel and food stalls
+a state fair with a livestock barn and tractor pulls
+a parade with marching bands moving down a closed street
+a fourth of july parade with kids on decorated bikes
+a halloween street with carved pumpkins on every step
+a christmas street with wreaths on every door
+a chinese new year parade with a long dragon
+a diwali festival with lit lamps lining a street
+a new years celebration with fireworks over a river
+a wedding procession leaving a small church
+a funeral procession crossing a quiet square
+a protest march with handmade signs filling a wide avenue
+a cyclists' rally crossing a downtown intersection
+a film crew shooting a scene on a closed city street
+
+# ---------------------------------------------------------------------------
+# Food & drink (~150)
+# ---------------------------------------------------------------------------
+a stack of pancakes with maple syrup running down the side
+a bowl of ramen with a soft-boiled egg on top
+a plate of spaghetti carbonara with cracked black pepper
+a margherita pizza fresh from a wood oven
+a hand-formed burger with melted cheese on a sesame bun
+a bowl of pho with herbs and lime on the side
+a sushi platter with assorted nigiri and rolls
+a single croissant on a small white plate
+a baguette torn open beside a wedge of brie
+a wheel of camembert with a knife stuck into it
+a charcuterie board with cured meats and olives
+a wooden board with sliced sourdough and butter
+a slice of new york cheesecake with strawberry sauce
+a chocolate lava cake with vanilla ice cream
+a tiramisu in a glass with cocoa powder on top
+a crème brûlée with the sugar top cracked
+a slice of carrot cake with cream cheese frosting
+a single donut with rainbow sprinkles
+a row of macarons in pastel colors
+a glass jar of peanut butter on a kitchen counter
+a wooden spoon stirring tomato sauce in a pan
+a frying pan with bacon mid-cook
+an omelette folded over with chives on top
+scrambled eggs on toast with avocado
+a poached egg on top of an english muffin
+a bowl of cereal with sliced banana
+a bowl of porridge with berries and honey
+a smoothie bowl topped with granola and chia
+an iced coffee with condensation on the glass
+a flat white with a leaf-shape latte art
+a cappuccino with cocoa dusted on the foam
+a tea pot with a curl of steam rising from the spout
+a glass of fresh-squeezed orange juice
+a pint of beer with a thick foam head
+a glass of red wine on a candlelit table
+a martini with a single olive on a toothpick
+a cocktail with a sprig of mint and a paper straw
+a hot chocolate with whipped cream and marshmallows
+a bowl of chili topped with sour cream and chives
+a steaming bowl of clam chowder with crackers on the side
+a tray of cinnamon buns fresh from the oven
+a slab of brisket on butcher paper with pickles
+a paper cone of fish and chips with vinegar
+a pulled pork sandwich on a brioche bun
+a mexican taco al pastor with pineapple
+a burrito wrapped in foil with one bite taken
+a quesadilla cut into wedges with salsa and guacamole
+a plate of nachos loaded with cheese and jalapeños
+a bowl of guacamole with tortilla chips
+a tray of sushi rolls beside soy sauce and wasabi
+a bento box with rice, fish, and pickled vegetables
+a steaming bowl of dumplings with a small dish of vinegar
+a plate of dim sum on a bamboo steamer
+a wok of stir-fried vegetables with steam rising
+a bowl of laksa with shrimp and bean sprouts
+a plate of pad thai with lime wedges and crushed peanuts
+a bowl of bibimbap with a raw egg yolk in the center
+a grilled cheese sandwich cut diagonally
+a club sandwich held together with toothpicks
+a lobster roll on a buttered split-top bun
+a hot dog with mustard and chopped onion
+a plate of fries with ketchup on the side
+a bag of popcorn at a movie theatre
+a roasted turkey on a platter with rosemary
+a glazed christmas ham with cloves
+a roast chicken with crisp skin and lemon
+a pan of lasagna fresh from the oven
+a casserole dish with the lid just lifted off
+a cast-iron skillet with cornbread inside
+a wooden cutting board with chopped onions and garlic
+a kitchen counter with a half-cut onion and tear stains
+a bowl of fresh basil leaves on a counter
+a cluster of cherry tomatoes on the vine
+a basket of fresh strawberries from a farm stand
+a bunch of carrots with the green tops still on
+a head of romaine lettuce being washed in a sink
+a wedge of watermelon on a hot summer day
+a pile of mangoes at a fruit stall
+a pyramid of oranges in a market display
+a bunch of bananas hanging from a hook
+a cluster of grapes still wet from washing
+a pomegranate cut open with seeds spilling out
+a halved avocado on a wooden board
+a single ripe peach on a windowsill
+a bowl of nuts on a coffee table
+a wedge of dark chocolate broken into squares
+a bar of chocolate with one bite taken
+a glass jar of jam on a sunlit table
+a glass of milk beside a plate of cookies
+a plate of warm chocolate-chip cookies
+a tray of brownies cut into squares
+a slice of apple pie with vanilla ice cream
+a slice of pumpkin pie with whipped cream
+a homemade pie with a lattice top fresh from the oven
+a tray of muffins cooling on a rack
+a tray of cupcakes with buttercream frosting
+a wedding cake with three tiers and fresh flowers
+a birthday cake with lit candles
+a sheet cake with a child's name in icing
+a fresh batch of bagels on a wooden board
+a glass jar of pickles on a kitchen shelf
+a tin of sardines opened on a piece of toast
+a bowl of olives with a small empty dish for pits
+a wedge of feta on a salad with cucumbers and tomatoes
+a bowl of hummus with olive oil pooled on top
+a plate of falafel with tahini and pickled turnips
+a plate of curry with basmati rice and naan
+a plate of biryani with raita
+a thali with several small bowls on a metal tray
+a bowl of dal makhani with butter melting on top
+a banana leaf with rice and curries served family-style
+a bowl of beef rendang with sticky rice
+a plate of sashimi with daikon shreds
+a bowl of miso soup with tofu and seaweed
+a plate of tempura with dipping sauce
+a tray of yakitori skewers with a wedge of lemon
+a bowl of pho with thinly sliced beef
+a plate of dumplings with chili oil on the side
+a steaming bao bun split open with pork inside
+a coconut split open with a straw inside
+a roadside stall selling fresh sugarcane juice
+a market stall stacked with dried chillies
+a stall selling spices in open burlap sacks
+a butcher counter with neat rows of meat
+a fishmonger's display with whole fish on ice
+a cheese counter with wheels and wedges labeled
+a wine cellar with bottles racked floor to ceiling
+a rustic farmhouse kitchen with a pot simmering
+a modern kitchen with a chef plating a dish
+a kitchen island covered with prep ingredients
+a cutting board with neatly chopped vegetables
+a meat thermometer reading the inside of a roast
+a stand mixer working a batch of dough
+a piping bag finishing a row of meringues
+a knife slicing a tomato into perfect rounds
+a kettle on a gas stove with steam pouring out
+a frying pan with butter foaming and just turning brown
+a pasta machine with a sheet of dough being rolled
+a crab boil dumped on a newspaper-covered table
+a crawfish boil with corn and potatoes
+a clam bake at the beach with a dug pit
+a backyard barbecue with steaks on the grill
+a backyard pizza oven with a fire glowing inside
+a smoker with brisket inside and smoke curling out
+a campfire with a dutch oven hanging above
+a hiker eating a granola bar on a rock
+a kid licking an ice-cream cone melting in the heat
+a popsicle dripping down a child's hand
+a slice of birthday cake on a paper plate at a party
+a half-eaten sandwich beside an open laptop at a desk
+a coffee mug with a heart drawn in the foam
+a glass of sparkling water with a slice of lemon
+a glass of iced tea on a wicker table
+a thermos of soup at a tailgate party
+
+# ---------------------------------------------------------------------------
+# Indoor — homes, offices, public spaces (~150)
+# ---------------------------------------------------------------------------
+a cozy living room with a fireplace and a sleeping cat
+a small studio apartment with a futon and a tiny kitchen
+a kitchen with morning light spilling onto the counter
+a kitchen island with hanging copper pots above
+a dining room set for a family dinner
+a bedroom with sunlight through sheer curtains
+an unmade bed with morning light through a window
+a child's bedroom with toys scattered across the floor
+a teenager's bedroom with posters and a desk in the corner
+a home office with two monitors and a coffee mug
+a study with bookshelves to the ceiling and a leather chair
+an attic with sloped ceilings and a single skylight
+a basement workshop with tools hung neatly on a pegboard
+a garage with a half-restored motorcycle
+a garage with a rake, snow shovel, and bicycles
+a laundry room with detergent shelves and a clothes line
+a bathroom with subway tiles and a clawfoot tub
+a small powder room with a vintage mirror
+a hallway with framed family photos along one wall
+a foyer with a wooden bench and rain boots underneath
+a pantry with mason jars labeled and stacked
+a walk-in closet with neatly hanging clothes
+a nursery with a wooden crib and a mobile of stars
+a playroom with a small table and crayons spilled across it
+a sunroom full of houseplants and wicker furniture
+a library inside an old country house
+a music room with a grand piano and sheet music on the stand
+a recording studio with foam walls and a microphone
+a photographer's studio with seamless paper backdrop
+a small art studio with paints and canvases stacked
+a potter's studio with clay-streaked aprons hanging on hooks
+a sewing room with bolts of fabric and a sewing machine
+a meditation room with cushions and a low table
+a yoga studio with hardwood floors and rolled mats
+a dance studio with mirrored walls and a barre
+a small home gym with weights and a treadmill
+a basement bar with a pool table and neon signs
+a man cave with a leather sofa and a big screen
+a finished basement set up as a theater room
+a gaming room with rgb lights and a dual-monitor setup
+a room mid-renovation with paint cans and drop cloths
+a freshly painted empty room with the smell of paint almost visible
+an empty apartment after a move-out with marks on the walls
+an open-plan office with rows of desks
+a small startup office with sit-stand desks and exposed beams
+a coworking space with a coffee bar and a few people on laptops
+a corporate boardroom with a long table and twelve chairs
+a doctor's waiting room with magazines on a side table
+a hospital corridor with fluorescent lighting at night
+an emergency room reception desk with people waiting
+a dentist's chair with the overhead light on
+a vet's exam room with a metal table and a scale
+a hospital room with a single occupied bed and a window
+an x-ray room with a film viewer wall lit up
+a chemistry lab with rows of glassware on shelves
+a biology lab with microscopes and a centrifuge
+a clean room with workers in full bunny suits
+a server room with rows of blinking racks
+a control room with screens covering one wall
+a courtroom with empty benches and a wooden bar
+a police station's front desk on a quiet night
+a fire station bay with a fire truck inside
+a post office sorting room with parcels everywhere
+a bank's main hall with marble floors and tellers
+an old-fashioned bank vault door open
+a museum hallway with art on the walls and benches
+an art gallery with track lighting on a single painting
+a natural history museum with a dinosaur skeleton
+a children's museum with a hands-on exhibit
+a planetarium with a domed ceiling lit up
+a music hall with red velvet seats empty before a show
+a concert hall with the stage lit and the audience seated
+a small theater stage with set still being built
+a black-box theater with chairs in a circle around the actors
+a movie theater with the screen lit and seats half full
+a multiplex lobby with a popcorn counter
+a bowling alley with neon and pinspotters at the far end
+an arcade with rows of cabinets glowing
+a roller rink with kids skating in a circle
+a swimming pool indoor with lap lanes
+a hotel lobby with marble floors and a chandelier
+a hotel room with the bed turned down and a chocolate on the pillow
+a hostel dormitory with bunk beds and shared cubbies
+a bed and breakfast living room with a fire and tea tray
+a small inn with timber beams and a low ceiling
+a coffee shop interior with mismatched chairs and a bookshelf
+a bakery interior with breads displayed in baskets
+a butcher shop interior with cuts of meat in a display case
+an old-fashioned ice cream parlor with red booths
+a diner interior with chrome stools at a counter
+a fast-food restaurant at midnight with a single customer
+a fine-dining restaurant with white tablecloths and candles
+a sushi bar with chefs working behind a glass case
+a dim sum restaurant with carts being pushed between tables
+a noodle shop with steam rising from open kitchen
+a tapas bar with small plates and bottles of wine on shelves
+a pub interior with a dart board and a fire
+a small wine bar with bottles racked behind the counter
+a hipster cocktail bar with brass fixtures and a jazz singer
+a karaoke room with neon and a leather couch
+a billiards hall with green felt tables and overhead lights
+a small bookstore with a reading nook and a sleeping cat
+a record shop with crates of vinyl and posters on the walls
+a vintage clothing store with mannequins and racks
+a department store with displays under bright lighting
+a supermarket aisle with shelves stocked floor to ceiling
+a corner store with a clerk reading behind the counter
+a hardware store interior with rows of paint
+a pharmacy interior with a queue at the prescription counter
+a post office with a wall of small boxes
+a gym interior with weights and rubber flooring
+a martial arts dojo with mats and a kanji on the wall
+a climbing gym interior with a tall bouldering wall
+a swimming pool changing room with lockers and benches
+a sauna interior with wooden benches and a heater
+a hammam interior with steam and stone benches
+a spa treatment room with a massage table and dim lighting
+a salon interior with chairs and mirrored walls
+a barbershop interior with a striped pole and big mirrors
+a tattoo parlor interior with portfolios on a coffee table
+a piercing studio with a clean steel chair
+an ice rink with the surface freshly resurfaced
+a roller derby practice on an outdoor banked track
+a children's classroom with low tables and bright walls
+a high school classroom with desks in rows and a projector
+a university lecture hall with sloped seating
+a school library with study carrels and a globe
+a school cafeteria with trays on a steel rail
+a school gym with a basketball hoop and bleachers
+a school auditorium with a grand piano on stage
+a kindergarten with painted handprints on the wall
+a nursery with cribs and a rocking chair
+an after-school program with kids playing chess
+a preschool art table with finger paints and aprons
+a co-op kindergarten with parents helping out
+a children's reading area at a public library
+a community center hall set up for a wedding
+a town hall meeting with chairs in rows
+a polling station with privacy booths in a school gym
+a senior center with bingo cards on tables
+a homeless shelter with cots in rows
+a soup kitchen with volunteers serving from a long table
+a food bank with shelves of canned goods and pasta
+a community garden's tool shed with neatly hung tools
+a children's hospital play area with toys and a small slide
+a hospice room with a hand-knit blanket on the bed
+a small dental office reception with a fish tank
+a busy emergency department triage area
+a rural medical clinic with a wood-paneled waiting room
+an emergency operations center with maps and screens
+
+# ---------------------------------------------------------------------------
+# Objects & still life (~150)
+# ---------------------------------------------------------------------------
+a single red apple on a wooden table
+a worn leather wallet on a kitchen counter
+a set of keys on a small dish in an entryway
+a smartphone face down on a desk
+a cracked iphone screen on a bed
+a laptop with a coffee ring on the lid
+a wireless headphone case open on a desk
+a pair of running shoes by a front door
+a pair of leather boots beside a hearth
+a pair of high heels at the bottom of a staircase
+a stack of books on a bedside table
+a book open with reading glasses on top
+a fountain pen and a half-written letter
+a set of artist's brushes in a glass jar
+a tube of paint with the cap off
+a sketchbook open to a half-finished drawing
+a wristwatch with the second hand visible
+a vintage pocket watch on a velvet cloth
+a ring in a small velvet box
+a simple silver chain on a dressing table
+a pair of glasses folded on an open book
+a stack of mail on a hallway table
+a crumpled receipt on a kitchen counter
+a coffee mug with a chipped rim
+a tea cup with a small crack along the side
+a stainless steel water bottle on a desk
+a battered thermos beside a hiking pack
+a cast-iron skillet hanging on a hook
+a wooden cutting board with chef's knife
+a bunch of keys with a dozen keychains
+a cluttered desk with notebooks and pens
+an empty water glass on a windowsill
+a vase with three drooping tulips
+a vase with fresh peonies just placed
+an old typewriter on a writing desk
+a dial telephone in a green hallway
+a transistor radio on a kitchen shelf
+an old polaroid camera on a dresser
+a film camera with the lens cap off
+a digital slr on a tripod in a studio
+a pair of binoculars hanging on a hook
+a magnifying glass on top of a stamp album
+an open jewelry box with mismatched earrings
+a small metal tin with old buttons inside
+an iron with the cord wrapped around it
+a sewing kit open on a coffee table
+a knitting basket with half-finished sock
+a spool of thread and a sewing needle
+a pair of scissors on a paper-strewn desk
+a stapler open with a strip of staples
+a roll of tape and a half-cut sheet of wrapping paper
+a bunch of crayons spilled across a table
+a child's drawing taped to a fridge
+a fridge magnet collection of vacation souvenirs
+a coffee maker drip-brewing into a glass pot
+a kettle whistling on a gas stove
+a toaster with two slices of bread popped up
+a microwave with a covered bowl spinning inside
+a blender mid-pulse with smoothie inside
+a stand mixer mid-knead with dough wrapping the hook
+a cast iron pan with a fried egg
+a pressure cooker with steam venting from the top
+a slow cooker with a stew bubbling
+a sushi rolling mat with rice and seaweed
+a chef's knife on a wooden block
+a bread knife mid-slice on a sourdough loaf
+a meat thermometer stuck into a roast
+a wooden rolling pin dusted with flour
+a measuring jug with milk
+a measuring cup overflowing with sugar
+a kitchen scale with a pat of butter on it
+a glass jar of dry pasta on a shelf
+a row of mason jars with grains and beans
+a basket of fresh eggs on a counter
+a wooden bowl of nuts on a coffee table
+a bowl of fresh fruit on a kitchen island
+a paper bag of groceries on a doorstep
+a brown bag lunch with a sandwich peeking out
+a metal lunch box with stickers
+a child's backpack with a stuffed animal sticking out
+a school backpack on a hallway bench
+a satchel hanging from a coat rack
+a leather briefcase on an office chair
+a backpack with hiking pole strapped to the side
+a duffel bag on a hotel bed
+a suitcase with a luggage tag in an airport
+a vintage trunk at the foot of a bed
+a stack of vintage suitcases as side tables
+a guitar case beside a leather couch
+a violin case open with the violin inside
+a saxophone on a stand in a corner
+a harmonica on a wooden table
+a kazoo and a recorder on a child's bed
+a vinyl record on a turntable mid-spin
+a stack of vinyl records beside a turntable
+a cassette tape with the tape pulled out
+an old cd jewel case open on a desk
+a boom box from the eighties on a shelf
+a tube tv with rabbit ears on top
+a flat-screen tv mounted above a fireplace
+a video game controller on a couch
+a chess board mid-game with a king tipped over
+a rubik's cube nearly solved
+a deck of cards spread on a table
+a backgammon board with dice on the side
+a monopoly board mid-game with houses everywhere
+a jigsaw puzzle half-completed on a coffee table
+a single lego brick on a hardwood floor
+a box of crayons with several broken
+a child's wooden train set on a rug
+a doll house with a half-finished living room
+a stuffed bear sitting on a child's bed
+a wooden rocking horse in a sunlit nursery
+a toolbox open with tools spread out on a workbench
+a hammer and a row of nails on a wooden bench
+a level and a tape measure on a kitchen counter
+a paint roller in a tray of paint
+a stepladder under a half-painted wall
+a power drill with a few bits beside it
+a chainsaw on a tarp beside a stack of wood
+a wheelbarrow full of garden mulch
+a garden hose coiled on a hook
+a watering can beside a row of seedlings
+a pair of garden gloves on a dirt-stained workbench
+a pair of pruners on a kitchen counter
+a stack of garden trays with seedlings
+a basket of fresh eggs from the chicken coop
+a wooden honeycomb frame dripping with honey
+a glass jar of homemade jam with a checked cloth top
+a hand-knit scarf in progress with needles and yarn
+a cross-stitched piece in a hoop
+a half-finished crochet blanket on a couch
+a wooden whittling project with shavings on the floor
+a mason jar of pickles fermenting with a glass weight
+an open envelope with a handwritten letter inside
+a stack of postcards from various cities
+a leather journal with a fountain pen on top
+a passport open to a stamp page
+a wedding ring on a velvet cushion
+a baby's first shoes on a dresser
+a wedding bouquet pressed under a glass dome
+a christening gown in tissue paper
+a graduation cap on a kitchen counter
+a single trophy on a mantel
+a sports medal hanging from a doorknob
+a pair of skates with the laces still tied
+a hockey stick leaning by a back door
+a baseball mitt and ball on a porch
+a basketball on a driveway
+a soccer ball on wet grass
+a tennis racket against a court fence
+
+# ---------------------------------------------------------------------------
+# Activities, sports, hobbies (~120)
+# ---------------------------------------------------------------------------
+two friends hiking up a forest trail with backpacks
+a couple paddling a tandem kayak across a lake
+a lone backpacker setting up a tent at dusk
+a climber on a sheer rock face with a rope
+a group of skiers on a chairlift in a snowstorm
+a snowboarder mid-jump in a halfpipe
+a cross-country skier on a wooded trail
+a snowshoer crossing a clearing in deep snow
+a fly fisherman casting in a rocky stream
+a boy fishing off a wooden dock
+a kayaker paddling down a slow river
+a rafter mid-rapid with water spray
+a sailboat heeling over in a strong breeze
+a windsurfer on a choppy bay
+a kitesurfer mid-jump above the waves
+a surfer carving the face of a wave
+a paddleboarder on a glassy lake at dawn
+a swimmer doing laps in an outdoor pool
+a runner pacing on a wooded trail at sunrise
+a marathoner crossing a finish line in pouring rain
+a cyclist standing on the pedals climbing a hill
+a mountain biker mid-jump on a forest trail
+a bmx rider doing a trick at a skate park
+a skateboarder grinding a rail in a downtown plaza
+a roller skater dancing in a city park
+a parkour runner leaping between rooftops
+a basketball player shooting a free throw
+a soccer player about to take a corner kick
+a tennis player serving from the baseline
+a golfer driving from a tee on a sunny course
+a baseball pitcher mid-windup in a stadium
+a football quarterback handing off to a running back
+a rugby player breaking a tackle
+a hockey player checking another into the boards
+a lacrosse player scoring a goal
+a fencer in mid-lunge in a sport hall
+a boxer hitting a heavy bag in a gym
+a karate student breaking a board with a kick
+a judo throw being executed on a mat
+a kickboxer training with pads
+a powerlifter mid-deadlift in a competition
+a weightlifter performing a clean and jerk
+a gymnast on a balance beam mid-routine
+a diver mid-flip from a high platform
+a synchronized swimming team in formation
+a ballet dancer rehearsing in front of a mirror
+a tap dancer practicing on a wooden floor
+a salsa couple dancing on a city sidewalk
+a hip-hop crew rehearsing in a studio
+a folk dancer in traditional costume at a festival
+a knitter at a craft fair with a basket of yarn
+a quilter at her sewing machine
+a wood carver mid-project on a porch
+a jeweler at a workbench setting a stone
+a calligrapher mid-stroke on a long piece of paper
+a watercolor painter at an easel outdoors
+a plein-air oil painter on a cliff
+a sculptor chipping at a marble block
+a printmaker pulling a print from an etching press
+a glassblower shaping a vase at a furnace
+a blacksmith hammering hot metal on an anvil
+a leatherworker stamping a wallet on a workbench
+a beekeeper inspecting a frame in protective gear
+a beekeeper extracting honey from a comb
+a vintner pruning vines in winter
+a vintner tasting wine from a barrel in a cellar
+a brewer stirring a mash tun with a long paddle
+a brewer pulling a sample from a fermenter
+a baker shaping a sourdough loaf
+a baker scoring a loaf with a razor blade
+a chef breaking down a whole fish
+a chef plating a tasting menu course
+a barista pulling latte art on a flat white
+a barista weighing beans for a brew
+a bartender shaking a cocktail at a polished bar
+a bartender lighting a flame on top of a cocktail
+a sommelier pouring wine into a tasting glass
+a butcher breaking down a side of beef
+a fisherman pulling a crab pot up from the dock
+a clammer working at low tide with a rake
+an oysterman shucking oysters at a market stall
+a hunter glassing a ridge with binoculars
+a falconer with a bird of prey on a gloved hand
+a dog trainer working with a malinois on a long line
+a horse trainer working a young horse on a lunge line
+a rancher branding a calf at a roundup
+a sheep shearer mid-shear with wool piling up
+a farmer driving a tractor through a wheat field
+a farmer harvesting potatoes by hand
+a flower farmer cutting blooms for market
+a beekeeper preparing a smoker
+a forager gathering mushrooms on a forest floor
+a gardener pruning roses in summer
+a gardener mulching a vegetable bed
+a kid learning to ride a bike with training wheels
+a kid learning to swim with arm floaties
+a kid learning to read with a parent
+a kid practicing piano with sheet music
+a kid drawing with chalk on a driveway
+a kid building a fort out of couch cushions
+a kid jumping into a pile of leaves
+a kid running through sprinklers in a backyard
+a kid riding a sled down a snowy hill
+a kid eating watermelon with juice running down the chin
+a teenager learning to drive in a parking lot
+a teenager studying late at night with a single lamp
+a teenager moving boxes into a college dorm
+a couple painting their first apartment together
+a couple assembling furniture from instructions
+a couple cooking together in a small kitchen
+a couple making sushi at home with a rolling mat
+a couple sitting on a porch swing on a summer evening
+a couple walking on a beach at low tide
+a couple laughing at a joke at a kitchen table
+a group of friends playing board games at a coffee table
+a group of friends playing poker around a kitchen table
+a group of friends watching a sports game at a bar
+a group of friends going camping in a forest
+a group of friends paddling canoes on a lake
+a group of friends having a backyard barbecue
+a group of friends at a karaoke bar
+a group of friends taking a holiday photo
+a family eating dinner together at a long table
+a family decorating a christmas tree together
+a family carving pumpkins on the porch
+a family visiting a pumpkin patch
+a family hiking a national park trail
+a family at the beach with sandcastles and umbrellas
+a family on a camping trip with a fire and marshmallows
+a family on a road trip with snacks in the back seat
+a family on a ski trip getting lift tickets
+a family doing chores together on a saturday morning
+
+# ---------------------------------------------------------------------------
+# Travel, transport, landmarks (~130)
+# ---------------------------------------------------------------------------
+a backpacker waiting at a quiet train station at dawn
+a tourist looking up at a cathedral ceiling
+a couple posing in front of the eiffel tower
+a lone traveler watching the sun set over angkor wat
+a tour group photographing the colosseum
+a tourist holding a map in front of the brandenburg gate
+a hiker on a switchback trail in patagonia
+a monk lighting candles inside a tibetan monastery
+a farmer ploughing a field in tuscany
+a fisherman mending nets in a portuguese harbor
+a couple walking through a flower market in amsterdam
+a tram crossing a square in lisbon
+a vespa parked outside a roman cafe
+a paris cafe table with two espressos and a newspaper
+a london bus crossing tower bridge
+a black cab waiting at a piccadilly intersection
+a new york yellow cab pulling up to a curb
+a chicago el train passing between buildings
+a san francisco cable car climbing a steep hill
+a las vegas strip at night with neon
+a route 66 motel sign at sunset
+a roadside diner along a desert highway
+a long highway stretching toward distant mountains
+a pickup truck stopped at an overlook on a mountain pass
+a campervan parked at a forest clearing
+a vintage volkswagen bus on a coastal road
+an old wood-paneled station wagon on a country road
+a rented car parked at a national park visitor center
+a mototaxi waiting at a thai night market
+a tuk-tuk weaving through bangkok traffic
+a rickshaw on a quiet calcutta street
+a horse-drawn carriage on a cobblestone street
+a steam train pulling out of a small heritage station
+a sleeper train carriage at night with curtains drawn
+a high-speed train passing a rural station too fast to see
+a freight train crossing a long iron bridge over a river
+a mountain railway with a single carriage on a steep grade
+a cable car ascending a snowy peak
+a gondola crossing a venetian canal at dusk
+a riverboat docked beside an old town
+a houseboat on a quiet canal
+a yacht anchored in a turquoise bay
+a fishing trawler returning to port at dawn
+a container ship steaming past a coastline
+a ferry crossing a misty fjord
+a lighthouse keeper's cottage on a rocky promontory
+a coastal village with white houses and red roofs
+a swiss village with timber chalets and cow bells
+a moroccan medina with narrow alleys and lanterns
+a kyoto street with wooden tea houses
+a kyoto bamboo grove with sunlight filtering through
+a tokyo crossing at rush hour from above
+a tokyo neon alley with a small ramen shop
+a hong kong skyline at night from victoria peak
+a singapore marina at dusk with the cityscape behind
+a sydney opera house at sunrise from the harbor
+a sydney harbor bridge with ferries crossing below
+a melbourne laneway with cafes and street art
+a perth beach at sunset
+a queenstown lake at dawn with mountains
+a maori meeting house at the edge of a forest
+a buenos aires tango couple in a plaza
+a brazilian carnival float covered in feathers
+a colombian coffee farm on a steep hillside
+a peruvian terraced field high in the andes
+a mexican market with bright textiles and pottery
+a guatemalan village square on market day
+a cuban classic car on a havana street
+a jamaican beach with reggae playing softly
+a kenyan safari jeep at a watering hole
+a tanzanian sunrise behind kilimanjaro
+a moroccan sahara camp at sunset
+an egyptian temple wall with hieroglyphs
+a bedouin tent in a saharan dune field
+a syrian souk with copper pots stacked
+a turkish carpet shop with rolls stacked to the ceiling
+a greek island village with white houses and blue domes
+a santorini sunset from a cliffside terrace
+an italian trattoria with checkered tablecloths
+a venetian gondola on a quiet canal at dawn
+a tuscan farmhouse with cypress trees lining the drive
+a provence lavender field at full bloom
+a scottish highlands sheep crossing a single-track road
+an irish coastal cliff with a stone tower
+a welsh stone bridge over a rocky stream
+a prague rooftop view at sunrise
+a budapest bath house at dusk
+a vienna coffeehouse with marble tables
+a swiss train passing a snow-covered village
+a norwegian fjord with a small village at the head
+a icelandic geothermal pool at sunset
+a finnish sauna by a lake at dusk
+a danish hygge living room with candles
+a dutch canal house with a bicycle in front
+a belgian beer cafe with a dozen taps
+a polish town square with christmas market stalls
+a russian orthodox church with onion domes
+a transsiberian train carriage at night
+a mongolian ger in a wide grassland
+a japanese fish auction at dawn
+a south korean palace courtyard with autumn leaves
+a thai beach with longtail boats
+a vietnamese rice terrace at sunrise
+a cambodian river village with stilt houses
+a balinese temple with offerings at the gate
+a malaysian street food stall at night
+a singaporean hawker center at lunchtime
+an australian outback road with red dirt
+an alaskan fishing village with snowy peaks
+a canadian rockies hike with a glacier in view
+a new england lighthouse on a stormy day
+a southern usa porch with rocking chairs and sweet tea
+a midwest small town water tower with the town name
+a route 1 viewpoint along the california coast
+a great smoky mountain trail in autumn
+a yellowstone hot spring with steam rising
+a yosemite valley view at golden hour
+a grand canyon viewpoint with tourists at the rim
+a monument valley road stretching toward the buttes
+a death valley salt flat at noon
+a niagara falls boat in mist
+a maine lobster shack with red picnic tables
+a florida beach pier at sunset
+a new orleans balcony with iron railing and flowers
+a key west sunset crowd at a pier
+a chicago lakefront beach with the skyline behind
+a seattle pike place market with vendors
+a portland food cart pod
+a banff lake with mountains reflected
+a quebec city old town in winter
+a montreal jazz club exterior with a small marquee
+a halifax harbor with a tall ship docked
+a hawaiian luau at sunset with hula dancers
+a fiji overwater bungalow at dusk
+a tahitian lagoon with a small outrigger canoe
+a maldives white-sand beach at high noon
+a galapagos beach with sea lions on the sand
+a patagonian glacier face calving into a bay
+
+# ---------------------------------------------------------------------------
+# Macro / close-up / textures (~80)
+# ---------------------------------------------------------------------------
+close-up of dew on a single blade of grass
+close-up of a honeybee on a sunflower
+close-up of an ant carrying a leaf fragment
+close-up of a snail's eye stalks
+close-up of a butterfly wing showing scales
+close-up of frost crystals on a window
+close-up of a single snowflake on a black wool glove
+close-up of raindrops on a spider web
+close-up of the texture of weathered wood
+close-up of peeling paint on an old door
+close-up of rust on a metal gate
+close-up of moss growing on a stone wall
+close-up of lichen on a tree branch
+close-up of mushrooms growing from a fallen log
+close-up of cracked earth in a dry riverbed
+close-up of pebbles on a beach with seawater receding
+close-up of beach sand showing tiny shells
+close-up of fingerprints on a wine glass
+close-up of a thumbprint in soft clay
+close-up of stitching on a pair of jeans
+close-up of the weave of a wool sweater
+close-up of a single feather on a forest floor
+close-up of an eye showing the iris
+close-up of a baby's hand grasping an adult finger
+close-up of an old hand holding a teacup
+close-up of fingertips covered in flour
+close-up of a paint brush dipped in paint
+close-up of pencil shavings on a desk
+close-up of a typewriter key mid-strike
+close-up of a sewing needle and thread
+close-up of a piano keyboard with one key pressed
+close-up of guitar strings being plucked
+close-up of a violin's f-hole
+close-up of a vinyl record's grooves
+close-up of a clock's mechanism with gears visible
+close-up of a watch's movement under a loupe
+close-up of a magnifying glass over text
+close-up of a fountain pen nib touching paper
+close-up of a candle flame with the wick visible
+close-up of a match igniting against the strike strip
+close-up of bubbles in a glass of carbonated water
+close-up of espresso pouring from a portafilter
+close-up of milk being steamed in a pitcher
+close-up of latte art being poured
+close-up of a wine glass with light through the rim
+close-up of a tea bag tag in a hot cup
+close-up of bread crust with deep cracks
+close-up of a freshly cut sourdough showing crumb
+close-up of melting butter on warm toast
+close-up of a runny egg yolk being broken
+close-up of citrus zest being grated
+close-up of garlic cloves being crushed
+close-up of fresh basil being torn
+close-up of a steaming bowl of soup with herbs floating
+close-up of melted cheese stretching off a slice of pizza
+close-up of soy sauce being poured into sushi rolls
+close-up of caramel being drizzled
+close-up of chocolate being broken from a bar
+close-up of fresh strawberries with droplets of water
+close-up of a peeled lychee
+close-up of a halved fig showing the flesh
+close-up of a halved pomegranate
+close-up of a wedge of brie at room temperature
+close-up of a cluster of barley in a field
+close-up of a corn cob with kernels visible
+close-up of a coffee cherry on a branch
+close-up of a tea leaf being plucked
+close-up of a cocoa pod cut open
+close-up of a vanilla pod sliced lengthwise
+close-up of a cinnamon stick with curling layers
+close-up of saffron threads on a white plate
+close-up of black peppercorns in a small mortar
+close-up of fresh ginger root being grated
+close-up of sesame seeds being toasted
+close-up of bubbling kombucha in a glass jar
+close-up of a sourdough starter with bubbles
+close-up of yarn winding off a spinning wheel
+close-up of a knitting needle pulling a stitch
+close-up of a bicycle chain with a few drops of oil
+close-up of a brass instrument's valve being pressed
+
+# ---------------------------------------------------------------------------
+# Casual snapshots — phone photos, mundane scenes (~100)
+# ---------------------------------------------------------------------------
+a slightly blurry phone snapshot of a coffee cup on a desk
+a slightly out-of-focus photo of a friend laughing across a table
+a casual snapshot of feet in flip-flops on a beach
+a casual snapshot of a dog asleep with one paw twitching
+a phone photo of the contents of a fridge
+a phone photo of a strangely formed potato
+a phone photo of a parking ticket on a windshield
+a phone photo of a long supermarket queue
+a phone photo of a leaning street sign after a storm
+a phone photo of a dropped ice cream cone on a sidewalk
+a phone photo of a half-eaten sandwich at a desk
+a phone photo of a flat tire on a country road
+a phone photo of a beautiful sunset taken through a windshield
+a phone photo of a flower bouquet held up against a brick wall
+a phone photo of a dog in a stroller
+a phone photo of a cat that just woke up looking annoyed
+a phone photo of a backyard chicken caught mid-step
+a phone photo of a ridiculous traffic jam
+a phone photo of a small boy showing his missing tooth
+a phone photo of a child's drawing held up to a fridge
+a casual photo of a friend asleep on a long flight
+a casual photo of a sunburned arm on a beach towel
+a casual photo of a half-built ikea bookshelf with manual
+a casual photo of a neighbor's cat sitting on a fence
+a casual photo of muddy boots after a hike
+a casual photo of an unmade bed with a curled-up dog
+a phone photo of a sock missing its match in a laundry basket
+a phone photo of a plate of leftovers in a fridge
+a phone photo of a bag of potato chips popped open
+a phone photo of a coffee spilled on a notebook
+a phone photo of a friend asleep in a hammock
+a phone photo of the inside of an oven mid-roast
+a phone photo of a fresh haircut taken in a barbershop mirror
+a phone photo of a new tattoo wrapped in cling film
+a phone photo of a bandaged finger
+a phone photo of a sunburn line at the edge of a sock
+a phone photo of an old sweatshirt with paint stains
+a casual photo of a backyard tomato plant with the first ripe fruit
+a casual photo of a half-eaten birthday cake
+a casual photo of a child's lego creation
+a casual photo of a garage with a half-built shelving unit
+a casual photo of a dog wearing a toddler's hat
+a casual photo of a cat sitting in an empty cardboard box
+a phone photo of a plant slowly recovering after being almost dead
+a phone photo of a herb garden on a kitchen window
+a phone photo of a freshly baked loaf cooling on a rack
+a phone photo of a soup pot half-emptied
+a phone photo of a takeout bag from a favorite restaurant
+a phone photo of a strange sky just before a storm
+a phone photo of a moon barely visible through clouds
+a phone photo of a friend's hand holding up a beer at a bar
+a phone photo of a karaoke screen mid-song
+a phone photo of a wrong turn on a winding country road
+a phone photo of a parking lot full of identical white cars
+a casual photo of three friends squeezed into a photobooth
+a casual photo of a wedding ring on a kitchen counter
+a casual photo of an empty wine bottle and two glasses
+a casual photo of a half-eaten pizza in the box
+a casual photo of a fridge covered in magnets and notes
+a casual photo of a child sleeping in a car seat
+a casual photo of two siblings arguing at a kitchen table
+a casual photo of a dog with food crumbs on its nose
+a casual photo of a cat crashing into a glass door
+a phone photo of a friend mid-blink looking goofy
+a phone photo of three people awkwardly mid-cheers
+a phone photo of a bad parking job in a tight spot
+a phone photo of a road sign reading something funny
+a phone photo of a car odometer reading a milestone number
+a casual photo of a grandmother kissing a baby
+a casual photo of a grandfather teaching a child to fish
+a phone photo of a child's first lost tooth in a tissue
+a phone photo of a runner's bib pinned to a shirt
+a phone photo of a marathon medal on a kitchen counter
+a phone photo of a sunburn line on a back
+a phone photo of a melted candle on a birthday cake
+a phone photo of a tipped over wine glass with a stain
+a phone photo of a popped balloon on a kitchen floor
+a phone photo of a fresh layer of snow on a parked car
+a phone photo of a leaf stuck to the windshield
+a phone photo of a deer crossing a suburban street
+a phone photo of a raccoon caught in the porch light
+a phone photo of a bird's nest with eggs in a hedge
+a phone photo of a stranger's cute dog on a walk
+a phone photo of someone's elaborate latte art
+a phone photo of a half-empty plate at a fancy restaurant
+a phone photo of an unidentifiable food at a buffet
+a phone photo of a bag of groceries spilling in a parking lot
+a phone photo of an unfortunate haircut taken in a mirror
+a phone photo of a sock with a hole in the toe
+a phone photo of a hand-drawn sign at a roadside stand
+a phone photo of a freshly washed car in a driveway
+a phone photo of an old photo found in an attic box
+a phone photo of a half-completed jigsaw puzzle
+a phone photo of a jar with the lid stuck shut
+a phone photo of a pile of unopened mail on a doormat
+a phone photo of a child's first attempt at a haircut
+a phone photo of a homemade cake that didn't quite work
+a phone photo of a tent set up in the backyard
+a phone photo of a campfire with marshmallows on sticks
+a phone photo of a wave that just splashed the camera
+a phone photo of a bug on a windshield
+a phone photo of a flat soda fizz in a glass

scripts/dataset/pull_stage3a_data.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# Pull Stage 3A AI-generated images and manifests from the RunPod
+# generation box to the local training laptop.
+#
+# Run from Git Bash on Windows (not PowerShell — rsync line continuations
+# and POSIX paths are messy in PowerShell).
+#
+# Usage:
+#   bash scripts/dataset/pull_stage3a_data.sh <phase>
+#
+# Phases:
+#   stable           Pull SDXL + SD 3.5 Medium. Safe to run anytime; those
+#                    dirs are stable and not being written to. Run NOW
+#                    while AuraFlow is still generating to save time.
+#   auraflow         Pull AuraFlow. Verifies generation has stopped first
+#                    by polling the file count twice; refuses if still
+#                    growing.
+#   await_auraflow   Wait (polling pod every 60s) until the auto_stop
+#                    watcher logs "STOP signal sent", then automatically
+#                    run the auraflow phase. Use this to walk away and
+#                    have the AuraFlow pull happen unattended overnight.
+#   verify           Compare counts + sizes on pod and laptop. Read-only.
+#   help             Show this message.
+#
+# All rsync transfers are resumable: if the network drops or you Ctrl-C,
+# re-run the same command and it picks up from the last fully-written
+# file.
+#
+# Override defaults with env vars: POD_HOST, POD_PORT, POD, SSH_KEY,
+# POD_SRC, DEST.
+
+set -euo pipefail
+
+POD_HOST=${POD_HOST:-195.26.233.56}
+POD_PORT=${POD_PORT:-57837}
+SSH_KEY=${SSH_KEY:-/c/Users/Dulip/.ssh/id_ed25519}
+POD=${POD:-root@$POD_HOST}
+POD_SRC=${POD_SRC:-/workspace/deepfakescanner/data/raw/ai_generated}
+DEST=${DEST:-/c/Data/Personal/work/Projects/deepfakescanner/data/raw/ai_generated}
+
+ssh_cmd="ssh -o ConnectTimeout=10 -p $POD_PORT -i $SSH_KEY"
+phase=${1:-help}
+
+pull_dir() {
+  local name=$1
+  echo
+  echo "=== Pulling $name ==="
+  rsync -avh --partial --info=progress2 -e "$ssh_cmd" \
+    "$POD:$POD_SRC/$name/" "$DEST/$name/"
+}
+
+pull_files() {
+  echo
+  echo "=== Pulling manifests: $* ==="
+  for f in "$@"; do
+    rsync -avh -e "$ssh_cmd" "$POD:$POD_SRC/$f" "$DEST/"
+  done
+}
+
+await_watcher_fired() {
+  # Polls the pod's auto_stop.log every 60s for the "STOP signal sent"
+  # line written by auto_stop.sh when the AuraFlow output dir reaches
+  # 10,000 files. Prints a periodic heartbeat with the current AuraFlow
+  # count so the operator has evidence the watcher loop is alive.
+  #
+  # If the SSH check fails transiently (network blip, pod restart, etc.)
+  # the if-block treats it as "not yet" and the loop simply retries.
+  echo "=== Watching auto_stop.log for STOP signal (polling pod every 60s) ==="
+  local n=0 cur
+  while true; do
+    if $ssh_cmd "$POD" "grep -q 'STOP signal sent' /workspace/deepfakescanner/auto_stop.log 2>/dev/null"; then
+      echo
+      echo "[$(date +%H:%M:%S)] Watcher fired."
+      return 0
+    fi
+    n=$((n + 1))
+    if [ $((n % 5)) -eq 0 ]; then
+      cur=$($ssh_cmd "$POD" "ls $POD_SRC/auraflow-v0.3 2>/dev/null | wc -l" 2>/dev/null || echo '?')
+      echo "[$(date +%H:%M:%S)] still waiting — AuraFlow count: $cur / 10000"
+    fi
+    sleep 60
+  done
+}
+
+verify_auraflow_stopped() {
+  echo "=== Confirming AuraFlow generation has stopped ==="
+  local c1 c2
+  c1=$($ssh_cmd "$POD" "ls $POD_SRC/auraflow-v0.3 2>/dev/null | wc -l")
+  sleep 10
+  c2=$($ssh_cmd "$POD" "ls $POD_SRC/auraflow-v0.3 2>/dev/null | wc -l")
+  if [ "$c1" != "$c2" ]; then
+    echo "ERROR: AuraFlow still generating ($c1 -> $c2 in 10s)."
+    echo "Wait for the auto_stop watcher to fire — check the log:"
+    echo "    $ssh_cmd $POD 'tail /workspace/deepfakescanner/auto_stop.log'"
+    echo "Then re-run: $0 auraflow"
+    exit 2
+  fi
+  echo "Stable at $c1 images."
+}
+
+verify_counts() {
+  echo
+  echo "=== Verification: pod vs laptop ==="
+  echo "--- POD ---"
+  $ssh_cmd "$POD" \
+    "cd $POD_SRC && for d in */; do printf '%-22s %6d files  %s\n' \"\$d\" \"\$(ls \"\$d\" | wc -l)\" \"\$(du -sh \"\$d\" | cut -f1)\"; done"
+  echo "--- LAPTOP ---"
+  for d in "$DEST"/*/; do
+    [ -d "$d" ] || continue
+    printf '%-22s %6d files  %s\n' "$(basename "$d")/" "$(ls "$d" | wc -l)" "$(du -sh "$d" | cut -f1)"
+  done
+}
+
+mkdir -p "$DEST"
+
+case "$phase" in
+  stable)
+    pull_dir sdxl
+    pull_dir sd35-medium
+    pull_files sdxl_manifest.csv sd35_medium_manifest.csv
+    verify_counts
+    ;;
+  auraflow)
+    verify_auraflow_stopped
+    pull_dir auraflow-v0.3
+    pull_files auraflow_manifest.csv
+    verify_counts
+    ;;
+  await_auraflow)
+    await_watcher_fired
+    verify_auraflow_stopped
+    pull_dir auraflow-v0.3
+    pull_files auraflow_manifest.csv
+    verify_counts
+    ;;
+  verify)
+    verify_counts
+    ;;
+  help|--help|-h)
+    sed -n '2,30p' "$0"
+    ;;
+  *)
+    echo "Unknown phase: $phase" >&2
+    sed -n '2,30p' "$0" >&2
+    exit 1
+    ;;
+esac

scripts/dataset/rebuild_auraflow_manifest.py

@@ -0,0 +1,226 @@
+"""
+Reconstruct the full AuraFlow manifest from the images on disk.
+
+Why this exists
+---------------
+During Stage 3A generation, `generate_auraflow_synthetic.py` flushed manifest
+rows to CSV in batches. A SIGINT stopped the run before the next flush, so
+`data/raw/ai_generated/auraflow_manifest.csv` was left as a 982-byte stub
+holding only the first 2 of 10,000 rows. This was noticed at the time and
+judged "not blocking training" on the basis that per-image generator labels
+come from the directory structure.
+
+That judgement was wrong. `split.py` builds splits from the *manifests*, not
+from the directory tree, so 9,998 AuraFlow images were silently excluded from
+every split — and therefore from training. See
+`docs/stage3a-implementation.md` (2026-08-02 Handoff Log entry).
+
+How the reconstruction works (and why it is exact, not a guess)
+--------------------------------------------------------------
+Output filenames are content-addressed from the (prompt, seed) pair by
+`generation_utils.py`, and every generator in Stage 3A consumed the same
+`prompts.txt` with the same deterministic seed derivation. Consequence: the
+set of AuraFlow filenames is an exact subset of the SDXL and SD 3.5 filename
+sets, and a given filename denotes the same (prompt, seed) in all three.
+
+So for each AuraFlow image on disk we recover prompt/seed/width/height by
+joining on basename against a sibling manifest, and take the AuraFlow-specific
+columns (source, license, model_id, generation params) from the surviving stub
+rows. `sha256` is computed from the actual file, never copied.
+
+**Self-check:** the two genuine stub rows are re-derived by the same code path
+and compared field-by-field against the originals. If the reconstruction
+method were wrong, this check fails and the script refuses to write.
+
+Usage
+-----
+    python scripts/dataset/rebuild_auraflow_manifest.py            # dry run
+    python scripts/dataset/rebuild_auraflow_manifest.py --write
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import sys
+from pathlib import Path
+
+MANIFEST_FIELDS = [
+    "path",
+    "class",
+    "source",
+    "license",
+    "license_url",
+    "sha256",
+    "generator",
+    "model_family",
+    "model_id",
+    "prompt",
+    "seed",
+    "width",
+    "height",
+    "generation_params_json",
+]
+
+# Columns that are identical for every row of a single generator's run.
+_CONSTANT_FIELDS = [
+    "class",
+    "source",
+    "license",
+    "license_url",
+    "generator",
+    "model_family",
+    "model_id",
+    "generation_params_json",
+]
+# Columns recovered from a sibling generator's manifest via the (prompt, seed)
+# identity encoded in the content-addressed filename.
+_JOINED_FIELDS = ["prompt", "seed", "width", "height"]
+
+
+def _sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _read_rows(path: Path) -> list[dict[str, str]]:
+    with path.open(newline="", encoding="utf-8") as fh:
+        return list(csv.DictReader(fh))
+
+
+def _build_row(
+    image: Path,
+    repo_root: Path,
+    constants: dict[str, str],
+    joined: dict[str, str],
+) -> dict[str, str]:
+    rel = image.relative_to(repo_root / "data").as_posix()
+    row = {"path": rel, "sha256": _sha256(image)}
+    row.update(constants)
+    row.update(joined)
+    return {f: row.get(f, "") for f in MANIFEST_FIELDS}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--write", action="store_true", help="write the manifest")
+    parser.add_argument(
+        "--repo-root", type=Path, default=Path(__file__).resolve().parents[2]
+    )
+    parser.add_argument("--generator-dir", default="auraflow-v0.3")
+    parser.add_argument("--stub", default="auraflow_manifest.csv")
+    parser.add_argument(
+        "--reference",
+        default="sdxl_manifest.csv",
+        help="sibling manifest to recover prompt/seed from",
+    )
+    args = parser.parse_args()
+
+    ai_dir = args.repo_root / "data" / "raw" / "ai_generated"
+    img_dir = ai_dir / args.generator_dir
+    stub_path = ai_dir / args.stub
+    ref_path = ai_dir / args.reference
+
+    for p in (img_dir, stub_path, ref_path):
+        if not p.exists():
+            print(f"ERROR: missing {p}", file=sys.stderr)
+            return 1
+
+    stub_rows = _read_rows(stub_path)
+    if not stub_rows:
+        print(f"ERROR: {stub_path} has no rows to learn constants from", file=sys.stderr)
+        return 1
+
+    # Constant columns must agree across every surviving stub row, else the
+    # "one generator run = one set of constants" assumption is false.
+    constants: dict[str, str] = {}
+    for field in _CONSTANT_FIELDS:
+        values = {r.get(field, "") for r in stub_rows}
+        if len(values) != 1:
+            print(
+                f"ERROR: stub rows disagree on constant field {field!r}: {values}",
+                file=sys.stderr,
+            )
+            return 1
+        constants[field] = values.pop()
+
+    ref_by_name = {
+        Path(r["path"]).name: {f: r.get(f, "") for f in _JOINED_FIELDS}
+        for r in _read_rows(ref_path)
+    }
+
+    images = sorted(img_dir.glob("*.png"))
+    if not images:
+        print(f"ERROR: no PNGs under {img_dir}", file=sys.stderr)
+        return 1
+
+    missing = [p.name for p in images if p.name not in ref_by_name]
+    if missing:
+        print(
+            f"ERROR: {len(missing)} images have no match in {args.reference} "
+            f"(first: {missing[:3]}). Reconstruction would be a guess — refusing.",
+            file=sys.stderr,
+        )
+        return 1
+
+    # ---- Self-check against the genuine stub rows -----------------------
+    print(f"Self-check: re-deriving {len(stub_rows)} known-good stub row(s)...")
+    for original in stub_rows:
+        name = Path(original["path"]).name
+        rebuilt = _build_row(
+            img_dir / name, args.repo_root, constants, ref_by_name[name]
+        )
+        for field in MANIFEST_FIELDS:
+            if rebuilt[field] != original.get(field, ""):
+                print(
+                    f"ERROR: self-check FAILED on {name} field {field!r}\n"
+                    f"  original : {original.get(field, '')!r}\n"
+                    f"  rebuilt  : {rebuilt[field]!r}\n"
+                    "The reconstruction method does not reproduce known-good "
+                    "rows. Refusing to write.",
+                    file=sys.stderr,
+                )
+                return 1
+        print(f"  OK {name}")
+
+    print(f"\nRebuilding {len(images)} rows (hashing {len(images)} files)...")
+    rows = []
+    for i, image in enumerate(images, 1):
+        rows.append(_build_row(image, args.repo_root, constants, ref_by_name[image.name]))
+        if i % 1000 == 0:
+            print(f"  {i}/{len(images)}")
+
+    shas = {r["sha256"] for r in rows}
+    if len(shas) != len(rows):
+        print(
+            f"WARNING: {len(rows) - len(shas)} duplicate sha256 values — "
+            "build_manifest.py will drop these as duplicates.",
+            file=sys.stderr,
+        )
+
+    print(f"\n  rows        : {len(rows)}")
+    print(f"  unique sha  : {len(shas)}")
+    print(f"  stub had    : {len(stub_rows)}")
+
+    if not args.write:
+        print("\nDry run - pass --write to overwrite the stub manifest.")
+        return 0
+
+    backup = stub_path.with_suffix(".csv.stub-backup")
+    if not backup.exists():
+        backup.write_bytes(stub_path.read_bytes())
+        print(f"\n  backed up original stub -> {backup}")
+
+    with stub_path.open("w", newline="", encoding="utf-8") as fh:
+        writer = csv.DictWriter(fh, fieldnames=MANIFEST_FIELDS)
+        writer.writeheader()
+        writer.writerows(rows)
+    print(f"  wrote {stub_path} ({len(rows)} rows)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/dataset/run_stage3a_smoke.py

@@ -0,0 +1,472 @@
+"""
+Run a tiny local Stage 3A smoke pipeline.
+
+This creates fixture images and manifest fragments, then runs the real
+manifest builder, generator-aware splitter, augmentation script, CLIP-head
+trainer, and checkpoint evaluator. Embeddings are deterministic mocked
+512-dimensional vectors so this smoke test proves script compatibility without
+downloading or running CLIP.
+
+Usage
+-----
+    python scripts/dataset/run_stage3a_smoke.py \
+        --work-dir /tmp/deepfakescanner-stage3a-smoke
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from PIL import Image
+
+
+CLASS_TO_LABEL = {
+    "authentic": 0,
+    "ai_generated": 1,
+}
+
+FIELDNAMES = [
+    "path",
+    "class",
+    "source",
+    "license",
+    "license_url",
+    "sha256",
+    "generator",
+    "model_family",
+    "model_id",
+    "prompt",
+    "seed",
+    "width",
+    "height",
+    "generation_params_json",
+]
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def _reset_work_dir(work_dir: Path) -> None:
+    if not work_dir.exists():
+        return
+    marker = work_dir.name.lower()
+    if "smoke" not in marker:
+        raise ValueError(
+            f"Refusing to clear existing non-smoke work dir: {work_dir}. "
+            "Choose a scratch path with 'smoke' in the final directory name "
+            "or pass --keep-existing."
+        )
+    shutil.rmtree(work_dir)
+
+
+def _sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _write_image(path: Path, color: tuple[int, int, int]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    image = Image.new("RGB", (32, 24), color=color)
+    image.save(path, format="PNG")
+
+
+def _write_csv(path: Path, rows: list[dict[str, str]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=FIELDNAMES)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def _fixture_row(
+    data_root: Path,
+    relative_path: str,
+    *,
+    cls: str,
+    source: str,
+    license_name: str,
+    license_url: str,
+    color: tuple[int, int, int],
+    generator: str = "",
+    model_family: str = "",
+    model_id: str = "",
+    prompt: str = "",
+    seed: int | None = None,
+) -> dict[str, str]:
+    image_path = data_root / relative_path
+    _write_image(image_path, color)
+    return {
+        "path": relative_path,
+        "class": cls,
+        "source": source,
+        "license": license_name,
+        "license_url": license_url,
+        "sha256": _sha256_file(image_path),
+        "generator": generator,
+        "model_family": model_family,
+        "model_id": model_id,
+        "prompt": prompt,
+        "seed": "" if seed is None else str(seed),
+        "width": "32",
+        "height": "24",
+        "generation_params_json": (
+            ""
+            if seed is None
+            else json.dumps({"smoke": True, "seed": seed}, sort_keys=True)
+        ),
+    }
+
+
+def _create_fixtures(data_root: Path) -> None:
+    raw = data_root / "raw"
+    real_rows: list[dict[str, str]] = []
+    flux_rows: list[dict[str, str]] = []
+    sdxl_rows: list[dict[str, str]] = []
+
+    for i in range(6):
+        real_rows.append(
+            _fixture_row(
+                data_root,
+                f"raw/real/real_{i}.png",
+                cls="authentic",
+                source="open_images_v7",
+                license_name="CC-BY-2.0",
+                license_url="https://creativecommons.org/licenses/by/2.0/",
+                color=(30 + i * 10, 80 + i * 5, 120 + i * 3),
+            )
+        )
+        flux_rows.append(
+            _fixture_row(
+                data_root,
+                f"raw/ai_generated/flux/flux_{i}.png",
+                cls="ai_generated",
+                source="flux.1-schnell",
+                license_name="Apache-2.0",
+                license_url="https://www.apache.org/licenses/LICENSE-2.0",
+                color=(120 + i * 8, 40 + i * 6, 70 + i * 9),
+                prompt=f"stage3a smoke flux prompt {i}",
+                seed=1000 + i,
+            )
+        )
+
+    for i in range(4):
+        sdxl_rows.append(
+            _fixture_row(
+                data_root,
+                f"raw/ai_generated/sdxl/sdxl_{i}.png",
+                cls="ai_generated",
+                source="sdxl",
+                license_name="CreativeML Open RAIL++-M",
+                license_url=(
+                    "https://huggingface.co/stabilityai/"
+                    "stable-diffusion-xl-base-1.0/blob/main/LICENSE.md"
+                ),
+                color=(60 + i * 13, 110 + i * 4, 30 + i * 15),
+                generator="sdxl",
+                model_family="diffusion_unet",
+                model_id="stabilityai/stable-diffusion-xl-base-1.0",
+                prompt=f"stage3a smoke sdxl prompt {i}",
+                seed=2000 + i,
+            )
+        )
+
+    _write_csv(raw / "real_manifest.csv", real_rows)
+    _write_csv(raw / "ai_generated" / "flux_manifest.csv", flux_rows)
+    _write_csv(raw / "ai_generated" / "sdxl_manifest.csv", sdxl_rows)
+
+
+def _run(repo_root: Path, command: list[str]) -> subprocess.CompletedProcess[str]:
+    printable = " ".join(command)
+    print(f"\n$ {printable}")
+    result = subprocess.run(
+        command,
+        cwd=repo_root,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    if result.stdout.strip():
+        print(result.stdout.rstrip())
+    if result.stderr.strip():
+        print(result.stderr.rstrip(), file=sys.stderr)
+    return result
+
+
+def _read_csv(path: Path) -> list[dict[str, str]]:
+    with path.open() as fh:
+        return list(csv.DictReader(fh))
+
+
+def _mock_embedding(row: dict[str, str], index: int) -> np.ndarray:
+    label = CLASS_TO_LABEL[row["class"]]
+    vector = np.zeros(512, dtype=np.float32)
+    vector[label] = 1.0
+    vector[10 + (index % 32)] = 0.05
+    if row.get("augmentation"):
+        vector[80 + (index % 32)] = 0.02
+    norm = np.linalg.norm(vector)
+    return vector / norm
+
+
+def _write_mock_embeddings(csv_path: Path, out_path: Path) -> None:
+    rows = _read_csv(csv_path)
+    embeddings = np.stack(
+        [_mock_embedding(row, index) for index, row in enumerate(rows)],
+        axis=0,
+    ).astype(np.float32)
+    labels = np.asarray([CLASS_TO_LABEL[row["class"]] for row in rows], dtype=np.int8)
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    np.savez_compressed(
+        out_path,
+        embeddings=embeddings,
+        labels=labels,
+        paths=np.asarray([row["path"] for row in rows]),
+        sources=np.asarray([row.get("source", "") for row in rows]),
+        generators=np.asarray([row.get("generator", "") for row in rows]),
+        model_families=np.asarray([row.get("model_family", "") for row in rows]),
+        augmentations=np.asarray([row.get("augmentation", "") for row in rows]),
+        original_paths=np.asarray([row.get("original_path", "") for row in rows]),
+    )
+
+
+def _write_eval_mock_embeddings(splits_dir: Path, emb_dir: Path) -> None:
+    split_files = {
+        "val": splits_dir / "val.csv",
+        "test": splits_dir / "test.csv",
+        "heldout": splits_dir / "heldout.csv",
+        "test_augmented": splits_dir / "test_augmented.csv",
+    }
+    for split, csv_path in split_files.items():
+        _write_mock_embeddings(csv_path, emb_dir / f"{split}.npz")
+        print(f"  wrote mocked embeddings: {emb_dir / f'{split}.npz'}")
+
+
+def _load_train_head(repo_root: Path) -> Any:
+    import importlib.util
+
+    module_path = repo_root / "scripts" / "train_head.py"
+    spec = importlib.util.spec_from_file_location("train_head", module_path)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Could not load {module_path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _write_constant_baseline(repo_root: Path, out_path: Path) -> None:
+    import torch
+
+    train_head = _load_train_head(repo_root)
+    head = train_head._build_head()
+    for param in head.parameters():
+        param.data.zero_()
+    head[-1].bias.data[0] = 1.0
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    torch.save(head.state_dict(), out_path)
+    print(f"  wrote constant-authentic baseline: {out_path}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--work-dir",
+        type=Path,
+        required=True,
+        help="Scratch directory for generated fixtures and reports.",
+    )
+    parser.add_argument(
+        "--keep-existing",
+        action="store_true",
+        help="Do not clear --work-dir before running.",
+    )
+    args = parser.parse_args()
+
+    repo_root = _repo_root()
+    work_dir = args.work_dir.resolve()
+    data_root = work_dir / "data"
+    splits_dir = data_root / "splits"
+    emb_dir = data_root / "embeddings"
+    checkpoints_dir = data_root / "checkpoints"
+    reports_dir = data_root / "reports"
+
+    if not args.keep_existing:
+        _reset_work_dir(work_dir)
+    work_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Stage 3A smoke work dir: {work_dir}")
+    _create_fixtures(data_root)
+
+    _run(
+        repo_root,
+        [
+            sys.executable,
+            "scripts/dataset/build_manifest.py",
+            "--input-dir",
+            str(data_root / "raw"),
+            "--out",
+            str(data_root / "manifest.csv"),
+        ],
+    )
+    _run(
+        repo_root,
+        [
+            sys.executable,
+            "scripts/dataset/split.py",
+            "--manifest",
+            str(data_root / "manifest.csv"),
+            "--out-dir",
+            str(splits_dir),
+            "--val",
+            "0.25",
+            "--test",
+            "0.25",
+            "--seed",
+            "17",
+            "--stratify-by",
+            "class-generator",
+            "--holdout-generator",
+            "sdxl",
+        ],
+    )
+    _run(
+        repo_root,
+        [
+            sys.executable,
+            "scripts/dataset/augment_images.py",
+            "--manifest",
+            str(splits_dir / "train.csv"),
+            "--data-root",
+            str(data_root),
+            "--out-dir",
+            str(data_root / "augmented" / "train"),
+            "--out-manifest",
+            str(splits_dir / "train_augmented.csv"),
+            "--copies",
+            "1",
+            "--seed",
+            "17",
+        ],
+    )
+    _run(
+        repo_root,
+        [
+            sys.executable,
+            "scripts/dataset/augment_images.py",
+            "--manifest",
+            str(splits_dir / "test.csv"),
+            "--data-root",
+            str(data_root),
+            "--out-dir",
+            str(data_root / "augmented" / "test"),
+            "--out-manifest",
+            str(splits_dir / "test_augmented.csv"),
+            "--copies",
+            "1",
+            "--seed",
+            "23",
+        ],
+    )
+
+    print("\nWriting deterministic mocked embeddings...")
+    train_rows = _read_csv(splits_dir / "train.csv") + _read_csv(
+        splits_dir / "train_augmented.csv"
+    )
+    combined_train = splits_dir / "train_with_augmented.csv"
+    with combined_train.open("w", newline="") as fh:
+        writer = csv.DictWriter(
+            fh,
+            fieldnames=FIELDNAMES
+            + [
+                "original_path",
+                "augmentation",
+                "augmentation_seed",
+                "augmentation_params_json",
+            ],
+        )
+        writer.writeheader()
+        writer.writerows(train_rows)
+    _write_eval_mock_embeddings(splits_dir, emb_dir)
+    _write_mock_embeddings(combined_train, emb_dir / "train.npz")
+    print(f"  wrote mocked embeddings: {emb_dir / 'train.npz'}")
+
+    _run(
+        repo_root,
+        [
+            sys.executable,
+            "scripts/train_head.py",
+            "--emb-dir",
+            str(emb_dir),
+            "--out",
+            str(checkpoints_dir / "head_smoke.pt"),
+            "--epochs",
+            "20",
+            "--batch-size",
+            "4",
+            "--patience",
+            "5",
+            "--lr",
+            "0.05",
+            "--seed",
+            "17",
+            "--eval-split",
+            "heldout",
+            "--eval-split",
+            "test_augmented",
+            "--report-out",
+            str(reports_dir / "head_smoke_train.json"),
+        ],
+    )
+
+    baseline_path = checkpoints_dir / "head_constant_authentic.pt"
+    _write_constant_baseline(repo_root, baseline_path)
+
+    _run(
+        repo_root,
+        [
+            sys.executable,
+            "scripts/evaluate_head.py",
+            "--emb-dir",
+            str(emb_dir),
+            "--candidate",
+            str(checkpoints_dir / "head_smoke.pt"),
+            "--baseline",
+            str(baseline_path),
+            "--split",
+            "test",
+            "--split",
+            "heldout",
+            "--split",
+            "test_augmented",
+            "--report-out",
+            str(reports_dir / "head_smoke_eval.json"),
+            "--uncertainty-threshold",
+            "0.6",
+        ],
+    )
+
+    print("\nStage 3A smoke pipeline passed.")
+    print(f"Manifest: {data_root / 'manifest.csv'}")
+    print(f"Splits: {splits_dir}")
+    print(f"Train report: {reports_dir / 'head_smoke_train.json'}")
+    print(f"Evaluation report: {reports_dir / 'head_smoke_eval.json'}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/simulate_shared_copy.py

@@ -0,0 +1,107 @@
+"""
+Produce "shared copy" variants of eval images: what survives being passed around.
+
+Why
+---
+Every P0 measurement so far is on *untouched downloads*, where a C2PA manifest
+decides the verdict and detection is 100%. That is not how images reach a
+visitor. A messaging app, a social upload or a re-save downscales the image,
+recompresses it as JPEG, and discards the metadata — including the signature.
+At that point the model is alone.
+
+This script isolates exactly that: same pixels, minus the manifest, plus the
+recompression an ordinary sharing path applies. Comparing the two conditions
+gives the cost of losing provenance, measured rather than assumed.
+
+What this is NOT
+----------------
+**Not the screenshot arm.** A screenshot adds display rescaling, a second
+round of resampling, possible UI chrome and colour-profile conversion. This is
+the milder, cleaner case: metadata stripped and one recompression. Treat the
+numbers as an OPTIMISTIC bound on the sharing path — real screenshots should be
+no better and are likely worse. Owner-collected screenshots remain the thing
+that settles it; see the P0 spec.
+
+Output is written to `ai/<tool>/reencoded_sim/`, deliberately suffixed `_sim`
+so no reader mistakes a generated condition for collected evidence.
+
+Usage
+-----
+    python scripts/dataset/simulate_shared_copy.py --root data/closed_eval
+    python scripts/dataset/simulate_shared_copy.py --root data/closed_eval --force
+"""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from PIL import Image
+
+# Chosen to mimic a mainstream messaging/social pipeline rather than a worst
+# case: most resize the long edge to roughly this and recompress at high-ish
+# quality. A harsher setting would flatter the argument for P1 by making the
+# baseline look worse than reality.
+LONG_EDGE = 1080
+JPEG_QUALITY = 85
+
+
+def shared_copy(src: Path, dst: Path) -> None:
+    """Downscale, recompress as JPEG, and drop every metadata block."""
+    with Image.open(src) as im:
+        im = im.convert("RGB")
+        w, h = im.size
+        scale = LONG_EDGE / max(w, h)
+        if scale < 1.0:
+            im = im.resize((round(w * scale), round(h * scale)), Image.LANCZOS)
+        # A fresh image object carries no info dict, so the C2PA chunk, EXIF and
+        # ICC profile are all left behind — the same way a share pipeline does it.
+        out = Image.new("RGB", im.size)
+        out.putdata(list(im.getdata()))
+        out.save(dst, "JPEG", quality=JPEG_QUALITY, optimize=True)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--root", type=Path, default=Path("data/closed_eval"))
+    ap.add_argument("--force", action="store_true",
+                    help="Regenerate even if the output folder already has files.")
+    args = ap.parse_args()
+
+    ai_root = args.root / "ai"
+    if not ai_root.is_dir():
+        raise SystemExit(f"{ai_root} does not exist.")
+
+    markers = (b"caBX", b"jumbf", b"c2pa")
+    total = 0
+    for tool_dir in sorted(p for p in ai_root.iterdir() if p.is_dir()):
+        src_dir = tool_dir / "original"
+        if not src_dir.is_dir():
+            continue
+        dst_dir = tool_dir / "reencoded_sim"
+        dst_dir.mkdir(parents=True, exist_ok=True)
+        existing = list(dst_dir.glob("*.jpg"))
+        if existing and not args.force:
+            print(f"{tool_dir.name}: {len(existing)} already present, skipping "
+                  f"(use --force to regenerate)")
+            continue
+
+        made = leaked = 0
+        for src in sorted(src_dir.glob("*.png"), key=lambda p: p.stem):
+            dst = dst_dir / f"{src.stem}.jpg"
+            shared_copy(src, dst)
+            # Verify the point of the exercise actually happened. If a manifest
+            # survived, the condition is not what it claims to be.
+            if any(m in dst.read_bytes() for m in markers):
+                leaked += 1
+            made += 1
+        total += made
+        status = "all manifests stripped" if not leaked else f"{leaked} STILL CARRY C2PA"
+        print(f"{tool_dir.name}: wrote {made} shared copies -> {dst_dir}  ({status})")
+
+    print(f"\n{total} images written. Re-run the eval to compare conditions:")
+    print("  python scripts/eval_closed_generators.py --dir data/closed_eval")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset/split.py

@@ -0,0 +1,304 @@
+"""
+Produce stratified train/val/test splits from the master manifest.
+
+Stratification preserves the per-class ratio across splits — important when
+the classes are imbalanced (and they always are, eventually).
+
+Determinism: a fixed seed plus sha256 sorting means re-running on the same
+manifest produces identical splits. This matters for reproducibility.
+
+Usage
+-----
+    python scripts/dataset/split.py \
+        --manifest data/manifest.csv \
+        --out-dir data \
+        --val 0.1 --test 0.1
+
+Generator-aware Stage 3A split:
+
+    python scripts/dataset/split.py \
+        --manifest data/manifest.csv \
+        --out-dir data \
+        --stratify-by class-generator
+
+Held-out generator evaluation split:
+
+    python scripts/dataset/split.py \
+        --manifest data/manifest.csv \
+        --out-dir data \
+        --stratify-by class-generator \
+        --holdout-generator sdxl
+
+Content-disjoint held-out split (recommended for generalisation claims):
+
+    python scripts/dataset/split.py \
+        --manifest data/manifest.csv \
+        --out-dir data/splits_v2 \
+        --stratify-by class-generator \
+        --holdout-generator sdxl \
+        --group-by content-id
+
+Why --group-by content-id exists
+--------------------------------
+Output filenames are content-addressed from the (prompt, seed) pair, and every
+Stage 3A generator consumed the same `prompts.txt` with the same deterministic
+seed derivation. So `<stem>.png` denotes the same prompt and seed in the SDXL,
+SD 3.5 and AuraFlow sets.
+
+`--holdout-generator` alone takes *every* row of that generator, which means
+the held-out set can depict prompts that a sibling generator already
+contributed to training. That is exactly what happened in Stage 3A: all 20,000
+held-out SDXL images shared their (prompt, seed) with an SD 3.5 image in
+train/val/test, so the split measured "unseen generator, **seen** content"
+while being described as a true generalisation test.
+
+With `--group-by content-id`, a (prompt, seed) identity is assigned to exactly
+one partition, and two held-out files are written:
+
+  heldout.csv               unseen generator AND unseen content (honest test)
+  heldout_seen_content.csv  unseen generator, seen content (legacy-comparable)
+
+Rows of non-held-out generators that fall in the reserved held-out content
+partition are dropped from train/val/test and reported, since including them
+would reintroduce the leak.
+
+Default remains `--group-by none`, which reproduces the legacy behaviour
+byte-for-byte.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import random
+from collections import defaultdict
+from pathlib import Path
+
+
+def _content_id(row: dict[str, str]) -> str:
+    """Identity of the (prompt, seed) pair behind a content-addressed filename.
+
+    The same stem denotes the same prompt and seed across every Stage 3A
+    generator, so this groups sibling renderings of identical content.
+    """
+    return Path(row["path"]).stem
+
+
+def _generator_label(row: dict[str, str]) -> str:
+    """Return the best available generator label for an AI row."""
+    return row.get("generator") or row.get("source") or "unknown"
+
+
+def _split_key(row: dict[str, str], stratify_by: str) -> str:
+    """Return the grouping key used for stratified splitting."""
+    cls = row["class"]
+    if stratify_by == "class-generator" and cls == "ai_generated":
+        return f"{cls}:{_generator_label(row)}"
+    return cls
+
+
+def _write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, str]]) -> None:
+    with path.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def _split_content_disjoint(
+    args: argparse.Namespace,
+    rows: list[dict[str, str]],
+    fieldnames: list[str],
+    holdout_generators: set[str],
+) -> None:
+    """Partition by (prompt, seed) identity so held-out content is truly unseen."""
+    ai_rows = [r for r in rows if r["class"] == "ai_generated"]
+    real_rows = [r for r in rows if r["class"] != "ai_generated"]
+
+    groups: dict[str, list[dict[str, str]]] = defaultdict(list)
+    for r in ai_rows:
+        groups[_content_id(r)].append(r)
+
+    group_ids = sorted(groups)
+    rng = random.Random(args.seed)
+    rng.shuffle(group_ids)
+
+    n_groups = len(group_ids)
+    n_holdout = int(round(n_groups * args.holdout_frac)) if holdout_generators else 0
+    holdout_ids = set(group_ids[:n_holdout])
+    remaining = group_ids[n_holdout:]
+
+    n_test = int(round(len(remaining) * args.test))
+    n_val = int(round(len(remaining) * args.val))
+    partition = {gid: "test" for gid in remaining[:n_test]}
+    partition.update({gid: "val" for gid in remaining[n_test : n_test + n_val]})
+    partition.update({gid: "train" for gid in remaining[n_test + n_val :]})
+
+    buckets: dict[str, list[dict[str, str]]] = defaultdict(list)
+    dropped: dict[str, int] = defaultdict(int)
+    for gid, items in groups.items():
+        in_holdout_content = gid in holdout_ids
+        for r in items:
+            is_holdout_gen = _generator_label(r) in holdout_generators
+            if is_holdout_gen:
+                buckets["heldout" if in_holdout_content else "heldout_seen_content"].append(r)
+            elif in_holdout_content:
+                # Including this would put the held-out prompt back into
+                # training via a sibling generator — the exact Stage 3A leak.
+                dropped[_generator_label(r)] += 1
+            else:
+                buckets[partition[gid]].append(r)
+
+    # Real photos share no content identity with generated images, so they are
+    # split by the ordinary stratified fractions.
+    real_sorted = sorted(real_rows, key=lambda r: r["sha256"])
+    rng_real = random.Random(args.seed)
+    rng_real.shuffle(real_sorted)
+    n = len(real_sorted)
+    n_test_r = int(round(n * args.test))
+    n_val_r = int(round(n * args.val))
+    buckets["test"].extend(real_sorted[:n_test_r])
+    buckets["val"].extend(real_sorted[n_test_r : n_test_r + n_val_r])
+    buckets["train"].extend(real_sorted[n_test_r + n_val_r :])
+
+    print(f"content-id groups: {n_groups}")
+    print(f"  reserved for held-out content: {len(holdout_ids)}")
+    print(f"  train/val/test groups: {len(remaining) - n_test - n_val}/{n_val}/{n_test}")
+    if dropped:
+        total_dropped = sum(dropped.values())
+        print(
+            f"  dropped {total_dropped} non-held-out rows whose content is "
+            "reserved (prevents leakage):"
+        )
+        for gen, count in sorted(dropped.items()):
+            print(f"    {gen}: {count}")
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    for name in ("train", "val", "test", "heldout", "heldout_seen_content"):
+        rs = buckets.get(name, [])
+        if not rs:
+            continue
+        rs.sort(key=lambda r: (r["sha256"], r["path"]))
+        path = args.out_dir / f"{name}.csv"
+        _write_csv(path, fieldnames, rs)
+        counts: dict[str, int] = defaultdict(int)
+        for r in rs:
+            key = (
+                _generator_label(r) if r["class"] == "ai_generated" else "authentic"
+            )
+            counts[key] += 1
+        breakdown = "  ".join(f"{k}={v}" for k, v in sorted(counts.items()))
+        print(f"  wrote {path} ({len(rs)} rows)  {breakdown}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--val", type=float, default=0.1)
+    parser.add_argument("--test", type=float, default=0.1)
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument(
+        "--stratify-by",
+        choices=["class", "class-generator"],
+        default="class",
+        help=(
+            "Stratification mode. 'class' preserves legacy behavior; "
+            "'class-generator' additionally balances AI generators."
+        ),
+    )
+    parser.add_argument(
+        "--holdout-generator",
+        action="append",
+        default=[],
+        help=(
+            "AI generator to remove from train/val/test and write to "
+            "heldout.csv. Can be passed multiple times."
+        ),
+    )
+    parser.add_argument(
+        "--group-by",
+        choices=["none", "content-id"],
+        default="none",
+        help=(
+            "'none' (default) reproduces legacy behaviour. 'content-id' keeps "
+            "every generator's rendering of the same (prompt, seed) in one "
+            "partition, so the held-out split is content-disjoint as well as "
+            "generator-disjoint."
+        ),
+    )
+    parser.add_argument(
+        "--holdout-frac",
+        type=float,
+        default=0.1,
+        help=(
+            "Fraction of content-id groups reserved for the held-out content "
+            "partition. Only used with --group-by content-id."
+        ),
+    )
+    args = parser.parse_args()
+
+    if args.val + args.test >= 1.0:
+        raise ValueError("val + test must be < 1.0")
+
+    with args.manifest.open() as fh:
+        reader = csv.DictReader(fh)
+        rows = list(reader)
+        fieldnames = reader.fieldnames or []
+
+    holdout_generators = set(args.holdout_generator)
+
+    if args.group_by == "content-id":
+        return _split_content_disjoint(args, rows, fieldnames, holdout_generators)
+
+    split_candidates: list[dict[str, str]] = []
+    holdout_rows: list[dict[str, str]] = []
+    for r in rows:
+        if r["class"] == "ai_generated" and _generator_label(r) in holdout_generators:
+            holdout_rows.append(r)
+        else:
+            split_candidates.append(r)
+
+    if holdout_generators and not holdout_rows:
+        raise ValueError(
+            "No rows matched --holdout-generator values: "
+            f"{sorted(holdout_generators)}"
+        )
+
+    by_group: dict[str, list[dict[str, str]]] = defaultdict(list)
+    for r in split_candidates:
+        by_group[_split_key(r, args.stratify_by)].append(r)
+
+    rng = random.Random(args.seed)
+    train_rows, val_rows, test_rows = [], [], []
+    for group, items in sorted(by_group.items()):
+        # Sort by sha256 for determinism, then shuffle with seeded RNG.
+        items.sort(key=lambda r: r["sha256"])
+        rng.shuffle(items)
+        n = len(items)
+        n_test = int(round(n * args.test))
+        n_val = int(round(n * args.val))
+        test_rows.extend(items[:n_test])
+        val_rows.extend(items[n_test : n_test + n_val])
+        train_rows.extend(items[n_test + n_val :])
+        print(
+            f"  {group}: total={n}  "
+            f"train={n - n_test - n_val}  val={n_val}  test={n_test}"
+        )
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    for name, rs in [("train", train_rows), ("val", val_rows), ("test", test_rows)]:
+        path = args.out_dir / f"{name}.csv"
+        _write_csv(path, fieldnames, rs)
+        print(f"  wrote {path} ({len(rs)} rows)")
+
+    if holdout_generators:
+        path = args.out_dir / "heldout.csv"
+        holdout_rows.sort(key=lambda r: (r["sha256"], r["path"]))
+        _write_csv(path, fieldnames, holdout_rows)
+        print(
+            f"  wrote {path} ({len(holdout_rows)} rows) "
+            f"for held-out generators: {sorted(holdout_generators)}"
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/download_weights.py

@@ -0,0 +1,27 @@
+"""
+Pre-download CLIP weights to the HuggingFace cache.
+
+Run during Docker image build or first deploy so the first user-facing
+request isn't blocked by a multi-hundred-MB download.
+
+Usage
+-----
+    python scripts/download_weights.py
+"""
+from __future__ import annotations
+
+
+def main() -> None:
+    from transformers import CLIPModel, CLIPProcessor
+
+    from deepfake_scanner.config import settings
+
+    name = settings.clip_model_name
+    print(f"Downloading {name} (Apache-2.0 transformers, MIT-licensed weights)...")
+    CLIPProcessor.from_pretrained(name)
+    CLIPModel.from_pretrained(name)
+    print("Done. Weights cached.")
+
+
+if __name__ == "__main__":
+    main()

scripts/eval_closed_generators.py

@@ -0,0 +1,421 @@
+"""
+Evaluate the live detector on closed-generator images AND real photos (P0 benchmark).
+
+Why this exists
+---------------
+We deliberately never *train* on closed-API generator outputs (ToS rule,
+docs/decisions.md 2026-05-14), so we detect them only by family resemblance to
+the open generators we can train on. That leaves us partly blind to how well we
+do on the consumer tools people actually use. This script closes that gap on
+the MEASUREMENT side.
+
+The May 2026 run was 11 screenshotted AI images with **no real-photo arm**, so
+it could report a detection rate but not a false-positive rate — the exact
+number that separates a useful detector from one that just says "AI" a lot.
+This version reports both, per tool and per condition, plus a threshold sweep
+so the production 0.55 cutoff can be re-checked against a set large enough to
+justify it. Spec: docs/closed-generator-position-review-2026-07-30.md section 6.
+
+IMPORTANT — licensing / privacy guardrails
+-------------------------------------------
+- These images are for EVALUATION ONLY. They must never be added to a training
+  manifest or `data/splits*/`. Training reads only the split CSVs, so images
+  placed under the eval folder cannot leak into training by accident.
+- The default eval folder lives under `data/` which is fully .gitignored, so
+  ToS-encumbered images are never committed to the repo.
+- Nothing is uploaded anywhere; this runs the detector locally.
+
+Folder layout
+-------------
+    data/closed_eval/
+      ai/<tool>/original/      untouched downloads   (C2PA path testable)
+      ai/<tool>/screenshot/    screenshots of those  (the real sharing path)
+      ai/<tool>/logo_intact/   generator logo left in frame
+      real/phone_original/     own camera photos
+      real/phone_screenshot/   the SAME photos, screenshotted  <- key control
+      real/edited/             filtered / edited real photos
+      real/stock/              web / stock real photos
+
+Everything under `ai/` is scored as known-synthetic; everything under `real/`
+as known-authentic. Folders starting with `_` (e.g. `_legacy_may2026/`) are
+skipped unless --include-legacy is passed.
+
+If neither `ai/` nor `real/` exists, the target is treated as a flat set of AI
+images (the pre-P0 behaviour) and a warning is printed — no false-positive
+rate can be computed in that mode.
+
+Usage
+-----
+    # whole benchmark
+    python scripts/eval_closed_generators.py --dir data/closed_eval
+
+    # one condition only
+    python scripts/eval_closed_generators.py --dir data/closed_eval/ai/chatgpt/original
+
+    # freeze a report to diff against after a model change
+    python scripts/eval_closed_generators.py --dir data/closed_eval \
+        --report-out data/reports/closed_eval_v0.4.0.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+_IMG_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp"}
+_PROD_THRESHOLD = 0.55
+_SWEEP = [0.50, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95]
+
+# Byte markers for an embedded C2PA manifest. Used only to sanity-check the
+# provenance reader against the files themselves — see the self-check below.
+_C2PA_MARKERS = (b"caBX", b"jumbf", b"c2pa")
+
+_NAN = float("nan")
+
+
+@dataclass
+class Group:
+    """One reporting bucket: an (arm, tool, condition) triple of images."""
+
+    name: str
+    arm: str  # "ai" (known synthetic) | "real" (known authentic)
+    paths: list[Path]
+    tool: str | None = None
+    condition: str | None = None
+    records: list[dict] = field(default_factory=list)
+    errors: int = 0
+
+
+def _images(d: Path) -> list[Path]:
+    return [p for p in sorted(d.rglob("*")) if p.suffix.lower() in _IMG_EXTS]
+
+
+def _skipped(d: Path) -> list[Path]:
+    """Image-like files the scanner cannot read (e.g. iPhone .HEIC).
+
+    Reported loudly rather than dropped: a folder that looks full but is half
+    unreadable produces a rate computed on a silently different sample. This is
+    the same failure shape as the 2026-08 C2PA outage - see docs/decisions.md.
+    """
+    return [
+        p for p in sorted(d.rglob("*"))
+        if p.is_file()
+        and p.suffix.lower() not in _IMG_EXTS
+        and p.suffix.lower() in {".heic", ".heif", ".avif", ".tif", ".tiff", ".gif", ".dng", ".raw"}
+    ]
+
+
+def _loose(d: Path) -> list[Path]:
+    return [
+        p for p in sorted(d.iterdir())
+        if p.is_file() and p.suffix.lower() in _IMG_EXTS
+    ]
+
+
+def _subdirs(d: Path, include_legacy: bool) -> list[Path]:
+    return [
+        p for p in sorted(d.iterdir())
+        if p.is_dir() and (include_legacy or not p.name.startswith("_"))
+    ]
+
+
+def _discover(root: Path, include_legacy: bool) -> tuple[list[Group], list[str]]:
+    """Map the folder tree onto labelled groups. Returns (groups, warnings)."""
+    groups: list[Group] = []
+    warnings: list[str] = []
+    ai_root, real_root = root / "ai", root / "real"
+
+    if not ai_root.is_dir() and not real_root.is_dir():
+        # Pre-P0 flat layout, or a single condition folder passed directly.
+        imgs = _images(root)
+        if imgs:
+            groups.append(Group(name=root.name or str(root), arm="ai", paths=imgs))
+        warnings.append(
+            "No ai/ or real/ subfolder found - treating every image as known-AI. "
+            "No false-positive rate can be computed in this mode."
+        )
+        return groups, warnings
+
+    if ai_root.is_dir():
+        for tool_dir in _subdirs(ai_root, include_legacy):
+            for cond_dir in _subdirs(tool_dir, include_legacy):
+                imgs = _images(cond_dir)
+                if imgs:
+                    groups.append(Group(
+                        name=f"ai/{tool_dir.name}/{cond_dir.name}", arm="ai",
+                        paths=imgs, tool=tool_dir.name, condition=cond_dir.name,
+                    ))
+            stray = _loose(tool_dir)
+            if stray:
+                groups.append(Group(
+                    name=f"ai/{tool_dir.name}/(loose)", arm="ai", paths=stray,
+                    tool=tool_dir.name, condition="(loose)",
+                ))
+                warnings.append(
+                    f"ai/{tool_dir.name}/ has {len(stray)} loose image(s) outside a "
+                    f"condition subfolder - they cannot be paired original<->screenshot."
+                )
+
+    if real_root.is_dir():
+        for bucket_dir in _subdirs(real_root, include_legacy):
+            imgs = _images(bucket_dir)
+            if imgs:
+                groups.append(Group(
+                    name=f"real/{bucket_dir.name}", arm="real", paths=imgs,
+                    condition=bucket_dir.name,
+                ))
+
+    if include_legacy:
+        for d in sorted(root.iterdir()):
+            if d.is_dir() and d.name.startswith("_") and _images(d):
+                groups.append(Group(name=d.name, arm="ai", paths=_images(d)))
+
+    for d in (ai_root, real_root):
+        if not d.is_dir():
+            continue
+        skipped = _skipped(d)
+        if skipped:
+            kinds = sorted({s.suffix.lower() for s in skipped})
+            warnings.append(
+                f"{len(skipped)} file(s) under {d.name}/ are NOT readable by the "
+                f"scanner and were EXCLUDED from every number below "
+                f"({', '.join(kinds)}). They are not counted as collected - the "
+                f"rates come from the readable files only."
+            )
+
+    if not any(g.arm == "real" for g in groups):
+        warnings.append(
+            "Real arm is empty - a detection rate alone cannot tell you whether the "
+            "detector is accurate or merely biased toward 'AI'. Populate "
+            "real/phone_original/ and real/phone_screenshot/ before drawing "
+            "conclusions."
+        )
+    return groups, warnings
+
+
+def _classify(p_ai: float, threshold: float) -> str:
+    """Reproduce the production rule: the top class wins only if it clears the bar."""
+    if p_ai >= threshold:
+        return "ai_generated"
+    if (1.0 - p_ai) >= threshold:
+        return "authentic"
+    return "uncertain"
+
+
+def _rates(records: list[dict], arm: str, threshold: float) -> dict:
+    n = len(records)
+    if n == 0:
+        return {"n": 0}
+    calls = [_classify(r["p_ai"], threshold) for r in records]
+    said_ai = sum(c == "ai_generated" for c in calls)
+    said_real = sum(c == "authentic" for c in calls)
+    unsure = sum(c == "uncertain" for c in calls)
+    out = {
+        "n": n,
+        "uncertain_rate": unsure / n,
+        "mean_p_ai": sum(r["p_ai"] for r in records) / n,
+    }
+    if arm == "ai":
+        out["detect_rate"] = said_ai / n
+        out["miss_rate"] = said_real / n           # known-AI image called real
+    else:
+        out["false_positive_rate"] = said_ai / n   # real photo called AI
+        out["correct_rate"] = said_real / n
+    return out
+
+
+def _headline(r: dict, arm: str) -> str:
+    if arm == "ai":
+        return (f"detected_AI={r['detect_rate']:.1%}  "
+                f"missed_as_real={r['miss_rate']:.1%}")
+    return (f"FALSE_POSITIVE={r['false_positive_rate']:.1%}  "
+            f"correct={r['correct_rate']:.1%}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--dir", type=Path, default=Path("data/closed_eval"),
+        help="Benchmark root (expects ai/ and real/ subfolders).",
+    )
+    parser.add_argument("--report-out", type=Path, default=None)
+    parser.add_argument(
+        "--include-legacy", action="store_true",
+        help="Include _-prefixed folders such as _legacy_may2026/. Off by default: "
+             "that sample is 11 screenshot-only images with no real arm and must "
+             "not be merged into P0 counts.",
+    )
+    parser.add_argument(
+        "--threshold", type=float, default=_PROD_THRESHOLD,
+        help="Decision threshold for the headline numbers (default: production 0.55).",
+    )
+    args = parser.parse_args()
+
+    if not args.dir.exists():
+        raise SystemExit(
+            f"Eval folder {args.dir} does not exist. See this script's header."
+        )
+
+    # Import here so --help works without loading torch/CLIP.
+    from deepfake_scanner.config import settings
+    from deepfake_scanner.detectors.ensemble import run_ensemble
+    # The SAME decoder the API uses. Calling Image.open().convert("RGB") here
+    # instead measured a preprocessing path production does not run: 31 of the
+    # 100 real benchmark photographs carry an EXIF rotation tag, so the
+    # benchmark scored them sideways and reported 21% where production-
+    # equivalent decoding gives 24%. Found in review 2026-09-14. Never decode
+    # independently of the service in this script.
+    from deepfake_scanner.preprocess import decode
+    from deepfake_scanner.provenance.c2pa import check_c2pa
+
+    groups, warnings = _discover(args.dir, args.include_legacy)
+    if not groups:
+        raise SystemExit(f"No images found under {args.dir}.")
+
+    print(f"Detector model_version: {settings.model_version}")
+    print(f"Decision threshold: {args.threshold}\n")
+    for w in warnings:
+        print(f"  ! {w}")
+    if warnings:
+        print()
+
+    # --- inference ---------------------------------------------------------
+    c2pa_bytes_seen = c2pa_read_ok = 0
+    for g in groups:
+        for path in g.paths:
+            try:
+                raw = path.read_bytes()
+                out = run_ensemble(decode(raw))
+            except Exception as e:  # corrupt/unsupported file - count, don't crash
+                g.errors += 1
+                print(f"  [skip] {path.name}: {e}")
+                continue
+            prov = check_c2pa(raw)
+            has_marker = any(m in raw for m in _C2PA_MARKERS)
+            c2pa_bytes_seen += int(has_marker)
+            c2pa_read_ok += int(prov.present)
+            g.records.append({
+                "file": path.name,
+                "p_ai": out.probabilities.ai_generated,
+                "verdict": out.verdict,
+                "c2pa_present": prov.present,
+                "c2pa_issuer": prov.issuer,
+                "c2pa_marker_in_bytes": has_marker,
+                # A validated chain declaring AI content decides the verdict in
+                # production (see verdict.py), so the benchmark measures the
+                # same combination the service actually ships.
+                "provenance_says_ai": bool(prov.present and prov.valid
+                                           and prov.ai_declared),
+            })
+
+    report: dict = {
+        "model_version": settings.model_version,
+        "threshold": args.threshold,
+        "groups": {},
+        "warnings": warnings,
+    }
+
+    # --- per-group report --------------------------------------------------
+    for arm, title in (("ai", "AI ARM (known synthetic)"),
+                       ("real", "REAL ARM (known authentic)")):
+        members = [g for g in groups if g.arm == arm and g.records]
+        if not members:
+            continue
+        print(title)
+        print("-" * len(title))
+        for g in members:
+            r = _rates(g.records, arm, args.threshold)
+            c2pa_n = sum(rec["c2pa_present"] for rec in g.records)
+            report["groups"][g.name] = {
+                **r, "arm": arm, "tool": g.tool, "condition": g.condition,
+                "errors": g.errors, "c2pa_present_n": c2pa_n,
+            }
+            print(f"  {g.name:36s} n={r['n']:<4d} {_headline(r, arm)}  "
+                  f"uncertain={r['uncertain_rate']:.1%}  "
+                  f"mean_p_ai={r['mean_p_ai']:.3f}  c2pa={c2pa_n}/{r['n']}")
+        pooled = [rec for g in members for rec in g.records]
+        r = _rates(pooled, arm, args.threshold)
+        report[f"overall_{arm}"] = r
+        print(f"  {'ALL ' + arm.upper():36s} n={r['n']:<4d} {_headline(r, arm)}  "
+              f"uncertain={r['uncertain_rate']:.1%}\n")
+
+    # --- threshold sweep ---------------------------------------------------
+    ai_recs = [rec for g in groups if g.arm == "ai" for rec in g.records]
+    real_recs = [rec for g in groups if g.arm == "real" for rec in g.records]
+    both = ai_recs + real_recs
+
+    print("THRESHOLD SWEEP")
+    print("-" * 15)
+    print(f"  {'thresh':>6s}  {'AI detected':>11s}  {'AI missed':>9s}  "
+          f"{'real FPR':>8s}  {'real correct':>12s}  {'uncertain(all)':>14s}")
+    sweep = {}
+    for t in _SWEEP:
+        a = _rates(ai_recs, "ai", t) if ai_recs else {}
+        b = _rates(real_recs, "real", t) if real_recs else {}
+        unc = sum(_classify(r["p_ai"], t) == "uncertain" for r in both) / max(len(both), 1)
+        sweep[f"{t:.2f}"] = {"ai": a, "real": b, "uncertain_rate_all": unc}
+        print(f"  {t:>6.2f}  {a.get('detect_rate', _NAN):>10.1%}  "
+              f"{a.get('miss_rate', _NAN):>8.1%}  "
+              f"{b.get('false_positive_rate', _NAN):>7.1%}  "
+              f"{b.get('correct_rate', _NAN):>11.1%}  {unc:>13.1%}")
+    report["threshold_sweep"] = sweep
+    if not real_recs:
+        print("\n  (real FPR columns are nan because the real arm is empty)")
+
+    # --- production routing: provenance overriding the model ----------------
+    # verdict.py lets a validated C2PA chain that declares AI content override
+    # the classifier, one-directionally. Report what the shipped combination
+    # scores, not just the model in isolation.
+    print()
+    print("WITH PROVENANCE ROUTING (what production actually returns)")
+    print("-" * 57)
+    routed = {}
+    for arm, members in (("ai", [g for g in groups if g.arm == "ai" and g.records]),
+                         ("real", [g for g in groups if g.arm == "real" and g.records])):
+        for g in members:
+            n = len(g.records)
+            model_ai = sum(_classify(r["p_ai"], args.threshold) == "ai_generated"
+                           for r in g.records)
+            both = sum(_classify(r["p_ai"], args.threshold) == "ai_generated"
+                       or r["provenance_says_ai"] for r in g.records)
+            rescued = sum(r["provenance_says_ai"]
+                          and _classify(r["p_ai"], args.threshold) != "ai_generated"
+                          for r in g.records)
+            label = "detected" if arm == "ai" else "FALSE POSITIVE"
+            routed[g.name] = {"n": n, "model_only": model_ai / n, "routed": both / n,
+                              "rescued_by_provenance": rescued}
+            print(f"  {g.name:36s} n={n:<4d} {label}: model {model_ai/n:5.1%} "
+                  f"-> routed {both/n:5.1%}   ({rescued} decided by signature)")
+    report["provenance_routing"] = routed
+    if any(g.arm == "real" for g in groups):
+        print("  NOTE: on the real arm the routed column is the false-positive rate;")
+        print("        provenance can only push toward ai_generated, so it can only rise.")
+
+    # --- provenance self-check ---------------------------------------------
+    # A silently broken C2PA reader is indistinguishable from "no image carried a
+    # manifest", so compare what the reader found against what is in the bytes.
+    print(f"\nC2PA: reader found {c2pa_read_ok} manifest(s); "
+          f"{c2pa_bytes_seen} file(s) contain an embedded C2PA marker.")
+    if c2pa_bytes_seen and not c2pa_read_ok:
+        print("  ! The provenance reader found NOTHING in files that demonstrably "
+              "carry a C2PA manifest.")
+        print("  ! Treat every c2pa column above as UNMEASURED, not as absent.")
+    report["c2pa_selfcheck"] = {
+        "reader_found": c2pa_read_ok,
+        "marker_in_bytes": c2pa_bytes_seen,
+    }
+
+    if args.report_out is not None:
+        args.report_out.parent.mkdir(parents=True, exist_ok=True)
+        report["per_image"] = {g.name: g.records for g in groups if g.records}
+        with args.report_out.open("w", encoding="utf-8") as fh:
+            json.dump(report, fh, indent=2, sort_keys=True)
+        print(f"\nsaved report to {args.report_out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/evaluate_head.py

@@ -0,0 +1,291 @@
+"""
+Evaluate CLIP head checkpoints on cached embedding splits.
+
+This is the Stage 3A checkpoint-comparison script. It loads one candidate head,
+optionally a Stage 2 baseline head, and reports overall, per-generator,
+per-source, per-model-family, and per-augmentation metrics for any requested
+embedding splits.
+
+Usage
+-----
+    python scripts/evaluate_head.py \\
+        --emb-dir data/embeddings \\
+        --candidate data/checkpoints/head_v3a.pt \\
+        --baseline data/checkpoints/head_stage2.pt \\
+        --split test \\
+        --split heldout \\
+        --split test_augmented \\
+        --report-out data/reports/head_v3a_eval.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from train_head import SplitEmbeddings, _build_head, _load_split
+
+
+def _softmax(logits):
+    import torch
+
+    return torch.softmax(logits, dim=-1)
+
+
+def _rate(num: int, den: int) -> float:
+    return float(num / den) if den else 0.0
+
+
+def _threshold_metrics(logits, labels, uncertainty_threshold: float) -> dict[str, Any]:
+    probs = _softmax(logits)
+    max_probs, argmax = probs.max(dim=-1)
+    certain = max_probs >= uncertainty_threshold
+
+    authentic = labels == 0
+    ai = labels == 1
+    pred_authentic = argmax == 0
+    pred_ai = argmax == 1
+
+    tp = int((certain & pred_ai & ai).sum().item())
+    tn = int((certain & pred_authentic & authentic).sum().item())
+    fp = int((certain & pred_ai & authentic).sum().item())
+    fn = int((certain & pred_authentic & ai).sum().item())
+    uncertain_authentic = int((~certain & authentic).sum().item())
+    uncertain_ai = int((~certain & ai).sum().item())
+    uncertain = uncertain_authentic + uncertain_ai
+    n = int(len(labels))
+
+    argmax_correct = int((argmax == labels).sum().item())
+    certain_correct = int(((argmax == labels) & certain).sum().item())
+    certain_n = int(certain.sum().item())
+
+    return {
+        "n": n,
+        "argmax_accuracy": _rate(argmax_correct, n),
+        "coverage_accuracy": _rate(certain_correct, certain_n),
+        "coverage_rate": _rate(certain_n, n),
+        "uncertainty_rate": _rate(uncertain, n),
+        "false_positive_rate": _rate(fp, fp + tn + uncertain_authentic),
+        "false_negative_rate": _rate(fn, fn + tp + uncertain_ai),
+        "confusion": {
+            "tp": tp,
+            "tn": tn,
+            "fp": fp,
+            "fn": fn,
+            "uncertain_authentic": uncertain_authentic,
+            "uncertain_ai": uncertain_ai,
+        },
+        "mean_confidence": float(max_probs.mean().item()) if n else 0.0,
+    }
+
+
+def _indices_for(values: np.ndarray, label: str) -> list[int]:
+    return [i for i, value in enumerate(values) if str(value) == label]
+
+
+def _group_metrics(
+    logits,
+    labels,
+    values: np.ndarray,
+    uncertainty_threshold: float,
+) -> dict[str, dict[str, Any]]:
+    import torch
+
+    result: dict[str, dict[str, Any]] = {}
+    group_labels = sorted({str(v) for v in values if str(v)})
+    for label in group_labels:
+        idx = _indices_for(values, label)
+        if not idx:
+            continue
+        tensor_idx = torch.as_tensor(idx, dtype=torch.long, device=logits.device)
+        result[label] = _threshold_metrics(
+            logits.index_select(0, tensor_idx),
+            labels.index_select(0, tensor_idx),
+            uncertainty_threshold,
+        )
+    return result
+
+
+def _generator_values(split: SplitEmbeddings) -> np.ndarray:
+    values: list[str] = []
+    labels = split.y.cpu().numpy()
+    for i, label in enumerate(labels):
+        if label != 1:
+            values.append("")
+            continue
+        values.append(str(split.generators[i] or split.sources[i]))
+    return np.asarray(values)
+
+
+def _augmentation_values(split: SplitEmbeddings) -> np.ndarray:
+    return np.asarray(
+        [str(value) if str(value) else "clean" for value in split.augmentations]
+    )
+
+
+def _split_report(
+    split: SplitEmbeddings,
+    logits,
+    uncertainty_threshold: float,
+) -> dict[str, Any]:
+    report: dict[str, Any] = {
+        "overall": _threshold_metrics(logits, split.y, uncertainty_threshold)
+    }
+
+    groups = {
+        "by_source": split.sources,
+        "by_generator": _generator_values(split),
+        "by_model_family": split.model_families,
+    }
+    if any(str(value) for value in split.augmentations):
+        groups["by_augmentation"] = _augmentation_values(split)
+
+    for name, values in groups.items():
+        metrics = _group_metrics(logits, split.y, values, uncertainty_threshold)
+        if metrics:
+            report[name] = metrics
+
+    return report
+
+
+def _load_head(checkpoint: Path):
+    import torch
+
+    head = _build_head()
+    state = torch.load(checkpoint, map_location="cpu")
+    head.load_state_dict(state)
+    head.eval()
+    return head
+
+
+def _evaluate_checkpoint(
+    checkpoint: Path,
+    splits: dict[str, SplitEmbeddings],
+    uncertainty_threshold: float,
+) -> dict[str, Any]:
+    import torch
+
+    head = _load_head(checkpoint)
+    report: dict[str, Any] = {
+        "checkpoint": str(checkpoint),
+        "splits": {},
+    }
+    with torch.no_grad():
+        for name, split in splits.items():
+            logits = head(split.x)
+            report["splits"][name] = _split_report(
+                split,
+                logits,
+                uncertainty_threshold,
+            )
+    return report
+
+
+def _comparison(candidate: dict[str, Any], baseline: dict[str, Any] | None) -> dict:
+    if baseline is None:
+        return {}
+
+    result: dict[str, dict[str, float]] = {}
+    for split, candidate_report in candidate["splits"].items():
+        if split not in baseline["splits"]:
+            continue
+        candidate_overall = candidate_report["overall"]
+        baseline_overall = baseline["splits"][split]["overall"]
+        result[split] = {
+            "argmax_accuracy_delta": (
+                candidate_overall["argmax_accuracy"]
+                - baseline_overall["argmax_accuracy"]
+            ),
+            "uncertainty_rate_delta": (
+                candidate_overall["uncertainty_rate"]
+                - baseline_overall["uncertainty_rate"]
+            ),
+            "false_positive_rate_delta": (
+                candidate_overall["false_positive_rate"]
+                - baseline_overall["false_positive_rate"]
+            ),
+            "false_negative_rate_delta": (
+                candidate_overall["false_negative_rate"]
+                - baseline_overall["false_negative_rate"]
+            ),
+        }
+    return result
+
+
+def _print_summary(report: dict[str, Any]) -> None:
+    print(f"Candidate: {report['candidate']['checkpoint']}")
+    if report.get("baseline"):
+        print(f"Baseline: {report['baseline']['checkpoint']}")
+    for split, metrics in report["candidate"]["splits"].items():
+        overall = metrics["overall"]
+        print(
+            f"  {split}: n={overall['n']} "
+            f"acc={overall['argmax_accuracy']:.4f} "
+            f"uncertain={overall['uncertainty_rate']:.4f} "
+            f"fpr={overall['false_positive_rate']:.4f} "
+            f"fnr={overall['false_negative_rate']:.4f}"
+        )
+        for group_name in ["by_generator", "by_augmentation"]:
+            if group_name not in metrics:
+                continue
+            print(f"    {group_name}:")
+            for label, group_metrics in metrics[group_name].items():
+                print(
+                    f"      {label}: n={group_metrics['n']} "
+                    f"acc={group_metrics['argmax_accuracy']:.4f} "
+                    f"uncertain={group_metrics['uncertainty_rate']:.4f}"
+                )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--emb-dir", type=Path, required=True)
+    parser.add_argument("--candidate", type=Path, required=True)
+    parser.add_argument("--baseline", type=Path, default=None)
+    parser.add_argument(
+        "--split",
+        action="append",
+        default=[],
+        help="Embedding split name to evaluate, without .npz. Defaults to test.",
+    )
+    parser.add_argument("--report-out", type=Path, required=True)
+    parser.add_argument("--uncertainty-threshold", type=float, default=0.6)
+    args = parser.parse_args()
+
+    split_names = args.split or ["test"]
+    splits = {name: _load_split(args.emb_dir, name) for name in split_names}
+
+    candidate = _evaluate_checkpoint(
+        args.candidate,
+        splits,
+        args.uncertainty_threshold,
+    )
+    baseline = (
+        _evaluate_checkpoint(args.baseline, splits, args.uncertainty_threshold)
+        if args.baseline is not None
+        else None
+    )
+
+    report = {
+        "uncertainty_threshold": args.uncertainty_threshold,
+        "candidate": candidate,
+        "baseline": baseline,
+        "comparison": _comparison(candidate, baseline),
+    }
+
+    args.report_out.parent.mkdir(parents=True, exist_ok=True)
+    with args.report_out.open("w", encoding="utf-8") as fh:
+        json.dump(report, fh, indent=2, sort_keys=True)
+
+    _print_summary(report)
+    print(f"\nSaved evaluation report to {args.report_out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/precompute_embeddings.py

@@ -0,0 +1,488 @@
+"""
+Pre-compute CLIP image embeddings for the training set.
+
+Why a separate step?
+--------------------
+CLIP forward passes are the expensive part of Stage 2 training. The CLIP
+backbone is FROZEN — only the small classification head defined in
+`src/deepfake_scanner/detectors/clip_classifier.py` is being trained.
+Encoding ~100k images on CPU takes 1-4 hours; training the head on the
+cached embeddings then takes seconds per epoch and can be iterated freely
+without re-encoding.
+
+Local feasibility
+-----------------
+Yes — this script runs fine on the dev laptop (i7-1195G7, 32 GB).
+With the default batch size of 16, peak RAM is ~1 GB. Throughput is
+roughly 50-150 ms per image depending on thread count.
+
+Same script runs ~50-100x faster on a CUDA GPU. Use `--device cuda` only
+if you happen to be on a GPU box already (e.g. the Flux generation box,
+before tearing it down).
+
+Output format
+-------------
+One `.npz` file per split, in `--out-dir`:
+  - `embeddings`: float32 array, shape (N, 512), L2-normalised
+  - `labels`:     int8 array,    shape (N,)        0 = authentic, 1 = ai_generated
+  - `paths`:      unicode array, shape (N,)        source image paths
+  - `sources`:    unicode array, shape (N,)        manifest source values
+  - `generators`: unicode array, shape (N,)        AI generator values or ""
+  - `model_families`: unicode array, shape (N,)    generator family values or ""
+  - `augmentations`: unicode array, shape (N,)     augmentation version or ""
+  - `original_paths`: unicode array, shape (N,)    original path for augmented rows
+
+The extra metadata arrays are ignored by `train_head.py` today, but they make
+the cached embeddings auditable and give Stage 3A evaluation code enough
+context to separate clean rows from augmented rows.
+
+Resumability
+------------
+Long runs survive crashes, lid-closes, and Ctrl-C. Each split is encoded in
+small shards (`{split}_shards/shard_NNNNNNNN.npz`); on restart we count the
+contiguous shards already written from index 0 and resume from there. At
+the end the shards are concatenated into the final `{split}.npz` and the
+shard directory is removed.
+
+Idempotent: an existing final `.npz` is skipped unless `--force` is passed.
+`--force` also wipes any leftover shard directory.
+
+Usage
+-----
+    python scripts/precompute_embeddings.py \\
+        --splits-dir data \\
+        --out-dir data/embeddings \\
+        --data-root data
+
+Include augmented training rows:
+
+    python scripts/precompute_embeddings.py \\
+        --splits-dir data \\
+        --out-dir data/embeddings \\
+        --data-root data \\
+        --train-augment-manifest data/train_augmented.csv
+
+Encode a separate augmented evaluation split:
+
+    python scripts/precompute_embeddings.py \\
+        --splits-dir data \\
+        --out-dir data/embeddings \\
+        --data-root data \\
+        --extra-split test_augmented=data/test_augmented.csv
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import shutil
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+
+
+# Order MUST match the head's output ordering in
+# src/deepfake_scanner/detectors/clip_classifier.py (logits[..., 0] = authentic,
+# logits[..., 1] = ai_generated). Drift between train and inference here
+# would silently invert predictions.
+CLASS_TO_LABEL: dict[str, int] = {
+    "authentic": 0,
+    "ai_generated": 1,
+}
+
+# Images per shard. 500 × 512 × 4 bytes = 1 MB on disk; small enough that a
+# crash loses at most ~25 seconds of CPU work, large enough that shard I/O
+# overhead is negligible.
+SHARD_SIZE = 500
+
+
+@dataclass(frozen=True)
+class EmbeddingItem:
+    image_path: Path
+    label: int
+    manifest_path: str
+    source: str = ""
+    generator: str = ""
+    model_family: str = ""
+    augmentation: str = ""
+    original_path: str = ""
+
+
+def _read_manifest(path: Path, data_root: Path) -> list[EmbeddingItem]:
+    items: list[EmbeddingItem] = []
+    with path.open() as fh:
+        reader = csv.DictReader(fh)
+        for row in reader:
+            cls = row["class"]
+            if cls not in CLASS_TO_LABEL:
+                raise ValueError(f"{path}: unknown class {cls!r}")
+            manifest_path = row["path"]
+            items.append(
+                EmbeddingItem(
+                    image_path=data_root / manifest_path,
+                    label=CLASS_TO_LABEL[cls],
+                    manifest_path=manifest_path,
+                    source=row.get("source", ""),
+                    generator=row.get("generator", ""),
+                    model_family=row.get("model_family", ""),
+                    augmentation=row.get("augmentation", ""),
+                    original_path=row.get("original_path", ""),
+                )
+            )
+    return items
+
+
+def _read_split(
+    path: Path,
+    data_root: Path,
+    extra_manifests: list[Path] | None = None,
+) -> list[EmbeddingItem]:
+    items = _read_manifest(path, data_root)
+    for extra_manifest in extra_manifests or []:
+        items.extend(_read_manifest(extra_manifest, data_root))
+
+    seen: set[str] = set()
+    duplicates: list[str] = []
+    for item in items:
+        if item.manifest_path in seen:
+            duplicates.append(item.manifest_path)
+        seen.add(item.manifest_path)
+    if duplicates:
+        raise ValueError(
+            f"Duplicate image paths in split inputs for {path}: {duplicates[:5]}"
+        )
+
+    return items
+
+
+def _resume_cursor(shard_dir: Path) -> int:
+    """How many items have been encoded already, by counting the contiguous
+    shard prefix starting at index 0. Corrupt shards are deleted in passing."""
+    if not shard_dir.exists():
+        return 0
+    by_start: dict[int, int] = {}
+    for p in sorted(shard_dir.glob("shard_*.npz")):
+        try:
+            with np.load(p) as z:
+                start = int(z["start_idx"])
+                end = int(z["end_idx"])
+                required = [
+                    "embeddings",
+                    "labels",
+                    "paths",
+                    "sources",
+                    "generators",
+                    "model_families",
+                    "augmentations",
+                    "original_paths",
+                ]
+                for key in required:
+                    if key not in z:
+                        raise KeyError(key)
+        except Exception:
+            print(f"  removing corrupt shard {p.name}")
+            p.unlink()
+            continue
+        by_start[start] = end
+    cursor = 0
+    while cursor in by_start:
+        cursor = by_start[cursor]
+    return cursor
+
+
+def _save_shard_atomic(
+    shard_path: Path,
+    embeddings: np.ndarray,
+    labels: np.ndarray,
+    paths: np.ndarray,
+    sources: np.ndarray,
+    generators: np.ndarray,
+    model_families: np.ndarray,
+    augmentations: np.ndarray,
+    original_paths: np.ndarray,
+    start_idx: int,
+    end_idx: int,
+) -> None:
+    """Write to .tmp first, then atomic rename. Avoids partial-write corruption
+    if the process is killed mid-save."""
+    tmp = shard_path.with_suffix(".tmp.npz")
+    np.savez_compressed(
+        tmp,
+        embeddings=embeddings,
+        labels=labels,
+        paths=paths,
+        sources=sources,
+        generators=generators,
+        model_families=model_families,
+        augmentations=augmentations,
+        original_paths=original_paths,
+        start_idx=np.asarray(start_idx),
+        end_idx=np.asarray(end_idx),
+    )
+    tmp.replace(shard_path)
+
+
+def _concat_shards(shard_dir: Path, out_path: Path, expected_total: int) -> None:
+    paths = sorted(shard_dir.glob("shard_*.npz"))
+    embeds_list: list[np.ndarray] = []
+    labels_list: list[np.ndarray] = []
+    paths_list: list[np.ndarray] = []
+    sources_list: list[np.ndarray] = []
+    generators_list: list[np.ndarray] = []
+    model_families_list: list[np.ndarray] = []
+    augmentations_list: list[np.ndarray] = []
+    original_paths_list: list[np.ndarray] = []
+    for p in paths:
+        with np.load(p) as z:
+            embeds_list.append(z["embeddings"].copy())
+            labels_list.append(z["labels"].copy())
+            paths_list.append(z["paths"].copy())
+            sources_list.append(z["sources"].copy())
+            generators_list.append(z["generators"].copy())
+            model_families_list.append(z["model_families"].copy())
+            augmentations_list.append(z["augmentations"].copy())
+            original_paths_list.append(z["original_paths"].copy())
+    embeddings = np.concatenate(embeds_list, axis=0)
+    labels = np.concatenate(labels_list, axis=0)
+    manifest_paths = np.concatenate(paths_list, axis=0)
+    sources = np.concatenate(sources_list, axis=0)
+    generators = np.concatenate(generators_list, axis=0)
+    model_families = np.concatenate(model_families_list, axis=0)
+    augmentations = np.concatenate(augmentations_list, axis=0)
+    original_paths = np.concatenate(original_paths_list, axis=0)
+    if len(embeddings) != expected_total:
+        raise RuntimeError(
+            f"Shard concat produced {len(embeddings)} rows but split expected "
+            f"{expected_total}. Refusing to write {out_path}."
+        )
+    np.savez_compressed(
+        out_path,
+        embeddings=embeddings,
+        labels=labels,
+        paths=manifest_paths,
+        sources=sources,
+        generators=generators,
+        model_families=model_families,
+        augmentations=augmentations,
+        original_paths=original_paths,
+    )
+    shutil.rmtree(shard_dir)
+
+
+def _process_split(
+    split: str,
+    split_csv: Path,
+    out_dir: Path,
+    data_root: Path,
+    batch_size: int,
+    force: bool,
+    extra_manifests: list[Path],
+    model,
+    processor,
+    device: str,
+    torch_module,
+    Image,
+) -> None:
+    out_path = out_dir / f"{split}.npz"
+    shard_dir = out_dir / f"{split}_shards"
+
+    if force:
+        if out_path.exists():
+            out_path.unlink()
+        if shard_dir.exists():
+            shutil.rmtree(shard_dir)
+
+    if out_path.exists():
+        print(
+            f"Skipping {split} — {out_path} already exists. "
+            "Use --force to recompute."
+        )
+        return
+
+    if not split_csv.exists():
+        print(f"Skipping {split} — {split_csv} not found.")
+        return
+
+    items = _read_split(split_csv, data_root, extra_manifests)
+    n = len(items)
+    print(f"\n{split}: {n} images")
+    if extra_manifests:
+        print(
+            f"  includes {len(extra_manifests)} extra manifest(s): "
+            + ", ".join(str(p) for p in extra_manifests)
+        )
+
+    cursor = _resume_cursor(shard_dir)
+    if cursor > 0:
+        print(f"  resuming from item {cursor}/{n} ({cursor / n:.1%} done)")
+    shard_dir.mkdir(parents=True, exist_ok=True)
+
+    start_t = time.time()
+    items_at_start = cursor
+
+    while cursor < n:
+        shard_end = min(cursor + SHARD_SIZE, n)
+        shard_n = shard_end - cursor
+        shard_embeds = np.empty((shard_n, 512), dtype=np.float32)
+        shard_items = items[cursor:shard_end]
+        shard_labels = np.asarray(
+            [item.label for item in shard_items],
+            dtype=np.int8,
+        )
+        shard_paths = np.asarray([item.manifest_path for item in shard_items])
+        shard_sources = np.asarray([item.source for item in shard_items])
+        shard_generators = np.asarray([item.generator for item in shard_items])
+        shard_model_families = np.asarray(
+            [item.model_family for item in shard_items]
+        )
+        shard_augmentations = np.asarray(
+            [item.augmentation for item in shard_items]
+        )
+        shard_original_paths = np.asarray(
+            [item.original_path for item in shard_items]
+        )
+
+        pos = 0
+        i = cursor
+        while i < shard_end:
+            batch_end = min(i + batch_size, shard_end)
+            batch_paths = [item.image_path for item in items[i:batch_end]]
+            batch_imgs = []
+            for p in batch_paths:
+                with Image.open(p) as img:
+                    img_rgb = img.convert("RGB")
+                    img_rgb.load()
+                    batch_imgs.append(img_rgb.copy())
+
+            inputs = processor(images=batch_imgs, return_tensors="pt").to(device)
+            with torch_module.no_grad():
+                feats = model.get_image_features(**inputs)
+            # L2-normalise to match the inference path
+            # (clip_classifier.py:94). Drift here breaks predictions.
+            feats = feats / feats.norm(p=2, dim=-1, keepdim=True)
+            shard_embeds[pos : pos + len(batch_imgs)] = (
+                feats.cpu().numpy().astype(np.float32)
+            )
+            pos += len(batch_imgs)
+            i = batch_end
+
+        shard_path = shard_dir / f"shard_{cursor:08d}.npz"
+        _save_shard_atomic(
+            shard_path,
+            shard_embeds,
+            shard_labels,
+            shard_paths,
+            shard_sources,
+            shard_generators,
+            shard_model_families,
+            shard_augmentations,
+            shard_original_paths,
+            cursor,
+            shard_end,
+        )
+        cursor = shard_end
+
+        elapsed = time.time() - start_t
+        done_this_run = cursor - items_at_start
+        rate = done_this_run / max(elapsed, 1e-6)
+        eta = (n - cursor) / max(rate, 1e-6)
+        print(
+            f"  {cursor}/{n}  ({rate:.1f} img/s, "
+            f"eta {eta / 60:.1f} min, shard saved)"
+        )
+
+    print(f"  concatenating shards into {out_path}...")
+    _concat_shards(shard_dir, out_path, expected_total=n)
+    print(f"  wrote {out_path}  ({out_path.stat().st_size / 1e6:.1f} MB)")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--splits-dir", type=Path, required=True,
+                        help="Directory containing train.csv / val.csv / test.csv")
+    parser.add_argument("--out-dir", type=Path, required=True,
+                        help="Where to write {train,val,test}.npz")
+    parser.add_argument("--data-root", type=Path, required=True,
+                        help="Root that the manifest 'path' column is relative to")
+    parser.add_argument("--batch-size", type=int, default=16)
+    parser.add_argument("--model", default="openai/clip-vit-base-patch32",
+                        help="HF model name. MUST match the inference config.")
+    parser.add_argument("--device", default="cpu", choices=["cpu", "cuda"])
+    parser.add_argument("--force", action="store_true",
+                        help="Recompute even if output .npz / shards already exist")
+    parser.add_argument(
+        "--train-augment-manifest",
+        type=Path,
+        action="append",
+        default=[],
+        help=(
+            "Augmented manifest to append to train.csv only. Can be passed "
+            "multiple times."
+        ),
+    )
+    parser.add_argument(
+        "--extra-split",
+        action="append",
+        default=[],
+        metavar="NAME=CSV",
+        help=(
+            "Additional explicit split to encode, for example "
+            "test_augmented=data/test_augmented.csv. Can be passed multiple times."
+        ),
+    )
+    args = parser.parse_args()
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Loading CLIP model {args.model} on {args.device}...")
+    import torch
+    from PIL import Image
+    from transformers import CLIPModel, CLIPProcessor
+
+    processor = CLIPProcessor.from_pretrained(args.model)
+    model = CLIPModel.from_pretrained(args.model).to(args.device)
+    model.eval()
+    for p in model.parameters():
+        p.requires_grad = False
+
+    extra_splits: list[tuple[str, Path]] = []
+    for spec in args.extra_split:
+        if "=" not in spec:
+            raise ValueError(f"--extra-split must be NAME=CSV, got {spec!r}")
+        name, csv_path = spec.split("=", 1)
+        if not name or "/" in name or "\\" in name:
+            raise ValueError(f"Invalid extra split name: {name!r}")
+        extra_splits.append((name, Path(csv_path)))
+
+    split_specs = [
+        ("train", args.splits_dir / "train.csv", args.train_augment_manifest),
+        ("val", args.splits_dir / "val.csv", []),
+        ("test", args.splits_dir / "test.csv", []),
+        *[(name, path, []) for name, path in extra_splits],
+    ]
+
+    for split, split_csv, extra_manifests in split_specs:
+        _process_split(
+            split=split,
+            split_csv=split_csv,
+            out_dir=args.out_dir,
+            data_root=args.data_root,
+            batch_size=args.batch_size,
+            force=args.force,
+            extra_manifests=extra_manifests,
+            model=model,
+            processor=processor,
+            device=args.device,
+            torch_module=torch,
+            Image=Image,
+        )
+
+    print("\nDone.")
+
+
+if __name__ == "__main__":
+    main()

scripts/precompute_frequency_features.py

@@ -0,0 +1,263 @@
+"""
+Pre-compute Stage 3B frequency features for the dataset splits.
+
+Mirrors `scripts/precompute_embeddings.py` in shape (same split CSVs, same
+`--data-root` resolution, same label map, resumable shards) but emits the
+15-dim `freq-v1` feature vector from
+`deepfake_scanner.detectors.frequency_features` instead of CLIP embeddings.
+
+Why a separate offline step?
+----------------------------
+Feature extraction is pure-numpy and CPU-cheap (~a few ms/image), but the
+full multi-generator dataset is ~100k+ images, so caching the features once
+lets the classifier (S3B-07) iterate in seconds without re-reading images.
+
+Output format
+-------------
+One `{split}_freq.npz` per split in `--out-dir`:
+  - features:        float32 (N, 16)   freq-v1 feature vectors
+  - labels:          int8    (N,)      0 = authentic, 1 = ai_generated
+  - paths:           unicode (N,)      manifest-relative image paths
+  - sources:         unicode (N,)
+  - generators:      unicode (N,)
+  - model_families:  unicode (N,)
+  - augmentations:   unicode (N,)
+  - feature_schema:  scalar str        FEATURE_SCHEMA_VERSION ("freq-v1")
+  - feature_names:   unicode (16,)     FEATURE_NAMES, for validation
+
+Resumability
+------------
+Each split is encoded in shards (`{split}_freq_shards/shard_NNNNNNNN.npz`);
+on restart we resume from the first missing contiguous shard. Final files
+are concatenated at the end and the shard dir removed. Idempotent: an
+existing final `.npz` is skipped unless `--force`.
+
+Usage
+-----
+    python scripts/precompute_frequency_features.py \\
+        --splits-dir data/splits \\
+        --out-dir data/freq_features \\
+        --data-root data
+
+    # Include the augmented splits used for robustness training/eval:
+    python scripts/precompute_frequency_features.py \\
+        --splits-dir data/splits \\
+        --out-dir data/freq_features \\
+        --data-root data \\
+        --splits train val test heldout test_augmented train_augmented
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import shutil
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+from PIL import Image
+
+from deepfake_scanner.detectors.frequency_features import (
+    FEATURE_NAMES,
+    FEATURE_SCHEMA_VERSION,
+    N_FEATURES,
+    extract_features,
+)
+
+# Keep label mapping identical to precompute_embeddings.py / train_head.py:
+# logits[..., 1] = ai_generated.
+_LABEL_MAP = {"authentic": 0, "ai_generated": 1}
+
+# Images per shard. Feature extraction is cheap, so use a larger shard than
+# the embeddings script; a crash loses at most a few seconds of work.
+SHARD_SIZE = 2000
+
+
+@dataclass(frozen=True)
+class FeatureItem:
+    label: int
+    manifest_path: str
+    image_path: Path
+    source: str
+    generator: str
+    model_family: str
+    augmentation: str
+
+
+def _read_split(csv_path: Path, data_root: Path) -> list[FeatureItem]:
+    items: list[FeatureItem] = []
+    with csv_path.open(encoding="utf-8") as fh:
+        reader = csv.DictReader(fh)
+        for row in reader:
+            label = _LABEL_MAP[row["class"]]
+            manifest_path = row["path"]
+            items.append(
+                FeatureItem(
+                    label=label,
+                    manifest_path=manifest_path,
+                    image_path=data_root / manifest_path,
+                    source=row.get("source", ""),
+                    generator=row.get("generator", ""),
+                    model_family=row.get("model_family", ""),
+                    augmentation=row.get("augmentation", ""),
+                )
+            )
+    return items
+
+
+def _resume_cursor(shard_dir: Path) -> int:
+    """Count the contiguous valid shard prefix; delete a trailing corrupt one."""
+    if not shard_dir.exists():
+        return 0
+    cursor = 0
+    for p in sorted(shard_dir.glob("shard_*.npz")):
+        try:
+            with np.load(p) as z:
+                _ = z["features"], z["labels"]
+            cursor += int(np.load(p)["features"].shape[0])
+        except Exception:
+            print(f"  removing corrupt shard {p.name}")
+            p.unlink()
+            break
+    return cursor
+
+
+def _save_shard_atomic(shard_path: Path, **arrays) -> None:
+    tmp = shard_path.with_suffix(".tmp.npz")
+    np.savez_compressed(tmp, **arrays)
+    tmp.replace(shard_path)
+
+
+def _concat_shards(shard_dir: Path, out_path: Path, expected_total: int) -> None:
+    paths = sorted(shard_dir.glob("shard_*.npz"))
+    keys = [
+        "features",
+        "labels",
+        "paths",
+        "sources",
+        "generators",
+        "model_families",
+        "augmentations",
+    ]
+    acc: dict[str, list[np.ndarray]] = {k: [] for k in keys}
+    for p in paths:
+        with np.load(p) as z:
+            for k in keys:
+                acc[k].append(z[k].copy())
+    merged = {k: np.concatenate(acc[k], axis=0) for k in keys}
+    if len(merged["labels"]) != expected_total:
+        raise RuntimeError(
+            f"Concatenated {len(merged['labels'])} rows but expected "
+            f"{expected_total}. Refusing to write {out_path}."
+        )
+    np.savez_compressed(
+        out_path,
+        feature_schema=np.asarray(FEATURE_SCHEMA_VERSION),
+        feature_names=np.asarray(FEATURE_NAMES),
+        **merged,
+    )
+    shutil.rmtree(shard_dir)
+
+
+def _encode_split(
+    split: str,
+    splits_dir: Path,
+    out_dir: Path,
+    data_root: Path,
+    force: bool,
+) -> None:
+    csv_path = splits_dir / f"{split}.csv"
+    if not csv_path.exists():
+        print(f"Skipping {split} — {csv_path} not found.")
+        return
+
+    out_path = out_dir / f"{split}_freq.npz"
+    shard_dir = out_dir / f"{split}_freq_shards"
+
+    if force:
+        if out_path.exists():
+            out_path.unlink()
+        if shard_dir.exists():
+            shutil.rmtree(shard_dir)
+
+    if out_path.exists():
+        print(f"Skipping {split} — {out_path} already exists. Use --force.")
+        return
+
+    items = _read_split(csv_path, data_root)
+    n = len(items)
+    print(f"{split}: {n} images")
+
+    cursor = _resume_cursor(shard_dir)
+    if cursor:
+        print(f"  resuming from {cursor}/{n}")
+    shard_dir.mkdir(parents=True, exist_ok=True)
+
+    t0 = time.time()
+    while cursor < n:
+        shard_end = min(cursor + SHARD_SIZE, n)
+        chunk = items[cursor:shard_end]
+        feats = np.empty((len(chunk), N_FEATURES), dtype=np.float32)
+        for i, item in enumerate(chunk):
+            with Image.open(item.image_path) as img:
+                feats[i] = extract_features(img)
+        _save_shard_atomic(
+            shard_dir / f"shard_{cursor:08d}.npz",
+            features=feats,
+            labels=np.asarray([it.label for it in chunk], dtype=np.int8),
+            paths=np.asarray([it.manifest_path for it in chunk]),
+            sources=np.asarray([it.source for it in chunk]),
+            generators=np.asarray([it.generator for it in chunk]),
+            model_families=np.asarray([it.model_family for it in chunk]),
+            augmentations=np.asarray([it.augmentation for it in chunk]),
+        )
+        cursor = shard_end
+        elapsed = time.time() - t0
+        rate = cursor / elapsed if elapsed else 0.0
+        eta = (n - cursor) / rate if rate else 0.0
+        print(
+            f"  {cursor}/{n}  ({rate:.0f} img/s, eta {eta / 60:.1f} min, "
+            f"shard saved)"
+        )
+
+    print(f"  concatenating shards into {out_path}...")
+    _concat_shards(shard_dir, out_path, expected_total=n)
+    print(f"  wrote {out_path}  ({out_path.stat().st_size / 1e6:.1f} MB)")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--splits-dir", type=Path, required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--data-root", type=Path, required=True)
+    parser.add_argument(
+        "--splits",
+        nargs="+",
+        default=["train", "val", "test", "heldout", "test_augmented"],
+        help="Split names (without .csv) to encode.",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Recompute even if an output .npz already exists.",
+    )
+    args = parser.parse_args()
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    print(f"Feature schema: {FEATURE_SCHEMA_VERSION} ({N_FEATURES} features)")
+    for split in args.splits:
+        _encode_split(
+            split=split,
+            splits_dir=args.splits_dir,
+            out_dir=args.out_dir,
+            data_root=args.data_root,
+            force=args.force,
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/report_frequency_features.py

@@ -0,0 +1,133 @@
+"""
+QA report for cached Stage 3B frequency features (S3B-06).
+
+Reads the `{split}_freq.npz` files written by
+`scripts/precompute_frequency_features.py` and produces a small JSON+text
+report so we can sanity-check the features BEFORE training a classifier:
+
+  - per-feature mean/std, split by class (authentic vs ai_generated)
+  - per-feature mean by generator (where present)
+  - NaN/Inf counts (must be zero — the extractor forces finite, this verifies)
+  - extreme-outlier counts (|z| > 8 against the split's own mean/std)
+  - a crude per-feature separability score: |mean_ai - mean_real| / pooled_std,
+    which flags which features carry class signal at all (a fast pre-check of
+    whether the frequency approach has any chance before we train).
+
+No new dependencies (numpy only). Read-only over the feature cache.
+
+Usage
+-----
+    python scripts/report_frequency_features.py \\
+        --feat-dir data/freq_features \\
+        --splits train val test heldout test_augmented \\
+        --report-out data/reports/freq_v1_feature_qa.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+
+from deepfake_scanner.detectors.frequency_features import (
+    FEATURE_NAMES,
+    FEATURE_SCHEMA_VERSION,
+)
+
+
+def _load(feat_dir: Path, split: str):
+    path = feat_dir / f"{split}_freq.npz"
+    if not path.exists():
+        return None
+    with np.load(path, allow_pickle=False) as z:
+        return {
+            "features": z["features"].astype(np.float64),
+            "labels": z["labels"].astype(np.int64),
+            "generators": np.asarray([str(g) for g in z["generators"]]),
+            "schema": str(z["feature_schema"]),
+        }
+
+
+def _separability(feats: np.ndarray, labels: np.ndarray) -> list[float]:
+    """|mean_ai - mean_real| / pooled_std per feature. Higher = more signal."""
+    real = feats[labels == 0]
+    ai = feats[labels == 1]
+    if len(real) == 0 or len(ai) == 0:
+        return [0.0] * feats.shape[1]
+    pooled = np.sqrt((real.var(axis=0) + ai.var(axis=0)) / 2.0) + 1e-8
+    return (np.abs(ai.mean(axis=0) - real.mean(axis=0)) / pooled).tolist()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--feat-dir", type=Path, required=True)
+    parser.add_argument(
+        "--splits",
+        nargs="+",
+        default=["train", "val", "test", "heldout", "test_augmented"],
+    )
+    parser.add_argument("--report-out", type=Path, default=None)
+    args = parser.parse_args()
+
+    report: dict = {"feature_schema": FEATURE_SCHEMA_VERSION, "splits": {}}
+
+    for split in args.splits:
+        data = _load(args.feat_dir, split)
+        if data is None:
+            print(f"skip {split} (no feature file)")
+            continue
+        if data["schema"] != FEATURE_SCHEMA_VERSION:
+            raise ValueError(
+                f"{split}: schema {data['schema']!r} != {FEATURE_SCHEMA_VERSION!r}"
+            )
+        x = data["features"]
+        y = data["labels"]
+        n_nan = int(np.isnan(x).sum())
+        n_inf = int(np.isinf(x).sum())
+        mean = x.mean(axis=0)
+        std = x.std(axis=0) + 1e-8
+        z = np.abs((x - mean) / std)
+        n_outliers = int((z > 8).sum())
+        sep = _separability(x, y)
+
+        split_rep = {
+            "n": int(len(y)),
+            "class_counts": {
+                "authentic": int((y == 0).sum()),
+                "ai_generated": int((y == 1).sum()),
+            },
+            "nan_count": n_nan,
+            "inf_count": n_inf,
+            "extreme_outlier_count": n_outliers,
+            "per_feature": {
+                FEATURE_NAMES[i]: {
+                    "mean": float(mean[i]),
+                    "std": float(std[i] - 1e-8),
+                    "separability": float(sep[i]),
+                }
+                for i in range(len(FEATURE_NAMES))
+            },
+        }
+        report["splits"][split] = split_rep
+
+        # Console summary: top separating features on this split.
+        order = np.argsort(sep)[::-1]
+        top = ", ".join(f"{FEATURE_NAMES[i]}={sep[i]:.2f}" for i in order[:5])
+        print(
+            f"[{split}] n={len(y)} nan={n_nan} inf={n_inf} "
+            f"outliers={n_outliers} | top-sep: {top}"
+        )
+
+    if args.report_out is not None:
+        args.report_out.parent.mkdir(parents=True, exist_ok=True)
+        with args.report_out.open("w", encoding="utf-8") as fh:
+            json.dump(report, fh, indent=2, sort_keys=True)
+        print(f"saved report to {args.report_out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_stage3a_pipeline.py

@@ -0,0 +1,627 @@
+"""
+Run the full Stage 3A dataset, training, evaluation, and optional publish flow.
+
+This is the production counterpart to `scripts/dataset/run_stage3a_smoke.py`.
+It is intentionally orchestration-only: generator scripts, manifest building,
+splitting, augmentation, embedding precompute, training, and evaluation remain
+owned by their focused scripts.
+
+Typical GPU-box generation and local training flow:
+
+    python scripts/run_stage3a_pipeline.py --dry-run
+    python scripts/run_stage3a_pipeline.py --stop-after generate
+
+    # After copying the data directory back to the training machine:
+    python scripts/run_stage3a_pipeline.py \\
+        --skip-generation \\
+        --baseline data/checkpoints/head_v1.pt
+
+Publishing is opt-in and gated by the evaluation report:
+
+    python scripts/run_stage3a_pipeline.py \\
+        --skip-generation \\
+        --baseline data/checkpoints/head_v1.pt \\
+        --publish-if-accepted
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+STEPS = [
+    "generate",
+    "manifest",
+    "split",
+    "augment",
+    "precompute",
+    "train",
+    "evaluate",
+    "recommend",
+    "publish",
+]
+
+
+@dataclass(frozen=True)
+class CommandStep:
+    name: str
+    command: list[str]
+    log_name: str
+
+
+@dataclass(frozen=True)
+class Thresholds:
+    min_test_accuracy: float
+    min_heldout_accuracy: float
+    min_augmented_accuracy: float
+    max_false_positive_rate: float
+    max_false_negative_rate: float
+    max_accuracy_regression: float
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[1]
+
+
+def _step_range(start_at: str, stop_after: str) -> set[str]:
+    start = STEPS.index(start_at)
+    stop = STEPS.index(stop_after)
+    if stop < start:
+        raise ValueError("--stop-after must be the same as or after --start-at")
+    return set(STEPS[start : stop + 1])
+
+
+def _python(args: argparse.Namespace) -> str:
+    return str(args.python)
+
+
+def _stage3a_paths(args: argparse.Namespace) -> dict[str, Path]:
+    data_root = args.data_root
+    splits_dir = data_root / "splits"
+    return {
+        "raw": data_root / "raw",
+        "manifest": data_root / "manifest.csv",
+        "splits": splits_dir,
+        "train_augmented": splits_dir / "train_augmented.csv",
+        "test_augmented": splits_dir / "test_augmented.csv",
+        "embeddings": data_root / "embeddings",
+        "candidate": args.candidate,
+        "baseline": args.baseline,
+        "train_report": args.train_report,
+        "eval_report": args.eval_report,
+        "recommendation": args.recommendation_out,
+        "logs": args.logs_dir,
+    }
+
+
+def _generation_steps(args: argparse.Namespace) -> list[CommandStep]:
+    specs = [
+        (
+            "flux",
+            args.flux_count,
+            "scripts/dataset/generate_flux_synthetic.py",
+            args.data_root / "raw" / "ai_generated",
+            ["--steps", str(args.flux_steps)],
+        ),
+        (
+            "sdxl",
+            args.sdxl_count,
+            "scripts/dataset/generate_sdxl_synthetic.py",
+            args.data_root / "raw" / "ai_generated" / "sdxl",
+            ["--steps", str(args.sdxl_steps)],
+        ),
+        (
+            "sd35",
+            args.sd35_count,
+            "scripts/dataset/generate_sd35_synthetic.py",
+            args.data_root / "raw" / "ai_generated" / "sd35-medium",
+            ["--steps", str(args.sd35_steps)],
+        ),
+        (
+            "auraflow",
+            args.auraflow_count,
+            "scripts/dataset/generate_auraflow_synthetic.py",
+            args.data_root / "raw" / "ai_generated" / "auraflow-v0.3",
+            ["--steps", str(args.auraflow_steps)],
+        ),
+    ]
+
+    steps: list[CommandStep] = []
+    for name, count, script, out_dir, extra in specs:
+        if count <= 0:
+            continue
+        command = [
+            _python(args),
+            script,
+            "--out",
+            str(out_dir),
+            "--count",
+            str(count),
+            "--prompts",
+            str(args.prompts),
+            "--seed",
+            str(args.seed),
+            *extra,
+        ]
+        if name != "flux":
+            command.extend(["--data-root", str(args.data_root)])
+        steps.append(CommandStep("generate", command, f"generate_{name}.log"))
+    return steps
+
+
+def _pipeline_steps(args: argparse.Namespace) -> list[CommandStep]:
+    paths = _stage3a_paths(args)
+    holdout_args: list[str] = []
+    for generator in args.holdout_generator:
+        holdout_args.extend(["--holdout-generator", generator])
+
+    steps = [
+        CommandStep(
+            "manifest",
+            [
+                _python(args),
+                "scripts/dataset/build_manifest.py",
+                "--input-dir",
+                str(paths["raw"]),
+                "--out",
+                str(paths["manifest"]),
+            ],
+            "build_manifest.log",
+        ),
+        CommandStep(
+            "split",
+            [
+                _python(args),
+                "scripts/dataset/split.py",
+                "--manifest",
+                str(paths["manifest"]),
+                "--out-dir",
+                str(paths["splits"]),
+                "--val",
+                str(args.val_fraction),
+                "--test",
+                str(args.test_fraction),
+                "--seed",
+                str(args.seed),
+                "--stratify-by",
+                "class-generator",
+                *holdout_args,
+            ],
+            "split.log",
+        ),
+        CommandStep(
+            "augment",
+            [
+                _python(args),
+                "scripts/dataset/augment_images.py",
+                "--manifest",
+                str(paths["splits"] / "train.csv"),
+                "--data-root",
+                str(args.data_root),
+                "--out-dir",
+                str(args.data_root / "augmented" / "train"),
+                "--out-manifest",
+                str(paths["train_augmented"]),
+                "--copies",
+                str(args.train_augmentation_copies),
+                "--seed",
+                str(args.seed),
+            ],
+            "augment_train.log",
+        ),
+        CommandStep(
+            "augment",
+            [
+                _python(args),
+                "scripts/dataset/augment_images.py",
+                "--manifest",
+                str(paths["splits"] / "test.csv"),
+                "--data-root",
+                str(args.data_root),
+                "--out-dir",
+                str(args.data_root / "augmented" / "test"),
+                "--out-manifest",
+                str(paths["test_augmented"]),
+                "--copies",
+                str(args.test_augmentation_copies),
+                "--seed",
+                str(args.seed + 1),
+            ],
+            "augment_test.log",
+        ),
+        CommandStep(
+            "precompute",
+            [
+                _python(args),
+                "scripts/precompute_embeddings.py",
+                "--splits-dir",
+                str(paths["splits"]),
+                "--out-dir",
+                str(paths["embeddings"]),
+                "--data-root",
+                str(args.data_root),
+                "--batch-size",
+                str(args.embedding_batch_size),
+                "--device",
+                args.embedding_device,
+                "--train-augment-manifest",
+                str(paths["train_augmented"]),
+                "--extra-split",
+                f"heldout={paths['splits'] / 'heldout.csv'}",
+                "--extra-split",
+                f"test_augmented={paths['test_augmented']}",
+                # Force rebuild so stale .npz files from a previous (e.g.
+                # Stage 2) run don't get reused against new split CSVs.
+                # precompute_embeddings.py is idempotent by default —
+                # it skips splits whose final .npz already exists, which
+                # bit us on 2026-05-30 when Stage 3A train/val/test
+                # split CSVs were silently evaluated against stale
+                # Stage 2 train/val/test .npz files. Within a single
+                # run, shard-based resumability still works (shards are
+                # wiped on --force only at start, then built up
+                # incrementally).
+                "--force",
+            ],
+            "precompute_embeddings.log",
+        ),
+        CommandStep(
+            "train",
+            [
+                _python(args),
+                "scripts/train_head.py",
+                "--emb-dir",
+                str(paths["embeddings"]),
+                "--out",
+                str(paths["candidate"]),
+                "--epochs",
+                str(args.epochs),
+                "--batch-size",
+                str(args.train_batch_size),
+                "--lr",
+                str(args.learning_rate),
+                "--weight-decay",
+                str(args.weight_decay),
+                "--patience",
+                str(args.patience),
+                "--seed",
+                str(args.seed),
+                "--eval-split",
+                "heldout",
+                "--eval-split",
+                "test_augmented",
+                "--report-out",
+                str(paths["train_report"]),
+            ],
+            "train_head.log",
+        ),
+        CommandStep(
+            "evaluate",
+            [
+                _python(args),
+                "scripts/evaluate_head.py",
+                "--emb-dir",
+                str(paths["embeddings"]),
+                "--candidate",
+                str(paths["candidate"]),
+                "--baseline",
+                str(paths["baseline"]),
+                "--split",
+                "test",
+                "--split",
+                "heldout",
+                "--split",
+                "test_augmented",
+                "--report-out",
+                str(paths["eval_report"]),
+                "--uncertainty-threshold",
+                str(args.uncertainty_threshold),
+            ],
+            "evaluate_head.log",
+        ),
+    ]
+    return steps
+
+
+def _planned_steps(args: argparse.Namespace) -> list[CommandStep]:
+    selected = _step_range(args.start_at, args.stop_after)
+    steps: list[CommandStep] = []
+    if "generate" in selected and not args.skip_generation:
+        steps.extend(_generation_steps(args))
+    steps.extend(step for step in _pipeline_steps(args) if step.name in selected)
+    return steps
+
+
+def _has_cuda() -> bool:
+    try:
+        import torch
+    except Exception:
+        return False
+    return bool(torch.cuda.is_available())
+
+
+def _validate_before_run(args: argparse.Namespace, selected: set[str]) -> None:
+    paths = _stage3a_paths(args)
+    if (
+        "generate" in selected
+        and not args.skip_generation
+        and not args.dry_run
+        and not _has_cuda()
+    ):
+        raise RuntimeError(
+            "CUDA is not available. Run generation on a GPU box, or pass "
+            "--skip-generation after copying/generated data into --data-root."
+        )
+    if "generate" in selected and not args.skip_generation and not args.prompts.exists():
+        raise FileNotFoundError(f"Prompts file does not exist: {args.prompts}")
+    if "evaluate" in selected and not args.dry_run and not paths["baseline"].exists():
+        raise FileNotFoundError(
+            f"Baseline checkpoint does not exist: {paths['baseline']}. "
+            "Pass --baseline pointing at the Stage 2 head."
+        )
+
+
+def _run_step(repo_root: Path, step: CommandStep, log_dir: Path, dry_run: bool) -> None:
+    printable = " ".join(step.command)
+    print(f"\n[{step.name}] $ {printable}")
+    if dry_run:
+        return
+
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_path = log_dir / step.log_name
+    with log_path.open("w", encoding="utf-8") as log:
+        log.write(f"$ {printable}\n\n")
+        result = subprocess.run(
+            step.command,
+            cwd=repo_root,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+        log.write(result.stdout)
+    print(f"  log: {log_path}")
+    if result.returncode != 0:
+        raise subprocess.CalledProcessError(result.returncode, step.command)
+
+
+def _metric(report: dict[str, Any], split: str, key: str) -> float:
+    return float(report["candidate"]["splits"][split]["overall"][key])
+
+
+def build_recommendation(
+    report: dict[str, Any],
+    thresholds: Thresholds,
+) -> dict[str, Any]:
+    comparison = report.get("comparison") or {}
+    checks = {
+        "test_accuracy": (
+            _metric(report, "test", "argmax_accuracy"),
+            ">=",
+            thresholds.min_test_accuracy,
+        ),
+        "heldout_accuracy": (
+            _metric(report, "heldout", "argmax_accuracy"),
+            ">=",
+            thresholds.min_heldout_accuracy,
+        ),
+        "augmented_accuracy": (
+            _metric(report, "test_augmented", "argmax_accuracy"),
+            ">=",
+            thresholds.min_augmented_accuracy,
+        ),
+        "test_false_positive_rate": (
+            _metric(report, "test", "false_positive_rate"),
+            "<=",
+            thresholds.max_false_positive_rate,
+        ),
+        "test_false_negative_rate": (
+            _metric(report, "test", "false_negative_rate"),
+            "<=",
+            thresholds.max_false_negative_rate,
+        ),
+    }
+    for split, deltas in comparison.items():
+        checks[f"{split}_accuracy_delta"] = (
+            float(deltas["argmax_accuracy_delta"]),
+            ">=",
+            -thresholds.max_accuracy_regression,
+        )
+
+    failures: list[dict[str, Any]] = []
+    for name, (actual, op, expected) in checks.items():
+        passed = actual >= expected if op == ">=" else actual <= expected
+        if not passed:
+            failures.append(
+                {
+                    "check": name,
+                    "actual": actual,
+                    "operator": op,
+                    "expected": expected,
+                }
+            )
+
+    return {
+        "accepted": not failures,
+        "failures": failures,
+        "checks": {
+            name: {"actual": actual, "operator": op, "expected": expected}
+            for name, (actual, op, expected) in checks.items()
+        },
+    }
+
+
+def _write_recommendation(args: argparse.Namespace) -> dict[str, Any]:
+    paths = _stage3a_paths(args)
+    with paths["eval_report"].open(encoding="utf-8") as fh:
+        report = json.load(fh)
+
+    recommendation = build_recommendation(
+        report,
+        Thresholds(
+            min_test_accuracy=args.min_test_accuracy,
+            min_heldout_accuracy=args.min_heldout_accuracy,
+            min_augmented_accuracy=args.min_augmented_accuracy,
+            max_false_positive_rate=args.max_false_positive_rate,
+            max_false_negative_rate=args.max_false_negative_rate,
+            max_accuracy_regression=args.max_accuracy_regression,
+        ),
+    )
+    recommendation["candidate"] = str(paths["candidate"])
+    recommendation["baseline"] = str(paths["baseline"])
+    recommendation["eval_report"] = str(paths["eval_report"])
+    recommendation["model_version"] = args.model_version
+
+    paths["recommendation"].parent.mkdir(parents=True, exist_ok=True)
+    with paths["recommendation"].open("w", encoding="utf-8") as fh:
+        json.dump(recommendation, fh, indent=2, sort_keys=True)
+
+    verdict = "ACCEPT" if recommendation["accepted"] else "DO NOT SHIP"
+    print(f"\n[recommend] {verdict}: {paths['recommendation']}")
+    for failure in recommendation["failures"]:
+        print(
+            "  failed {check}: {actual:.4f} {operator} {expected:.4f}".format(
+                **failure
+            )
+        )
+    return recommendation
+
+
+def _publish(args: argparse.Namespace, recommendation: dict[str, Any] | None) -> None:
+    if not args.publish_if_accepted:
+        return
+    if recommendation is None:
+        recommendation = _write_recommendation(args)
+    if not recommendation["accepted"]:
+        raise RuntimeError("Refusing to publish because the ship gate did not pass.")
+    if not os.getenv("HF_TOKEN"):
+        raise RuntimeError("HF_TOKEN is not set; cannot publish to a private HF repo.")
+
+    from huggingface_hub import upload_file
+
+    paths = _stage3a_paths(args)
+    upload_file(
+        path_or_fileobj=str(paths["candidate"]),
+        path_in_repo=args.hf_filename,
+        repo_id=args.hf_repo,
+        repo_type="model",
+    )
+    print(f"\n[publish] uploaded {paths['candidate']} to {args.hf_repo}/{args.hf_filename}")
+    print("Runtime env after publish:")
+    print(f"  MODEL_VERSION={args.model_version}")
+    print(f"  HEAD_CHECKPOINT_HF_REPO={args.hf_repo}")
+    print(f"  HEAD_CHECKPOINT_HF_FILENAME={args.hf_filename}")
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--data-root", type=Path, default=Path("data"))
+    parser.add_argument(
+        "--prompts",
+        type=Path,
+        default=Path("scripts/dataset/prompts.txt"),
+    )
+    parser.add_argument("--python", type=Path, default=Path(sys.executable))
+    parser.add_argument("--logs-dir", type=Path, default=Path("data/logs/stage3a"))
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--skip-generation", action="store_true")
+    parser.add_argument("--start-at", choices=STEPS, default="generate")
+    parser.add_argument("--stop-after", choices=STEPS, default="recommend")
+    parser.add_argument("--seed", type=int, default=0)
+
+    parser.add_argument("--flux-count", type=int, default=0)
+    parser.add_argument("--sdxl-count", type=int, default=20_000)
+    parser.add_argument("--sd35-count", type=int, default=20_000)
+    parser.add_argument("--auraflow-count", type=int, default=20_000)
+    parser.add_argument("--flux-steps", type=int, default=4)
+    parser.add_argument("--sdxl-steps", type=int, default=30)
+    parser.add_argument("--sd35-steps", type=int, default=28)
+    parser.add_argument("--auraflow-steps", type=int, default=30)
+
+    parser.add_argument("--val-fraction", type=float, default=0.1)
+    parser.add_argument("--test-fraction", type=float, default=0.1)
+    parser.add_argument("--holdout-generator", action="append", default=None)
+    parser.add_argument("--train-augmentation-copies", type=int, default=1)
+    parser.add_argument("--test-augmentation-copies", type=int, default=1)
+
+    parser.add_argument("--embedding-device", choices=["cpu", "cuda"], default="cpu")
+    parser.add_argument("--embedding-batch-size", type=int, default=16)
+    parser.add_argument(
+        "--candidate",
+        type=Path,
+        default=Path("data/checkpoints/head_v3a.pt"),
+    )
+    parser.add_argument(
+        "--baseline",
+        type=Path,
+        default=Path("data/checkpoints/head_v1.pt"),
+    )
+    parser.add_argument(
+        "--train-report",
+        type=Path,
+        default=Path("data/reports/head_v3a_train.json"),
+    )
+    parser.add_argument(
+        "--eval-report",
+        type=Path,
+        default=Path("data/reports/head_v3a_eval.json"),
+    )
+    parser.add_argument(
+        "--recommendation-out",
+        type=Path,
+        default=Path("data/reports/head_v3a_recommendation.json"),
+    )
+    parser.add_argument("--epochs", type=int, default=30)
+    parser.add_argument("--train-batch-size", type=int, default=512)
+    parser.add_argument("--learning-rate", type=float, default=1e-3)
+    parser.add_argument("--weight-decay", type=float, default=1e-4)
+    parser.add_argument("--patience", type=int, default=5)
+    parser.add_argument("--uncertainty-threshold", type=float, default=0.6)
+
+    parser.add_argument("--min-test-accuracy", type=float, default=0.90)
+    parser.add_argument("--min-heldout-accuracy", type=float, default=0.80)
+    parser.add_argument("--min-augmented-accuracy", type=float, default=0.85)
+    parser.add_argument("--max-false-positive-rate", type=float, default=0.05)
+    parser.add_argument("--max-false-negative-rate", type=float, default=0.12)
+    parser.add_argument("--max-accuracy-regression", type=float, default=0.01)
+
+    parser.add_argument("--publish-if-accepted", action="store_true")
+    parser.add_argument("--hf-repo", default="Veridicate/scanner-head-v1")
+    parser.add_argument("--hf-filename", default="head_v3a.pt")
+    parser.add_argument("--model-version", default="v0.4.0-stage3a")
+    args = parser.parse_args()
+    if args.holdout_generator is None:
+        args.holdout_generator = ["sdxl"]
+    return args
+
+
+def main() -> None:
+    args = _parse_args()
+    repo_root = _repo_root()
+    selected = _step_range(args.start_at, args.stop_after)
+    _validate_before_run(args, selected)
+
+    steps = _planned_steps(args)
+    print("Stage 3A pipeline plan:")
+    for step in steps:
+        print(f"  - {step.name}: {' '.join(step.command)}")
+
+    for step in steps:
+        _run_step(repo_root, step, args.logs_dir, args.dry_run)
+
+    recommendation: dict[str, Any] | None = None
+    if "recommend" in selected and not args.dry_run:
+        recommendation = _write_recommendation(args)
+    if "publish" in selected and not args.dry_run:
+        _publish(args, recommendation)
+
+
+if __name__ == "__main__":
+    main()

scripts/train_frequency_head.py

@@ -0,0 +1,243 @@
+"""
+Train the Stage 3B frequency classifier on cached freq-v1 features.
+
+Mirrors `scripts/train_head.py` in idiom (torch, seeded, early stop, JSON
+report) but operates on the 16-dim frequency features from
+`scripts/precompute_frequency_features.py` instead of CLIP embeddings.
+
+Model
+-----
+A standardised linear logistic-regression head (the "classical first model"
+required by S3B-07): feature standardisation (mean/std from train) baked into
+the saved checkpoint, then `Linear(16 -> 2)`. Interpretable, tiny, no new
+deps (torch only). A small MLP is available via --hidden if the linear model
+underperforms, but linear is the default per the tracker's guidance.
+
+Checkpoint format (torch.save dict)
+-----------------------------------
+  - feature_schema:  "freq-v1"  (must match the features at inference)
+  - feature_mean:    float32 (16,)
+  - feature_std:     float32 (16,)
+  - state_dict:      the classifier head weights
+  - hidden:          int (0 = linear logreg)
+  - classes:         ["authentic", "ai_generated"]
+
+Usage
+-----
+    python scripts/train_frequency_head.py \\
+        --feat-dir data/freq_features \\
+        --out data/checkpoints/freq_v1.pt \\
+        --report-out data/reports/freq_v1_train.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+import numpy as np
+
+FEATURE_SCHEMA_VERSION = "freq-v1"
+_CLASSES = ["authentic", "ai_generated"]
+
+
+def _load_feat(feat_dir: Path, split: str):
+    path = feat_dir / f"{split}_freq.npz"
+    with np.load(path, allow_pickle=False) as z:
+        schema = str(z["feature_schema"])
+        if schema != FEATURE_SCHEMA_VERSION:
+            raise ValueError(
+                f"{path}: feature_schema {schema!r} != {FEATURE_SCHEMA_VERSION!r}"
+            )
+        return {
+            "features": z["features"].astype(np.float32),
+            "labels": z["labels"].astype(np.int64),
+            "generators": z["generators"],
+            "sources": z["sources"],
+        }
+
+
+def _rates(pred: np.ndarray, y: np.ndarray) -> dict:
+    tp = int(((pred == 1) & (y == 1)).sum())
+    tn = int(((pred == 0) & (y == 0)).sum())
+    fp = int(((pred == 1) & (y == 0)).sum())
+    fn = int(((pred == 0) & (y == 1)).sum())
+    return {
+        "n": int(len(y)),
+        "accuracy": float((pred == y).mean()) if len(y) else 0.0,
+        "confusion": {"tp": tp, "tn": tn, "fp": fp, "fn": fn},
+        "false_positive_rate": float(fp / (fp + tn)) if (fp + tn) else 0.0,
+        "false_negative_rate": float(fn / (fn + tp)) if (fn + tp) else 0.0,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--feat-dir", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--report-out", type=Path, default=None)
+    parser.add_argument("--epochs", type=int, default=200)
+    parser.add_argument("--batch-size", type=int, default=512)
+    parser.add_argument("--lr", type=float, default=1e-2)
+    parser.add_argument("--weight-decay", type=float, default=1e-4)
+    parser.add_argument("--patience", type=int, default=15)
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument(
+        "--hidden",
+        type=int,
+        default=0,
+        help="Hidden units (0 = linear logistic regression).",
+    )
+    parser.add_argument(
+        "--eval-split",
+        action="append",
+        default=[],
+        help="Extra split(s) to evaluate, e.g. --eval-split heldout.",
+    )
+    args = parser.parse_args()
+
+    import torch
+    import torch.nn as nn
+
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+
+    train = _load_feat(args.feat_dir, "train")
+    val = _load_feat(args.feat_dir, "val")
+    test = _load_feat(args.feat_dir, "test")
+    extra = {name: _load_feat(args.feat_dir, name) for name in args.eval_split}
+
+    # Standardisation from train only (baked into the checkpoint).
+    mean = train["features"].mean(axis=0)
+    std = train["features"].std(axis=0) + 1e-8
+
+    def prep(split):
+        x = (split["features"] - mean) / std
+        return torch.from_numpy(x.astype(np.float32)), torch.from_numpy(
+            split["labels"]
+        )
+
+    x_tr, y_tr = prep(train)
+    x_val, y_val = prep(val)
+
+    n_feat = x_tr.shape[1]
+    if args.hidden > 0:
+        head = nn.Sequential(
+            nn.Linear(n_feat, args.hidden),
+            nn.ReLU(),
+            nn.Linear(args.hidden, 2),
+        )
+    else:
+        head = nn.Linear(n_feat, 2)
+
+    # Class weights to counter the authentic/ai imbalance in the splits.
+    counts = np.bincount(train["labels"], minlength=2).astype(np.float64)
+    w = counts.sum() / (2.0 * np.maximum(counts, 1.0))
+    class_weight = torch.tensor(w, dtype=torch.float32)
+    loss_fn = nn.CrossEntropyLoss(weight=class_weight)
+    opt = torch.optim.Adam(
+        head.parameters(), lr=args.lr, weight_decay=args.weight_decay
+    )
+
+    print(
+        f"train={tuple(x_tr.shape)} val={tuple(x_val.shape)} "
+        f"class_counts={counts.tolist()} hidden={args.hidden}"
+    )
+
+    best_val = float("inf")
+    best_state = None
+    no_improve = 0
+    for epoch in range(1, args.epochs + 1):
+        head.train()
+        perm = torch.randperm(len(x_tr))
+        losses = []
+        for i in range(0, len(x_tr), args.batch_size):
+            idx = perm[i : i + args.batch_size]
+            opt.zero_grad()
+            loss = loss_fn(head(x_tr[idx]), y_tr[idx])
+            loss.backward()
+            opt.step()
+            losses.append(loss.item())
+        head.eval()
+        with torch.no_grad():
+            vloss = loss_fn(head(x_val), y_val).item()
+            vacc = float((head(x_val).argmax(-1) == y_val).float().mean())
+        if epoch % 20 == 0 or epoch == 1:
+            print(
+                f"  epoch {epoch:3d} train_loss={np.mean(losses):.4f} "
+                f"val_loss={vloss:.4f} val_acc={vacc:.4f}"
+            )
+        if vloss < best_val - 1e-4:
+            best_val = vloss
+            best_state = {k: v.clone() for k, v in head.state_dict().items()}
+            no_improve = 0
+        else:
+            no_improve += 1
+            if no_improve >= args.patience:
+                print(f"  early stop at epoch {epoch}")
+                break
+
+    if best_state is not None:
+        head.load_state_dict(best_state)
+    head.eval()
+
+    def evaluate(split) -> dict:
+        x = (split["features"] - mean) / std
+        with torch.no_grad():
+            logits = head(torch.from_numpy(x.astype(np.float32)))
+            pred = logits.argmax(-1).numpy()
+        y = split["labels"]
+        rep = _rates(pred, y)
+        # Per-generator breakdown over AI rows.
+        gens = np.asarray([str(g) for g in split["generators"]])
+        by_gen = {}
+        for g in sorted(set(gens) - {""}):
+            m = gens == g
+            by_gen[g] = _rates(pred[m], y[m])
+        if by_gen:
+            rep["by_generator"] = by_gen
+        return rep
+
+    report = {
+        "feature_schema": FEATURE_SCHEMA_VERSION,
+        "hidden": args.hidden,
+        "splits": {
+            "val": evaluate(val),
+            "test": evaluate(test),
+            **{name: evaluate(s) for name, s in extra.items()},
+        },
+    }
+    for split_name in ("test", *args.eval_split):
+        r = report["splits"][split_name]
+        print(
+            f"  [{split_name}] acc={r['accuracy']:.4f} "
+            f"fpr={r['false_positive_rate']:.4f} fnr={r['false_negative_rate']:.4f}"
+        )
+
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    torch.save(
+        {
+            "feature_schema": FEATURE_SCHEMA_VERSION,
+            "feature_mean": mean.astype(np.float32),
+            "feature_std": std.astype(np.float32),
+            "state_dict": head.state_dict(),
+            "hidden": args.hidden,
+            "classes": _CLASSES,
+        },
+        args.out,
+    )
+    print(f"\nsaved frequency head to {args.out}")
+
+    if args.report_out is not None:
+        args.report_out.parent.mkdir(parents=True, exist_ok=True)
+        with args.report_out.open("w", encoding="utf-8") as fh:
+            json.dump(report, fh, indent=2, sort_keys=True)
+        print(f"saved report to {args.report_out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/train_head.py

@@ -0,0 +1,367 @@
+"""
+Train the CLIP classifier head on cached embeddings.
+
+Loads precomputed embeddings (from `scripts/precompute_embeddings.py`),
+trains the head architecture from `clip_classifier.py`, and saves a
+state_dict that can be loaded by `ClipClassifier.load_head_weights()`.
+
+The head is intentionally tiny (~130k params), so training is laptop-fast
+even on CPU — each epoch over 100k cached embeddings takes a few seconds.
+That makes it practical to iterate on hyperparameters without re-encoding
+the dataset.
+
+Usage
+-----
+    python scripts/train_head.py \\
+        --emb-dir data/embeddings \\
+        --out data/checkpoints/head_v1.pt \\
+        --epochs 30
+
+Save a Stage 3A metrics report:
+
+    python scripts/train_head.py \\
+        --emb-dir data/embeddings \\
+        --out data/checkpoints/head_v3a.pt \\
+        --report-out data/reports/head_v3a_metrics.json \\
+        --eval-split test_augmented
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class SplitEmbeddings:
+    x: Any
+    y: Any
+    paths: np.ndarray
+    sources: np.ndarray
+    generators: np.ndarray
+    model_families: np.ndarray
+    augmentations: np.ndarray
+    original_paths: np.ndarray
+
+
+# Mirror src/deepfake_scanner/detectors/clip_classifier.py:78-83 exactly.
+# If that architecture changes, this MUST change too — checkpoints won't
+# load otherwise.
+def _build_head():
+    import torch.nn as nn
+
+    return nn.Sequential(
+        nn.Linear(512, 256),
+        nn.ReLU(),
+        nn.Dropout(0.2),
+        nn.Linear(256, 2),
+    )
+
+
+def _load_split(emb_dir: Path, name: str):
+    import torch
+
+    with np.load(emb_dir / f"{name}.npz") as z:
+        embeddings = z["embeddings"].copy()
+        labels = z["labels"].copy()
+        n = len(labels)
+        x = torch.from_numpy(embeddings).float()
+        y = torch.from_numpy(labels).long()
+        return SplitEmbeddings(
+            x=x,
+            y=y,
+            paths=_optional_str_array(z, "paths", n),
+            sources=_optional_str_array(z, "sources", n),
+            generators=_optional_str_array(z, "generators", n),
+            model_families=_optional_str_array(z, "model_families", n),
+            augmentations=_optional_str_array(z, "augmentations", n),
+            original_paths=_optional_str_array(z, "original_paths", n),
+        )
+
+
+def _optional_str_array(z, key: str, n: int) -> np.ndarray:
+    if key not in z:
+        return np.asarray([""] * n)
+    values = np.asarray(z[key]).astype(str)
+    if len(values) != n:
+        raise ValueError(f"{key} has {len(values)} rows but labels has {n}")
+    return values
+
+
+def _accuracy(logits, y) -> float:
+    return float((logits.argmax(dim=-1) == y).float().mean().item())
+
+
+def _confusion(logits, y) -> dict:
+    pred = logits.argmax(dim=-1)
+    tp = int(((pred == 1) & (y == 1)).sum().item())
+    tn = int(((pred == 0) & (y == 0)).sum().item())
+    fp = int(((pred == 1) & (y == 0)).sum().item())
+    fn = int(((pred == 0) & (y == 1)).sum().item())
+    return {"tp": tp, "tn": tn, "fp": fp, "fn": fn}
+
+
+def _rate(num: int, den: int) -> float:
+    return float(num / den) if den else 0.0
+
+
+def _metrics(logits, y, loss_fn=None) -> dict[str, Any]:
+    cm = _confusion(logits, y)
+    result: dict[str, Any] = {
+        "n": int(len(y)),
+        "accuracy": _accuracy(logits, y),
+        "confusion": cm,
+        "false_positive_rate": _rate(cm["fp"], cm["fp"] + cm["tn"]),
+        "false_negative_rate": _rate(cm["fn"], cm["fn"] + cm["tp"]),
+    }
+    if loss_fn is not None:
+        result["loss"] = float(loss_fn(logits, y).item())
+    return result
+
+
+def _group_metrics(logits, y, values: np.ndarray, loss_fn=None) -> dict[str, dict]:
+    import torch
+
+    result: dict[str, dict] = {}
+    labels = sorted({str(v) for v in values if str(v)})
+    for label in labels:
+        idx = [i for i, value in enumerate(values) if str(value) == label]
+        if not idx:
+            continue
+        tensor_idx = torch.as_tensor(idx, dtype=torch.long, device=logits.device)
+        result[label] = _metrics(
+            logits.index_select(0, tensor_idx),
+            y.index_select(0, tensor_idx),
+            loss_fn,
+        )
+    return result
+
+
+def _augmentation_values(split: SplitEmbeddings) -> np.ndarray:
+    values = np.asarray(split.augmentations).astype(str)
+    return np.asarray([value if value else "clean" for value in values])
+
+
+def _generator_values(split: SplitEmbeddings) -> np.ndarray:
+    values: list[str] = []
+    labels = split.y.cpu().numpy()
+    for i, label in enumerate(labels):
+        if label != 1:
+            values.append("")
+            continue
+        generator = split.generators[i] or split.sources[i]
+        values.append(str(generator))
+    return np.asarray(values)
+
+
+def _split_report(name: str, split: SplitEmbeddings, logits, loss_fn) -> dict[str, Any]:
+    report: dict[str, Any] = {"overall": _metrics(logits, split.y, loss_fn)}
+
+    by_source = _group_metrics(logits, split.y, split.sources, loss_fn)
+    if by_source:
+        report["by_source"] = by_source
+
+    by_generator = _group_metrics(logits, split.y, _generator_values(split), loss_fn)
+    if by_generator:
+        report["by_generator"] = by_generator
+
+    by_model_family = _group_metrics(logits, split.y, split.model_families, loss_fn)
+    if by_model_family:
+        report["by_model_family"] = by_model_family
+
+    if any(str(value) for value in split.augmentations):
+        report["by_augmentation"] = _group_metrics(
+            logits,
+            split.y,
+            _augmentation_values(split),
+            loss_fn,
+        )
+
+    print(
+        f"\n{name}:  loss={report['overall']['loss']:.4f}  "
+        f"acc={report['overall']['accuracy']:.4f}"
+    )
+    cm = report["overall"]["confusion"]
+    print(
+        f"  confusion matrix: tp={cm['tp']}  tn={cm['tn']}  "
+        f"fp={cm['fp']}  fn={cm['fn']}"
+    )
+    for group_name in [
+        "by_generator",
+        "by_source",
+        "by_model_family",
+        "by_augmentation",
+    ]:
+        if group_name not in report:
+            continue
+        print(f"  {group_name}:")
+        for label, metrics in report[group_name].items():
+            print(
+                f"    {label}: n={metrics['n']} "
+                f"acc={metrics['accuracy']:.4f} "
+                f"fpr={metrics['false_positive_rate']:.4f} "
+                f"fnr={metrics['false_negative_rate']:.4f}"
+            )
+
+    return report
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--emb-dir", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--epochs", type=int, default=30)
+    parser.add_argument("--batch-size", type=int, default=512)
+    parser.add_argument("--lr", type=float, default=1e-3)
+    parser.add_argument("--weight-decay", type=float, default=1e-4)
+    parser.add_argument("--patience", type=int, default=5,
+                        help="Early-stop after N epochs without val-loss improvement")
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument(
+        "--eval-split",
+        action="append",
+        default=[],
+        help=(
+            "Additional embedding split name to evaluate, without .npz. "
+            "Example: --eval-split test_augmented"
+        ),
+    )
+    parser.add_argument(
+        "--report-out",
+        type=Path,
+        default=None,
+        help="Optional JSON path for overall and grouped metrics",
+    )
+    args = parser.parse_args()
+
+    import torch
+    import torch.nn as nn
+
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+
+    print(f"Loading embeddings from {args.emb_dir}...")
+    train = _load_split(args.emb_dir, "train")
+    val = _load_split(args.emb_dir, "val")
+    test = _load_split(args.emb_dir, "test")
+    extra_evals = {
+        name: _load_split(args.emb_dir, name)
+        for name in args.eval_split
+    }
+    x_train, y_train = train.x, train.y
+    x_val, y_val = val.x, val.y
+    print(f"  train: {train.x.shape}, val: {val.x.shape}, test: {test.x.shape}")
+    for name, split in extra_evals.items():
+        print(f"  {name}: {split.x.shape}")
+
+    cls_counts = torch.bincount(y_train, minlength=2)
+    print(f"  train class counts: authentic={cls_counts[0].item()}  "
+          f"ai_generated={cls_counts[1].item()}")
+
+    head = _build_head()
+    optimizer = torch.optim.Adam(
+        head.parameters(), lr=args.lr, weight_decay=args.weight_decay,
+    )
+    loss_fn = nn.CrossEntropyLoss()
+
+    best_val_loss = float("inf")
+    best_state = None
+    epochs_no_improve = 0
+
+    for epoch in range(1, args.epochs + 1):
+        t0 = time.time()
+        head.train()
+        perm = torch.randperm(len(x_train))
+        train_losses: list[float] = []
+        for i in range(0, len(x_train), args.batch_size):
+            idx = perm[i : i + args.batch_size]
+            xb, yb = x_train[idx], y_train[idx]
+            optimizer.zero_grad()
+            logits = head(xb)
+            loss = loss_fn(logits, yb)
+            loss.backward()
+            optimizer.step()
+            train_losses.append(loss.item())
+
+        head.eval()
+        with torch.no_grad():
+            val_logits = head(x_val)
+            val_loss = loss_fn(val_logits, y_val).item()
+            val_acc = _accuracy(val_logits, y_val)
+
+        train_loss = float(np.mean(train_losses))
+        dt = time.time() - t0
+        print(
+            f"  epoch {epoch:3d}  train_loss={train_loss:.4f}  "
+            f"val_loss={val_loss:.4f}  val_acc={val_acc:.4f}  ({dt:.1f}s)"
+        )
+
+        if val_loss < best_val_loss - 1e-4:
+            best_val_loss = val_loss
+            best_state = {k: v.clone() for k, v in head.state_dict().items()}
+            epochs_no_improve = 0
+        else:
+            epochs_no_improve += 1
+            if epochs_no_improve >= args.patience:
+                print(
+                    f"  early stop at epoch {epoch} "
+                    f"(no val-loss improvement for {args.patience} epochs)"
+                )
+                break
+
+    if best_state is not None:
+        head.load_state_dict(best_state)
+
+    head.eval()
+    with torch.no_grad():
+        validation_logits = head(val.x)
+        eval_logits = {
+            "test": head(test.x),
+            **{name: head(split.x) for name, split in extra_evals.items()},
+        }
+
+    report: dict[str, Any] = {
+        "train": {
+            "n": int(len(train.y)),
+            "class_counts": {
+                "authentic": int(cls_counts[0].item()),
+                "ai_generated": int(cls_counts[1].item()),
+            },
+        },
+        "validation": _split_report("validation", val, validation_logits, loss_fn),
+        "evaluation": {
+            "test": _split_report("test", test, eval_logits["test"], loss_fn),
+        },
+    }
+    for name, split in extra_evals.items():
+        report["evaluation"][name] = _split_report(
+            name,
+            split,
+            eval_logits[name],
+            loss_fn,
+        )
+
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    torch.save(head.state_dict(), args.out)
+    print(f"\nsaved head to {args.out}")
+    if args.report_out is not None:
+        args.report_out.parent.mkdir(parents=True, exist_ok=True)
+        with args.report_out.open("w", encoding="utf-8") as fh:
+            json.dump(report, fh, indent=2, sort_keys=True)
+        print(f"saved metrics report to {args.report_out}")
+    print("To use this checkpoint at inference time:")
+    print("  from deepfake_scanner.detectors.clip_classifier import ClipClassifier")
+    print("  c = ClipClassifier()")
+    print(f"  c.load_head_weights('{args.out}')")
+
+
+if __name__ == "__main__":
+    main()

src/deepfake_scanner/__init__.py

@@ -0,0 +1,3 @@
+"""DeepFakeScanner — commercial deepfake / AI-generated image detection service."""
+
+__version__ = "0.2.0"

src/deepfake_scanner/api/schemas.py

@@ -0,0 +1,97 @@
+"""
+API schemas (v1).
+
+These are the public contract for the /v1/scan/image endpoint. Treat any change
+to a field name, type, or required-ness as a breaking change.
+
+Forward-compatibility notes
+---------------------------
+• `probabilities` is a fixed 4-class vector. Stage 1 only meaningfully populates
+  `authentic` and `ai_generated`; `deepfake` and `edited` start at 0.0 until
+  the corresponding detectors come online in Stage 2. Clients should treat 0.0
+  as "not assessed" for an unsigned-classifier slot — but the field is always
+  present so the schema itself never changes.
+
+• `signals` is a list, currently with a single entry for the CLIP detector.
+  More detectors (frequency, face-swap) append to this list in later stages.
+
+• `provenance` is always present. When C2PA is disabled or unavailable,
+  `c2pa_present` is False and `c2pa_valid` is None.
+"""
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+# The 4-class taxonomy. New classes must NOT be added without bumping API to v2.
+Verdict = Literal["authentic", "ai_generated", "deepfake", "edited", "uncertain"]
+
+
+class Probabilities(BaseModel):
+    """Probability mass over the 4 mutually-exclusive classes.
+
+    Sums to 1.0 (allowing for floating-point rounding within ±1e-3).
+    """
+
+    authentic: float = Field(ge=0.0, le=1.0)
+    ai_generated: float = Field(ge=0.0, le=1.0)
+    deepfake: float = Field(ge=0.0, le=1.0)
+    edited: float = Field(ge=0.0, le=1.0)
+
+
+class DetectorSignal(BaseModel):
+    """Per-detector contribution, surfaced for transparency / debugging.
+
+    `score` is the detector's own internal "fakeness" estimate in [0, 1];
+    its meaning depends on the detector. Aggregation into `probabilities`
+    happens in the ensemble layer.
+    """
+
+    name: str
+    score: float = Field(ge=0.0, le=1.0)
+    notes: str | None = None
+
+
+class Provenance(BaseModel):
+    """C2PA / Content Credentials check.
+
+    A trustworthy C2PA manifest from a known camera or generator can short-
+    circuit the model — those signals are currently advisory only.
+    """
+
+    c2pa_present: bool
+    c2pa_valid: bool | None = None  # None → not checked / unverifiable
+    issuer: str | None = None  # e.g. "Sony", "OpenAI", "Adobe"
+    claim_generator: str | None = None  # raw claim_generator string from the manifest
+    # True when a manifest anywhere in the chain declares an AI digital source
+    # type (IPTC trainedAlgorithmicMedia). This is what can override the model.
+    ai_declared: bool = False
+
+
+class ScanResponse(BaseModel):
+    """The /v1/scan/image response. This is the public contract."""
+
+    verdict: Verdict
+    confidence: float = Field(ge=0.0, le=1.0)
+    probabilities: Probabilities
+    signals: list[DetectorSignal]
+    provenance: Provenance
+    # "model"      — the verdict is the classifier's own output
+    # "provenance" — a validated C2PA chain declared the image AI-generated and
+    #                overrode the classifier. Read this before treating
+    #                `probabilities` as the explanation for `verdict`: when the
+    #                source is provenance the probabilities still show what the
+    #                model thought, which may disagree.
+    verdict_source: str = "model"
+    model_version: str
+    service_version: str
+    scan_id: str
+    latency_ms: float
+
+
+class ErrorResponse(BaseModel):
+    """Uniform error shape for non-2xx responses."""
+
+    error: str
+    detail: str | None = None

src/deepfake_scanner/api/v1.py

@@ -0,0 +1,219 @@
+"""
+v1 API.
+
+Endpoints
+---------
+GET  /health                  → liveness probe
+GET  /v1/info                 → model + config metadata
+POST /v1/scan/image           → scan an uploaded image
+
+Privacy
+-------
+Image bytes live in-memory only for the duration of the request. They are
+never written to disk or to blob storage in Stage 1. After the response is
+returned the bytes go out of scope and are garbage-collected.
+"""
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+
+from fastapi import FastAPI, File, HTTPException, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+
+from ..config import settings
+from ..detectors.ensemble import _DETECTORS, run_ensemble
+from ..preprocess import InvalidImageError, decode
+from ..provenance import check_c2pa
+from ..verdict import decide
+from ..storage import record_scan
+from ..storage.db import ScanRecord
+from .schemas import (
+    DetectorSignal,
+    ErrorResponse,
+    Probabilities,
+    Provenance,
+    ScanResponse,
+)
+
+log = logging.getLogger(__name__)
+
+
+def _detectors_for_info():
+    """Live detector instances, for reporting real loaded state."""
+    return _DETECTORS
+
+
+_ALLOWED_TYPES = {"image/jpeg", "image/png", "image/webp"}
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title=settings.api_title,
+        version=settings.api_version,
+        description=(
+            "Detect AI-generated and manipulated images. "
+            "Visitor uploads are processed in-memory and never stored."
+        ),
+    )
+
+    # Browser preflight allowlist for the static-HTML frontend. Credentialed
+    # requests are off (no cookies); the upload is the only state we accept.
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origin_regex=settings.cors_allow_origin_regex,
+        allow_methods=["GET", "POST", "OPTIONS"],
+        allow_headers=["*"],
+        allow_credentials=False,
+    )
+
+    # HEAD is accepted on read-only endpoints so uptime monitors and CDN
+    # validators can ping cheaply without a body. Per RFC 7231, any URL
+    # that accepts GET should also accept HEAD.
+    @app.api_route("/health", methods=["GET", "HEAD"])
+    def health() -> dict:
+        # service_version here so a single curl identifies the running build.
+        return {"status": "ok", "service_version": settings.service_version}
+
+    @app.api_route("/v1/info", methods=["GET", "HEAD"])
+    def info() -> dict:
+        # head_loaded reports what is ACTUALLY serving, not what is configured.
+        # model_version is a config string and stays the same whether or not the
+        # weights loaded - which is exactly how a random head can look healthy.
+        head_loaded = None
+        for det in _detectors_for_info():
+            if getattr(det, "name", None) == "clip_classifier":
+                head_loaded = bool(getattr(det, "is_trained", False))
+        return {
+            "api_version": settings.api_version,
+            "service_version": settings.service_version,
+            "model_version": settings.model_version,
+            "head_loaded": head_loaded,
+            "detectors": {
+                "clip_classifier": settings.enable_clip_detector,
+                "frequency_artifacts": settings.enable_frequency_detector,
+                "face_swap": settings.enable_face_swap_detector,
+            },
+            "c2pa_check_enabled": settings.enable_c2pa_check,
+            "max_upload_bytes": settings.max_upload_bytes,
+            "store_uploads": settings.store_uploads,
+        }
+
+    @app.post(
+        "/v1/scan/image",
+        response_model=ScanResponse,
+        responses={
+            413: {"model": ErrorResponse},
+            415: {"model": ErrorResponse},
+            422: {"model": ErrorResponse},
+        },
+    )
+    async def scan_image(file: UploadFile = File(...)) -> ScanResponse:
+        if file.content_type not in _ALLOWED_TYPES:
+            raise HTTPException(
+                status_code=415,
+                detail=(
+                    f"Unsupported media type '{file.content_type}'. "
+                    "Use JPEG, PNG, or WebP."
+                ),
+            )
+
+        image_bytes = await file.read()
+
+        if len(image_bytes) > settings.max_upload_bytes:
+            raise HTTPException(
+                status_code=413,
+                detail=(
+                    f"File exceeds the {settings.max_upload_bytes // (1024*1024)} MB limit."
+                ),
+            )
+
+        t0 = time.perf_counter()
+
+        # 1. Provenance check (advisory — does not yet override the model).
+        provenance_result = check_c2pa(image_bytes)
+
+        # 2. Decode image.
+        try:
+            image = decode(image_bytes)
+        except InvalidImageError as exc:
+            raise HTTPException(status_code=422, detail=str(exc)) from exc
+
+        # 3. Run detector ensemble.
+        ensemble_out = run_ensemble(image)
+
+        # 3b. Let validated provenance override the model. One-directional:
+        # it can only push toward `ai_generated`, never toward `authentic`.
+        # See verdict.py for why a camera signature is not trusted in reverse.
+        decision = decide(
+            ensemble_out.verdict, ensemble_out.confidence, provenance_result
+        )
+
+        latency_ms = (time.perf_counter() - t0) * 1000.0
+        scan_id = f"scn_{uuid.uuid4().hex[:24]}"
+
+        response = ScanResponse(
+            verdict=decision.verdict,
+            confidence=round(decision.confidence, 4),
+            verdict_source=decision.source,
+            probabilities=Probabilities(
+                authentic=round(ensemble_out.probabilities.authentic, 4),
+                ai_generated=round(ensemble_out.probabilities.ai_generated, 4),
+                deepfake=round(ensemble_out.probabilities.deepfake, 4),
+                edited=round(ensemble_out.probabilities.edited, 4),
+            ),
+            signals=(
+                ensemble_out.signals
+                + [DetectorSignal(name="c2pa_provenance", score=1.0,
+                                  notes=decision.reason)]
+                if decision.source == "provenance"
+                else ensemble_out.signals
+            ),
+            provenance=Provenance(
+                c2pa_present=provenance_result.present,
+                c2pa_valid=provenance_result.valid,
+                issuer=provenance_result.issuer,
+                claim_generator=provenance_result.claim_generator,
+                ai_declared=provenance_result.ai_declared,
+            ),
+            model_version=settings.model_version,
+            service_version=settings.service_version,
+            scan_id=scan_id,
+            latency_ms=round(latency_ms, 1),
+        )
+
+        # 4. Persist METADATA only — never the image.
+        record_scan(
+            ScanRecord(
+                scan_id=scan_id,
+                verdict=response.verdict,
+                confidence=response.confidence,
+                model_version=response.model_version,
+                latency_ms=response.latency_ms,
+                c2pa_present=provenance_result.present,
+            )
+        )
+
+        # 5. Drop the image bytes ASAP. (Local var goes out of scope when the
+        # function returns; the explicit del documents the privacy intent.)
+        del image_bytes
+
+        return response
+
+    @app.exception_handler(HTTPException)
+    async def http_exception_handler(_, exc: HTTPException) -> JSONResponse:
+        return JSONResponse(
+            status_code=exc.status_code,
+            content=ErrorResponse(
+                error=f"HTTP {exc.status_code}",
+                detail=str(exc.detail),
+            ).model_dump(),
+        )
+
+    return app
+
+
+# Default app instance for `uvicorn deepfake_scanner.api.v1:app`.
+app = create_app()

src/deepfake_scanner/config.py

@@ -0,0 +1,142 @@
+"""
+Central configuration for DeepFakeScanner.
+
+All runtime knobs are read from environment variables with sensible defaults
+so the service can run unchanged from local dev → HF Spaces → Cloud Run.
+
+Read this once at import; do NOT scatter os.getenv() calls throughout the code.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        return default
+
+
+@dataclass(frozen=True)
+class Settings:
+    # ------------------------------------------------------------------ API
+    api_title: str = "DeepFakeScanner"
+    api_version: str = "0.2.0"
+
+    # Build identifier for the deployed service. Three versions travel in the
+    # API response and they answer different questions:
+    #   api_version     — the response CONTRACT. Bump only on a breaking change.
+    #   model_version   — which trained head is serving (see below).
+    #   service_version — which BUILD of the application code is live.
+    # The third exists because a deploy can change behaviour without touching
+    # either of the other two: the 2026-08-13 release fixed a dead C2PA reader,
+    # so provenance fields went from always-absent to populated, with an
+    # identical contract and an identical model. Bump this on every deploy that
+    # changes runtime behaviour, and note it in CHANGELOG.md.
+    #   v0.5.0 — C2PA provenance reading repaired (2026-08-13)
+    #   v0.6.0 — validated provenance now overrides the model, one-directionally
+    #            (68% -> 100% on the 100-image closed-generator benchmark)
+    #   v0.7.0 — EXIF orientation applied on decode; portrait phone photos were
+    #            being scanned rotated 90 degrees
+    #   v0.8.0 — C2PA validity fails closed (Untrusted/Unknown no longer read as
+    #            valid); refuse to start without a trained head; /v1/info
+    #            reports head_loaded
+    service_version: str = os.getenv("SERVICE_VERSION", "v0.8.0")
+    max_upload_bytes: int = _env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)  # 10 MB
+
+    # --------------------------------------------------------- Image policy
+    # Privacy-by-default: visitor uploads are NEVER persisted.
+    # Logged-in paid users will be able to opt in later via a separate flag.
+    store_uploads: bool = _env_bool("STORE_UPLOADS", False)
+
+    # ---------------------------------------------------------- Detectors
+    # Stage 1: only the CLIP-based classifier is wired up.
+    # Stage 2 will flip these on as the detectors are trained / verified.
+    enable_clip_detector: bool = _env_bool("ENABLE_CLIP_DETECTOR", True)
+    enable_frequency_detector: bool = _env_bool("ENABLE_FREQUENCY_DETECTOR", False)
+    enable_face_swap_detector: bool = _env_bool("ENABLE_FACE_SWAP_DETECTOR", False)
+
+    # The CLIP backbone we use for feature extraction.
+    # OpenAI CLIP ViT-B/32 weights are MIT-licensed (commercially safe).
+    clip_model_name: str = os.getenv("CLIP_MODEL_NAME", "openai/clip-vit-base-patch32")
+
+    # Path to the trained classifier-head checkpoint. ClipClassifier auto-
+    # loads this in __init__ if the file exists. The same env var works
+    # locally (dev machine) and inside the inference Docker image.
+    # Stage 3A: head_v3a.pt (multi-generator: Flux+SDXL+SD3.5+AuraFlow).
+    head_checkpoint_path: str = os.getenv(
+        "HEAD_CHECKPOINT_PATH", "data/checkpoints/head_v3a.pt"
+    )
+
+    # Fallback for production: if the local checkpoint file isn't present,
+    # ClipClassifier downloads the weights from this HF Hub model repo at
+    # startup. The repo is PRIVATE — the container authenticates with the
+    # HF_TOKEN secret set in the HF Space settings (huggingface_hub reads
+    # HF_TOKEN from the environment automatically). Setting this to an
+    # empty string disables the fallback (used in tests).
+    # Stage 3A: head_v3a.pt. The Stage 2 head_v1.pt remains in the same
+    # private repo for rollback — set HEAD_CHECKPOINT_HF_FILENAME=head_v1.pt
+    # (and MODEL_VERSION=v0.3.0-stage2) to revert without a redeploy.
+    head_checkpoint_hf_repo: str = os.getenv(
+        "HEAD_CHECKPOINT_HF_REPO", "Veridicate/scanner-head-v1"
+    )
+    head_checkpoint_hf_filename: str = os.getenv(
+        "HEAD_CHECKPOINT_HF_FILENAME", "head_v3a.pt"
+    )
+
+    # Refuse to start if the trained head cannot be loaded. Scaffold mode runs
+    # a RANDOMLY INITIALISED head: it answers every request, reports the
+    # configured model_version, and /health returns ok - so a failed weight
+    # download looks exactly like a healthy service returning nonsense. That is
+    # the same shape as the 2026-08 C2PA outage, and it is why this defaults to
+    # True. Tests and local scaffold work set REQUIRE_TRAINED_HEAD=false
+    # explicitly (see tests/conftest.py).
+    require_trained_head: bool = _env_bool("REQUIRE_TRAINED_HEAD", True)
+
+    # ---------------------------------------------------------- Provenance
+    enable_c2pa_check: bool = _env_bool("ENABLE_C2PA_CHECK", True)
+
+    # ------------------------------------------------------------ Storage
+    # Postgres URL for scan records (None → in-memory fallback for local dev).
+    database_url: str | None = os.getenv("DATABASE_URL")
+
+    # S3-compatible blob storage (e.g. Cloudflare R2). Only used when
+    # store_uploads is True AND a paid user opts in.
+    blob_endpoint: str | None = os.getenv("BLOB_ENDPOINT")
+    blob_bucket: str | None = os.getenv("BLOB_BUCKET")
+
+    # ---------------------------------------------------- Model identifier
+    # Surfaced in API responses so clients can pin behaviour to a version.
+    # v0.4.0-stage3a = CLIP head trained on a multi-generator dataset
+    # (50k Flux + 20k SDXL + 20k SD 3.5 + 10k AuraFlow + matched authentic).
+    # In-distribution test accuracy 98.43%; +5.55 pp vs the Stage 2 head on
+    # the SDXL hold-out (a generator never seen in training). See
+    # docs/stage3a-implementation.md.
+    # Previous: v0.3.0-stage2 (Flux-only, head_v1.pt) — still available in the
+    # HF Hub repo for rollback (see head_checkpoint_hf_filename above).
+    model_version: str = os.getenv("MODEL_VERSION", "v0.4.0-stage3a")
+
+    # --------------------------------------------------------------- CORS
+    # Browser preflight allowlist for the static-HTML frontend. The default
+    # covers the production domain, any Cloudflare Pages preview, and
+    # localhost (any port) for local dev. Override via env var if a new
+    # origin needs access.
+    cors_allow_origin_regex: str = os.getenv(
+        "CORS_ALLOW_ORIGIN_REGEX",
+        r"^(https://(www\.)?veridicate\.com|https://[a-z0-9-]+\.pages\.dev|https://[a-z0-9-]+\.dulipcf\.workers\.dev|http://localhost(:\d+)?|http://127\.0\.0\.1(:\d+)?)$",
+    )
+
+
+settings = Settings()

src/deepfake_scanner/detectors/__init__.py

@@ -0,0 +1,5 @@
+"""Image-forensic detectors. Each detector implements the Detector ABC."""
+
+from .base import Detector, DetectorResult
+
+__all__ = ["Detector", "DetectorResult"]

src/deepfake_scanner/detectors/base.py

@@ -0,0 +1,57 @@
+"""
+Detector interface.
+
+Every detector is a plug-in that takes a PIL image and returns a
+DetectorResult. The ensemble layer aggregates results from all enabled
+detectors into a single Probabilities vector.
+
+Adding a new detector = subclassing Detector + registering it in ensemble.py.
+The /v1 API response shape never changes.
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+from PIL import Image
+
+
+@dataclass
+class DetectorResult:
+    """Output of a single detector run.
+
+    Attributes
+    ----------
+    name :
+        Stable identifier for this detector (used in API `signals`).
+    score :
+        The detector's own "fakeness" estimate in [0, 1].
+        1.0 means "definitely synthetic"; 0.0 means "definitely authentic".
+    contributions :
+        Optional per-class hints in [0, 1]. Keys must be a subset of the
+        4-class taxonomy: "authentic", "ai_generated", "deepfake", "edited".
+        Detectors that only know "real vs. fake" leave this empty and let
+        the ensemble splat their score across the relevant classes.
+    notes :
+        Free-form human-readable note (surfaced in API for debugging).
+    """
+
+    name: str
+    score: float
+    contributions: dict[str, float]
+    notes: str | None = None
+
+
+class Detector(ABC):
+    """Base class for all forensic detectors.
+
+    Detectors must be safe to instantiate once at process start and reused
+    across requests — load model weights in __init__, not in run().
+    """
+
+    name: str = "abstract"
+
+    @abstractmethod
+    def run(self, image: Image.Image) -> DetectorResult:
+        """Score a single PIL image. Must not mutate the image."""
+        raise NotImplementedError

src/deepfake_scanner/detectors/clip_classifier.py

@@ -0,0 +1,174 @@
+"""
+CLIP-based binary classifier.
+
+Strategy
+--------
+Use a frozen pretrained CLIP image encoder to produce a 512-d feature vector,
+then a small linear head maps it to [P(authentic), P(ai_generated)].
+
+Why CLIP?
+~~~~~~~~~
+CLIP's pretraining on 400M+ image-text pairs has been shown (UniversalFakeDetect,
+Ojha et al. 2023, and follow-up work) to generalise far better across unseen
+generators than ImageNet-pretrained CNNs. This is the single biggest lever for
+cross-generator robustness in a commercial product.
+
+Stage 1 status
+--------------
+The classification head is RANDOMLY INITIALIZED. This file establishes the
+correct architecture and pipeline; predictions are not meaningful until the
+head is trained on the curated dataset (see `scripts/dataset/`).
+
+Licensing
+---------
+- HuggingFace `transformers` — Apache-2.0 (commercially safe)
+- OpenAI CLIP ViT-B/32 weights — MIT (commercially safe)
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import torch
+import torch.nn as nn
+from PIL import Image
+
+from ..config import settings
+from .base import Detector, DetectorResult
+
+
+class _LazyClipBackbone:
+    """Defers loading transformers + downloading CLIP weights until first use.
+
+    Keeps `import` cheap so tests that don't need the model don't pay the cost.
+    """
+
+    def __init__(self) -> None:
+        self._model: Any | None = None
+        self._processor: Any | None = None
+
+    def get(self) -> tuple[Any, Any]:
+        if self._model is None or self._processor is None:
+            # Local import — transformers is heavy.
+            from transformers import CLIPModel, CLIPProcessor
+
+            name = settings.clip_model_name
+            self._processor = CLIPProcessor.from_pretrained(name)
+            self._model = CLIPModel.from_pretrained(name)
+            self._model.eval()
+            for p in self._model.parameters():
+                p.requires_grad = False
+        return self._model, self._processor
+
+
+_clip_singleton = _LazyClipBackbone()
+
+
+class ClipClassifier(Detector):
+    """Frozen CLIP image encoder + small trainable head."""
+
+    name = "clip_classifier"
+
+    # CLIP ViT-B/32 image-embedding dim is 512.
+    _EMBED_DIM = 512
+
+    def __init__(self) -> None:
+        # The trainable head. Two outputs: logits for [authentic, ai_generated].
+        # Architecture mirrored exactly in scripts/train_head.py — keep them
+        # in sync or checkpoint loads will fail.
+        self._head: nn.Module = nn.Sequential(
+            nn.Linear(self._EMBED_DIM, 256),
+            nn.ReLU(),
+            nn.Dropout(0.2),
+            nn.Linear(256, 2),
+        )
+        self._head.eval()
+        self._trained: bool = False
+
+        # Stage 2 head loading. Priority:
+        #   1. Local file at settings.head_checkpoint_path (dev / pre-baked).
+        #   2. HF Hub model repo (settings.head_checkpoint_hf_repo).
+        #   3. Fall through to scaffold mode (random head) — what tests use.
+        # Errors here are caught and logged so the container always starts.
+        ckpt = Path(settings.head_checkpoint_path)
+        if not ckpt.is_file() and settings.head_checkpoint_hf_repo:
+            try:
+                from huggingface_hub import hf_hub_download
+
+                downloaded = hf_hub_download(
+                    repo_id=settings.head_checkpoint_hf_repo,
+                    filename=settings.head_checkpoint_hf_filename,
+                )
+                ckpt = Path(downloaded)
+            except Exception as exc:  # noqa: BLE001 — never crash on startup
+                print(
+                    f"WARNING: Stage 2 head download failed ({type(exc).__name__}: "
+                    f"{exc}). Falling back to scaffold mode.",
+                    flush=True,
+                )
+
+        if ckpt.is_file():
+            self.load_head_weights(str(ckpt))
+
+        if not self._trained:
+            msg = (
+                "Trained head could not be loaded (looked for "
+                f"{settings.head_checkpoint_path!r} and HF repo "
+                f"{settings.head_checkpoint_hf_repo!r}). The head is randomly "
+                "initialised and its output is meaningless."
+            )
+            if settings.require_trained_head:
+                # Fail closed. Serving random predictions while /health reports
+                # ok is worse than not serving: the failure is invisible to
+                # monitoring and indistinguishable from a working detector.
+                raise RuntimeError(
+                    msg + " Refusing to start. Set REQUIRE_TRAINED_HEAD=false "
+                    "to run in scaffold mode for development or tests."
+                )
+            print(f"WARNING: {msg} Running in scaffold mode.", flush=True)
+
+    @property
+    def is_trained(self) -> bool:
+        """True when real weights loaded; False in scaffold mode."""
+        return self._trained
+
+    @torch.no_grad()
+    def run(self, image: Image.Image) -> DetectorResult:
+        model, processor = _clip_singleton.get()
+
+        # Processor handles resize, center-crop, normalize — same as CLIP's
+        # original training preprocessing.
+        inputs = processor(images=image, return_tensors="pt")
+        features = model.get_image_features(**inputs)  # [1, 512]
+        features = features / features.norm(p=2, dim=-1, keepdim=True)
+
+        logits = self._head(features)  # [1, 2]
+        probs = torch.softmax(logits, dim=-1)
+        p_ai = float(probs[0, 1].item())
+
+        return DetectorResult(
+            name=self.name,
+            score=p_ai,
+            # Stage 2: trained on Open Images V7 (real) + Flux.1-schnell (AI).
+            # Only authentic vs. ai_generated. Other classes left empty —
+            # the ensemble will not assign mass to them.
+            contributions={
+                "authentic": 1.0 - p_ai,
+                "ai_generated": p_ai,
+            },
+            notes=(
+                None
+                if self._trained
+                else (
+                    "Stage 1 scaffold — classification head is randomly initialised. "
+                    "Output is not meaningful until Stage 2 fine-tuning."
+                )
+            ),
+        )
+
+    def load_head_weights(self, checkpoint_path: str) -> None:
+        """Load fine-tuned head weights produced by Stage 2 training."""
+        state = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
+        self._head.load_state_dict(state)
+        self._head.eval()
+        self._trained = True

src/deepfake_scanner/detectors/ensemble.py

@@ -0,0 +1,121 @@
+"""
+Detector ensemble.
+
+Holds the registry of enabled detectors and aggregates their per-class
+contributions into the final 4-class probability vector returned to the API.
+
+Aggregation strategy (Stage 1)
+------------------------------
+Simple averaging of the per-class contributions across detectors that emit
+non-empty contributions, then renormalisation. When only one detector is
+active (Stage 1), the output is just its contributions (with the absent
+classes filled with 0.0 and a tiny epsilon to keep softmax-style downstream
+math safe).
+
+Stage 2+ may switch to:
+  • weighted averaging (per-detector confidence / calibration)
+  • a small meta-classifier that takes detector scores as input
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from PIL import Image
+
+from ..api.schemas import DetectorSignal, Probabilities, Verdict
+from ..config import settings
+from .base import Detector, DetectorResult
+from .clip_classifier import ClipClassifier
+from .face_swap import FaceSwapDetector
+from .frequency import FrequencyDetector
+
+# The fixed taxonomy. Order matters — used for deterministic argmax tie-break.
+_CLASSES: tuple[str, ...] = ("authentic", "ai_generated", "deepfake", "edited")
+
+
+@dataclass
+class EnsembleOutput:
+    probabilities: Probabilities
+    signals: list[DetectorSignal]
+    verdict: Verdict
+    confidence: float
+
+
+def _build_detectors() -> list[Detector]:
+    """Instantiate the enabled detectors at process start."""
+    detectors: list[Detector] = []
+    if settings.enable_clip_detector:
+        detectors.append(ClipClassifier())
+    if settings.enable_frequency_detector:
+        detectors.append(FrequencyDetector())
+    if settings.enable_face_swap_detector:
+        detectors.append(FaceSwapDetector())
+    if not detectors:
+        raise RuntimeError(
+            "No detectors enabled. At minimum, ENABLE_CLIP_DETECTOR must be true."
+        )
+    return detectors
+
+
+# Module-level singleton — built once at import.
+_DETECTORS: list[Detector] = _build_detectors()
+
+
+def _aggregate(results: list[DetectorResult]) -> Probabilities:
+    """Average per-class contributions across detectors, renormalise."""
+    sums = {c: 0.0 for c in _CLASSES}
+    counts = {c: 0 for c in _CLASSES}
+
+    for r in results:
+        for cls, val in r.contributions.items():
+            if cls in sums:
+                sums[cls] += val
+                counts[cls] += 1
+
+    averaged = {
+        c: (sums[c] / counts[c]) if counts[c] > 0 else 0.0 for c in _CLASSES
+    }
+    total = sum(averaged.values())
+    if total <= 0.0:
+        # No detector contributed — default to maximum uncertainty over the
+        # two Stage 1 classes (graceful degradation).
+        averaged = {"authentic": 0.5, "ai_generated": 0.5, "deepfake": 0.0, "edited": 0.0}
+    else:
+        averaged = {c: v / total for c, v in averaged.items()}
+
+    return Probabilities(**averaged)
+
+
+def _verdict_from(probs: Probabilities) -> tuple[Verdict, float]:
+    """Pick the top class as the verdict; return it with its probability.
+
+    If the top probability is below 0.55, return 'uncertain' to discourage
+    callers from over-trusting low-confidence outputs.
+    """
+    items = [
+        ("authentic", probs.authentic),
+        ("ai_generated", probs.ai_generated),
+        ("deepfake", probs.deepfake),
+        ("edited", probs.edited),
+    ]
+    items.sort(key=lambda kv: kv[1], reverse=True)
+    top_class, top_prob = items[0]
+    if top_prob < 0.55:
+        return "uncertain", top_prob
+    return top_class, top_prob  # type: ignore[return-value]
+
+
+def run_ensemble(image: Image.Image) -> EnsembleOutput:
+    """Run all enabled detectors and aggregate their outputs."""
+    results = [d.run(image) for d in _DETECTORS]
+    probs = _aggregate(results)
+    signals = [
+        DetectorSignal(name=r.name, score=r.score, notes=r.notes) for r in results
+    ]
+    verdict, confidence = _verdict_from(probs)
+    return EnsembleOutput(
+        probabilities=probs,
+        signals=signals,
+        verdict=verdict,
+        confidence=confidence,
+    )

src/deepfake_scanner/detectors/face_swap.py

@@ -0,0 +1,26 @@
+"""
+Face-swap / deepfake detector (Stage 2).
+
+Targets the deepfake class specifically (real photo + swapped face), as
+opposed to fully synthetic AI-generated images. Typically combines:
+  • a face detector (commercially-licensed — e.g. MediaPipe, Apache-2.0)
+  • a CNN trained on face-swap artefacts (blend boundaries, eye consistency,
+    skin texture mismatches)
+
+NOT IMPLEMENTED in Stage 1.
+"""
+from __future__ import annotations
+
+from PIL import Image
+
+from .base import Detector, DetectorResult
+
+
+class FaceSwapDetector(Detector):
+    name = "face_swap"
+
+    def run(self, image: Image.Image) -> DetectorResult:  # noqa: ARG002
+        raise NotImplementedError(
+            "FaceSwapDetector is reserved for Stage 2. "
+            "Set ENABLE_FACE_SWAP_DETECTOR=false in Stage 1."
+        )

src/deepfake_scanner/detectors/frequency.py

@@ -0,0 +1,26 @@
+"""
+Frequency-domain detector (Stage 2).
+
+The intuition: many generators leave characteristic artefacts in the frequency
+spectrum (DCT/FFT) that are not visible to humans but are highly diagnostic.
+This detector complements CLIP-based semantic detection — when the generator
+fools CLIP, frequency artefacts often still betray it.
+
+NOT IMPLEMENTED in Stage 1. Wire the implementation in when the dataset is
+curated and we can validate the approach on held-out generators.
+"""
+from __future__ import annotations
+
+from PIL import Image
+
+from .base import Detector, DetectorResult
+
+
+class FrequencyDetector(Detector):
+    name = "frequency_artifacts"
+
+    def run(self, image: Image.Image) -> DetectorResult:  # noqa: ARG002
+        raise NotImplementedError(
+            "FrequencyDetector is reserved for Stage 2. "
+            "Set ENABLE_FREQUENCY_DETECTOR=false in Stage 1."
+        )

src/deepfake_scanner/detectors/frequency_features.py

@@ -0,0 +1,253 @@
+"""
+Frequency-domain feature extraction for the Stage 3B detector.
+
+Why this exists
+---------------
+The CLIP classifier (Stage 2/3A) reads images the way humans do — semantic
+content. It has two measured weak spots (see
+docs/stage3b-implementation.md S3B-01):
+  - test_augmented FNR 19.2% — JPEG/resize/crop degrade the semantic signal
+  - heldout SDXL FNR 13.5% — unseen generators it misses as "real"
+Frequency artefacts are an ORTHOGONAL, largely generator-agnostic signal:
+diffusion/DiT upsampling leaves periodic spectral fingerprints that survive
+metadata stripping and many transforms. This module turns an image into a
+small, fixed-length, interpretable feature vector capturing those artefacts.
+
+Design constraints
+------------------
+- **No new dependencies**: pure NumPy + Pillow (both already runtime deps).
+  The DCT is implemented on top of `numpy.fft` (no SciPy).
+- **Deterministic**: same image bytes -> identical feature vector.
+- **Runtime-safe**: bounded work (image is resized to a fixed analysis
+  size first), guaranteed-finite output (no NaN/Inf reaches the caller).
+- **Versioned**: FEATURE_SCHEMA_VERSION + a fixed FEATURE_NAMES order so
+  offline-precomputed feature files can be validated against the runtime
+  extractor.
+
+Feature schema (freq-v1, 16 features, fixed order = FEATURE_NAMES)
+-----------------------------------------------------------------
+  radial_bin_0 .. radial_bin_7  (8) : log radial power spectrum of the
+       luminance 2D FFT, averaged into 8 concentric annuli (low->high freq),
+       each normalised by total spectral power. Captures the overall
+       low-vs-high frequency energy distribution.
+  mid_high_ratio          : energy in mid annuli / energy in high annuli.
+  high_energy_frac        : fraction of spectral power in the top half of
+                            frequencies (diffusion images often differ here).
+  spectral_slope          : slope of log(power) vs log(radius) — natural
+                            images follow a characteristic 1/f^a falloff;
+                            generated images deviate.
+  dct_highfreq_mean_abs   : mean |coeff| in the high-frequency quadrant of a
+                            whole-image DCT-II (log1p-scaled).
+  dct_highfreq_energy_frac: high-freq DCT energy / total DCT energy.
+  jpeg_blockiness_8       : 8-pixel-grid blockiness (boundary vs interior
+                            gradient ratio) — compression / generation grids.
+  highpass_energy         : log energy of a high-pass residual (image minus
+                            a box-blurred copy).
+  highpass_kurtosis       : kurtosis of the high-pass residual — real-sensor
+                            noise vs synthesised micro-texture differ.
+"""
+from __future__ import annotations
+
+import numpy as np
+from PIL import Image
+
+# Bump this whenever FEATURE_NAMES or the extraction math changes, so stale
+# precomputed feature files are rejected rather than silently mixed.
+FEATURE_SCHEMA_VERSION = "freq-v1"
+
+_N_RADIAL_BINS = 8
+
+FEATURE_NAMES: tuple[str, ...] = (
+    *[f"radial_bin_{i}" for i in range(_N_RADIAL_BINS)],
+    "mid_high_ratio",
+    "high_energy_frac",
+    "spectral_slope",
+    "dct_highfreq_mean_abs",
+    "dct_highfreq_energy_frac",
+    "jpeg_blockiness_8",
+    "highpass_energy",
+    "highpass_kurtosis",
+)
+
+N_FEATURES = len(FEATURE_NAMES)
+
+# Fixed analysis size. Resizing first makes FFT bins comparable across images
+# of any input size, and bounds the runtime cost on large uploads. Must be a
+# multiple of 8 so the JPEG-blockiness grid lines up.
+_ANALYSIS_SIZE = 256
+_EPS = 1e-8
+
+
+def _to_luminance_array(image: Image.Image) -> np.ndarray:
+    """RGB/any PIL image -> float64 luminance array at the fixed analysis size."""
+    # "L" = ITU-R 601-2 luma transform; deterministic. Resize with a fixed
+    # resample filter so output is reproducible.
+    gray = image.convert("L").resize(
+        (_ANALYSIS_SIZE, _ANALYSIS_SIZE), Image.BILINEAR
+    )
+    arr = np.asarray(gray, dtype=np.float64)
+    return arr
+
+
+def _radial_power_features(lum: np.ndarray) -> tuple[np.ndarray, float, float, float]:
+    """Radial power-spectrum bins + mid/high ratio, high fraction, slope."""
+    # 2D FFT magnitude spectrum, DC shifted to centre.
+    f = np.fft.fftshift(np.fft.fft2(lum))
+    power = np.abs(f) ** 2
+    total = power.sum() + _EPS
+
+    h, w = power.shape
+    cy, cx = h / 2.0, w / 2.0
+    yy, xx = np.ogrid[:h, :w]
+    r = np.sqrt((yy - cy) ** 2 + (xx - cx) ** 2)
+    r_max = r.max() + _EPS
+    # Bin index 0..N-1 by normalised radius.
+    bin_idx = np.minimum(
+        (r / r_max * _N_RADIAL_BINS).astype(int), _N_RADIAL_BINS - 1
+    )
+
+    bins = np.zeros(_N_RADIAL_BINS, dtype=np.float64)
+    for i in range(_N_RADIAL_BINS):
+        mask = bin_idx == i
+        if mask.any():
+            bins[i] = power[mask].mean()
+    # Normalise by total mean power, then log1p for scale stability.
+    bins_norm = np.log1p(bins / (bins.sum() + _EPS))
+
+    # Mid (bins 3-4) vs high (bins 6-7) energy ratio.
+    mid = bins[3:5].sum()
+    high = bins[6:8].sum()
+    mid_high_ratio = float(np.log1p(mid / (high + _EPS)))
+
+    # Fraction of power in the top-half frequency bins.
+    high_energy_frac = float(bins[_N_RADIAL_BINS // 2 :].sum() / (bins.sum() + _EPS))
+
+    # Spectral slope: linear fit of log(power) vs log(radius) over bins.
+    radii = np.arange(1, _N_RADIAL_BINS + 1, dtype=np.float64)
+    log_r = np.log(radii)
+    log_p = np.log(bins + _EPS)
+    # least-squares slope
+    slope = float(np.polyfit(log_r, log_p, 1)[0])
+
+    return bins_norm, mid_high_ratio, high_energy_frac, slope
+
+
+def _dct2(block: np.ndarray) -> np.ndarray:
+    """Separable 2D DCT-II implemented via numpy.fft (no SciPy).
+
+    Uses the standard even-symmetric-extension trick so we depend only on
+    numpy. Returns the 2D DCT-II of the input array.
+    """
+
+    def dct1(x: np.ndarray, axis: int) -> np.ndarray:
+        n = x.shape[axis]
+        # Even extension along axis: [x, reversed(x)]
+        x_ext = np.concatenate([x, np.flip(x, axis=axis)], axis=axis)
+        X = np.fft.fft(x_ext, axis=axis)
+        # Keep first n, apply the DCT-II phase factor.
+        sl = [slice(None)] * x.ndim
+        sl[axis] = slice(0, n)
+        Xn = X[tuple(sl)]
+        k = np.arange(n)
+        phase = np.exp(-1j * np.pi * k / (2 * n))
+        shape = [1] * x.ndim
+        shape[axis] = n
+        phase = phase.reshape(shape)
+        return np.real(Xn * phase)
+
+    return dct1(dct1(block, axis=0), axis=1)
+
+
+def _dct_features(lum: np.ndarray) -> tuple[float, float]:
+    """High-frequency DCT statistics over the whole (resized) image."""
+    coeffs = _dct2(lum)
+    abs_c = np.abs(coeffs)
+    n = abs_c.shape[0]
+    half = n // 2
+    # High-frequency quadrant = bottom-right (both axes high).
+    high = abs_c[half:, half:]
+    total_energy = (abs_c ** 2).sum() + _EPS
+    high_mean_abs = float(np.log1p(high.mean()))
+    high_energy_frac = float((high ** 2).sum() / total_energy)
+    return high_mean_abs, high_energy_frac
+
+
+def _blockiness_8(lum: np.ndarray) -> float:
+    """8-grid blockiness: mean gradient ON 8-px boundaries vs OFF them.
+
+    JPEG and many generator upsamplers leave an 8x8 grid signature: gradients
+    across block boundaries are systematically larger than interior gradients.
+    Ratio ~1.0 = no grid; >1.0 = grid present.
+    """
+    # Vertical boundaries every 8 columns (cols 8,16,...).
+    col_diff = np.abs(np.diff(lum, axis=1))
+    bound_cols = col_diff[:, 7::8]
+    interior_cols = np.delete(col_diff, np.s_[7::8], axis=1)
+    # Horizontal boundaries every 8 rows.
+    row_diff = np.abs(np.diff(lum, axis=0))
+    bound_rows = row_diff[7::8, :]
+    interior_rows = np.delete(row_diff, np.s_[7::8], axis=0)
+
+    bound = (bound_cols.mean() + bound_rows.mean()) / 2.0
+    interior = (interior_cols.mean() + interior_rows.mean()) / 2.0 + _EPS
+    return float(bound / interior)
+
+
+def _highpass_features(lum: np.ndarray) -> tuple[float, float]:
+    """Energy + kurtosis of a high-pass residual (image minus box blur)."""
+    # 3x3 box blur via cumulative-sum trick would be overkill; use a simple
+    # separable mean filter through FFT-free convolution with np.
+    k = 3
+    pad = k // 2
+    padded = np.pad(lum, pad, mode="reflect")
+    # Separable box blur.
+    cs = np.cumsum(np.cumsum(padded, axis=0), axis=1)
+    cs = np.pad(cs, ((1, 0), (1, 0)), mode="constant")
+    h, w = lum.shape
+    y0, x0 = np.arange(h), np.arange(w)
+    Y1, X1 = np.meshgrid(y0, x0, indexing="ij")
+    Y2, X2 = Y1 + k, X1 + k
+    box = cs[Y2, X2] - cs[Y1, X2] - cs[Y2, X1] + cs[Y1, X1]
+    blur = box / (k * k)
+    resid = lum - blur
+
+    energy = float(np.log1p((resid ** 2).mean()))
+    std = resid.std() + _EPS
+    kurt = float(((resid - resid.mean()) ** 4).mean() / (std ** 4))
+    return energy, kurt
+
+
+def extract_features(image: Image.Image) -> np.ndarray:
+    """Extract the fixed-length freq-v1 feature vector from a PIL image.
+
+    Returns a float32 array of shape (N_FEATURES,). Output is guaranteed
+    finite (NaN/Inf are mapped to 0.0) so downstream code never sees bad
+    values regardless of degenerate input (constant image, 1x1, etc.).
+    """
+    lum = _to_luminance_array(image)
+
+    radial, mid_high, high_frac, slope = _radial_power_features(lum)
+    dct_mean_abs, dct_high_frac = _dct_features(lum)
+    blockiness = _blockiness_8(lum)
+    hp_energy, hp_kurt = _highpass_features(lum)
+
+    feats = np.concatenate(
+        [
+            radial,
+            np.array(
+                [
+                    mid_high,
+                    high_frac,
+                    slope,
+                    dct_mean_abs,
+                    dct_high_frac,
+                    blockiness,
+                    hp_energy,
+                    hp_kurt,
+                ],
+                dtype=np.float64,
+            ),
+        ]
+    )
+    feats = np.nan_to_num(feats, nan=0.0, posinf=0.0, neginf=0.0)
+    return feats.astype(np.float32)

src/deepfake_scanner/preprocess.py

@@ -0,0 +1,48 @@
+"""
+Image decoding.
+
+In the new architecture, each detector does its own tensor preprocessing
+(CLIP needs CLIP's specific resize/normalise; frequency detectors don't
+resize at all). This module just turns raw bytes into a validated PIL Image
+that all detectors can consume.
+"""
+from __future__ import annotations
+
+from io import BytesIO
+
+from PIL import Image, ImageOps, UnidentifiedImageError
+
+# Pillow's default decompression-bomb guard is ~89 MP. Tighten it: a 25 MP
+# image is more than enough for any consumer photo, and rejecting larger
+# inputs caps memory usage on the inference server.
+Image.MAX_IMAGE_PIXELS = 25_000_000
+
+
+class InvalidImageError(ValueError):
+    """Raised when the bytes do not decode as a usable image."""
+
+
+def decode(image_bytes: bytes) -> Image.Image:
+    """Decode raw bytes into an RGB PIL image. Raises InvalidImageError on failure."""
+    try:
+        image = Image.open(BytesIO(image_bytes))
+        image.load()  # Force decode now so we catch errors here, not later.
+    except (UnidentifiedImageError, OSError) as exc:
+        raise InvalidImageError(f"Could not decode image: {exc}") from exc
+    except Image.DecompressionBombError as exc:
+        raise InvalidImageError(f"Image too large: {exc}") from exc
+
+    # Phone cameras almost always store the sensor readout unrotated and record
+    # the intended rotation in the EXIF Orientation tag — 9 of the first 13 real
+    # test photos (Galaxy S23 Ultra) came through as orientation 6, i.e. stored
+    # sideways. Every viewer applies that tag, so a user who uploads a portrait
+    # photo sees it upright and assumes we scanned it upright. Without this call
+    # the detector was seeing it rotated 90°, which is both wrong for users and
+    # would have silently corrupted the P0 false-positive measurement.
+    # exif_transpose is a no-op when the tag is absent or already upright.
+    image = ImageOps.exif_transpose(image)
+
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+
+    return image

src/deepfake_scanner/provenance/__init__.py

@@ -0,0 +1,3 @@
+from .c2pa import check_c2pa
+
+__all__ = ["check_c2pa"]

src/deepfake_scanner/provenance/c2pa.py

@@ -0,0 +1,346 @@
+"""
+C2PA / Content Credentials verification.
+
+C2PA is a cryptographic provenance standard adopted by:
+  • Cameras: Sony, Nikon, Leica, Canon (rolling out)
+  • AI tools: OpenAI (DALL-E, GPT image), Adobe Firefly, Microsoft, Google
+  • Editors: Photoshop, Lightroom
+
+A valid C2PA manifest gives strong evidence about an image's origin. We use
+this as a FIRST-PASS check before the ML model runs:
+
+  • valid manifest from a known camera     → strong evidence of "authentic"
+  • valid manifest from a known generator  → strong evidence of "ai_generated"
+  • no manifest / invalid manifest         → fall through to the model
+
+This module returns advisory data only — the routing logic lives in api/v1.py.
+
+Library: `c2pa-python` is Apache-2.0 licensed and the official Python bindings
+to the Content Authenticity Initiative SDK.
+
+Graceful degradation, and its trap
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The c2pa package has native dependencies and may not be installable in every
+environment (e.g. Hugging Face Spaces ARM builds). If import fails, the service
+still works — provenance is just always reported as "not present".
+
+That fallback hid a real outage between 2026-05 and 2026-08-09: the module
+called `Reader.from_stream(...)`, which was removed in c2pa-python 0.3x, and
+`requirements.txt` pinned only `>=0.5.0`. Every call raised `AttributeError`,
+the blanket `except` swallowed it, and provenance was reported absent for every
+upload — including images carrying a valid OpenAI manifest. "No manifest found"
+and "the reader is broken" looked identical from the outside.
+
+Two defences against a repeat:
+  1. `_build_reader()` tries each known constructor generation and records
+     which one worked, so an API change degrades to a warning, not silence.
+  2. `check_c2pa()` looks for C2PA byte markers in the payload. If a marker is
+     present but the reader returned nothing, that is logged at WARNING —
+     the one case that must never pass quietly.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from io import BytesIO
+
+from ..config import settings
+
+log = logging.getLogger(__name__)
+
+# Try to import c2pa lazily; degrade gracefully if unavailable.
+try:
+    import c2pa  # type: ignore
+    _C2PA_AVAILABLE = True
+except Exception as exc:  # pragma: no cover - environment-dependent
+    log.info("c2pa-python not available (%s); provenance check disabled.", exc)
+    c2pa = None  # type: ignore
+    _C2PA_AVAILABLE = False
+
+# Byte signatures of an embedded C2PA manifest. `caBX` is the JUMBF box type
+# used in PNG/JPEG; `jumbf`/`c2pa` appear in the box headers of most formats.
+_C2PA_MARKERS = (b"caBX", b"jumbf", b"c2pa")
+
+# Magic-number → mime. The reader needs the real format: passing "image/jpeg"
+# for a PNG makes it fail to find a manifest that is demonstrably there.
+_MAGIC: tuple[tuple[bytes, str], ...] = (
+    (b"\x89PNG\r\n\x1a\n", "image/png"),
+    (b"\xff\xd8\xff", "image/jpeg"),
+    (b"GIF87a", "image/gif"),
+    (b"GIF89a", "image/gif"),
+    (b"BM", "image/bmp"),
+    (b"II*\x00", "image/tiff"),
+    (b"MM\x00*", "image/tiff"),
+)
+
+
+# IPTC digital-source-type codes that mean "a generative model made this".
+# Keyed on the IPTC vocabulary rather than on issuer names, because issuer
+# alone proves nothing about how the pixels were made: Adobe signs Photoshop
+# edits of ordinary photographs with the same certificate it would use for
+# Firefly output. The source type is the standards-defined answer.
+_AI_SOURCE_TYPES = frozenset({
+    "trainedAlgorithmicMedia",           # created by a generative model
+    "compositeWithTrainedAlgorithmicMedia",  # real content combined with AI content
+})
+
+
+@dataclass
+class C2paResult:
+    present: bool
+    valid: bool | None
+    issuer: str | None
+    claim_generator: str | None
+    # True when some manifest in the chain declares an AI digital source type.
+    ai_declared: bool = False
+    # "active"  — the active manifest itself declares it (e.g. ChatGPT)
+    # "chain"   — only an ancestor declares it (e.g. Gemini, whose active
+    #             manifest describes watermarking and reports `composite`,
+    #             while every parent declares trainedAlgorithmicMedia)
+    ai_declared_scope: str | None = None
+    # Whether the signing certificate chains to an accepted trust anchor.
+    # SEPARATE from `valid`: a manifest can be intact and cryptographically
+    # sound while signed by anyone at all. None = could not establish.
+    signer_trusted: bool | None = None
+
+
+_NOT_PRESENT = C2paResult(present=False, valid=None, issuer=None, claim_generator=None)
+
+
+def _ai_source_types(manifest: dict) -> set[str]:
+    """Digital source types declared by a single manifest's actions."""
+    found: set[str] = set()
+    for assertion in manifest.get("assertions") or []:
+        if not str(assertion.get("label", "")).startswith("c2pa.actions"):
+            continue
+        for action in (assertion.get("data") or {}).get("actions") or []:
+            dst = action.get("digitalSourceType")
+            if dst and str(dst).rsplit("/", 1)[-1] in _AI_SOURCE_TYPES:
+                found.add(str(dst).rsplit("/", 1)[-1])
+    return found
+
+
+def _scan_for_ai(data: dict, active_id: str | None) -> tuple[bool, str | None]:
+    """Walk EVERY manifest in the store looking for an AI source-type claim.
+
+    Checking only the active manifest is not enough. Gemini's active manifest
+    covers the final watermarking step and reports `composite`; the claim that
+    a model generated the pixels sits in the parent manifests. Reading only the
+    active one would miss all 50 Gemini images in the P0 benchmark.
+    """
+    manifests = data.get("manifests") or {}
+    if active_id and active_id in manifests:
+        if _ai_source_types(manifests[active_id]):
+            return True, "active"
+    for mid, manifest in manifests.items():
+        if mid != active_id and _ai_source_types(manifest):
+            return True, "chain"
+    return False, None
+
+
+def sniff_mime(image_bytes: bytes) -> str:
+    """Best-effort mime type from magic bytes; falls back to image/jpeg."""
+    for magic, mime in _MAGIC:
+        if image_bytes.startswith(magic):
+            return mime
+    # RIFF....WEBP — the size field sits between the two markers.
+    if image_bytes[:4] == b"RIFF" and image_bytes[8:12] == b"WEBP":
+        return "image/webp"
+    # ISO-BMFF family (HEIC/AVIF): "....ftyp<brand>".
+    if image_bytes[4:8] == b"ftyp":
+        brand = image_bytes[8:12]
+        if brand.startswith(b"avif"):
+            return "image/avif"
+        return "image/heic"
+    return "image/jpeg"
+
+
+def _build_reader(mime: str, image_bytes: bytes):
+    """Construct a c2pa Reader across API generations.
+
+    0.3x   → Reader(format, stream)
+    0.5-0.2x → Reader.from_stream(format, stream)
+    Raises the last error if no constructor works.
+    """
+    last_exc: Exception | None = None
+    for attempt in ("constructor", "from_stream"):
+        try:
+            if attempt == "constructor":
+                return c2pa.Reader(mime, BytesIO(image_bytes))  # type: ignore[misc]
+            factory = getattr(c2pa.Reader, "from_stream", None)  # type: ignore[union-attr]
+            if factory is None:
+                continue
+            return factory(mime, BytesIO(image_bytes))
+        except AttributeError as exc:
+            # Wrong API generation — try the next one.
+            last_exc = exc
+        except Exception:
+            # The constructor exists but this payload has no readable manifest.
+            raise
+    raise last_exc or RuntimeError("no usable c2pa Reader constructor")
+
+
+def _claim_generator(active: dict) -> str | None:
+    """Read the generator name across manifest schema versions.
+
+    Older manifests carry a `claim_generator` string. Newer ones (C2PA 1.3+,
+    which is what OpenAI writes today) carry `claim_generator_info`, a list of
+    {name, version} entries — `claim_generator` is then absent.
+    """
+    direct = active.get("claim_generator")
+    if direct:
+        return str(direct)
+
+    info = active.get("claim_generator_info")
+    if isinstance(info, dict):
+        info = [info]
+    if isinstance(info, list):
+        names = []
+        for entry in info:
+            if isinstance(entry, dict) and entry.get("name"):
+                version = entry.get("version")
+                names.append(f"{entry['name']} {version}" if version else str(entry["name"]))
+            elif isinstance(entry, str):
+                names.append(entry)
+        if names:
+            return ", ".join(names)
+    return None
+
+
+# Exact validation states, never substring tests. `"untrusted"` CONTAINS
+# `"trusted"`, and a substring check therefore read an untrusted signature as a
+# trusted one — the same trap as `"Invalid".endswith("valid")`, reintroduced in
+# the other branch of this function and caught by review on 2026-09-14.
+_STATE_VALID = frozenset({"valid", "trusted"})
+_STATE_INVALID = frozenset({"invalid", "untrusted"})
+
+
+def _validity(reader, data: dict, active: dict) -> bool | None:
+    """Prefer the SDK's own validation state; fall back to manifest fields.
+
+    Returns True only for a state we positively recognise as good, False for one
+    we recognise as bad, and **None for anything else** — unknown states, an
+    unreadable state, and an absent fallback all mean *we could not establish
+    this*, which must never be spelled the same way as *this is fine*. Callers
+    treat None as indeterminate and leave the model in charge.
+    """
+    getter = getattr(reader, "get_validation_state", None)
+    if callable(getter):
+        try:
+            state = getter()
+        except Exception as exc:  # pragma: no cover - SDK-dependent
+            log.debug("C2PA validation_state unavailable: %s", exc)
+        else:
+            if state is not None:
+                text = str(state).strip().lower()
+                if text in _STATE_INVALID:
+                    return False
+                if text in _STATE_VALID:
+                    return True
+                log.warning(
+                    "C2PA reported an unrecognised validation state %r; treating "
+                    "it as indeterminate rather than valid.", state,
+                )
+                return None
+
+    # `validation_status` is an EMPTY LIST when verification passes and a
+    # populated one when it does not. Test key presence rather than truthiness:
+    # `[] or active.get(...)` collapses "present and empty" (a pass) into
+    # "absent" (unknown), which is the opposite meaning.
+    for source in (data, active):
+        if "validation_status" in source:
+            return not bool(source["validation_status"])
+    # Neither source said anything at all — fail closed.
+    return None
+
+
+def _signer_trusted(state: object, data: dict, active: dict) -> bool | None:
+    """Whether the signing certificate chains to an accepted trust anchor.
+
+    This is a SEPARATE question from `_validity`. A manifest can be perfectly
+    intact and cryptographically sound while being signed by anybody at all —
+    including an attacker. Only a `trusted` state proves the signer chains to a
+    trust list.
+
+    We ship no trust anchors (adding a trust list is a third-party asset and
+    needs a licensing-tracker entry first), so in practice this returns False or
+    None today: even genuine OpenAI images report `Valid` alongside a
+    `signingCredential.untrusted` status. Surfaced so the gap is visible in the
+    API response and in logs rather than silently assumed away.
+    """
+    if state is not None and str(state).strip().lower() == "trusted":
+        return True
+
+    status = data.get("validation_status") or active.get("validation_status") or []
+    if isinstance(status, list):
+        for entry in status:
+            if isinstance(entry, dict) and str(
+                entry.get("code", "")
+            ).startswith("signingCredential.untrusted"):
+                return False
+    return None
+
+
+def check_c2pa(image_bytes: bytes) -> C2paResult:
+    """Return whatever C2PA evidence we can extract from the raw image bytes."""
+    if not settings.enable_c2pa_check or not _C2PA_AVAILABLE:
+        return _NOT_PRESENT
+
+    has_marker = any(m in image_bytes for m in _C2PA_MARKERS)
+    mime = sniff_mime(image_bytes)
+
+    try:
+        reader = _build_reader(mime, image_bytes)
+        manifest_json = reader.json()
+    except Exception as exc:
+        # Either no manifest at all, or the reader could not handle this
+        # payload. Those are indistinguishable to the caller — but if the
+        # bytes clearly contain a manifest, the reader is at fault and that
+        # must be loud. This is the exact failure that went unnoticed for
+        # three months.
+        if has_marker:
+            log.warning(
+                "C2PA manifest markers present in a %s payload but the reader "
+                "failed (%s: %s). Provenance is UNMEASURED, not absent — check "
+                "the installed c2pa-python API.",
+                mime, type(exc).__name__, exc,
+            )
+        else:
+            log.debug("C2PA read failed for %s: %s", mime, exc)
+        return _NOT_PRESENT
+
+    try:
+        data = json.loads(manifest_json)
+    except json.JSONDecodeError:
+        log.warning("C2PA reader returned non-JSON for a %s payload.", mime)
+        return _NOT_PRESENT
+
+    active_id = data.get("active_manifest")
+    manifests = data.get("manifests", {})
+    active = manifests.get(active_id, {}) if active_id else {}
+
+    issuer = None
+    sig_info = active.get("signature_info") or {}
+    if isinstance(sig_info, dict):
+        issuer = sig_info.get("issuer")
+
+    ai_declared, scope = _scan_for_ai(data, active_id)
+
+    raw_state = None
+    getter = getattr(reader, "get_validation_state", None)
+    if callable(getter):
+        try:
+            raw_state = getter()
+        except Exception:  # pragma: no cover - SDK-dependent
+            raw_state = None
+
+    return C2paResult(
+        present=True,
+        valid=_validity(reader, data, active),
+        issuer=issuer,
+        claim_generator=_claim_generator(active),
+        ai_declared=ai_declared,
+        ai_declared_scope=scope,
+        signer_trusted=_signer_trusted(raw_state, data, active),
+    )

src/deepfake_scanner/storage/__init__.py

@@ -0,0 +1,3 @@
+from .db import record_scan
+
+__all__ = ["record_scan"]

src/deepfake_scanner/storage/blob.py

@@ -0,0 +1,35 @@
+"""
+Optional image storage (S3 / Cloudflare R2).
+
+Privacy contract
+----------------
+Visitor uploads are NEVER persisted by default. This module is only invoked
+when ALL of the following are true:
+
+  1. settings.store_uploads is True (env: STORE_UPLOADS=true)
+  2. The request comes from an authenticated user
+  3. That user has explicitly opted in to scan history / forensic reports
+
+None of this is wired up in Stage 1. The module exists to make the
+architecture visible and to lock in the privacy default at the type level.
+"""
+from __future__ import annotations
+
+
+def store_image(scan_id: str, image_bytes: bytes) -> str:  # noqa: ARG001
+    """Upload image to blob storage; return the object key.
+
+    Raises
+    ------
+    PermissionError :
+        If called when settings.store_uploads is False, as a defence-in-depth
+        check against a future bug accidentally persisting visitor uploads.
+    """
+    from ..config import settings
+
+    if not settings.store_uploads:
+        raise PermissionError(
+            "Image storage is disabled by default. Enable explicitly via "
+            "STORE_UPLOADS=true AND user opt-in."
+        )
+    raise NotImplementedError("Blob storage wired up in Stage 2.")

src/deepfake_scanner/storage/db.py

@@ -0,0 +1,76 @@
+"""
+Scan-record persistence.
+
+Important
+---------
+This module persists ONLY metadata (scan_id, verdict, model_version, latency,
+timestamp). The image itself is NEVER stored here. Image storage is a separate,
+opt-in path handled by `storage.blob`.
+
+Stage 1 ships an in-memory fallback so the service runs out of the box with
+no database. Set DATABASE_URL to switch to Postgres (Neon, Supabase, etc.).
+"""
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+
+from ..config import settings
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class ScanRecord:
+    scan_id: str
+    verdict: str
+    confidence: float
+    model_version: str
+    latency_ms: float
+    c2pa_present: bool
+    timestamp_unix: float = field(default_factory=time.time)
+
+
+# In-memory store — bounded to last N records to avoid leaking memory.
+_MEMORY_CAP = 1000
+_memory_store: list[ScanRecord] = []
+
+
+def record_scan(record: ScanRecord) -> None:
+    """Persist a scan record. No-op-safe; never raises into the request path."""
+    try:
+        if settings.database_url:
+            _record_postgres(record)
+        else:
+            _record_memory(record)
+    except Exception as exc:
+        # Persistence failures must NEVER break the user-facing scan response.
+        log.warning("Failed to persist scan record: %s", exc)
+
+
+def _record_memory(record: ScanRecord) -> None:
+    _memory_store.append(record)
+    if len(_memory_store) > _MEMORY_CAP:
+        del _memory_store[: len(_memory_store) - _MEMORY_CAP]
+
+
+def _record_postgres(record: ScanRecord) -> None:  # pragma: no cover - infra
+    """Lazy import — SQLAlchemy is heavy and not needed in dev/tests."""
+    # Implementation intentionally deferred until Stage 2 — schema below.
+    #
+    #   CREATE TABLE scans (
+    #       scan_id        TEXT PRIMARY KEY,
+    #       verdict        TEXT NOT NULL,
+    #       confidence     REAL NOT NULL,
+    #       model_version  TEXT NOT NULL,
+    #       latency_ms     REAL NOT NULL,
+    #       c2pa_present   BOOLEAN NOT NULL,
+    #       created_at     TIMESTAMPTZ NOT NULL DEFAULT now()
+    #   );
+    raise NotImplementedError("Postgres persistence wired up in Stage 2.")
+
+
+def memory_store_snapshot() -> list[ScanRecord]:
+    """Test/debug helper — returns a copy of the in-memory store."""
+    return list(_memory_store)

src/deepfake_scanner/verdict.py

@@ -0,0 +1,87 @@
+"""
+Final verdict routing: cryptographic provenance first, model second.
+
+Why provenance outranks the model
+---------------------------------
+The CLIP head is a probabilistic guess. A validated C2PA manifest declaring
+an AI digital source type is a signed statement by the generator's own vendor
+that a model produced the pixels. On the P0 benchmark the model identifies
+68/100 closed-generator images; the provenance chain identifies 100/100. When
+both are available and they disagree, the signature is the better evidence.
+
+What this deliberately does NOT do
+----------------------------------
+1. **A camera manifest never forces `authentic`.** A C2PA-signing camera
+   pointed at a screen showing an AI image produces a perfectly valid capture
+   manifest for synthetic content, so "signed by a camera" cannot be trusted
+   to mean "real". The override is one-directional: provenance can only push a
+   verdict TOWARD `ai_generated`, never away from it.
+2. **Absence of a manifest means nothing.** Most real photographs carry no
+   C2PA data at all, and any re-encode — a screenshot, a messaging app, a
+   social upload — strips it. No manifest simply hands the decision to the
+   model, exactly as before.
+3. **An invalid manifest is ignored.** If the signature does not validate, the
+   claim is worthless and the model decides.
+
+Consequence worth stating plainly: this raises accuracy only on *untouched
+downloads*. The moment an image is screenshotted or re-encoded the manifest is
+gone and the model is on its own again — which is why the screenshot arm of
+the P0 benchmark is the one that measures the detector itself.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from .api.schemas import Verdict
+from .provenance.c2pa import C2paResult
+
+# Confidence reported when a validated signature decides the verdict. Not 1.0:
+# the signature proves the declared origin, not that our reading of every edge
+# case is perfect.
+PROVENANCE_CONFIDENCE = 0.99
+
+VerdictSource = Literal["model", "provenance"]
+
+
+@dataclass(frozen=True)
+class VerdictDecision:
+    verdict: Verdict
+    confidence: float
+    source: VerdictSource
+    reason: str | None = None
+
+
+def decide(
+    model_verdict: Verdict,
+    model_confidence: float,
+    provenance: C2paResult,
+) -> VerdictDecision:
+    """Combine the model's verdict with C2PA evidence.
+
+    Returns the model's own verdict unchanged unless a *validated* manifest in
+    the provenance chain declares the content to be model-generated.
+    """
+    trustworthy = (
+        provenance.present
+        and provenance.valid is True
+        and provenance.ai_declared
+    )
+    if not trustworthy:
+        return VerdictDecision(model_verdict, model_confidence, "model")
+
+    where = (
+        "the active manifest"
+        if provenance.ai_declared_scope == "active"
+        else "a parent manifest in the provenance chain"
+    )
+    issuer = provenance.issuer or "an unnamed issuer"
+    return VerdictDecision(
+        verdict="ai_generated",
+        confidence=PROVENANCE_CONFIDENCE,
+        source="provenance",
+        reason=(
+            f"Content Credentials signed by {issuer} declare this image as "
+            f"AI-generated in {where}."
+        ),
+    )