Update SynthScope Space from the GitHub repository

.env.example

@@ -0,0 +1,17 @@
+# SynthScope secrets template — copy to `.env` (gitignored) and fill in.
+# The backend auto-loads `.env` at startup (interface/backend/config.py:_load_dotenv).
+# Real shell env vars / secrets-manager values OVERRIDE this file.
+
+# SOTA image-generator API keys (for the "SOTA vs Detectors" demo). Each is optional;
+# a generator is greyed-out in the UI if its key is unset.
+GEMINI_API_KEY=        # Nano Banana (Gemini 2.5 Flash Image)  -> aistudio.google.com/apikey
+OPENAI_API_KEY=        # GPT Image (gpt-image-1)                -> platform.openai.com/api-keys
+XAI_API_KEY=           # Grok (grok-2-image)                    -> console.x.ai
+FAL_KEY=               # FLUX img2img via fal.ai                -> fal.ai/dashboard/keys
+
+# Ideogram 4.0 runs LOCALLY (open-weight) — no paid API key. HF_TOKEN is only needed ONCE to
+# download the license-gated weights (accept the license at hf.co/ideogram-ai/ideogram-4-nf4-diffusers).
+HF_TOKEN=              # free Hugging Face token, for gated open-weight downloads
+
+# Optional path overrides (defaults assume the cn023 node layout)
+# SYNTHSCOPE_CKPT_DIR=/hddstore/ssalehi/SynthScope/checkpoints

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+docs/images/datasets.jpg filter=lfs diff=lfs merge=lfs -text
+docs/images/detect.jpg filter=lfs diff=lfs merge=lfs -text
+docs/images/genbench.jpg filter=lfs diff=lfs merge=lfs -text
+docs/images/generator.jpg filter=lfs diff=lfs merge=lfs -text
+docs/images/leaderboard.jpg filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,39 @@
+name: Bug report
+description: Something doesn't work as expected
+labels: ["bug"]
+body:
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: A clear description of the bug, including the full error/traceback.
+      placeholder: Paste the traceback here.
+    validations:
+      required: true
+  - type: textarea
+    id: repro
+    attributes:
+      label: How to reproduce
+      description: The exact command or minimal code snippet (model, dataset, overrides).
+      placeholder: |
+        bash experiments/aide.sh
+        # or
+        import synthscope; synthscope.load("fat_former").predict("x.png")
+      render: shell
+    validations:
+      required: true
+  - type: input
+    id: model
+    attributes:
+      label: Detector / dataset
+      description: Which `model_type` and/or dataset config is involved?
+  - type: textarea
+    id: env
+    attributes:
+      label: Environment
+      description: OS, Python, PyTorch + CUDA versions, GPU, and `synthscope` / commit hash.
+      placeholder: |
+        python -c "import torch, synthscope; print(synthscope.__version__, torch.__version__, torch.version.cuda)"
+      render: shell
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Questions & discussion
+    url: https://github.com/SMSD75/SynthScope/discussions
+    about: Ask usage questions, share results, or propose ideas before opening an issue.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,27 @@
+name: Feature request / new detector or dataset
+description: Propose a new detector, dataset, or capability
+labels: ["enhancement"]
+body:
+  - type: dropdown
+    id: kind
+    attributes:
+      label: Type
+      options:
+        - New detector (method)
+        - New dataset / benchmark
+        - API / usability
+        - Other
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: What would you like?
+      description: Describe the method/dataset/feature. For a new detector, link the paper and official repo.
+    validations:
+      required: true
+  - type: textarea
+    id: value
+    attributes:
+      label: Why is it useful?
+      description: What does it add to the benchmark (e.g. a generator family or modality not yet covered)?

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,24 @@
+<!-- Thanks for contributing to SynthScope! -->
+
+## Summary
+<!-- What does this PR do and why? -->
+
+## Type of change
+- [ ] New detector (`model_type`)
+- [ ] New dataset / benchmark
+- [ ] Bug fix
+- [ ] Docs
+- [ ] Other
+
+## Checklist
+- [ ] Ran the affected experiment/eval and pasted the resulting `Test Metrics` below.
+- [ ] For a new detector: added a `models/<modality>/<name>/` folder with a method `README.md`,
+      `conf/model/...` + `conf/trainer/...` groups, an `experiments/*.sh` recipe, and (if it can
+      run from the API) a registry entry in `interface/backend/registry.py`.
+- [ ] `ruff check .` passes and `python -m compileall` is clean.
+- [ ] Updated the reproduction table / docs if results changed.
+
+## Results
+<!-- Paste the relevant logged metrics so reviewers can see the numbers reproduce. -->
+```
+```

.github/workflows/ci.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+# Lightweight, always-green-on-day-one checks. We deliberately do NOT build the heavy model
+# stack (torch + large backbones + dataset downloads) on GitHub runners; functional smoke tests
+# run on a GPU machine. See tests/ for the local smoke suite.
+jobs:
+  compile:
+    name: Byte-compile (syntax)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Compile all Python sources
+        run: python -m compileall -q synthscope synthscope interface models benchmark *.py
+
+  leaderboard:
+    name: Benchmark submissions + leaderboard
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      # Pure-stdlib check: validates every submissions/*.json and that LEADERBOARD.md is current.
+      - run: python benchmark/leaderboard.py --check
+
+  lint:
+    name: Ruff (errors only)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install "ruff>=0.4"
+      # Lint the maintained library surface (config + ignores in pyproject.toml [tool.ruff]).
+      # Repo-wide syntax is covered by the byte-compile job above.
+      - run: ruff check synthscope synthscope
+
+  package:
+    name: Build + metadata check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install "build>=1" "twine>=5"
+      - run: python -m build
+      - run: twine check dist/*

.gitignore

@@ -0,0 +1,32 @@
+*.png
+*.pth
+.cursor
+__pycache__
+wandb/
+*.log
+.codex
+.vscode
+outputs/
+.cache/
+.claude/
+# packaging / tooling artifacts
+build/
+dist/
+*.egg-info/
+.ruff_cache/
+.pytest_cache/
+
+# Secrets — never commit; use .env.example as the template
+.env
+.env.local
+.env.*.local
+
+# Large shared HF/torch cache (node-local downloads)
+.cache_shared/
+
+# arXiv trending candidates (regenerated daily by fetch_arxiv_trending.py)
+repro/trending_arxiv_candidates.json
+
+# generator benchmark runs: keep manifest/results/RESULTS.md, not the (large) image sets
+benchmark/generator_runs/*/real/
+benchmark/generator_runs/*/fake/

AGENTS.md

@@ -0,0 +1,5 @@
+# Codex Project Rules
+
+- For this SynthScope project, always run Python/project commands inside the `SynthScope` conda environment, for example:
+  `/home/ssalehi/miniconda3/bin/conda run -n SynthScope <command>`.
+- For this SynthScope project, discuss the intended approach and changes first. Only apply file/code changes after the user explicitly asks to run or make the changes.

CITATION.cff

@@ -0,0 +1,24 @@
+cff-version: 1.2.0
+message: "If you use SynthScope in your research, please cite it."
+title: "SynthScope: A Unified Library and Benchmark for AI-Generated Image and Video Detection"
+abstract: >-
+  SynthScope re-implements a range of state-of-the-art AI-generated (synthetic) image and video
+  detectors behind a single configuration system, training loop, datasets, transforms, and
+  metrics, so that different methods can be trained and benchmarked under identical conditions,
+  with results reported per generator/source.
+type: software
+authors:
+  # TODO(maintainer): verify the author name(s) and affiliation, and add ORCID(s).
+  - family-names: "Salehi Dehnavi"
+    given-names: "Seyed"
+    email: "s.salehidehnavi@uva.nl"
+    affiliation: "University of Amsterdam"
+repository-code: "https://github.com/SMSD75/SynthScope"
+url: "https://github.com/SMSD75/SynthScope"
+license: Apache-2.0
+version: "0.1.0"
+keywords:
+  - AI-generated image detection
+  - synthetic media detection
+  - deepfake detection
+  - benchmark

CODE_OF_CONDUCT.md

@@ -0,0 +1,33 @@
+# Contributor Code of Conduct
+
+## Our pledge
+
+We as members, contributors, and maintainers pledge to make participation in the SynthScope community a
+harassment-free experience for everyone, regardless of age, body size, visible or invisible
+disability, ethnicity, sex characteristics, gender identity and expression, level of experience,
+education, socio-economic status, nationality, personal appearance, race, religion, or sexual
+identity and orientation.
+
+## Our standards
+
+Examples of behavior that contributes to a positive environment:
+
+- Being respectful of differing opinions, viewpoints, and experiences.
+- Giving and gracefully accepting constructive feedback.
+- Focusing on what is best for the community and the science.
+
+Unacceptable behavior includes harassment, insulting or derogatory comments, personal or political
+attacks, publishing others' private information, and other conduct that could reasonably be
+considered inappropriate in a professional setting.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project
+maintainers at **s.salehidehnavi@uva.nl**. All complaints will be reviewed and investigated
+promptly and fairly. Maintainers are responsible for clarifying and enforcing these standards and
+may take any action they deem appropriate, up to and including a temporary or permanent ban.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org),
+version 2.1.

CONTRIBUTING.md

@@ -0,0 +1,71 @@
+# Contributing to SynthScope
+
+Thanks for your interest! SynthScope aims to be the **unified, reproducible benchmark** for
+AI-generated image & video detection. The most valuable contributions are **new detectors**,
+**new datasets/generators**, and **independently reproduced results**.
+
+## Getting set up
+
+```bash
+git clone https://github.com/SMSD75/SynthScope.git
+cd SynthScope
+pip install -e ".[dev]"          # editable install + contributor tooling
+# PyTorch: install the build matching your CUDA from https://pytorch.org if needed.
+# Some detectors also need OpenAI CLIP:  pip install git+https://github.com/openai/CLIP.git
+```
+
+Before opening a PR:
+
+```bash
+ruff check synthscope                   # lint the library surface (errors only; see pyproject [tool.ruff])
+python -m compileall synthscope interface models *.py
+pytest tests/ -k smoke            # where data/weights are available
+```
+
+## Adding a new detector
+
+The framework is model-agnostic: the trainer only needs a `compute_loss(batch)` method, where a
+batch is `(image, label, folder_id)` and `folder_id` is the generator/source used for
+per-generator metrics. To add a method `foo`:
+
+1. **Model** — create `models/<image|video>/foo/foo.py` with an `nn.Module` exposing
+   `compute_loss(batch) -> {"loss": ..., "outputs": ...}`. Add a short method `README.md`.
+2. **Dispatch** — register `model_type=foo` in `SynthScopeModelTrainer._build_model` (`trainer.py`).
+3. **Config** — add `conf/model/<modality>/foo.yaml` and `conf/trainer/<modality>/foo.yaml`.
+4. **Recipe** — add `experiments/<modality>/foo.sh` with the env vars + launch command.
+5. **(Optional) API/demo** — add a `ModelSpec` to `interface/backend/registry.py` so the detector
+   is runnable via `synthscope.load("foo")`, the CLI, and the web demo (preprocessing, checkpoint +
+   any official-weights remap, and the forward → P(fake) convention).
+6. **Results** — run it and paste the logged `Test Metrics` in the PR. New numbers should trace to
+   a real run, never be estimated (see the reproduction table convention in the README).
+
+## Adding a dataset
+
+Add a `dataset` class + data module in `dataset.py` (auto-download → common `real`/`fake` layout)
+and a `conf/dataset/<modality>/<name>.yaml`. Keep paths overridable via config/env — no hard-coded
+absolute paths.
+
+## Submitting a benchmark result
+
+Add one JSON file under [`benchmark/submissions/`](benchmark/submissions/) (see
+[`benchmark/schema.json`](benchmark/schema.json) and the protocol in
+[`benchmark/README.md`](benchmark/README.md)). Prefer `verified` results produced by
+`benchmark/evaluate.py`; for `external` numbers, set `source`. Regenerate and commit the table:
+
+```bash
+python benchmark/evaluate.py --model <key> --track <track> --data <root> --out benchmark/submissions/<key>.json
+python benchmark/leaderboard.py        # updates LEADERBOARD.md
+python benchmark/leaderboard.py --check  # what CI runs
+```
+
+## Style
+
+- Match the surrounding code; keep imports and helper/class definitions at module top level.
+- Keep changes focused; one detector/dataset/fix per PR.
+- Be honest about faithfulness: if a re-implementation diverges from the paper, document it (as the
+  reproduction notes already do).
+
+## Reporting bugs / proposing features
+
+Use the issue templates. For questions, open a Discussion. By contributing you agree your work is
+licensed under the project's [Apache-2.0 license](LICENSE).

Dockerfile

@@ -0,0 +1,21 @@
+# SynthScope web page as a Hugging Face Docker Space (CPU hardware, e.g. "CPU Upgrade": 8 vCPU, 32 GB).
+# Build context = the repository root (push_space.py copies this file to the root of the Space repo).
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends git ffmpeg libgl1 libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+# Spaces run the container as user 1000
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user PATH=/home/user/.local/bin:$PATH PYTHONUNBUFFERED=1 \
+    SYNTHSCOPE_CKPT_DIR=/home/user/ckpts SYNTHSCOPE_CACHE_DIR=/home/user/cache HF_HOME=/home/user/cache/hf \
+    SYNTHSCOPE_MAX_UPLOAD_MB=20 SYNTHSCOPE_MAX_RESIDENT_MODELS=2 XFORMERS_DISABLED=1
+WORKDIR /home/user/app
+
+# CPU-only PyTorch first (much smaller than the CUDA build), then SynthScope with the web extras
+RUN pip install --no-cache-dir --user torch torchvision --index-url https://download.pytorch.org/whl/cpu
+COPY --chown=user . .
+RUN pip install --no-cache-dir --user -e ".[web]" "git+https://github.com/openai/CLIP.git"
+
+EXPOSE 7860
+CMD ["bash", "deploy/hf-space/start.sh"]

LEADERBOARD.md

@@ -0,0 +1,36 @@
+# SynthScope Leaderboard (v1)
+
+_Auto-generated by `benchmark/leaderboard.py` — do not edit by hand._ See [benchmark/README.md](benchmark/README.md) for the protocol and how to submit.
+
+This is the **standardized, versioned** table: a few tracks with a fixed protocol that anyone can submit to. The web page's **Leaderboard** tab is broader (every paper benchmark, 30+ groups) and the repo supports many more datasets, which download and organize themselves: see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
+
+Metrics are percentages. **mAP** = mean AP over generators. Each track is ranked by its primary metric (in bold).
+
+## ForenSynths — image (ProGAN-trained, cross-generator)
+
+| Rank | Method | Eval | Acc | **mAP** | AP | AUROC | Verified | Notes |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 1 | FatFormer | official-weights | 98.39 | 99.70 | — | — | ✓ | ForenSynths-8 subset; official released weights |
+| 2 | CO-SPY (official weights) | official-weights | 95.12 | 98.71 | — | — | ✓ | ForenSynths-8; vendored official architecture |
+| 3 | UnivFD / CLIP | frozen-probe | — | 97.20 | — | 97.00 | ✓ | ForenSynths; frozen CLIP ViT-L/14 probe (ProGAN) |
+| 4 | CO-SPY (from scratch) | from-scratch | — | 96.98 | — | 96.79 | ✓ | ForenSynths; SynthScope re-implementation trained on ProGAN |
+| 5 | CNNDetection | from-scratch | — | 96.80 | — | 96.60 | ✓ | ForenSynths; ResNet-50 fine-tune (ProGAN) |
+| 6 | DINOv2 probe | frozen-probe | — | 95.03 | — | 94.86 | ✓ | ForenSynths; frozen DINOv2 ViT-L/14 probe |
+| 7 | SAFE | from-scratch | — | 94.13 | — | 93.68 | ✓ | ForenSynths; trained from scratch |
+| 8 | AIDE | official-weights | — | 92.08 | — | — | ✓ | ForenSynths-8; mAP reproduces, acc@0.5 miscalibrated (single-view vs official 5-view DCT) |
+| 9 | NPR | official-weights | 91.83 | 87.09 | — | — | ✓ | ForenSynths-8; official ProGAN weights |
+
+## GenImage SD-v1.4 — image
+
+| Rank | Method | Eval | Acc | mAP | AP | **AUROC** | Verified | Notes |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 1 | VIB-Net | from-scratch | — | 99.33 | — | 99.33 | ✓ | GenImage SD-v1.4 single real/fake pool (mAP == AUROC by construction) |
+| 2 | SPAI (official weights) | official-weights | — | — | 94.01 | 98.45 | ✓ | GenImage SD-v1.4; vendored official architecture, fixed-res 2x2x224 |
+| 3 | SPAI (from scratch) | from-scratch | — | — | — | 97.66 | ✓ | DMID Latent-Diffusion vs COCO; SynthScope fixed-resolution variant |
+
+## GenVideo-Val — video
+
+| Rank | Method | Eval | Acc | mAP | **AP** | AUROC | Verified | Notes |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 1 | ReStraV | frozen-probe | — | — | 96.67 | 97.06 | ✓ | Frozen DINOv2 trajectory geometry + MLP |
+| 2 | D3 | training-free | — | — | 95.60 | 93.60 | ✓ | Training-free frozen CLIP 2nd-order trajectory |

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or Derivative
+          Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work, excluding
+          those notices that do not pertain to any part of the Derivative
+          Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Seyed Salehi Dehnavi and the SynthScope contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSES.md

@@ -0,0 +1,65 @@
+# Licenses and attribution
+
+The **SynthScope framework** — the training/eval pipeline, the unified config system, the dataset
+modules, the metrics/transform/logging stack, the `synthscope` Python API, and the web interface — is
+released under the **Apache License 2.0** (see [LICENSE](LICENSE)).
+
+SynthScope **re-implements** a range of published detectors and, for a few methods, **vendors** the
+authors' original network code so their released weights can be loaded faithfully. Those
+third-party components, and every **pretrained checkpoint / backbone** SynthScope loads, remain under
+**their original licenses**. SynthScope does not relicense them.
+
+> ⚠️ **Before redistributing any model weights** (e.g. via a public model zoo / Hugging Face Hub),
+> confirm that the upstream license permits redistribution and your intended use. Several
+> AI-generated-image-detection paper repos and some pretrained backbones are **research- or
+> non-commercial-only**, and some weights are **gated** (require accepting an upstream license).
+> The table below is a starting point — verify each entry against its source. (`⚠ verify` = please
+> confirm and fill in from the official repo before relying on it.)
+
+## Detectors
+
+| `model_type` | Method (venue) | Upstream code | Code license | Released weights SynthScope can load | Weights license |
+| --- | --- | --- | --- | --- | --- |
+| `cnn` | CNNDetection (CVPR'20) | PeterWang512/CNNDetection | ⚠ verify | ProGAN-trained (SynthScope-trained here) | ⚠ verify |
+| `clip` | UnivFD (CVPR'23) | WisconsinAIVision/UniversalFakeDetect | ⚠ verify | — (SynthScope-trained probe) | n/a |
+| `npr` | NPR (CVPR'24) | chuangchuangtan/NPR-DeepfakeDetection | ⚠ verify | **official** ProGAN weights | ⚠ verify |
+| `fat_former` | FatFormer (CVPR'24) | Michel-liu/FatFormer (verify) | ⚠ verify | **official** weights | ⚠ verify |
+| `aide` | AIDE (ICLR'25) | ⚠ verify (official repo) | ⚠ verify | **official** weights | ⚠ verify |
+| `co_spy` | CO-SPY (CVPR'25) | ⚠ verify (official repo) | ⚠ verify | **official** weights | ⚠ verify |
+| `spai` | SPAI (CVPR'25) | mever-team/spai (verify) | ⚠ verify | **official** weights | ⚠ verify |
+| `vib_net` | VIB-Net (CVPR'25) | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a |
+| `safe` | SAFE (KDD'25) | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a |
+| `pgc` | PGC | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a |
+| `cure` | CuRe (arXiv 2609.14316) | manic-cui/CuRe | Apache-2.0 (repo `LICENSE`) | **official** 25 MB adapter (`cure_adapter.pt`, loads as `cure.pth`) | Apache-2.0 (shipped in the repo; verify before redistributing) |
+| `dinov2` | DINOv2 probe | facebookresearch/dinov2 | Apache-2.0 (verify) | DINOv2 backbone | Apache-2.0 (verify) |
+| `dinov3` | DINOv3 probe | facebookresearch/dinov3 | DINOv3 License (gated) | DINOv3 backbone | **gated** — DINOv3 License |
+| `adapted_dino` | Adapter-tuned DINOv2 (SynthScope) | this repo | Apache-2.0 | — (SynthScope-trained) | n/a |
+| `d3` | D3 (ICCV'25) | ⚠ verify (official repo) | ⚠ verify | training-free (no weights) | n/a |
+| `restrav` | ReStraV | ⚠ verify (official repo) | ⚠ verify | — (SynthScope-trained) | n/a |
+| `stall` | STALL (CVPR'26) | ⚠ verify (official repo) | ⚠ verify | training-free; needs DINOv3 backbone | gated (DINOv3) |
+
+## Backbones / foundation models (downloaded on first use)
+
+| Backbone | Used by | License |
+| --- | --- | --- |
+| OpenAI CLIP (ViT-L/14) | `clip`, `fat_former`, `d3` | MIT (verify) |
+| OpenCLIP (ConvNeXt-XXL, LAION) | `aide` | code MIT; weights per LAION / model card (verify) |
+| DINOv2 (ViT-L/14, ViT-S/14) | `dinov2`, `adapted_dino`, `restrav` | Apache-2.0 (verify) |
+| DINOv3 (ViT-L/16) | `dinov3`, `stall` | **gated** — DINOv3 License (non-commercial; accept upstream) |
+| SigLIP | `co_spy` | Apache-2.0 (verify) |
+| MFM ViT-B/16 | `spai` | ⚠ verify |
+| Stable Diffusion VAE | `co_spy` | CreativeML Open RAIL-M (verify) |
+| Perception Encoder Core L/14-336 (`facebook/PE-Core-L14-336`) | `cure` | Apache-2.0 (Meta); `official_arch.py` vendors the CuRe/Meta backbone code — see its header |
+
+## Datasets
+
+The dataset auto-downloaders fetch third-party datasets (GenImage, ForenSynths/CNNDetection,
+AIGIBench, Co-Spy-Bench, DiTFake, UniversalFakeDetect, GenVideo, …). Each dataset is governed by
+its **own** terms / license at the source; SynthScope only automates retrieval and restructuring. Review
+and comply with each dataset's terms before use or redistribution.
+
+---
+
+*This file is a good-faith attribution map, not legal advice. Maintainers: replace every `⚠ verify`
+with the confirmed upstream repository URL and license before publishing a model zoo or any
+redistributed weights.*

README.md

@@ -1,10 +1,22 @@
 ---
 title: SynthScope
-emoji: 📚
-colorFrom: blue
-colorTo: red
+emoji: 🔎
+colorFrom: indigo
+colorTo: blue
 sdk: docker
+app_port: 7860
 pinned: false
+license: apache-2.0
+short_description: Is this image AI-generated? Compare published detectors.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# SynthScope
+
+Check whether an image or video is AI-generated with published detectors, see what each detector looks at, and compare how
+they hold up against today's image generators. This Space runs on CPU: a detector takes a few seconds per image.
+
+Code, benchmarks and documentation: **https://github.com/SMSD75/SynthScope**
+
+Each detector and its weights keep the license chosen by their authors (see the repository's `LICENSES.md`); this demo is
+free and non-commercial. If you are an author and would like your method removed or credited differently, please open an
+issue on GitHub.

TRENDING.md

@@ -0,0 +1,84 @@
+# 📈 What's Trending — AI-generated content detection
+
+*High-quality recent papers for image & video AIGC detection. High-quality = (1) a solid public GitHub repository, and (2) strong results versus previous methods. Auto-refreshed daily. Last updated: 2026-10-10.*
+
+Entries marked _in SynthScope_ are integrated in this repo; the rest are tracked, with the reason an integration is pending noted in their _Results_ line. See [docs/ADD_DETECTOR.md](docs/ADD_DETECTOR.md) for how a paper is integrated and reproduced.
+
+## Image detectors
+- **[CuRe: Learning Continuous Source Responses for Generalizable AI-Generated Image Detection](https://github.com/manic-cui/CuRe)** (manic-cui · arXiv · 2026) — [code](https://github.com/manic-cui/CuRe) · _in SynthScope: `cure`_  
+  Reframes backbone adaptation as regressing real-generated MIXING RATIOS instead of binary classification (finer supervision), with a compact source-response subspace that suppresses nuisance variation/shortcut cues. Frozen Perception-Encoder (PE-Core L/14-336) backbone + LoRA + subspace + linear head; the 25 MB adapter ships in the repo and infer.py scores a single image.  
+  _Results: 89.7% average balanced accuracy over ten public benchmarks, +5.2 pts over the second-best method, robust to common degradations (paper). [Integrated in SynthScope as `cure` (merged to main 2026-10-03): SynthScope's class is identical to the official model (395/395 state-dict tensors, logit diff 4.8e-6, 0 verdict flips). SynthScope-protocol results with the official adapter: UFD-8 93.9 / ForenSynths-13 82.0 / GenImage-8 85.2 balanced acc (SD-v1.4, its training set, 99.7). AIGIBench REPRODUCED: on the paper's own 20 full-image-synthesis subsets (full test data) the port scores 92.85 balanced acc / 97.48 AUROC vs the paper's 92.8 / 97.5, every subset within 0.10 pt (per-subset mean abs diff 0.03; the paper excludes the 5 face-swap subsets, which is why all 25 gives 85.95). Still unconfirmed: UnivFD (aggregation unspecified), AIGCDetect (data only on cn023, not run), and 7 of the paper's 10 benchmarks (not in SynthScope).]_
+- **[PGC: Peak-Guided Calibration for AI-Generated Image Detection](https://github.com/xiaoyu6868/PGC)** (Zhou et al. · 2026) — [code](https://github.com/xiaoyu6868/PGC) · _in SynthScope: `pgc`_  
+  LoRA-tuned DINOv2 RGB stream + a YCbCr-quantization-residual CNN, fused by a peak-guided (log-sum-exp) calibration. Strong cross-generator transfer.  
+  _Results: ForenSynths 96.25 mAP / GenImage 100 mAP (official weights)._
+- **[PROBE: Probing Generative Space for Generalizable AIGI Detection](https://github.com/Amamiya-C/PROBE-AIGI-Detection)** (arXiv · 2026) — [code](https://github.com/Amamiya-C/PROBE-AIGI-Detection) · _in SynthScope: `probe`_  
+  Boosts cross-generator generalization by augmenting training with images sampled from probed generative-space variations; the detector is a fine-tuned DINOv2 (sliding-patch) classifier. Released weights on ModelScope.  
+  _Results: PROBE-DINOv2 93.9% avg balanced-acc over 7 benchmarks (paper). SynthScope reproduces the official weights: GenImage 98.54 / Synthbuster 99.27 / UFD 95.95 mAP (mAUROC 96-99). [Integration on branch trending/probe, under review.]_
+- **[DiCoME: Divide-and-Conquer Multi-View Evidential Learning for Deepfake Detection](https://github.com/kxl0825/DiCoME)** (arXiv · 2026) — [code](https://github.com/kxl0825/DiCoME)  
+  Disentangles semantic vs structural-artifact views and fuses them with evidential learning to counter the 'semantic masking effect'. Released checkpoint (HF).  
+  _Results: SOTA cross-dataset deepfake generalization (DeepfakeBench protocol). [face-deepfake domain; SynthScope lacks those datasets locally — integration pending data.]_
+- **[Images as Tables: In-Context AIGI Detection with TabPFN](https://github.com/jpwalter30/Towards-Generalizable-Detection-of-AI-Generated-Images)** (arXiv · 2026) — [code](https://github.com/jpwalter30/Towards-Generalizable-Detection-of-AI-Generated-Images)  
+  Frozen DINOv3 CLS feature -> PCA(500) -> TabPFN in-context tabular classification; strong few-shot/low-data generalization with no task-specific training.  
+  _Results: Generalizable detection in the low-data regime (few labeled examples per new generator). [integration pending: in-context TabPFN paradigm — a clean SynthScope forward->logit needs the released fitted TabPFN+PCA+labeled-support-set checkpoint baked in (GDrive), plus an external version-sensitive tabpfn dependency; a different paradigm from the trained-detector template, and the headline is a low-data/few-shot niche (multi-multi acc 0.8765 / AUROC 0.9501 @625 samples) rather than full-data SOTA. Verified repo 2026-06-20: weights on GDrive, frozen DINOv3->PCA(500)->TabPFN.]_
+- **[Co-Spy: Fusing Semantic and Artifact Cues for Synthetic Image Detection](https://github.com/Megum1/Co-Spy)** (Cheng et al. · CVPR · 2025) — [code](https://github.com/Megum1/Co-Spy) · _in SynthScope: `co_spy`_  
+  Adaptive fusion of a SigLIP semantic stream and an SD-VAE artifact-residual stream.  
+  _Results: Co-Spy-Bench ~96 mAP; SynthScope retrain reaches 96.6 (beats released weights on that bench)._
+- **[SAFE: High-Frequency Artifact Detector with Targeted Augmentation](https://github.com/Ouxiang-Li/SAFE)** (Ouxiang Li et al. · KDD · 2025) — [code](https://github.com/Ouxiang-Li/SAFE) · _in SynthScope: `safe`_  
+  Lightweight DWT high-frequency artifact detector with a carefully designed augmentation recipe.  
+  _Results: SynthScope-trained reproduction (served): ForenSynths-8 97.79 / UFD-8 98.44 / GenImage-8 99.51 / DiTFake 99.96 mAP — beats the official checkpoint on GenImage._
+- **[AIDE: ConvNeXt Semantic + Low-Level DCT/SRM Cues](https://arxiv.org/abs/2406.19435)** (Yan et al. · ICLR · 2025) — [arXiv](https://arxiv.org/abs/2406.19435) · _in SynthScope: `aide`_  
+  Fuses ConvNeXt semantic features with low-level DCT/SRM frequency cues; native-resolution preprocessing.  
+  _Results: GenImage 97.12 mAP / AIGCDetect 97.94 mAP (official weights)._
+- **[SPAI: Spectral AI-Image Detection (any-resolution)](https://github.com/mever-team/spai)** (Karageorgiou et al. · CVPR · 2025) — [code](https://github.com/mever-team/spai) · _in SynthScope: `spai`_  
+  Spectral Reconstruction Similarity + Spectral Context Attention; detects at native/arbitrary resolution.  
+  _Results: SynthScope serves the official weights at native resolution (the faithful any-res path): GenImage 94.22 / Synthbuster 96.75 mAP — reproduces & beats the vendored-official 91.33._
+- **[FatFormer: Forgery-Aware Adaptive Transformer](https://github.com/Michel-liu/FatFormer)** (Liu et al. · CVPR · 2024) — [code](https://github.com/Michel-liu/FatFormer) · _in SynthScope: `fat_former`_  
+  Frozen CLIP + forgery-aware adapters + learnable text-prompt context.  
+  _Results: ForenSynths-8 99.7 mAP (official weights)._
+- **[NPR: Neighboring-Pixel Relationships](https://github.com/chuangchuangtan/NPR-DeepfakeDetection)** (Tan et al. · CVPR · 2024) — [code](https://github.com/chuangchuangtan/NPR-DeepfakeDetection) · _in SynthScope: `npr`_  
+  Small ResNet on neighboring-pixel residuals that expose generator up-sampling artifacts.  
+  _Results: GANGen 96.49 mAP / DiffusionForensics 99.4 mAP (official weights)._
+- **[UnivFD / CLIP: Towards Universal Fake Image Detectors](https://github.com/WisconsinAIVision/UniversalFakeDetect)** (Ojha et al. · CVPR · 2023) — [code](https://github.com/WisconsinAIVision/UniversalFakeDetect) · _in SynthScope: `clip`_  
+  Linear probe on frozen CLIP ViT-L/14 features — a simple, strong cross-generator baseline.  
+  _Results: UniversalFakeDetect 95.66 mAP; SynthScope-trained 96.44._
+- **[MoE-JEPA: Unifying Semantic Priors and High-Frequency Traces (V-JEPA for synthetic image forensics)](https://github.com/ALCOR-Lab-DIAG/MoE-JEPA)** (ALCOR-Lab-DIAG · ECCV'26 Workshop (AI for Multimedia Forensics) · 2026) — [code](https://github.com/ALCOR-Lab-DIAG/MoE-JEPA)  
+  Frozen V-JEPA 2 ViT-L backbone with a Residual Mixture-of-Experts injected into the last blocks, a Bayar-conv noise branch and Gated-Attention MIL pooling; uses JEPA 'world understanding' as a semantic prior.  
+  _Results: 95.54% acc / 94.21 F1 on SID-Set (300K images), outperforming much larger models (paper). [integration pending (assessed 2026-10-02): the authors' trained weights are NOT linked (only the frozen V-JEPA 2 backbone is downloaded separately) and there is no single-image inference entrypoint; can only be retrained. Re-checked 2026-10-05: unchanged.]_
+- **[RAID: Towards Robust AI-Generated Image Detection with Bit-Reversed Images](https://github.com/renxi-seu/RAID)** (renxi-seu · arXiv · 2026) — [code](https://github.com/renxi-seu/RAID)  
+  Detects via bit-planes: a bit-reversed image, gradient-based patch selection and a small convolutional classifier, with a theoretical analysis; ~100x faster than counterparts.  
+  _Results: Claims to outperform existing methods on 40+ benchmarks across cross-generator, cross-dataset and zero-shot settings (paper; README gives no numbers). [integration pending (assessed 2026-10-02): the SD-v1.4-trained weights are only shared via Baidu Cloud (not fetchable programmatically), and the README leaves the architecture and patch-score aggregation unspecified; revisit if a direct weight link appears. Re-checked 2026-10-05: unchanged.]_
+- **[Fleet: Few Shots Lead Effective AI-generated Image Detection](https://github.com/ICTMCG/Fleet)** (ICTMCG · arXiv · 2026) — [code](https://github.com/ICTMCG/Fleet)  
+  Dynamic few-shot adaptation: 'constrained routing correction' redirects novel AI samples away from non-AI-dominated routes in decoupled subspaces (DINOv3 backbone), so the detector adapts to rapidly-evolving/commercial generators from a handful of labeled examples. Introduces the Treasure benchmark (64 generators, 360k images, 20 closed-source commercial engines).  
+  _Results: On a modern generator (Doubao Seedream 4.0), 20.4% -> 73.1% with only 10-shot adaptation, where static SOTA collapses (paper). Repo now live with code (re-listed 2026-10-03; it had been pruned 2026-07-04 while an empty placeholder). [integration pending (assessed 2026-10-03): not a single-checkpoint detector — the Hugging Face repo wangjiaan/Fleet holds NINE per-target-generator few-shot adapted models (ADM, FLUX.2, GPT4O, HunyuanImage-3.0, NextStep, Qwen-Image, StarGAN, wan2.5, ...; 1.3 GB each), the generic stage-1 base is only on Baidu Pan, inference requires few-shot support data per target generator, and the Treasure benchmark is gated. No clean forward->logit that represents the method; revisit if a generic checkpoint appears on HF.]_
+
+## Video detectors
+- **[D3: Disentangled Detail Dynamics (training-free)](https://github.com/Zig-HS/D3)** (Zheng et al. · ICCV · 2025) — [code](https://github.com/Zig-HS/D3) · _in SynthScope: `d3`_  
+  Training-free: 2nd-order std of frozen per-frame features as the score.  
+  _Results: GenVideo-Val ~98 mAP (paper). SynthScope faithful protocol (X-CLIP-B/16 + 10% edge-crop): AUROC 95.95 / acc 91.7 (the paper's mAP isn't comparable on SynthScope's 10000:56 real:fake pool — AUROC is the robust metric)._
+- **[ReStraV: Representation-Trajectory Geometry](https://github.com/ChristianInterno/ReStraV)** (Internò et al. · 2025) — [code](https://github.com/ChristianInterno/ReStraV) · _in SynthScope: `restrav`_  
+  Frozen DINOv2 trajectory-geometry features + a standardized MLP head.  
+  _Results: GenVidBench Main ~93; SynthScope 92.7 mAP._
+- **[STALL: Spatio-Temporal whitened Log-Likelihoods (training-free)](https://github.com/OmerBenHayun/STALL)** (Ben Hayun et al. · CVPR · 2026) — [code](https://github.com/OmerBenHayun/STALL) · _in SynthScope: `stall`_  
+  Training-free spatial + temporal whitened-Gaussian log-likelihoods vs a real-video calibration.  
+  _Results: GenVideo-Val ~0.80 AUC (paper); SynthScope AUROC 83.5._
+- **[ATSS: Anomalous Temporal Self-Similarity for AI-Generated Video Detection](https://github.com/hwang-cs-ime/ATSS)** (Wang et al. (hwang-cs-ime) · arXiv · 2026) — [code](https://github.com/hwang-cs-ime/ATSS)  
+  Multimodal detector that exploits anomalous temporal self-similarity fingerprints across frames of AI-generated video. Inference code + a pretrained checkpoint (best.pt) are released; the data sampling/extraction pipeline is still being cleaned for release.  
+  _Results: Claims SOTA in AP/AUC/ACC over GenVideo, EvalCrafter, VideoPhy, and VidProM vs prior methods (e.g. DeMamba). Repo live 2026-06-21 with code+weights; per-generator numbers not yet in the README. [integration pending: the released checkpoint's forward() consumes PRECOMPUTED multimodal temporal-self-similarity tensors (sim_img/sim_txt/sim_cross, BLIP2-based), not raw video — and the feature-extraction/data-sampling front-end that produces them is 'still being cleaned for release (coming soon)'. SynthScope feeds raw video clips, so there is no faithful way to run ATSS until that pipeline is published; will revisit when released. Re-checked 2026-10-05: unchanged.]_
+- **[G2VD: Generalizable AI-Generated Video Detection via Counterfactual Intervention and Causal Disentanglement](https://github.com/DMOSCAR-98/G2VD)** (Du et al. (dumeng98) · arXiv · 2026) — [code](https://github.com/DMOSCAR-98/G2VD)  
+  Counterfactual-intervention + causal-disentanglement detector: a training-time CFIPipeline synthesizes controlled counterfactual samples (VAEs + frequency/pixel alignment), and a causal-disentanglement classifier with two domain-anchored branches and an HSIC independence constraint isolates the generalizable artifact signal from domain-specific content.  
+  _Results: GenVidBench cross-domain: >90% accuracy / AUC ~0.95 using only 10% of the training data (paper). Repo live 2026-07-07 (CLIP/XCLIP/DeMamba backbones — the same family as SynthScope's D3). [integration pending (re-checked 2026-10-05): repo moved from dumeng98/G2VD (now 404) to DMOSCAR-98/G2VD; checkpoints are still NOT released ('not stored in this repository', MODEL_ZOO.md lists no links) and the CFIPipeline VAE-pool is part of the staged pipeline. Revisit when checkpoints appear.]_
+- **[RIFT: Mind the Rift - Cross-Scale Coupling Mismatch for AI-Generated Video Detection](https://github.com/Litsay/RIFT)** (Litsay · arXiv (submitted ACM MM'26) · 2026) — [code](https://github.com/Litsay/RIFT)  
+  Treats a cross-scale coupling mismatch between macro temporal dynamics (differential geometry / persistent homology on frozen DINOv2 features) and micro residuals (steganalytic filtering + RAFT motion) as the forensic signal, scored by a coupling-divergence module.  
+  _Results: Own benchmark: 99.22% acc / 99.33 F1 / 99.96 AUC, 97.87% leave-one-generator-out (paper; no head-to-head against named prior detectors in the abstract, so not directly comparable). [integration pending (assessed 2026-10-02): trained weights are 'released upon acceptance' (not available), and it needs DINOv2 + RAFT features over ~120K training videos. Re-checked 2026-10-05: unchanged.]_
+
+## Benchmarks & curated lists
+- [GenImage: million-scale real-vs-fake benchmark](https://github.com/GenImage-Dataset/GenImage) — ImageNet-1k reals + ~equal synthetic from diverse generators.
+- [AIGIBench: robustness/generalization benchmark (23 fake subsets)](https://github.com/HorizonTEL/AIGIBench) — Multi-source generalization, degradation robustness, augmentation sensitivity, preprocessing impact.
+- [AI-GenBench: temporal (GAN→diffusion) generalization benchmark](https://github.com/MI-BioLab/AI-GenBench) — Incremental, historically-ordered training to test generalization to new generators (Verimedia @ IJCNN 2025).
+- [GenVidBench: 6M AI-generated video detection benchmark](https://arxiv.org/abs/2501.11340) — Large-scale benchmark for AI-generated video detection.
+- [DeepfakeBench: comprehensive deepfake-detection benchmark](https://github.com/SCLBD/DeepfakeBench) — Unified benchmark across many deepfake datasets/detectors.
+- [Awesome-AIGC-Image-Video-Detection (curated list)](https://github.com/ant-research/Awesome-AIGC-Image-Video-Detection) — Continuously-updated collection of image + video AIGC-detection papers and code.
+- [Awesome-Comprehensive-Deepfake-Detection (curated list)](https://github.com/qiqitao77/Awesome-Comprehensive-Deepfake-Detection) — Broad survey-style index of deepfake-detection research and repos.
+
+---
+*Auto-generated from `interface/backend/trending.json` by `repro/generate_trending_readme.py` — edit the JSON, not this file.*

_synthscope_env.py

@@ -0,0 +1,19 @@
+"""Backward compatibility for the pre-rename ``AGID_*`` environment variables.
+
+The project was renamed from AGID to SynthScope; every setting is now ``SYNTHSCOPE_<NAME>``. Old launch scripts that still
+export ``AGID_<NAME>`` keep working: importing this module copies each such variable to its new name unless the new
+name is already set. Dependency-free and idempotent, so any entry point can ``import _synthscope_env`` first.
+"""
+import os
+
+LEGACY_PREFIX = "AGID_"
+PREFIX = "SYNTHSCOPE_"
+
+
+def apply_legacy_env() -> None:
+    for key, value in list(os.environ.items()):
+        if key.startswith(LEGACY_PREFIX):
+            os.environ.setdefault(PREFIX + key[len(LEGACY_PREFIX):], value)
+
+
+apply_legacy_env()

accelerate_config.yaml

@@ -0,0 +1,19 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: DEEPSPEED
+mixed_precision: "no"
+num_machines: 1
+num_processes: 4            # number of GPUs
+machine_rank: 0
+main_training_function: main
+rdzv_backend: static
+same_network: true
+use_cpu: false
+
+deepspeed_config:
+  deepspeed_multinode_launcher: standard
+  gradient_accumulation_steps: 1
+  gradient_clipping: 1.0
+  offload_optimizer_device: none
+  offload_param_device: none
+  zero3_init_flag: false
+  zero_stage: 2

agid/__init__.py

@@ -0,0 +1,13 @@
+"""Deprecated alias of :mod:`synthscope` (the project was renamed from AGID to SynthScope).
+
+``import agid`` keeps working and returns the very same module object, so ``agid.load(...)``,
+``from agid.api import Detector`` and ``python -m agid`` behave exactly as before.
+"""
+import sys
+
+import synthscope
+from synthscope import api, cli, hub
+
+sys.modules[__name__] = synthscope
+for _name, _mod in (("api", api), ("cli", cli), ("hub", hub)):
+    sys.modules[f"{__name__}.{_name}"] = _mod

agid/__main__.py

@@ -0,0 +1,5 @@
+"""``python -m agid`` -> same as ``python -m synthscope`` (deprecated alias)."""
+from synthscope.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/README.md

@@ -0,0 +1,82 @@
+# SynthScope Benchmark & Leaderboard (v1)
+
+> Looking for the full setup guide (datasets, paper reproduction, generator benchmarks, run times)? See [`docs/BENCHMARKS.md`](../docs/BENCHMARKS.md).
+> This page covers the standardized, versioned submission track.
+
+A standardized, **versioned** evaluation protocol for AI-generated image & video detection, so that
+methods are comparable under identical data, metrics, and scoring. The leaderboard is generated
+from the submission files in [`submissions/`](submissions/) — see [LEADERBOARD.md](../LEADERBOARD.md).
+
+> **Why:** every detection paper ships its own repo, data format, and metric conventions, which
+> makes cross-method comparison unreliable. SynthScope fixes the protocol once and evaluates every
+> detector through the same pipeline.
+
+## Tracks (v1)
+
+| Track id | Modality | Test set | Train signal | Primary metric |
+| --- | --- | --- | --- | --- |
+| `forensynths` | image | ForenSynths real/fake, scored **per generator** | ProGAN-trained (cross-generator generalization) | **mAP** (mean AP over generators) |
+| `genimage-sdv1.4` | image | GenImage SD-v1.4 real/fake | in-distribution diffusion | **AUROC** |
+| `genvideo-val` | video | GenVideo validation split | mixed | **AP** |
+
+**These three tracks are only the start.** The repo supports 13 image and 8 video benchmark datasets (see the table in the
+[README](../README.md#-benchmarks)), and the web page's Leaderboard tab already compares detectors on 30+ benchmark groups.
+A dataset becomes a standardized track once its protocol (test set, grouping, metric) is fixed, so results stay comparable;
+adding one means a schema entry, an `evaluate.py` track and verified submissions, described in
+[`docs/BENCHMARKS.md`](../docs/BENCHMARKS.md). The datasets themselves download and organize themselves.
+
+Metrics are reported in **%**. `mAP` = mean Average Precision over generators; `AUROC`/`AP` use the
+detector's native fake-probability; `Acc` = mean balanced accuracy (mean of real- and fake-accuracy)
+over generators at threshold 0.5.
+
+## Protocol rules
+
+1. **No test-set training or tuning.** Thresholds/calibration must be fixed before evaluation.
+2. **Report per generator, then average.** A track's headline number is the unweighted mean over
+   generators (not a pooled number), which measures generalization rather than rewarding a few
+   large generators.
+3. **Declare provenance** via `evaluation`: `official-weights`, `from-scratch`, `frozen-probe`,
+   `training-free`, or `external` (numbers produced outside SynthScope).
+4. **Verified numbers trace to a run.** `verified: true` means the result came from a logged SynthScope
+   evaluation (set `source` to the log/run path). Reference seed entries are transcribed from the
+   repo's reproduction table.
+
+## Data layout
+
+The evaluator expects the SynthScope common layout — one directory per generator, each split into
+`real/` and `fake/`:
+
+```
+<benchmark_root>/
+  <generator_a>/real/*.png   <generator_a>/fake/*.png
+  <generator_b>/real/*.png   <generator_b>/fake/*.png
+```
+
+A single pooled set is just one generator directory (or `<root>/real`, `<root>/fake`).
+
+## Evaluate a detector and produce a submission
+
+```bash
+python benchmark/evaluate.py \
+  --model fat_former --track forensynths \
+  --data /path/to/forensynths_test \
+  --out benchmark/submissions/fat_former.json
+```
+
+This loads the detector via the `synthscope` API, scores every image (batched), computes per-generator +
+mean metrics, and writes a schema-valid submission JSON. Then regenerate the table:
+
+```bash
+python benchmark/leaderboard.py            # writes LEADERBOARD.md
+python benchmark/leaderboard.py --check     # CI: fail if LEADERBOARD.md is stale or a submission is invalid
+```
+
+## Submit a result (PR)
+
+1. Add one JSON file under `submissions/` (see [`schema.json`](schema.json) and existing entries).
+2. Run `python benchmark/leaderboard.py` and commit the updated `LEADERBOARD.md`.
+3. Open a PR. CI validates your submission and that the table is up to date. Prefer `verified`
+   results produced by `evaluate.py`; for `external` numbers, link the source.
+
+Versioning: this is **v1**. Protocol-breaking changes bump to v2 and start a new leaderboard so old
+numbers stay meaningful.

benchmark/aggregate_results.py

@@ -0,0 +1,102 @@
+"""Aggregate the self-recorded bench_eval results into a master reproduced-results table.
+
+Reads every repro/bench_results/<tag>.json (latest run per tag), prints a compact summary, writes
+repro/RESULTS_TABLE.md (model | weights | benchmark | n_gen | acc / mAP / mAUROC), and supports
+computing a paper-matching subset mean from a run's per-generator dict:
+
+    python benchmark/aggregate_results.py                       # refresh RESULTS_TABLE.md
+    python benchmark/aggregate_results.py --subset fatformer_official_forensynths progan,stylegan,...
+    python benchmark/aggregate_results.py --detail npr_official_diffusionforensics
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+REPRO = Path(__file__).resolve().parents[1] / "repro"
+RESULTS_DIR = REPRO / "bench_results"
+
+
+def load_runs() -> dict[str, dict]:
+    runs: dict[str, dict] = {}
+    for p in sorted(RESULTS_DIR.glob("*.json")):
+        if p.stem.startswith(("_validate", "_probe", "_detail")):
+            continue
+        try:
+            d = json.loads(p.read_text())
+        except Exception as e:  # noqa: BLE001
+            print(f"  skip {p.name}: {e}")
+            continue
+        tag = d.get("tag", p.stem)
+        if tag not in runs or d.get("date", "") >= runs[tag].get("date", ""):
+            runs[tag] = d
+    return runs
+
+
+def weights_of(tag: str) -> str:
+    if "_official_" in tag:
+        return "official"
+    if "_synthscope_" in tag:
+        return "synthscope"
+    return "other"
+
+
+def subset_mean(run: dict, gens: list[str]) -> dict:
+    pg = run.get("per_generator", {})
+    found = [g for g in gens if g in pg]
+    missing = [g for g in gens if g not in pg]
+    if not found:
+        return {"error": f"none of {gens} in {list(pg)}"}
+    def m(k):
+        return round(sum(pg[g][k] for g in found) / len(found), 2)
+    return {"n": len(found), "gens": found, "missing": missing,
+            "acc": m("acc"), "map": m("ap"), "mauroc": m("auroc")}
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--subset", nargs=2, metavar=("TAG", "GENS"),
+                    help="compute mean over a comma list of generators for one run tag")
+    ap.add_argument("--detail", metavar="TAG", help="print per-generator detail for one run tag")
+    args = ap.parse_args()
+    runs = load_runs()
+
+    if args.subset:
+        tag, gens = args.subset
+        if tag not in runs:
+            print(f"no run '{tag}'. have: {sorted(runs)}"); return 1
+        print(json.dumps(subset_mean(runs[tag], gens.split(",")), indent=2)); return 0
+
+    if args.detail:
+        if args.detail not in runs:
+            print(f"no run '{args.detail}'. have: {sorted(runs)}"); return 1
+        r = runs[args.detail]
+        print(f"# {args.detail}  ({r['model']}, {r['benchmark']}, {r.get('n_generators')} gens)  means={r['means']}")
+        for g, m in sorted(r["per_generator"].items()):
+            print(f"  {g:32s} acc={m['acc']:6.2f}  AP={m['ap']:6.2f}  AUROC={m['auroc']:6.2f}  "
+                  f"(real={m['real_acc']:.1f} fake={m['fake_acc']:.1f})")
+        return 0
+
+    # default: master table
+    rows = []
+    for tag, r in sorted(runs.items()):
+        me = r.get("means", {})
+        rows.append((r.get("model", "?"), weights_of(tag), r.get("benchmark", "?"),
+                     tag, r.get("n_generators", 0),
+                     me.get("acc"), me.get("map"), me.get("mauroc")))
+    rows.sort(key=lambda x: (x[0], x[1], x[3]))
+    lines = ["# SynthScope reproduced benchmark results (auto-generated from repro/bench_results/)\n",
+             "| model | weights | benchmark | tag | n_gen | acc | mAP | mAUROC |",
+             "| --- | --- | --- | --- | --- | --- | --- | --- |"]
+    for model, w, bench, tag, n, acc, mp, au in rows:
+        lines.append(f"| {model} | {w} | {bench} | `{tag}` | {n} | {acc} | {mp} | {au} |")
+    out = REPRO / "RESULTS_TABLE.md"
+    out.write_text("\n".join(lines) + "\n")
+    print("\n".join(lines))
+    print(f"\nWrote {out}  ({len(rows)} runs)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/bench_eval.py

@@ -0,0 +1,305 @@
+"""Unified benchmark eval for SynthScope detectors — reproduce a paper's per-generator table.
+
+Scores a detector over a benchmark's per-generator real/fake folders through the SAME verified
+synthscope inference path the web app uses (engine + registry + per-model preprocessing + fake-prob),
+computes per-generator + mean acc@0.5 / AP / AUROC, writes a JSON, and APPENDS a verified line to
+``repro/MASTER_RESULTS.jsonl`` so every reported number traces to this run + its job log.
+
+    python benchmark/bench_eval.py --model clip --benchmark forensynths \
+        --data /hddstore/ssalehi/SynthScope/CNNDetection/test --tag clip_synthscope_forensynths
+
+    # one step: download the dataset if needed (its built-in loader arranges it), pick the profile, evaluate
+    python benchmark/bench_eval.py --model npr --dataset commgen15 --root /data/CommGen15
+
+Benchmark profiles encode each dataset's correct generator grouping + real-pairing:
+  forensynths  CNNDetection/test    per top-level generator; reals+fakes collected recursively
+                                    (handles progan/<class>/{0_real,1_fake} nesting)
+  ufd          UniversalFakeDetect  fakes per generator; reals paired Ojha-style
+                                    (guided->imagenet, ldm/glide/dalle->laion)
+  pergen       freqnet/aigibench/   each <gen>/{0_real,1_fake}; optional shared --reals
+               cospybench/generic
+  pool         GenImage/single-pool one group from <root>/{real,fake} or {0_real,1_fake}
+
+Use --ckpt to override the checkpoint (official vs SynthScope-trained, for side-by-side runs).
+"""
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+from pathlib import Path
+
+import torch
+from PIL import Image, ImageFile
+
+import synthscope
+from models.image.cure.preprocess import SRC_FORMAT_KEY
+from datasets_util import ensure_dataset, find_eval_root
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff", ".JPEG"}
+REAL_DIRNAMES = ("0_real", "real", "reals")
+FAKE_DIRNAMES = ("1_fake", "fake", "fakes", "ai")
+SHARED_REAL_DIRS = {"imagenet", "laion", "_real_pool", "real", "0_real", "reals"}
+RESULTS_LOG = Path(__file__).resolve().parents[1] / "repro" / "MASTER_RESULTS.jsonl"
+# Ojha UniversalFakeDetect real pairing (verified against the official dataset_paths.py).
+UFD_REAL = {"guided": "imagenet"}  # everything else -> laion
+UFD_DEFAULT_REAL = "laion"
+
+
+def _images(d: Path) -> list[Path]:
+    if not d or not d.is_dir():
+        return []
+    return sorted(p for p in d.rglob("*") if p.suffix.lower() in {e.lower() for e in IMAGE_EXTS})
+
+
+def _collect(gen_dir: Path, dirnames: tuple[str, ...]) -> list[Path]:
+    """All images under any subdir of gen_dir whose name matches one of dirnames (any depth)."""
+    out: list[Path] = []
+    for name in dirnames:
+        for sub in gen_dir.rglob(name):
+            if sub.is_dir():
+                out += [p for p in sub.rglob("*") if p.suffix.lower() in {e.lower() for e in IMAGE_EXTS}]
+    # de-dup while keeping order
+    seen, uniq = set(), []
+    for p in out:
+        if p not in seen:
+            seen.add(p); uniq.append(p)
+    return sorted(uniq)
+
+
+def discover(benchmark: str, root: Path, shared_reals: Path | None, select: set[str] | None):
+    """Return [(generator, real_paths, fake_paths)] for the benchmark."""
+    shared = _images(shared_reals) if shared_reals else []
+
+    if benchmark == "pool":
+        reals = _collect(root, REAL_DIRNAMES) or shared
+        fakes = _collect(root, FAKE_DIRNAMES)
+        return [("all", reals, fakes)] if reals and fakes else []
+
+    if benchmark == "ufd":
+        out = []
+        gens = sorted(p.name for p in root.iterdir() if p.is_dir() and p.name not in SHARED_REAL_DIRS)
+        for gen in gens:
+            if select and gen not in select:
+                continue
+            real_src = root / UFD_REAL.get(gen, UFD_DEFAULT_REAL)
+            reals = _collect(real_src, REAL_DIRNAMES)
+            fakes = _collect(root / gen, FAKE_DIRNAMES)
+            if reals and fakes:
+                out.append((gen, reals, fakes))
+            else:
+                print(f"  skip {gen}: reals={len(reals)} fakes={len(fakes)}")
+        return out
+
+    if benchmark == "forensynths":
+        out = []
+        for gen_dir in sorted(p for p in root.iterdir() if p.is_dir()):
+            if select and gen_dir.name not in select:
+                continue
+            reals = _collect(gen_dir, REAL_DIRNAMES) or shared
+            fakes = _collect(gen_dir, FAKE_DIRNAMES)
+            if reals and fakes:
+                out.append((gen_dir.name, reals, fakes))
+            else:
+                print(f"  skip {gen_dir.name}: reals={len(reals)} fakes={len(fakes)}")
+        return out
+
+    # pergen / generic: each top-level dir (excluding shared-real dirs) is a generator
+    out = []
+    for gen_dir in sorted(p for p in root.iterdir() if p.is_dir()):
+        if gen_dir.name in SHARED_REAL_DIRS and gen_dir.name != root.name:
+            continue
+        if select and gen_dir.name not in select:
+            continue
+        reals = _collect(gen_dir, REAL_DIRNAMES) or shared
+        fakes = _collect(gen_dir, FAKE_DIRNAMES)
+        if reals and fakes:
+            out.append((gen_dir.name, reals, fakes))
+        else:
+            print(f"  skip {gen_dir.name}: reals={len(reals)} fakes={len(fakes)}")
+    return out
+
+
+def native_res_transform(tf):
+    """Drop the up-scaling Resize/CenterCrop from a model's eval transform, keeping
+    ToTensor + Normalize, so images are scored at NATIVE resolution.
+
+    Some methods (e.g. NPR on GANGen-Detection/freqnet) detect up-sampling artifacts and
+    MUST NOT be fed a bilinearly up-scaled image — resizing a 128x128 GAN crop to 256x256
+    injects the exact artifact they key on, collapsing the real class. The official NPR
+    GANGen recipe runs with ``no_resize=True``; this reproduces it. Safe only when every
+    image in a per-generator batch shares one resolution (true for freqnet's per-gen sets),
+    since bench_eval stacks a batch into a single tensor."""
+    import torchvision.transforms as T
+    if not isinstance(tf, T.Compose):
+        return tf
+    kept = [t for t in tf.transforms if not isinstance(t, (T.Resize, T.CenterCrop))]
+    return T.Compose(kept)
+
+
+@torch.no_grad()
+def _open_rgb(path: Path) -> Image.Image:
+    """Open ``path`` as RGB, recording the source file format in ``img.info`` so format-aware eval
+    transforms (CuRe's PNG->JPEG96 step) see what the official loader sees."""
+    source = Image.open(path)
+    source_format = source.format
+    image = source.convert("RGB")
+    image.info[SRC_FORMAT_KEY] = source_format
+    return image
+
+
+def score(det, model, spec, paths: list[Path], bs: int, transform=None) -> torch.Tensor:
+    device = det._engine.device
+    tf = transform if transform is not None else spec.transform
+    # Native-resolution specs (SPAI any-res) produce variable-size tensors that cannot be stacked,
+    # so they MUST be scored one image at a time.
+    if getattr(spec, "native_res", False):
+        bs = 1
+    probs = []
+    for i in range(0, len(paths), bs):
+        batch = []
+        for p in paths[i:i + bs]:
+            try:
+                batch.append(tf(_open_rgb(p)))
+            except Exception as e:  # noqa: BLE001
+                print(f"    bad image {p}: {e}")
+        if not batch:
+            continue
+        x = torch.stack(batch).to(device)
+        probs.append(det._engine._fake_prob(model, spec, x).detach().float().cpu())
+    return torch.cat(probs) if probs else torch.empty(0)
+
+
+def metrics(fake_probs: torch.Tensor, labels: torch.Tensor) -> dict | None:
+    from torchmetrics.functional import auroc, average_precision
+    if labels.unique().numel() < 2:
+        return None
+    ap = float(average_precision(fake_probs, labels, task="binary")) * 100
+    au = float(auroc(fake_probs, labels, task="binary")) * 100
+    preds = (fake_probs >= 0.5).int()
+    real_acc = float((preds[labels == 0] == 0).float().mean()) * 100
+    fake_acc = float((preds[labels == 1] == 1).float().mean()) * 100
+    return {"ap": round(ap, 2), "auroc": round(au, 2),
+            "real_acc": round(real_acc, 2), "fake_acc": round(fake_acc, 2),
+            "acc": round((real_acc + fake_acc) / 2, 2)}
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("-m", "--model", required=True, help="SynthScope detector key (see `synthscope list`)")
+    ap.add_argument("--benchmark", choices=["forensynths", "ufd", "pergen", "pool"],
+                    help="how the folders are grouped (chosen automatically with --dataset)")
+    ap.add_argument("--data", help="benchmark root (not needed with --dataset)")
+    ap.add_argument("--dataset", help="a supported dataset (see `python benchmark/download_dataset.py --list`): downloads it "
+                    "if missing, finds the folder to evaluate and the right --benchmark profile")
+    ap.add_argument("--root", help="with --dataset: where the dataset lives / should be downloaded (default: root_dir in its yaml)")
+    ap.add_argument("--no-download", action="store_true", help="with --dataset: fail instead of downloading missing data")
+    ap.add_argument("--dry-run", action="store_true", help="with --dataset: only show the resolved folder and profile")
+    ap.add_argument("--reals", default=None, help="shared real dir (pergen/forensynths fallback)")
+    ap.add_argument("--ckpt", default=None, help="checkpoint override (official vs SynthScope weights)")
+    ap.add_argument("--select", default=None, help="comma list: only these generators")
+    ap.add_argument("--limit", type=int, default=None, help="cap images per class per generator")
+    ap.add_argument("--bs", type=int, default=32)
+    ap.add_argument("--native-res", action="store_true",
+                    help="score at native resolution (strip up-scaling Resize/CenterCrop); "
+                         "for artifact-based methods like NPR on low-res GAN sets (freqnet)")
+    ap.add_argument("--device", default=None)
+    ap.add_argument("--tag", help="run tag (output filename + log); default <model>_<dataset>")
+    ap.add_argument("--out", default=None)
+    args = ap.parse_args(argv)
+
+    if args.dataset:
+        root = ensure_dataset(args.dataset, "image", args.root, download=not args.no_download and not args.dry_run)
+        eval_root, profile = find_eval_root(args.dataset, root)
+        args.data = args.data or str(eval_root)
+        args.benchmark = args.benchmark or profile
+        args.tag = args.tag or f"{args.model}_{args.dataset}"
+        print(f"[dataset] {args.dataset}: evaluating {args.data} with the '{args.benchmark}' profile", flush=True)
+        if args.dry_run:
+            return 0
+    if not (args.benchmark and args.data):
+        ap.error("give --dataset NAME, or both --benchmark and --data")
+    if not args.tag:
+        ap.error("--tag is required without --dataset")
+
+    load_kw = {}
+    if args.ckpt:
+        load_kw["checkpoint"] = args.ckpt
+    det = synthscope.load(args.model, device=args.device, **load_kw)
+    det._ensure_weights()
+    spec = det.spec
+    model = det._engine._get_model(spec)
+    eval_transform = native_res_transform(spec.transform) if args.native_res else spec.transform
+    if args.native_res:
+        print(f"[native-res] scoring at native resolution; transform={eval_transform}", flush=True)
+
+    root = Path(args.data).expanduser()
+    select = set(args.select.split(",")) if args.select else None
+    gens = discover(args.benchmark, root,
+                    Path(args.reals).expanduser() if args.reals else None, select)
+    if not gens:
+        print(f"No generators with real+fake under {root} (benchmark={args.benchmark})")
+        return 1
+
+    print(f"=== bench_eval {args.tag} | model={args.model} ckpt={args.ckpt or 'registry-default'} "
+          f"| {args.benchmark} | {len(gens)} generators | device={det._engine.device} ===", flush=True)
+
+    per_gen, agg = {}, {"ap": [], "auroc": [], "acc": []}
+    for name, reals, fakes in gens:
+        if args.limit:
+            reals, fakes = reals[:args.limit], fakes[:args.limit]
+        labels = torch.tensor([0] * len(reals) + [1] * len(fakes))
+        print(f"  {name}: {len(reals)} real + {len(fakes)} fake ...", flush=True)
+        probs = score(det, model, spec, reals + fakes, args.bs, transform=eval_transform)
+        if probs.numel() != labels.numel():
+            labels = labels[:probs.numel()]
+        m = metrics(probs, labels)
+        if m is None:
+            print(f"    skip {name}: one class only")
+            continue
+        per_gen[name] = m
+        for k in agg:
+            agg[k].append(m[k])
+        print(f"    {name}: acc={m['acc']} AP={m['ap']} AUROC={m['auroc']} "
+              f"(real_acc={m['real_acc']} fake_acc={m['fake_acc']})", flush=True)
+
+    if not per_gen:
+        print("No generator produced valid metrics.")
+        return 1
+
+    def mean(xs):
+        return round(sum(xs) / len(xs), 2)
+
+    means = {"acc": mean(agg["acc"]), "map": mean(agg["ap"]), "mauroc": mean(agg["auroc"])}
+    result = {
+        "tag": args.tag,
+        "model": args.model,
+        "ckpt": args.ckpt or "registry-default",
+        "benchmark": args.benchmark,
+        "data_root": str(root),
+        "n_generators": len(per_gen),
+        "means": means,
+        "per_generator": per_gen,
+        "date": datetime.datetime.now().isoformat(timespec="seconds"),
+        "synthscope_version": getattr(synthscope, "__version__", "?"),
+    }
+    out = Path(args.out).expanduser() if args.out else (
+        RESULTS_LOG.parent / "bench_results" / f"{args.tag}.json")
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(result, indent=2) + "\n")
+
+    RESULTS_LOG.parent.mkdir(parents=True, exist_ok=True)
+    with RESULTS_LOG.open("a") as f:
+        f.write(json.dumps({k: result[k] for k in
+                            ("date", "tag", "model", "ckpt", "benchmark", "data_root",
+                             "n_generators", "means")}) + "\n")
+
+    print(f"\n=== RESULT {args.tag}: mAP={means['map']} mAUROC={means['mauroc']} acc={means['acc']} "
+          f"over {len(per_gen)} generators -> {out} ===", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/bench_eval_video.py

@@ -0,0 +1,181 @@
+"""Unified VIDEO benchmark eval for SynthScope video detectors (d3, restrav, stall).
+
+Scores each video through the SAME verified synthscope engine path the web app uses
+(engine.predict_video_clip -> native clip pipeline + per-model norm + fake-prob), computes
+per-generator + mean acc@0.5 / AP / AUROC, and writes the SAME JSON schema as bench_eval.py
+(repro/bench_results/<tag>.json + append repro/MASTER_RESULTS.jsonl) so the aggregator ingests it.
+
+    python benchmark/bench_eval_video.py --model d3 --benchmark genvideo \
+        --data /hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val --limit 150 --tag d3_genvideo
+
+Benchmark layouts:
+  genvideo   <data>/Fake/<gen>/*.mp4 + <data>/Real/*.mp4 (shared real pool)   [GenVideo-Val]
+  fakeonly   <data>/<gen>/*.mp4 (all fake) + --reals DIR of real videos        [EvalCrafter]
+  pergen     <data>/<gen>/{0_real,1_fake}/*.mp4                                 [GenVidBench]
+"""
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+import sys
+from pathlib import Path
+
+import torch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "interface"))
+from backend.engine import get_engine          # noqa: E402
+from backend.registry import get_spec          # noqa: E402
+
+VIDEO_EXTS = {".mp4", ".avi", ".mov", ".mkv", ".webm", ".gif", ".m4v"}
+REPRO = Path(__file__).resolve().parents[1] / "repro"
+RESULTS_LOG = REPRO / "MASTER_RESULTS.jsonl"
+REAL_DIRNAMES = ("0_real", "real", "reals", "Real")
+FAKE_DIRNAMES = ("1_fake", "fake", "fakes", "Fake")
+
+
+def _videos(d: Path) -> list[Path]:
+    if not d or not d.is_dir():
+        return []
+    return sorted(p for p in d.rglob("*") if p.suffix.lower() in VIDEO_EXTS)
+
+
+def _find(parent: Path, names) -> Path | None:
+    for n in names:
+        if (parent / n).is_dir():
+            return parent / n
+    return None
+
+
+def discover(benchmark: str, root: Path, shared_reals: Path | None, select):
+    shared = _videos(shared_reals) if shared_reals else []
+    out = []
+    if benchmark == "genvideo":
+        fake_root = _find(root, FAKE_DIRNAMES)
+        real_root = _find(root, REAL_DIRNAMES)
+        reals = _videos(real_root) or shared
+        gens = sorted(p for p in fake_root.iterdir() if p.is_dir()) if fake_root else []
+        for g in gens:
+            if select and g.name not in select:
+                continue
+            fk = _videos(g)
+            if fk and reals:
+                out.append((g.name, reals, fk))
+        return out
+    if benchmark == "fakeonly":
+        gens = sorted(p for p in root.iterdir() if p.is_dir())
+        for g in gens:
+            if select and g.name not in select:
+                continue
+            fk = _videos(g)
+            if fk and shared:
+                out.append((g.name, shared, fk))
+        return out
+    # pergen
+    for g in sorted(p for p in root.iterdir() if p.is_dir()):
+        if select and g.name not in select:
+            continue
+        reals = _videos(_find(g, REAL_DIRNAMES)) or shared
+        fakes = _videos(_find(g, FAKE_DIRNAMES))
+        if reals and fakes:
+            out.append((g.name, reals, fakes))
+    return out
+
+
+def metrics(fake_probs: torch.Tensor, labels: torch.Tensor):
+    from torchmetrics.functional import auroc, average_precision
+    if labels.unique().numel() < 2:
+        return None
+    ap = float(average_precision(fake_probs, labels, task="binary")) * 100
+    au = float(auroc(fake_probs, labels, task="binary")) * 100
+    preds = (fake_probs >= 0.5).int()
+    real_acc = float((preds[labels == 0] == 0).float().mean()) * 100
+    fake_acc = float((preds[labels == 1] == 1).float().mean()) * 100
+    return {"ap": round(ap, 2), "auroc": round(au, 2), "real_acc": round(real_acc, 2),
+            "fake_acc": round(fake_acc, 2), "acc": round((real_acc + fake_acc) / 2, 2)}
+
+
+@torch.no_grad()
+def score_videos(engine, spec, paths):
+    probs = []
+    for p in paths:
+        try:
+            probs.append(engine.predict_video_clip(spec, str(p))["fake_probability"])
+        except Exception as e:  # noqa: BLE001
+            print(f"    bad video {p.name}: {e}")
+    return probs
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("-m", "--model", required=True, choices=["d3", "restrav", "stall"])
+    ap.add_argument("--benchmark", required=True, choices=["genvideo", "fakeonly", "pergen"])
+    ap.add_argument("--data", required=True)
+    ap.add_argument("--reals", default=None)
+    ap.add_argument("--select", default=None)
+    ap.add_argument("--limit", type=int, default=None, help="cap videos per class per generator")
+    ap.add_argument("--tag", required=True)
+    ap.add_argument("--out", default=None)
+    args = ap.parse_args(argv)
+
+    engine = get_engine()
+    spec = get_spec(args.model)
+    engine._get_model(spec)
+    root = Path(args.data)
+    select = set(args.select.split(",")) if args.select else None
+    gens = discover(args.benchmark, root, Path(args.reals) if args.reals else None, select)
+    if not gens:
+        print(f"No generators with real+fake videos under {root} ({args.benchmark})")
+        return 1
+    print(f"=== bench_eval_video {args.tag} | {args.model} | {args.benchmark} | {len(gens)} gens "
+          f"| device={engine.device} ===", flush=True)
+
+    per_gen, agg = {}, {"ap": [], "auroc": [], "acc": []}
+    # score the shared real pool once if it is identical across generators
+    real_cache = {}
+    for name, reals, fakes in gens:
+        if args.limit:
+            reals, fakes = reals[:args.limit], fakes[:args.limit]
+        rk = (id(reals) if args.limit is None else None)
+        print(f"  {name}: {len(reals)} real + {len(fakes)} fake ...", flush=True)
+        key = tuple(reals[:1]) + (len(reals),)
+        if key in real_cache:
+            rp = real_cache[key]
+        else:
+            rp = score_videos(engine, spec, reals)
+            real_cache[key] = rp
+        fp = score_videos(engine, spec, fakes)
+        probs = torch.tensor(rp + fp)
+        labels = torch.tensor([0] * len(rp) + [1] * len(fp))
+        m = metrics(probs, labels)
+        if m is None:
+            print(f"    skip {name}: one class"); continue
+        per_gen[name] = m
+        for k in agg:
+            agg[k].append(m[k])
+        print(f"    {name}: acc={m['acc']} AP={m['ap']} AUROC={m['auroc']}", flush=True)
+
+    if not per_gen:
+        print("No generator produced metrics."); return 1
+
+    def mean(xs):
+        return round(sum(xs) / len(xs), 2)
+    means = {"acc": mean(agg["acc"]), "map": mean(agg["ap"]), "mauroc": mean(agg["auroc"])}
+    result = {"tag": args.tag, "model": args.model, "ckpt": "registry-default",
+              "benchmark": f"video:{args.benchmark}", "data_root": str(root),
+              "n_generators": len(per_gen), "means": means, "per_generator": per_gen,
+              "date": datetime.datetime.now().isoformat(timespec="seconds"),
+              "synthscope_version": "video"}
+    out = Path(args.out) if args.out else (REPRO / "bench_results" / f"{args.tag}.json")
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(result, indent=2) + "\n")
+    with RESULTS_LOG.open("a") as f:
+        f.write(json.dumps({k: result[k] for k in
+                ("date", "tag", "model", "ckpt", "benchmark", "data_root", "n_generators", "means")}) + "\n")
+    print(f"\n=== RESULT {args.tag}: mAP={means['map']} mAUROC={means['mauroc']} acc={means['acc']} "
+          f"over {len(per_gen)} gens -> {out} ===", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/datasets_util.py

@@ -0,0 +1,99 @@
+"""Shared helpers for the benchmark scripts: download a dataset with its built-in loader and find where to evaluate it.
+
+``ensure_dataset`` composes the dataset's Hydra config (``conf/dataset/<modality>/<name>.yaml``), and, if the data is not
+on disk yet, constructs its dataset module, whose loader (``prefetch: true``) downloads the data and restructures it into
+``<generator>/{0_real,1_fake}``. ``find_eval_root`` then locates the folder to evaluate and picks the ``bench_eval.py``
+profile (forensynths / ufd / pergen / pool) from the layout.
+"""
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"}
+REAL_NAMES = {"0_real", "real", "reals"}
+FAKE_NAMES = {"1_fake", "fake", "fakes", "ai"}
+# datasets whose generator grouping / real pairing is special-cased by a bench_eval profile
+PROFILE_OVERRIDE = {"cnndetection": "forensynths", "cospy_subset": "forensynths", "spai_ldm": "forensynths",
+                    "universialfakedetection": "ufd"}
+
+
+def dataset_config(name: str, modality: str = "image", root: str | None = None):
+    """The composed Hydra config for ``conf/dataset/<modality>/<name>.yaml`` (``root`` overrides ``root_dir``)."""
+    from config_utils import compose_config
+    conf = ROOT / "conf" / "dataset"
+    if not (conf / modality / f"{name}.yaml").is_file():
+        other = [m for m in ("image", "video") if (conf / m / f"{name}.yaml").is_file()]
+        if other:
+            raise SystemExit(f"{name} is a {other[0]} dataset; use the {other[0]} evaluator "
+                             f"({'benchmark/bench_eval_video.py' if other[0] == 'video' else 'benchmark/bench_eval.py'})")
+        known = sorted(p.stem for p in (conf / modality).glob("*.yaml"))
+        raise SystemExit(f"unknown {modality} dataset '{name}'. Available: {', '.join(known)}")
+    overrides = [f"dataset@DatasetConfig={modality}/{name}"]
+    if root:
+        overrides.append(f"DatasetConfig.root_dir={root}")
+    return compose_config(overrides)
+
+
+def has_images(root: Path) -> bool:
+    """True as soon as one image file is found under ``root`` (cheap: stops at the first hit)."""
+    if not root.is_dir():
+        return False
+    for _, _, files in os.walk(root):
+        if any(os.path.splitext(f)[1].lower() in IMAGE_EXT for f in files):
+            return True
+    return False
+
+
+def download_dataset(name: str, modality: str = "image", root: str | None = None) -> Path:
+    """Run the dataset's loader (download + restructure). Returns the root directory."""
+    from dataset import AIGIDatasetModule
+    cfg = dataset_config(name, modality, root)
+    AIGIDatasetModule(cfg)        # constructing the datasets runs the loader (prefetch: true downloads + restructures)
+    return Path(cfg.DatasetConfig.root_dir)
+
+
+def ensure_dataset(name: str, modality: str = "image", root: str | None = None, download: bool = True) -> Path:
+    """The dataset's root directory, downloading it first when no images are there yet."""
+    cfg = dataset_config(name, modality, root)
+    path = Path(cfg.DatasetConfig.root_dir).expanduser()
+    if has_images(path):
+        return path
+    if not download:
+        raise SystemExit(f"{name}: no data under {path} (drop --no-download to fetch it, or pass --root)")
+    print(f"[dataset] {name}: nothing under {path}, downloading with the built-in loader ...", flush=True)
+    return download_dataset(name, modality, root)
+
+
+def _subdirs(d: Path) -> list[Path]:
+    try:
+        return sorted(c for c in d.iterdir() if c.is_dir() and not c.name.startswith((".", "_")))
+    except OSError:
+        return []
+
+
+def _has_real_fake(d: Path, depth: int = 3) -> bool:
+    names = {c.name.lower() for c in d.iterdir() if c.is_dir()} if d.is_dir() else set()
+    if names & REAL_NAMES and names & FAKE_NAMES:
+        return True
+    return depth > 0 and any(_has_real_fake(c, depth - 1) for c in _subdirs(d)[:60])
+
+
+def find_eval_root(name: str, root: Path) -> tuple[Path, str]:
+    """(folder to evaluate, bench_eval profile). Looks in ``root/test``, ``root/test_full`` and ``root`` itself."""
+    if name in PROFILE_OVERRIDE:       # fixed layouts handled by their own bench_eval profile (e.g. UFD: fakes and reals in separate folders)
+        for cand in (root / "test", root):
+            if cand.is_dir() and _subdirs(cand):
+                return cand, PROFILE_OVERRIDE[name]
+    for cand in (root / "test", root / "test_full", root):
+        if not cand.is_dir():
+            continue
+        names = {c.name.lower() for c in cand.iterdir() if c.is_dir()}
+        if names & REAL_NAMES and names & FAKE_NAMES:            # one pooled real/fake set
+            return cand, PROFILE_OVERRIDE.get(name, "pool")
+        if any(_has_real_fake(g) for g in _subdirs(cand)):       # one folder per generator
+            return cand, PROFILE_OVERRIDE.get(name, "pergen")
+    raise SystemExit(f"{name}: could not find real/fake folders under {root}; expected "
+                     "<root>/[test/]<generator>/{0_real,1_fake} (see docs/BENCHMARKS.md)")

benchmark/download_dataset.py

@@ -0,0 +1,60 @@
+"""Download and prepare one of SynthScope's benchmark datasets with its built-in loader.
+
+    python benchmark/download_dataset.py --list                       # datasets and where each comes from
+    python benchmark/download_dataset.py genimage --root /data/GenImage
+    python benchmark/download_dataset.py synthbuster --root /data/Synthbuster --dry-run   # only show what would happen
+
+The loaders download from the dataset's official source (Hugging Face, Zenodo, ModelScope or Google Drive; see
+``conf/dataset/image/<name>.yaml``) and restructure it into ``<root>/<generator>/{0_real,1_fake}``, which
+``benchmark/bench_eval.py`` reads. Some hosts rate-limit (Google Drive) or ask you to accept terms first; if a download
+fails, fetch the data by hand from the ``source_url`` and unpack it into ``--root`` in the same layout.
+"""
+import argparse
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from datasets_util import dataset_config, download_dataset
+
+SOURCE_KEYS = ("source_url", "hf_repo_id", "zenodo_record", "repo_id", "modelscope_id", "setting")
+
+
+def available() -> list[Path]:
+    return sorted((ROOT / "conf" / "dataset").glob("*/*.yaml"))
+
+
+def describe(path: Path) -> str:
+    """One line: modality/name and where the data comes from."""
+    from omegaconf import OmegaConf
+    cfg = OmegaConf.load(path)
+    src = ", ".join(f"{k}={cfg[k]}" for k in SOURCE_KEYS if k in cfg)
+    return f"{path.parent.name}/{path.stem:24s} {src or '(see the yaml)'}"
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("name", nargs="?", help="dataset config name, e.g. genimage, cnndetection, aigibench, synthbuster")
+    ap.add_argument("--root", help="where to put the data (default: root_dir from the dataset's yaml)")
+    ap.add_argument("--modality", default="image", choices=["image", "video"])
+    ap.add_argument("--list", action="store_true", help="list the available datasets and their sources")
+    ap.add_argument("--dry-run", action="store_true", help="only print the resolved config; download nothing")
+    args = ap.parse_args()
+
+    if args.list or not args.name:
+        print("\n".join(describe(p) for p in available()))
+        return
+    dc = dataset_config(args.name, args.modality, args.root).DatasetConfig
+    print(f"dataset {dc.name}: root_dir={dc.root_dir}  prefetch={dc.get('prefetch')}")
+    for k in SOURCE_KEYS:
+        if k in dc:
+            print(f"  {k}: {dc[k]}")
+    if args.dry_run:
+        return
+    path = download_dataset(args.name, args.modality, args.root)
+    print(f"done: {path}")
+
+
+if __name__ == "__main__":
+    main()

benchmark/evaluate.py

@@ -0,0 +1,191 @@
+"""Evaluate an SynthScope detector on a benchmark track and write a schema-valid submission JSON.
+
+    python benchmark/evaluate.py --model fat_former --track forensynths \
+        --data /path/to/forensynths_test --out benchmark/submissions/fat_former.json
+
+Reuses the verified ``synthscope`` inference path (model build + checkpoint load + per-model
+preprocessing + fake-probability), then computes per-generator + mean metrics with torchmetrics —
+the same metrics the trainer reports. Image tracks only in v1 (video harness is future work; the
+video seed entries come from their reproduction eval).
+
+Expected data layout (SynthScope common layout):
+
+    <data>/<generator>/real/*.png   <data>/<generator>/fake/*.png
+
+Use ``--reals DIR`` for benchmarks that share one real set across per-generator fake folders.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+from pathlib import Path
+
+import torch
+from PIL import Image, ImageFile
+
+import synthscope
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff"}
+REAL_NAMES = {"real", "0_real", "reals"}
+FAKE_NAMES = {"fake", "1_fake", "fakes", "ai"}
+
+
+def _images(d: Path) -> list[Path]:
+    return sorted(p for p in d.rglob("*") if p.suffix.lower() in IMAGE_EXTS) if d and d.is_dir() else []
+
+
+def _find_split(gen_dir: Path, names: set[str]) -> Path | None:
+    for n in names:
+        if (gen_dir / n).is_dir():
+            return gen_dir / n
+    return None
+
+
+def discover_generators(root: Path, shared_reals: Path | None):
+    """Return [(generator_name, real_paths, fake_paths)] from the benchmark root."""
+    shared = _images(shared_reals) if shared_reals else []
+    out = []
+    # Single pooled set: <root>/real, <root>/fake
+    if _find_split(root, REAL_NAMES) or _find_split(root, FAKE_NAMES):
+        reals = _images(_find_split(root, REAL_NAMES)) or shared
+        fakes = _images(_find_split(root, FAKE_NAMES))
+        if reals and fakes:
+            return [("all", reals, fakes)]
+    for gen_dir in sorted(p for p in root.iterdir() if p.is_dir()):
+        reals = _images(_find_split(gen_dir, REAL_NAMES)) or shared
+        fakes = _images(_find_split(gen_dir, FAKE_NAMES))
+        if reals and fakes:
+            out.append((gen_dir.name, reals, fakes))
+        else:
+            print(f"  skip generator '{gen_dir.name}': need both real and fake images")
+    return out
+
+
+@torch.no_grad()
+def score(det: "synthscope.Detector", paths: list[Path], batch_size: int) -> torch.Tensor:
+    """Return P(fake) for a list of image paths, using the detector's own preprocessing."""
+    det._ensure_weights()
+    model = det._engine._get_model(det.spec)          # build + load (verified path)
+    spec, device = det.spec, det._engine.device
+    probs = []
+    for i in range(0, len(paths), batch_size):
+        batch = []
+        for p in paths[i:i + batch_size]:
+            batch.append(spec.transform(Image.open(p).convert("RGB")))
+        x = torch.stack(batch).to(device)
+        probs.append(det._engine._fake_prob(model, spec, x).detach().float().cpu())
+    return torch.cat(probs) if probs else torch.empty(0)
+
+
+def _per_generator_metrics(fake_probs: torch.Tensor, labels: torch.Tensor) -> dict | None:
+    from torchmetrics.functional import auroc, average_precision
+
+    if labels.unique().numel() < 2:
+        return None  # AUROC/AP need both classes
+    ap = float(average_precision(fake_probs, labels, task="binary")) * 100
+    au = float(auroc(fake_probs, labels, task="binary")) * 100
+    preds = (fake_probs >= 0.5).int()
+    real_acc = float((preds[labels == 0] == 0).float().mean()) * 100
+    fake_acc = float((preds[labels == 1] == 1).float().mean()) * 100
+    return {"ap": round(ap, 2), "auroc": round(au, 2),
+            "real_acc": round(real_acc, 2), "fake_acc": round(fake_acc, 2),
+            "balanced_acc": round((real_acc + fake_acc) / 2, 2)}
+
+
+def _infer_eval_type(model_type: str) -> str:
+    from interface.backend.registry import get_spec
+
+    spec = get_spec(model_type)
+    if str(spec.ckpt_kind).startswith("official"):
+        return "official-weights"
+    if spec.ckpt_kind == "none":
+        return "training-free"
+    return "from-scratch"
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="Evaluate a detector and emit a benchmark submission.")
+    ap.add_argument("-m", "--model", required=True, help="SynthScope detector key (see `synthscope list`)")
+    ap.add_argument("--track", required=True, choices=["forensynths", "genimage-sdv1.4"],
+                    help="benchmark track (image tracks only in v1)")
+    ap.add_argument("--data", required=True, help="benchmark root (per-generator real/fake dirs)")
+    ap.add_argument("--reals", default=None, help="shared real-image dir (for shared-real layouts)")
+    ap.add_argument("--out", required=True, help="output submission JSON path")
+    ap.add_argument("--batch-size", type=int, default=32)
+    ap.add_argument("--limit", type=int, default=None, help="cap images per class per generator")
+    ap.add_argument("--device", default=None)
+    ap.add_argument("--method", default=None, help="display name (default: registry name)")
+    ap.add_argument("--eval-type", default=None,
+                    help="override provenance (official-weights/from-scratch/frozen-probe/...)")
+    ap.add_argument("--submitted-by", default="SynthScope")
+    args = ap.parse_args(argv)
+
+    det = synthscope.load(args.model, device=args.device)
+    root = Path(args.data).expanduser()
+    gens = discover_generators(root, Path(args.reals).expanduser() if args.reals else None)
+    if not gens:
+        print(f"No generators with real+fake images under {root}")
+        return 1
+
+    per_gen, means = {}, {"ap": [], "auroc": [], "balanced_acc": []}
+    for name, reals, fakes in gens:
+        if args.limit:
+            reals, fakes = reals[:args.limit], fakes[:args.limit]
+        paths = reals + fakes
+        labels = torch.tensor([0] * len(reals) + [1] * len(fakes))
+        print(f"  {name}: {len(reals)} real + {len(fakes)} fake ...", flush=True)
+        probs = score(det, paths, args.batch_size)
+        m = _per_generator_metrics(probs, labels)
+        if m is None:
+            print(f"    (skipped metrics: only one class)")
+            continue
+        per_gen[name] = m
+        for k in means:
+            means[k].append(m[k])
+
+    if not per_gen:
+        print("No generator produced valid metrics.")
+        return 1
+
+    def mean(xs):
+        return round(sum(xs) / len(xs), 2)
+
+    from interface.backend.registry import get_spec
+    spec = get_spec(args.model)
+    submission = {
+        "method": args.method or spec.name,
+        "track": args.track,
+        "modality": "image",
+        "model_type": args.model,
+        "paper": spec.paper,
+        "url": "https://github.com/SMSD75/SynthScope",
+        "evaluation": args.eval_type or _infer_eval_type(args.model),
+        "metrics": {
+            "acc": mean(means["balanced_acc"]),
+            "map": mean(means["ap"]),
+            "ap": None,
+            "auroc": mean(means["auroc"]),
+        },
+        "per_generator": per_gen,
+        "submitted_by": args.submitted_by,
+        "date": datetime.date.today().isoformat(),
+        "verified": True,
+        "source": f"synthscope evaluate.py on {root} ({len(per_gen)} generators)",
+        "protocol_notes": None,
+        "synthscope_version": synthscope.__version__,
+    }
+    out = Path(args.out).expanduser()
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(submission, indent=2) + "\n")
+    print(f"Wrote {out}")
+    print(f"  mAP={submission['metrics']['map']}  AUROC={submission['metrics']['auroc']}  "
+          f"Acc={submission['metrics']['acc']}  over {len(per_gen)} generators")
+    print("Run `python benchmark/leaderboard.py` to refresh LEADERBOARD.md.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/generator_bench.py

@@ -0,0 +1,208 @@
+"""Generator benchmark: how well do SynthScope's image detectors catch images from ONE image generator?
+
+For every real image a text prompt is written (BLIP caption + a photographic style suffix), the generator creates a
+new image from that prompt ALONE, and every available image detector scores the real and the generated images. The
+result is a ranking by AUROC, shown in the web page's "Generator benchmarks" tab.
+
+    # same 100 real images + prompts as the reference run, so generators are compared on identical content
+    python benchmark/generator_bench.py --generator flux2_klein --label "FLUX.2 [klein] 4B" \\
+        --reuse benchmark/generator_runs/reference/manifest.json --out benchmark/generator_runs/flux2_klein_4b
+    # (the reference real images are not in git: add --real-root <your GenImage and CNNDetection folders> to find them by name)
+
+    # build a fresh image set from your own real images
+    python benchmark/generator_bench.py --generator nanobanana --real-glob "/data/real/*.jpg" --n 100 \\
+        --out benchmark/generator_runs/nanobanana
+
+``--generator`` is any key of ``interface/backend/generators.py`` that can create an image from text. Local open-weight
+generators (FLUX.2 [klein], Qwen-Image-2.1) run in batch with one model load; hosted ones need their API key (``.env``).
+Outputs in ``--out``: manifest.json, results.json, RESULTS.md (committed) and real/ + fake/ images (git-ignored).
+Re-running skips images that already exist, so an interrupted run can be resumed. See docs/BENCHMARKS.md.
+"""
+import argparse
+import glob
+import json
+import os
+import random
+import shutil
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+import numpy as np
+from PIL import Image
+
+from interface.backend import generators as G
+from interface.backend.registry import list_specs
+
+IMG_EXT = (".jpg", ".jpeg", ".png", ".webp")
+
+
+def log(*a):
+    print(*a, flush=True)
+
+
+def build_pairs(args, real_dir: Path) -> list:
+    """[{idx, caption, prompt, real_src}] and the real images copied to ``real_dir`` as 000.png, 001.png, ..."""
+    pairs = []
+    if args.reuse:
+        src = json.load(open(args.reuse))
+        src_real = Path(args.reuse).parent / "real"
+        found = None                                   # file name -> path, built only if --real-root is needed
+        for p in src["pairs"][: args.n]:
+            png = src_real / f"{p['idx']:03d}.png"
+            dst = real_dir / f"{p['idx']:03d}.png"
+            if png.is_file():
+                shutil.copyfile(png, dst)
+            elif args.real_root:                       # the real images are not in git: find them by file name
+                if found is None:
+                    found = {f.name: f for root in args.real_root for f in Path(root).rglob("*")
+                             if f.suffix.lower() in IMG_EXT}
+                match = found.get(Path(p.get("real_src", "")).name)
+                if match is None:
+                    log(f"  [{p['idx']}] real image {Path(p.get('real_src', '')).name} not found under --real-root, skipped")
+                    continue
+                Image.open(match).convert("RGB").save(dst)
+            else:
+                continue
+            pairs.append({"idx": p["idx"], "caption": p.get("caption", ""), "prompt": p["prompt"],
+                          "real_src": p.get("real_src", "")})
+        log(f"[pairs] reusing {len(pairs)} prompts + real images from {args.reuse}")
+        return pairs
+    if not args.real_glob:
+        sys.exit("give --reuse MANIFEST or --real-glob PATTERN (real images to caption)")
+    cands = sorted({f for pat in args.real_glob for f in glob.glob(pat) if f.lower().endswith(IMG_EXT)})
+    random.Random(args.seed).shuffle(cands)
+    log(f"[pairs] {len(cands)} candidate real images; need {args.n} (people are skipped)")
+    for src in cands:
+        if len(pairs) >= args.n:
+            break
+        data = open(src, "rb").read()
+        cap = G._caption(data)
+        if not cap or G._PERSON_RE.search(cap):          # no photos of people in the benchmark
+            continue
+        i = len(pairs)
+        Image.open(src).convert("RGB").save(real_dir / f"{i:03d}.png")
+        pairs.append({"idx": i, "caption": cap, "prompt": cap + G.STYLE_SUFFIX, "real_src": src})
+    return pairs
+
+
+def generate(args, pairs: list, real_dir: Path, fake_dir: Path) -> list:
+    g = G.GENERATORS[args.generator]
+    if "recreate" not in g.modes:
+        sys.exit(f"{g.name} cannot create an image from text (modes: {g.modes})")
+    if not g.available():
+        sys.exit(f"{g.name} is not available (API key / local setup missing, see docs/BENCHMARKS.md)")
+    model_key = getattr(g.call, "model_key", None)
+    t0 = time.time()
+    if model_key:                                          # local open-weight model: one batch, one model load
+        items = [{"prompt": p["prompt"], "out": str(fake_dir / f"{p['idx']:03d}.png"), "seed": p["idx"],
+                  "size": f"{args.res}x{args.res}"} for p in pairs]
+        G.local_batch(model_key, items)
+    else:                                                  # hosted generator: one API call per image
+        for p in pairs:
+            out = fake_dir / f"{p['idx']:03d}.png"
+            if out.exists():
+                continue
+            try:
+                png = G.generate(args.generator, open(real_dir / f"{p['idx']:03d}.png", "rb").read(), p["prompt"], "recreate")
+                out.write_bytes(png)
+                log(f"  [{p['idx']}] {p['caption']!r}")
+            except Exception as e:                         # blocked by a safety filter, rate limit, ...
+                log(f"  [{p['idx']}] skipped: {type(e).__name__}: {e}")
+    kept = [p for p in pairs if (fake_dir / f"{p['idx']:03d}.png").is_file()]
+    for p in pairs:                                        # keep real and fake balanced
+        if p not in kept and (real_dir / f"{p['idx']:03d}.png").exists():
+            (real_dir / f"{p['idx']:03d}.png").unlink()
+    log(f"[generate] {len(kept)}/{len(pairs)} fakes ready ({time.time() - t0:.0f}s)")
+    return kept
+
+
+def score(args, real_dir: Path, fake_dir: Path, out: Path) -> list:
+    from sklearn.metrics import average_precision_score, roc_auc_score
+    from interface.backend.engine import get_engine
+    engine = get_engine()
+    reals = [Image.open(p).convert("RGB") for p in sorted(glob.glob(str(real_dir / "*.png")))]
+    fakes = [Image.open(p).convert("RGB") for p in sorted(glob.glob(str(fake_dir / "*.png")))]
+    log(f"[score] {len(reals)} real + {len(fakes)} fake")
+    y = np.array([0] * len(reals) + [1] * len(fakes))
+    keys = args.detectors.split(",") if args.detectors else None
+    rows = []
+    for spec in list_specs():
+        if spec.modality != "image" or (keys and spec.key not in keys):
+            continue
+        if not engine.is_available(spec):
+            log(f"  {spec.key}: no checkpoint here, skipped")
+            continue
+        try:
+            t0 = time.time()
+            probs = np.array([engine.predict_image(spec, im)["fake_probability"] for im in reals + fakes], dtype="float64")
+            rows.append({
+                "detector": spec.key, "name": spec.name,
+                "auroc": round(float(roc_auc_score(y, probs)) * 100, 2),
+                "ap": round(float(average_precision_score(y, probs)) * 100, 2),
+                "acc": round(float(((probs >= 0.5).astype(int) == y).mean()) * 100, 2),
+                "fake_detect_rate": round(float((probs[len(reals):] >= 0.5).mean()) * 100, 2),
+                "real_false_pos": round(float((probs[: len(reals)] >= 0.5).mean()) * 100, 2),
+                "mean_p_fake_on_fakes": round(float(probs[len(reals):].mean()), 4),
+                "mean_p_fake_on_reals": round(float(probs[: len(reals)].mean()), 4)})
+            r = rows[-1]
+            log(f"  {spec.key:12} AUROC={r['auroc']:6.2f} AP={r['ap']:6.2f} acc={r['acc']:6.2f}  ({time.time() - t0:.0f}s)")
+        except Exception as e:
+            log(f"  {spec.key}: ERROR {type(e).__name__}: {e}")
+    rows.sort(key=lambda r: r["auroc"], reverse=True)
+    json.dump(rows, open(out / "results.json", "w"), indent=2)
+    return rows
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--generator", required=True, help="key from interface/backend/generators.py, e.g. flux2_klein")
+    ap.add_argument("--label", help="name shown in the web page (default: the generator's name)")
+    ap.add_argument("--out", required=True, help="output directory, e.g. benchmark/generator_runs/flux2_klein_4b")
+    ap.add_argument("--reuse", help="manifest.json of an earlier run: reuse its real images + prompts (same content for every generator)")
+    ap.add_argument("--real-root", action="append", help="with --reuse: folder(s) to search for the reference run's real images by file "
+                    "name (e.g. your GenImage and CNNDetection downloads), because the images themselves are not in the repo")
+    ap.add_argument("--real-glob", action="append", help="glob of real images to caption (repeatable); needed without --reuse")
+    ap.add_argument("--n", type=int, default=100, help="number of real/fake pairs")
+    ap.add_argument("--res", type=int, default=1024, help="output side in pixels for local generators")
+    ap.add_argument("--seed", type=int, default=0)
+    ap.add_argument("--detectors", help="comma-separated detector keys (default: every available image detector)")
+    ap.add_argument("--skip-generate", action="store_true", help="only (re)score the images already in --out")
+    args = ap.parse_args()
+
+    out = Path(args.out).resolve()
+    real_dir, fake_dir = out / "real", out / "fake"
+    real_dir.mkdir(parents=True, exist_ok=True); fake_dir.mkdir(parents=True, exist_ok=True)
+    g = G.GENERATORS[args.generator]
+    manifest_path = out / "manifest.json"
+    if manifest_path.exists():                              # resume
+        pairs = json.load(open(manifest_path))["pairs"]
+        for p in pairs:                                     # real images may have been cleaned up after a partial run
+            src_png = Path(args.reuse).parent / "real" / f"{p['idx']:03d}.png" if args.reuse else None
+            if not (real_dir / f"{p['idx']:03d}.png").exists() and src_png and src_png.exists():
+                shutil.copyfile(src_png, real_dir / f"{p['idx']:03d}.png")
+    else:
+        pairs = build_pairs(args, real_dir)
+    if not args.skip_generate:
+        pairs = generate(args, pairs, real_dir, fake_dir)
+    rows = score(args, real_dir, fake_dir, out)
+    label = args.label or g.name
+    manifest = {"label": label, "generator": args.generator, "n": len(pairs), "res": args.res,
+                "settings": f"{args.res}px · text-to-image from a caption", "created": time.strftime("%Y-%m-%d"),
+                "reuse_from": args.reuse or "", "pairs": pairs}
+    json.dump(manifest, open(manifest_path, "w"), indent=2)
+    md = [f"# Detectors vs {label}", "",
+          f"{len(pairs)} real images (people excluded) + {len(pairs)} images generated by {label} from a BLIP caption of each "
+          f"real image ({args.res}px). Ranked by AUROC; accuracy uses the 0.5 threshold.", "",
+          "| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % |", "|---|---|---|---|---|---|---|"]
+    md += [f"| {i} | {r['name']} | {r['auroc']} | {r['ap']} | {r['acc']} | {r['fake_detect_rate']} | {r['real_false_pos']} |"
+           for i, r in enumerate(rows, 1)]
+    (out / "RESULTS.md").write_text("\n".join(md) + "\n")
+    log("\n".join(md))
+
+
+if __name__ == "__main__":
+    main()

benchmark/generator_runs/_test_qwen/RESULTS.md

@@ -0,0 +1,19 @@
+# Detectors vs Qwen-Image-2.1
+
+3 real images (people excluded) + 3 images generated by Qwen-Image-2.1 from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold.
+
+| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % |
+|---|---|---|---|---|---|---|
+| 1 | SAFE | 100.0 | 100.0 | 83.33 | 66.67 | 0.0 |
+| 2 | AIDE | 100.0 | 100.0 | 83.33 | 66.67 | 0.0 |
+| 3 | SPAI | 100.0 | 100.0 | 100.0 | 100.0 | 0.0 |
+| 4 | VIB-Net | 100.0 | 100.0 | 100.0 | 100.0 | 0.0 |
+| 5 | CO-SPY | 100.0 | 100.0 | 83.33 | 66.67 | 0.0 |
+| 6 | PGC | 100.0 | 100.0 | 66.67 | 33.33 | 0.0 |
+| 7 | CuRe | 100.0 | 100.0 | 66.67 | 33.33 | 0.0 |
+| 8 | NPR | 83.33 | 75.0 | 66.67 | 100.0 | 66.67 |
+| 9 | FatFormer | 77.78 | 86.67 | 83.33 | 66.67 | 0.0 |
+| 10 | CNNDetection (ResNet-50) | 72.22 | 83.33 | 50.0 | 0.0 | 0.0 |
+| 11 | Adapter-tuned DINOv2 | 72.22 | 83.33 | 50.0 | 0.0 | 0.0 |
+| 12 | UnivFD / CLIP | 44.44 | 55.56 | 50.0 | 0.0 | 0.0 |
+| 13 | DINOv2 probe | 44.44 | 53.33 | 50.0 | 0.0 | 0.0 |

benchmark/generator_runs/_test_qwen/manifest.json

@@ -0,0 +1,29 @@
+{
+  "label": "Qwen-Image-2.1",
+  "generator": "qwen_image_2_1",
+  "n": 3,
+  "res": 1024,
+  "settings": "1024px \u00b7 text-to-image from a caption",
+  "created": "2026-10-04",
+  "reuse_from": "benchmark/generator_runs/reference/manifest.json",
+  "pairs": [
+    {
+      "idx": 0,
+      "caption": "sheep is white",
+      "prompt": "sheep is white, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG"
+    },
+    {
+      "idx": 1,
+      "caption": "a bird sitting on a branch in a tree",
+      "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG"
+    },
+    {
+      "idx": 2,
+      "caption": "a green grasshopper on a leaf",
+      "prompt": "a green grasshopper on a leaf, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG"
+    }
+  ]
+}

benchmark/generator_runs/_test_qwen/results.json

@@ -0,0 +1,145 @@
+[
+  {
+    "detector": "safe",
+    "name": "SAFE",
+    "auroc": 100.0,
+    "ap": 100.0,
+    "acc": 83.33,
+    "fake_detect_rate": 66.67,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.5512,
+    "mean_p_fake_on_reals": 0.0758
+  },
+  {
+    "detector": "aide",
+    "name": "AIDE",
+    "auroc": 100.0,
+    "ap": 100.0,
+    "acc": 83.33,
+    "fake_detect_rate": 66.67,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.623,
+    "mean_p_fake_on_reals": 0.0486
+  },
+  {
+    "detector": "spai",
+    "name": "SPAI",
+    "auroc": 100.0,
+    "ap": 100.0,
+    "acc": 100.0,
+    "fake_detect_rate": 100.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.984,
+    "mean_p_fake_on_reals": 0.0026
+  },
+  {
+    "detector": "vib_net",
+    "name": "VIB-Net",
+    "auroc": 100.0,
+    "ap": 100.0,
+    "acc": 100.0,
+    "fake_detect_rate": 100.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.9034,
+    "mean_p_fake_on_reals": 0.0684
+  },
+  {
+    "detector": "co_spy",
+    "name": "CO-SPY",
+    "auroc": 100.0,
+    "ap": 100.0,
+    "acc": 83.33,
+    "fake_detect_rate": 66.67,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.7782,
+    "mean_p_fake_on_reals": 0.0484
+  },
+  {
+    "detector": "pgc",
+    "name": "PGC",
+    "auroc": 100.0,
+    "ap": 100.0,
+    "acc": 66.67,
+    "fake_detect_rate": 33.33,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.4674,
+    "mean_p_fake_on_reals": 0.2349
+  },
+  {
+    "detector": "cure",
+    "name": "CuRe",
+    "auroc": 100.0,
+    "ap": 100.0,
+    "acc": 66.67,
+    "fake_detect_rate": 33.33,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.5297,
+    "mean_p_fake_on_reals": 0.0
+  },
+  {
+    "detector": "npr",
+    "name": "NPR",
+    "auroc": 83.33,
+    "ap": 75.0,
+    "acc": 66.67,
+    "fake_detect_rate": 100.0,
+    "real_false_pos": 66.67,
+    "mean_p_fake_on_fakes": 1.0,
+    "mean_p_fake_on_reals": 0.6673
+  },
+  {
+    "detector": "fat_former",
+    "name": "FatFormer",
+    "auroc": 77.78,
+    "ap": 86.67,
+    "acc": 83.33,
+    "fake_detect_rate": 66.67,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.6629,
+    "mean_p_fake_on_reals": 0.0
+  },
+  {
+    "detector": "cnn",
+    "name": "CNNDetection (ResNet-50)",
+    "auroc": 72.22,
+    "ap": 83.33,
+    "acc": 50.0,
+    "fake_detect_rate": 0.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.0003,
+    "mean_p_fake_on_reals": 0.0
+  },
+  {
+    "detector": "adapted_dino",
+    "name": "Adapter-tuned DINOv2",
+    "auroc": 72.22,
+    "ap": 83.33,
+    "acc": 50.0,
+    "fake_detect_rate": 0.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.0,
+    "mean_p_fake_on_reals": 0.0
+  },
+  {
+    "detector": "clip",
+    "name": "UnivFD / CLIP",
+    "auroc": 44.44,
+    "ap": 55.56,
+    "acc": 50.0,
+    "fake_detect_rate": 0.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.1024,
+    "mean_p_fake_on_reals": 0.1369
+  },
+  {
+    "detector": "dinov2",
+    "name": "DINOv2 probe",
+    "auroc": 44.44,
+    "ap": 53.33,
+    "acc": 50.0,
+    "fake_detect_rate": 0.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.0132,
+    "mean_p_fake_on_reals": 0.0194
+  }
+]

benchmark/generator_runs/flux2_klein_4b/RESULTS.md

@@ -0,0 +1,20 @@
+# Detectors vs FLUX.2 [klein] 4B
+
+100 real images (people excluded) + 100 images generated by FLUX.2 [klein] 4B from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold.
+
+| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % |
+|---|---|---|---|---|---|---|
+| 1 | VIB-Net | 99.91 | 99.91 | 92.0 | 100.0 | 16.0 |
+| 2 | CO-SPY | 99.31 | 99.27 | 98.0 | 99.0 | 3.0 |
+| 3 | CuRe | 98.6 | 98.91 | 92.5 | 85.0 | 0.0 |
+| 4 | PGC | 97.33 | 96.57 | 53.5 | 7.0 | 0.0 |
+| 5 | AIDE | 94.08 | 90.5 | 81.0 | 69.0 | 7.0 |
+| 6 | SAFE | 92.04 | 82.32 | 87.0 | 84.0 | 10.0 |
+| 7 | NPR | 89.18 | 82.53 | 78.0 | 99.0 | 43.0 |
+| 8 | SPAI | 78.08 | 75.23 | 65.0 | 38.0 | 8.0 |
+| 9 | DINOv3 probe | 51.06 | 49.11 | 47.5 | 8.0 | 13.0 |
+| 10 | FatFormer | 46.01 | 48.06 | 50.5 | 5.0 | 4.0 |
+| 11 | Adapter-tuned DINOv2 | 42.62 | 47.55 | 50.0 | 0.0 | 0.0 |
+| 12 | CNNDetection (ResNet-50) | 40.84 | 45.42 | 50.5 | 2.0 | 1.0 |
+| 13 | DINOv2 probe | 38.52 | 43.2 | 47.0 | 7.0 | 13.0 |
+| 14 | UnivFD / CLIP | 26.8 | 36.87 | 47.0 | 1.0 | 7.0 |

benchmark/generator_runs/flux2_klein_4b/manifest.json

@@ -0,0 +1,611 @@
+{
+  "label": "FLUX.2 [klein] 4B",
+  "generator": "flux2_klein",
+  "n": 100,
+  "res": 1024,
+  "settings": "1024px \u00b7 text-to-image from a caption",
+  "created": "2026-10-04",
+  "reuse_from": "benchmark/generator_runs/reference/manifest.json",
+  "pairs": [
+    {
+      "idx": 0,
+      "caption": "sheep is white",
+      "prompt": "sheep is white, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG"
+    },
+    {
+      "idx": 1,
+      "caption": "a bird sitting on a branch in a tree",
+      "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG"
+    },
+    {
+      "idx": 2,
+      "caption": "a green grasshopper on a leaf",
+      "prompt": "a green grasshopper on a leaf, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG"
+    },
+    {
+      "idx": 3,
+      "caption": "the bell is black",
+      "prompt": "the bell is black, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG"
+    },
+    {
+      "idx": 4,
+      "caption": "a car driving down a city street in the middle of the day",
+      "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png"
+    },
+    {
+      "idx": 5,
+      "caption": "a dog is standing in the grass",
+      "prompt": "a dog is standing in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG"
+    },
+    {
+      "idx": 6,
+      "caption": "a small kitten walking across a rocky beach",
+      "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG"
+    },
+    {
+      "idx": 7,
+      "caption": "a mountain covered in snow",
+      "prompt": "a mountain covered in snow, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG"
+    },
+    {
+      "idx": 8,
+      "caption": "a sailer sailing in the ocean near a lighthouse",
+      "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG"
+    },
+    {
+      "idx": 9,
+      "caption": "a bowl of masoor",
+      "prompt": "a bowl of masoor, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG"
+    },
+    {
+      "idx": 10,
+      "caption": "a green bird house",
+      "prompt": "a green bird house, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG"
+    },
+    {
+      "idx": 11,
+      "caption": "a body of water",
+      "prompt": "a body of water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG"
+    },
+    {
+      "idx": 12,
+      "caption": "a dog in a pool",
+      "prompt": "a dog in a pool, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG"
+    },
+    {
+      "idx": 13,
+      "caption": "two tools, one with a wren and the other with a wren",
+      "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG"
+    },
+    {
+      "idx": 14,
+      "caption": "a black leather folder with a red trim",
+      "prompt": "a black leather folder with a red trim, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png"
+    },
+    {
+      "idx": 15,
+      "caption": "a small bird sitting on a wooden surface",
+      "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG"
+    },
+    {
+      "idx": 16,
+      "caption": "mouse mouse on mouse mouse mouse",
+      "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG"
+    },
+    {
+      "idx": 17,
+      "caption": "a small metal object on a wooden table",
+      "prompt": "a small metal object on a wooden table, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG"
+    },
+    {
+      "idx": 18,
+      "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour",
+      "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG"
+    },
+    {
+      "idx": 19,
+      "caption": "a white rabbit in a cage",
+      "prompt": "a white rabbit in a cage, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG"
+    },
+    {
+      "idx": 20,
+      "caption": "a grass bug on a green leaf",
+      "prompt": "a grass bug on a green leaf, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG"
+    },
+    {
+      "idx": 21,
+      "caption": "a field of grass",
+      "prompt": "a field of grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG"
+    },
+    {
+      "idx": 22,
+      "caption": "a black padlock with a key",
+      "prompt": "a black padlock with a key, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png"
+    },
+    {
+      "idx": 23,
+      "caption": "a group of balls",
+      "prompt": "a group of balls, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG"
+    },
+    {
+      "idx": 24,
+      "caption": "three dogs standing on a hill",
+      "prompt": "three dogs standing on a hill, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG"
+    },
+    {
+      "idx": 25,
+      "caption": "a dog standing in a field",
+      "prompt": "a dog standing in a field, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG"
+    },
+    {
+      "idx": 26,
+      "caption": "a pink toilet",
+      "prompt": "a pink toilet, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG"
+    },
+    {
+      "idx": 27,
+      "caption": "the sky is cloudy",
+      "prompt": "the sky is cloudy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG"
+    },
+    {
+      "idx": 28,
+      "caption": "a bird is standing on the ground",
+      "prompt": "a bird is standing on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG"
+    },
+    {
+      "idx": 29,
+      "caption": "a leopard sitting on a tree branch",
+      "prompt": "a leopard sitting on a tree branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG"
+    },
+    {
+      "idx": 30,
+      "caption": "an old barn with a broken roof",
+      "prompt": "an old barn with a broken roof, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG"
+    },
+    {
+      "idx": 31,
+      "caption": "a slug crawling on a concrete surface",
+      "prompt": "a slug crawling on a concrete surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG"
+    },
+    {
+      "idx": 32,
+      "caption": "the three bottles of the serums are shown in the image",
+      "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png"
+    },
+    {
+      "idx": 33,
+      "caption": "a bear cub",
+      "prompt": "a bear cub, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG"
+    },
+    {
+      "idx": 34,
+      "caption": "a street with a yellow line painted on the road",
+      "prompt": "a street with a yellow line painted on the road, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png"
+    },
+    {
+      "idx": 35,
+      "caption": "a lion fish swimming in the water",
+      "prompt": "a lion fish swimming in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG"
+    },
+    {
+      "idx": 36,
+      "caption": "a plastic container with food",
+      "prompt": "a plastic container with food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png"
+    },
+    {
+      "idx": 37,
+      "caption": "a car is driving down a street in a city",
+      "prompt": "a car is driving down a street in a city, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png"
+    },
+    {
+      "idx": 38,
+      "caption": "a wooden desk",
+      "prompt": "a wooden desk, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG"
+    },
+    {
+      "idx": 39,
+      "caption": "a bird standing in the water",
+      "prompt": "a bird standing in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG"
+    },
+    {
+      "idx": 40,
+      "caption": "two parking meters",
+      "prompt": "two parking meters, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG"
+    },
+    {
+      "idx": 41,
+      "caption": "a butterfly on a yellow flower",
+      "prompt": "a butterfly on a yellow flower, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG"
+    },
+    {
+      "idx": 42,
+      "caption": "a green snake on a rock",
+      "prompt": "a green snake on a rock, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG"
+    },
+    {
+      "idx": 43,
+      "caption": "a green snake in a tree",
+      "prompt": "a green snake in a tree, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG"
+    },
+    {
+      "idx": 44,
+      "caption": "purple and gold beaded bracelet",
+      "prompt": "purple and gold beaded bracelet, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG"
+    },
+    {
+      "idx": 45,
+      "caption": "a dog standing in the shade",
+      "prompt": "a dog standing in the shade, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG"
+    },
+    {
+      "idx": 46,
+      "caption": "a cliff with a cliff face",
+      "prompt": "a cliff with a cliff face, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG"
+    },
+    {
+      "idx": 47,
+      "caption": "a dog laying in the grass",
+      "prompt": "a dog laying in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG"
+    },
+    {
+      "idx": 48,
+      "caption": "a small dog sitting on a bench",
+      "prompt": "a small dog sitting on a bench, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG"
+    },
+    {
+      "idx": 49,
+      "caption": "a dog standing on a hill",
+      "prompt": "a dog standing on a hill, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG"
+    },
+    {
+      "idx": 50,
+      "caption": "a white plate with food on it",
+      "prompt": "a white plate with food on it, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG"
+    },
+    {
+      "idx": 51,
+      "caption": "a wooden chair",
+      "prompt": "a wooden chair, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG"
+    },
+    {
+      "idx": 52,
+      "caption": "a large church with a stained glass window",
+      "prompt": "a large church with a stained glass window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG"
+    },
+    {
+      "idx": 53,
+      "caption": "a dog running in the grass",
+      "prompt": "a dog running in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG"
+    },
+    {
+      "idx": 54,
+      "caption": "a hand holding a hot dog with lete and lete",
+      "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png"
+    },
+    {
+      "idx": 55,
+      "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish",
+      "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG"
+    },
+    {
+      "idx": 56,
+      "caption": "a brown dog with a long tongue",
+      "prompt": "a brown dog with a long tongue, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG"
+    },
+    {
+      "idx": 57,
+      "caption": "a wooden bench",
+      "prompt": "a wooden bench, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG"
+    },
+    {
+      "idx": 58,
+      "caption": "a white dog toy",
+      "prompt": "a white dog toy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG"
+    },
+    {
+      "idx": 59,
+      "caption": "a lake with trees and water in the background",
+      "prompt": "a lake with trees and water in the background, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG"
+    },
+    {
+      "idx": 60,
+      "caption": "a crab crawling on the ground",
+      "prompt": "a crab crawling on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG"
+    },
+    {
+      "idx": 61,
+      "caption": "a city street with a lot of buildings",
+      "prompt": "a city street with a lot of buildings, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png"
+    },
+    {
+      "idx": 62,
+      "caption": "old wooden barrels stock photo, old wooden barrels by andy",
+      "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG"
+    },
+    {
+      "idx": 63,
+      "caption": "a green beetle with red and green stripes",
+      "prompt": "a green beetle with red and green stripes, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG"
+    },
+    {
+      "idx": 64,
+      "caption": "a plate of food",
+      "prompt": "a plate of food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG"
+    },
+    {
+      "idx": 65,
+      "caption": "a fish in a tank",
+      "prompt": "a fish in a tank, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG"
+    },
+    {
+      "idx": 66,
+      "caption": "a street with a lot of luggage and palm trees",
+      "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png"
+    },
+    {
+      "idx": 67,
+      "caption": "a tree in the middle of a city",
+      "prompt": "a tree in the middle of a city, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png"
+    },
+    {
+      "idx": 68,
+      "caption": "a kitchen with a large white island and a large window",
+      "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png"
+    },
+    {
+      "idx": 69,
+      "caption": "a small brown and black dog standing on a white surface",
+      "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG"
+    },
+    {
+      "idx": 70,
+      "caption": "a train on the tracks",
+      "prompt": "a train on the tracks, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG"
+    },
+    {
+      "idx": 71,
+      "caption": "a large circular window",
+      "prompt": "a large circular window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG"
+    },
+    {
+      "idx": 72,
+      "caption": "a machine in a factory",
+      "prompt": "a machine in a factory, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG"
+    },
+    {
+      "idx": 73,
+      "caption": "a red boat docked in a harbor",
+      "prompt": "a red boat docked in a harbor, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG"
+    },
+    {
+      "idx": 74,
+      "caption": "a bird perched on a branch",
+      "prompt": "a bird perched on a branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG"
+    },
+    {
+      "idx": 75,
+      "caption": "a large gray elephant",
+      "prompt": "a large gray elephant, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG"
+    },
+    {
+      "idx": 76,
+      "caption": "a street with a star painted on the sidewalk",
+      "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png"
+    },
+    {
+      "idx": 77,
+      "caption": "a fish in a tank with a fish inside",
+      "prompt": "a fish in a tank with a fish inside, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG"
+    },
+    {
+      "idx": 78,
+      "caption": "a pot of soup",
+      "prompt": "a pot of soup, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG"
+    },
+    {
+      "idx": 79,
+      "caption": "a bunch of bubbles",
+      "prompt": "a bunch of bubbles, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG"
+    },
+    {
+      "idx": 80,
+      "caption": "a vase on display",
+      "prompt": "a vase on display, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG"
+    },
+    {
+      "idx": 81,
+      "caption": "a toilet with a toilet seat and a toilet roll",
+      "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG"
+    },
+    {
+      "idx": 82,
+      "caption": "a piece of food",
+      "prompt": "a piece of food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG"
+    },
+    {
+      "idx": 83,
+      "caption": "a car is parked in a street with a bus in the background",
+      "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png"
+    },
+    {
+      "idx": 84,
+      "caption": "a white car driving down a street next to a building",
+      "prompt": "a white car driving down a street next to a building, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png"
+    },
+    {
+      "idx": 85,
+      "caption": "a small canal in venice, italy",
+      "prompt": "a small canal in venice, italy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG"
+    },
+    {
+      "idx": 86,
+      "caption": "a black velvet jacket",
+      "prompt": "a black velvet jacket, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG"
+    },
+    {
+      "idx": 87,
+      "caption": "a bear is sitting in a tree branch",
+      "prompt": "a bear is sitting in a tree branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG"
+    },
+    {
+      "idx": 88,
+      "caption": "a goat standing in the woods",
+      "prompt": "a goat standing in the woods, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG"
+    },
+    {
+      "idx": 89,
+      "caption": "a military vehicle",
+      "prompt": "a military vehicle, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG"
+    },
+    {
+      "idx": 90,
+      "caption": "a cup with a bunch of pencils in it",
+      "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG"
+    },
+    {
+      "idx": 91,
+      "caption": "a jaguar in the water",
+      "prompt": "a jaguar in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG"
+    },
+    {
+      "idx": 92,
+      "caption": "a flower made out of felt and felt",
+      "prompt": "a flower made out of felt and felt, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG"
+    },
+    {
+      "idx": 93,
+      "caption": "a dog with a pink shirt on",
+      "prompt": "a dog with a pink shirt on, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG"
+    },
+    {
+      "idx": 94,
+      "caption": "a close up of a bug on a rock",
+      "prompt": "a close up of a bug on a rock, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG"
+    },
+    {
+      "idx": 95,
+      "caption": "a close up of a bird with a yellow eye",
+      "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG"
+    },
+    {
+      "idx": 96,
+      "caption": "a hyen walking down a dirt road",
+      "prompt": "a hyen walking down a dirt road, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG"
+    },
+    {
+      "idx": 97,
+      "caption": "a col of pictures of food including vegetables and vegetables",
+      "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG"
+    },
+    {
+      "idx": 98,
+      "caption": "a brown barn",
+      "prompt": "a brown barn, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG"
+    },
+    {
+      "idx": 99,
+      "caption": "a small black and white dog sitting on the ground",
+      "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG"
+    }
+  ]
+}

benchmark/generator_runs/flux2_klein_4b/results.json

@@ -0,0 +1,156 @@
+[
+  {
+    "detector": "vib_net",
+    "name": "VIB-Net",
+    "auroc": 99.91,
+    "ap": 99.91,
+    "acc": 92.0,
+    "fake_detect_rate": 100.0,
+    "real_false_pos": 16.0,
+    "mean_p_fake_on_fakes": 0.9752,
+    "mean_p_fake_on_reals": 0.2301
+  },
+  {
+    "detector": "co_spy",
+    "name": "CO-SPY",
+    "auroc": 99.31,
+    "ap": 99.27,
+    "acc": 98.0,
+    "fake_detect_rate": 99.0,
+    "real_false_pos": 3.0,
+    "mean_p_fake_on_fakes": 0.847,
+    "mean_p_fake_on_reals": 0.116
+  },
+  {
+    "detector": "cure",
+    "name": "CuRe",
+    "auroc": 98.6,
+    "ap": 98.91,
+    "acc": 92.5,
+    "fake_detect_rate": 85.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.8466,
+    "mean_p_fake_on_reals": 0.0129
+  },
+  {
+    "detector": "pgc",
+    "name": "PGC",
+    "auroc": 97.33,
+    "ap": 96.57,
+    "acc": 53.5,
+    "fake_detect_rate": 7.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.438,
+    "mean_p_fake_on_reals": 0.2672
+  },
+  {
+    "detector": "aide",
+    "name": "AIDE",
+    "auroc": 94.08,
+    "ap": 90.5,
+    "acc": 81.0,
+    "fake_detect_rate": 69.0,
+    "real_false_pos": 7.0,
+    "mean_p_fake_on_fakes": 0.6593,
+    "mean_p_fake_on_reals": 0.1102
+  },
+  {
+    "detector": "safe",
+    "name": "SAFE",
+    "auroc": 92.04,
+    "ap": 82.32,
+    "acc": 87.0,
+    "fake_detect_rate": 84.0,
+    "real_false_pos": 10.0,
+    "mean_p_fake_on_fakes": 0.7562,
+    "mean_p_fake_on_reals": 0.1425
+  },
+  {
+    "detector": "npr",
+    "name": "NPR",
+    "auroc": 89.18,
+    "ap": 82.53,
+    "acc": 78.0,
+    "fake_detect_rate": 99.0,
+    "real_false_pos": 43.0,
+    "mean_p_fake_on_fakes": 0.99,
+    "mean_p_fake_on_reals": 0.4365
+  },
+  {
+    "detector": "spai",
+    "name": "SPAI",
+    "auroc": 78.08,
+    "ap": 75.23,
+    "acc": 65.0,
+    "fake_detect_rate": 38.0,
+    "real_false_pos": 8.0,
+    "mean_p_fake_on_fakes": 0.368,
+    "mean_p_fake_on_reals": 0.0778
+  },
+  {
+    "detector": "dinov3",
+    "name": "DINOv3 probe",
+    "auroc": 51.06,
+    "ap": 49.11,
+    "acc": 47.5,
+    "fake_detect_rate": 8.0,
+    "real_false_pos": 13.0,
+    "mean_p_fake_on_fakes": 0.2746,
+    "mean_p_fake_on_reals": 0.2792
+  },
+  {
+    "detector": "fat_former",
+    "name": "FatFormer",
+    "auroc": 46.01,
+    "ap": 48.06,
+    "acc": 50.5,
+    "fake_detect_rate": 5.0,
+    "real_false_pos": 4.0,
+    "mean_p_fake_on_fakes": 0.0523,
+    "mean_p_fake_on_reals": 0.0412
+  },
+  {
+    "detector": "adapted_dino",
+    "name": "Adapter-tuned DINOv2",
+    "auroc": 42.62,
+    "ap": 47.55,
+    "acc": 50.0,
+    "fake_detect_rate": 0.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.0001,
+    "mean_p_fake_on_reals": 0.0
+  },
+  {
+    "detector": "cnn",
+    "name": "CNNDetection (ResNet-50)",
+    "auroc": 40.84,
+    "ap": 45.42,
+    "acc": 50.5,
+    "fake_detect_rate": 2.0,
+    "real_false_pos": 1.0,
+    "mean_p_fake_on_fakes": 0.0169,
+    "mean_p_fake_on_reals": 0.0083
+  },
+  {
+    "detector": "dinov2",
+    "name": "DINOv2 probe",
+    "auroc": 38.52,
+    "ap": 43.2,
+    "acc": 47.0,
+    "fake_detect_rate": 7.0,
+    "real_false_pos": 13.0,
+    "mean_p_fake_on_fakes": 0.1156,
+    "mean_p_fake_on_reals": 0.1751
+  },
+  {
+    "detector": "clip",
+    "name": "UnivFD / CLIP",
+    "auroc": 26.8,
+    "ap": 36.87,
+    "acc": 47.0,
+    "fake_detect_rate": 1.0,
+    "real_false_pos": 7.0,
+    "mean_p_fake_on_fakes": 0.0728,
+    "mean_p_fake_on_reals": 0.1656
+  }
+]

benchmark/generator_runs/flux2_klein_9b/RESULTS.md

@@ -0,0 +1,20 @@
+# Detectors vs FLUX.2 [klein] 9B
+
+100 real images (people excluded) + 100 images generated by FLUX.2 [klein] 9B from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold.
+
+| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % |
+|---|---|---|---|---|---|---|
+| 1 | VIB-Net | 99.73 | 99.73 | 92.0 | 100.0 | 16.0 |
+| 2 | CO-SPY | 99.34 | 99.3 | 98.0 | 99.0 | 3.0 |
+| 3 | PGC | 97.35 | 96.45 | 53.5 | 7.0 | 0.0 |
+| 4 | CuRe | 96.2 | 97.04 | 88.5 | 77.0 | 0.0 |
+| 5 | AIDE | 93.73 | 88.94 | 81.0 | 69.0 | 7.0 |
+| 6 | SAFE | 92.32 | 84.09 | 88.5 | 87.0 | 10.0 |
+| 7 | NPR | 88.61 | 81.74 | 78.5 | 100.0 | 43.0 |
+| 8 | SPAI | 66.54 | 63.78 | 56.0 | 20.0 | 8.0 |
+| 9 | FatFormer | 45.86 | 46.65 | 50.5 | 5.0 | 4.0 |
+| 10 | DINOv3 probe | 45.76 | 45.57 | 46.0 | 5.0 | 13.0 |
+| 11 | Adapter-tuned DINOv2 | 44.13 | 46.55 | 50.0 | 0.0 | 0.0 |
+| 12 | CNNDetection (ResNet-50) | 40.86 | 46.29 | 50.0 | 1.0 | 1.0 |
+| 13 | DINOv2 probe | 35.29 | 41.64 | 46.0 | 5.0 | 13.0 |
+| 14 | UnivFD / CLIP | 26.07 | 36.57 | 47.0 | 1.0 | 7.0 |

benchmark/generator_runs/flux2_klein_9b/manifest.json

@@ -0,0 +1,611 @@
+{
+  "label": "FLUX.2 [klein] 9B",
+  "generator": "flux2_klein_9b",
+  "n": 100,
+  "res": 1024,
+  "settings": "1024px \u00b7 text-to-image from a caption",
+  "created": "2026-10-04",
+  "reuse_from": "benchmark/generator_runs/reference/manifest.json",
+  "pairs": [
+    {
+      "idx": 0,
+      "caption": "sheep is white",
+      "prompt": "sheep is white, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG"
+    },
+    {
+      "idx": 1,
+      "caption": "a bird sitting on a branch in a tree",
+      "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG"
+    },
+    {
+      "idx": 2,
+      "caption": "a green grasshopper on a leaf",
+      "prompt": "a green grasshopper on a leaf, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG"
+    },
+    {
+      "idx": 3,
+      "caption": "the bell is black",
+      "prompt": "the bell is black, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG"
+    },
+    {
+      "idx": 4,
+      "caption": "a car driving down a city street in the middle of the day",
+      "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png"
+    },
+    {
+      "idx": 5,
+      "caption": "a dog is standing in the grass",
+      "prompt": "a dog is standing in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG"
+    },
+    {
+      "idx": 6,
+      "caption": "a small kitten walking across a rocky beach",
+      "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG"
+    },
+    {
+      "idx": 7,
+      "caption": "a mountain covered in snow",
+      "prompt": "a mountain covered in snow, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG"
+    },
+    {
+      "idx": 8,
+      "caption": "a sailer sailing in the ocean near a lighthouse",
+      "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG"
+    },
+    {
+      "idx": 9,
+      "caption": "a bowl of masoor",
+      "prompt": "a bowl of masoor, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG"
+    },
+    {
+      "idx": 10,
+      "caption": "a green bird house",
+      "prompt": "a green bird house, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG"
+    },
+    {
+      "idx": 11,
+      "caption": "a body of water",
+      "prompt": "a body of water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG"
+    },
+    {
+      "idx": 12,
+      "caption": "a dog in a pool",
+      "prompt": "a dog in a pool, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG"
+    },
+    {
+      "idx": 13,
+      "caption": "two tools, one with a wren and the other with a wren",
+      "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG"
+    },
+    {
+      "idx": 14,
+      "caption": "a black leather folder with a red trim",
+      "prompt": "a black leather folder with a red trim, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png"
+    },
+    {
+      "idx": 15,
+      "caption": "a small bird sitting on a wooden surface",
+      "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG"
+    },
+    {
+      "idx": 16,
+      "caption": "mouse mouse on mouse mouse mouse",
+      "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG"
+    },
+    {
+      "idx": 17,
+      "caption": "a small metal object on a wooden table",
+      "prompt": "a small metal object on a wooden table, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG"
+    },
+    {
+      "idx": 18,
+      "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour",
+      "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG"
+    },
+    {
+      "idx": 19,
+      "caption": "a white rabbit in a cage",
+      "prompt": "a white rabbit in a cage, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG"
+    },
+    {
+      "idx": 20,
+      "caption": "a grass bug on a green leaf",
+      "prompt": "a grass bug on a green leaf, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG"
+    },
+    {
+      "idx": 21,
+      "caption": "a field of grass",
+      "prompt": "a field of grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG"
+    },
+    {
+      "idx": 22,
+      "caption": "a black padlock with a key",
+      "prompt": "a black padlock with a key, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png"
+    },
+    {
+      "idx": 23,
+      "caption": "a group of balls",
+      "prompt": "a group of balls, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG"
+    },
+    {
+      "idx": 24,
+      "caption": "three dogs standing on a hill",
+      "prompt": "three dogs standing on a hill, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG"
+    },
+    {
+      "idx": 25,
+      "caption": "a dog standing in a field",
+      "prompt": "a dog standing in a field, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG"
+    },
+    {
+      "idx": 26,
+      "caption": "a pink toilet",
+      "prompt": "a pink toilet, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG"
+    },
+    {
+      "idx": 27,
+      "caption": "the sky is cloudy",
+      "prompt": "the sky is cloudy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG"
+    },
+    {
+      "idx": 28,
+      "caption": "a bird is standing on the ground",
+      "prompt": "a bird is standing on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG"
+    },
+    {
+      "idx": 29,
+      "caption": "a leopard sitting on a tree branch",
+      "prompt": "a leopard sitting on a tree branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG"
+    },
+    {
+      "idx": 30,
+      "caption": "an old barn with a broken roof",
+      "prompt": "an old barn with a broken roof, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG"
+    },
+    {
+      "idx": 31,
+      "caption": "a slug crawling on a concrete surface",
+      "prompt": "a slug crawling on a concrete surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG"
+    },
+    {
+      "idx": 32,
+      "caption": "the three bottles of the serums are shown in the image",
+      "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png"
+    },
+    {
+      "idx": 33,
+      "caption": "a bear cub",
+      "prompt": "a bear cub, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG"
+    },
+    {
+      "idx": 34,
+      "caption": "a street with a yellow line painted on the road",
+      "prompt": "a street with a yellow line painted on the road, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png"
+    },
+    {
+      "idx": 35,
+      "caption": "a lion fish swimming in the water",
+      "prompt": "a lion fish swimming in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG"
+    },
+    {
+      "idx": 36,
+      "caption": "a plastic container with food",
+      "prompt": "a plastic container with food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png"
+    },
+    {
+      "idx": 37,
+      "caption": "a car is driving down a street in a city",
+      "prompt": "a car is driving down a street in a city, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png"
+    },
+    {
+      "idx": 38,
+      "caption": "a wooden desk",
+      "prompt": "a wooden desk, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG"
+    },
+    {
+      "idx": 39,
+      "caption": "a bird standing in the water",
+      "prompt": "a bird standing in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG"
+    },
+    {
+      "idx": 40,
+      "caption": "two parking meters",
+      "prompt": "two parking meters, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG"
+    },
+    {
+      "idx": 41,
+      "caption": "a butterfly on a yellow flower",
+      "prompt": "a butterfly on a yellow flower, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG"
+    },
+    {
+      "idx": 42,
+      "caption": "a green snake on a rock",
+      "prompt": "a green snake on a rock, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG"
+    },
+    {
+      "idx": 43,
+      "caption": "a green snake in a tree",
+      "prompt": "a green snake in a tree, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG"
+    },
+    {
+      "idx": 44,
+      "caption": "purple and gold beaded bracelet",
+      "prompt": "purple and gold beaded bracelet, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG"
+    },
+    {
+      "idx": 45,
+      "caption": "a dog standing in the shade",
+      "prompt": "a dog standing in the shade, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG"
+    },
+    {
+      "idx": 46,
+      "caption": "a cliff with a cliff face",
+      "prompt": "a cliff with a cliff face, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG"
+    },
+    {
+      "idx": 47,
+      "caption": "a dog laying in the grass",
+      "prompt": "a dog laying in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG"
+    },
+    {
+      "idx": 48,
+      "caption": "a small dog sitting on a bench",
+      "prompt": "a small dog sitting on a bench, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG"
+    },
+    {
+      "idx": 49,
+      "caption": "a dog standing on a hill",
+      "prompt": "a dog standing on a hill, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG"
+    },
+    {
+      "idx": 50,
+      "caption": "a white plate with food on it",
+      "prompt": "a white plate with food on it, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG"
+    },
+    {
+      "idx": 51,
+      "caption": "a wooden chair",
+      "prompt": "a wooden chair, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG"
+    },
+    {
+      "idx": 52,
+      "caption": "a large church with a stained glass window",
+      "prompt": "a large church with a stained glass window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG"
+    },
+    {
+      "idx": 53,
+      "caption": "a dog running in the grass",
+      "prompt": "a dog running in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG"
+    },
+    {
+      "idx": 54,
+      "caption": "a hand holding a hot dog with lete and lete",
+      "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png"
+    },
+    {
+      "idx": 55,
+      "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish",
+      "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG"
+    },
+    {
+      "idx": 56,
+      "caption": "a brown dog with a long tongue",
+      "prompt": "a brown dog with a long tongue, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG"
+    },
+    {
+      "idx": 57,
+      "caption": "a wooden bench",
+      "prompt": "a wooden bench, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG"
+    },
+    {
+      "idx": 58,
+      "caption": "a white dog toy",
+      "prompt": "a white dog toy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG"
+    },
+    {
+      "idx": 59,
+      "caption": "a lake with trees and water in the background",
+      "prompt": "a lake with trees and water in the background, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG"
+    },
+    {
+      "idx": 60,
+      "caption": "a crab crawling on the ground",
+      "prompt": "a crab crawling on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG"
+    },
+    {
+      "idx": 61,
+      "caption": "a city street with a lot of buildings",
+      "prompt": "a city street with a lot of buildings, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png"
+    },
+    {
+      "idx": 62,
+      "caption": "old wooden barrels stock photo, old wooden barrels by andy",
+      "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG"
+    },
+    {
+      "idx": 63,
+      "caption": "a green beetle with red and green stripes",
+      "prompt": "a green beetle with red and green stripes, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG"
+    },
+    {
+      "idx": 64,
+      "caption": "a plate of food",
+      "prompt": "a plate of food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG"
+    },
+    {
+      "idx": 65,
+      "caption": "a fish in a tank",
+      "prompt": "a fish in a tank, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG"
+    },
+    {
+      "idx": 66,
+      "caption": "a street with a lot of luggage and palm trees",
+      "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png"
+    },
+    {
+      "idx": 67,
+      "caption": "a tree in the middle of a city",
+      "prompt": "a tree in the middle of a city, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png"
+    },
+    {
+      "idx": 68,
+      "caption": "a kitchen with a large white island and a large window",
+      "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png"
+    },
+    {
+      "idx": 69,
+      "caption": "a small brown and black dog standing on a white surface",
+      "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG"
+    },
+    {
+      "idx": 70,
+      "caption": "a train on the tracks",
+      "prompt": "a train on the tracks, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG"
+    },
+    {
+      "idx": 71,
+      "caption": "a large circular window",
+      "prompt": "a large circular window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG"
+    },
+    {
+      "idx": 72,
+      "caption": "a machine in a factory",
+      "prompt": "a machine in a factory, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG"
+    },
+    {
+      "idx": 73,
+      "caption": "a red boat docked in a harbor",
+      "prompt": "a red boat docked in a harbor, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG"
+    },
+    {
+      "idx": 74,
+      "caption": "a bird perched on a branch",
+      "prompt": "a bird perched on a branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG"
+    },
+    {
+      "idx": 75,
+      "caption": "a large gray elephant",
+      "prompt": "a large gray elephant, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG"
+    },
+    {
+      "idx": 76,
+      "caption": "a street with a star painted on the sidewalk",
+      "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png"
+    },
+    {
+      "idx": 77,
+      "caption": "a fish in a tank with a fish inside",
+      "prompt": "a fish in a tank with a fish inside, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG"
+    },
+    {
+      "idx": 78,
+      "caption": "a pot of soup",
+      "prompt": "a pot of soup, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG"
+    },
+    {
+      "idx": 79,
+      "caption": "a bunch of bubbles",
+      "prompt": "a bunch of bubbles, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG"
+    },
+    {
+      "idx": 80,
+      "caption": "a vase on display",
+      "prompt": "a vase on display, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG"
+    },
+    {
+      "idx": 81,
+      "caption": "a toilet with a toilet seat and a toilet roll",
+      "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG"
+    },
+    {
+      "idx": 82,
+      "caption": "a piece of food",
+      "prompt": "a piece of food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG"
+    },
+    {
+      "idx": 83,
+      "caption": "a car is parked in a street with a bus in the background",
+      "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png"
+    },
+    {
+      "idx": 84,
+      "caption": "a white car driving down a street next to a building",
+      "prompt": "a white car driving down a street next to a building, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png"
+    },
+    {
+      "idx": 85,
+      "caption": "a small canal in venice, italy",
+      "prompt": "a small canal in venice, italy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG"
+    },
+    {
+      "idx": 86,
+      "caption": "a black velvet jacket",
+      "prompt": "a black velvet jacket, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG"
+    },
+    {
+      "idx": 87,
+      "caption": "a bear is sitting in a tree branch",
+      "prompt": "a bear is sitting in a tree branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG"
+    },
+    {
+      "idx": 88,
+      "caption": "a goat standing in the woods",
+      "prompt": "a goat standing in the woods, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG"
+    },
+    {
+      "idx": 89,
+      "caption": "a military vehicle",
+      "prompt": "a military vehicle, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG"
+    },
+    {
+      "idx": 90,
+      "caption": "a cup with a bunch of pencils in it",
+      "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG"
+    },
+    {
+      "idx": 91,
+      "caption": "a jaguar in the water",
+      "prompt": "a jaguar in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG"
+    },
+    {
+      "idx": 92,
+      "caption": "a flower made out of felt and felt",
+      "prompt": "a flower made out of felt and felt, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG"
+    },
+    {
+      "idx": 93,
+      "caption": "a dog with a pink shirt on",
+      "prompt": "a dog with a pink shirt on, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG"
+    },
+    {
+      "idx": 94,
+      "caption": "a close up of a bug on a rock",
+      "prompt": "a close up of a bug on a rock, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG"
+    },
+    {
+      "idx": 95,
+      "caption": "a close up of a bird with a yellow eye",
+      "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG"
+    },
+    {
+      "idx": 96,
+      "caption": "a hyen walking down a dirt road",
+      "prompt": "a hyen walking down a dirt road, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG"
+    },
+    {
+      "idx": 97,
+      "caption": "a col of pictures of food including vegetables and vegetables",
+      "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG"
+    },
+    {
+      "idx": 98,
+      "caption": "a brown barn",
+      "prompt": "a brown barn, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG"
+    },
+    {
+      "idx": 99,
+      "caption": "a small black and white dog sitting on the ground",
+      "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG"
+    }
+  ]
+}

benchmark/generator_runs/flux2_klein_9b/results.json

@@ -0,0 +1,156 @@
+[
+  {
+    "detector": "vib_net",
+    "name": "VIB-Net",
+    "auroc": 99.73,
+    "ap": 99.73,
+    "acc": 92.0,
+    "fake_detect_rate": 100.0,
+    "real_false_pos": 16.0,
+    "mean_p_fake_on_fakes": 0.9684,
+    "mean_p_fake_on_reals": 0.2277
+  },
+  {
+    "detector": "co_spy",
+    "name": "CO-SPY",
+    "auroc": 99.34,
+    "ap": 99.3,
+    "acc": 98.0,
+    "fake_detect_rate": 99.0,
+    "real_false_pos": 3.0,
+    "mean_p_fake_on_fakes": 0.8509,
+    "mean_p_fake_on_reals": 0.116
+  },
+  {
+    "detector": "pgc",
+    "name": "PGC",
+    "auroc": 97.35,
+    "ap": 96.45,
+    "acc": 53.5,
+    "fake_detect_rate": 7.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.4389,
+    "mean_p_fake_on_reals": 0.2672
+  },
+  {
+    "detector": "cure",
+    "name": "CuRe",
+    "auroc": 96.2,
+    "ap": 97.04,
+    "acc": 88.5,
+    "fake_detect_rate": 77.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.7393,
+    "mean_p_fake_on_reals": 0.0129
+  },
+  {
+    "detector": "aide",
+    "name": "AIDE",
+    "auroc": 93.73,
+    "ap": 88.94,
+    "acc": 81.0,
+    "fake_detect_rate": 69.0,
+    "real_false_pos": 7.0,
+    "mean_p_fake_on_fakes": 0.6246,
+    "mean_p_fake_on_reals": 0.1102
+  },
+  {
+    "detector": "safe",
+    "name": "SAFE",
+    "auroc": 92.32,
+    "ap": 84.09,
+    "acc": 88.5,
+    "fake_detect_rate": 87.0,
+    "real_false_pos": 10.0,
+    "mean_p_fake_on_fakes": 0.7988,
+    "mean_p_fake_on_reals": 0.1425
+  },
+  {
+    "detector": "npr",
+    "name": "NPR",
+    "auroc": 88.61,
+    "ap": 81.74,
+    "acc": 78.5,
+    "fake_detect_rate": 100.0,
+    "real_false_pos": 43.0,
+    "mean_p_fake_on_fakes": 0.9986,
+    "mean_p_fake_on_reals": 0.4365
+  },
+  {
+    "detector": "spai",
+    "name": "SPAI",
+    "auroc": 66.54,
+    "ap": 63.78,
+    "acc": 56.0,
+    "fake_detect_rate": 20.0,
+    "real_false_pos": 8.0,
+    "mean_p_fake_on_fakes": 0.2118,
+    "mean_p_fake_on_reals": 0.0778
+  },
+  {
+    "detector": "fat_former",
+    "name": "FatFormer",
+    "auroc": 45.86,
+    "ap": 46.65,
+    "acc": 50.5,
+    "fake_detect_rate": 5.0,
+    "real_false_pos": 4.0,
+    "mean_p_fake_on_fakes": 0.0477,
+    "mean_p_fake_on_reals": 0.0412
+  },
+  {
+    "detector": "dinov3",
+    "name": "DINOv3 probe",
+    "auroc": 45.76,
+    "ap": 45.57,
+    "acc": 46.0,
+    "fake_detect_rate": 5.0,
+    "real_false_pos": 13.0,
+    "mean_p_fake_on_fakes": 0.2405,
+    "mean_p_fake_on_reals": 0.2792
+  },
+  {
+    "detector": "adapted_dino",
+    "name": "Adapter-tuned DINOv2",
+    "auroc": 44.13,
+    "ap": 46.55,
+    "acc": 50.0,
+    "fake_detect_rate": 0.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.0,
+    "mean_p_fake_on_reals": 0.0
+  },
+  {
+    "detector": "cnn",
+    "name": "CNNDetection (ResNet-50)",
+    "auroc": 40.86,
+    "ap": 46.29,
+    "acc": 50.0,
+    "fake_detect_rate": 1.0,
+    "real_false_pos": 1.0,
+    "mean_p_fake_on_fakes": 0.0116,
+    "mean_p_fake_on_reals": 0.0083
+  },
+  {
+    "detector": "dinov2",
+    "name": "DINOv2 probe",
+    "auroc": 35.29,
+    "ap": 41.64,
+    "acc": 46.0,
+    "fake_detect_rate": 5.0,
+    "real_false_pos": 13.0,
+    "mean_p_fake_on_fakes": 0.1056,
+    "mean_p_fake_on_reals": 0.1751
+  },
+  {
+    "detector": "clip",
+    "name": "UnivFD / CLIP",
+    "auroc": 26.07,
+    "ap": 36.57,
+    "acc": 47.0,
+    "fake_detect_rate": 1.0,
+    "real_false_pos": 7.0,
+    "mean_p_fake_on_fakes": 0.0715,
+    "mean_p_fake_on_reals": 0.1656
+  }
+]

benchmark/generator_runs/qwen_image_2_1/RESULTS.md

@@ -0,0 +1,20 @@
+# Detectors vs Qwen-Image-2.1
+
+100 real images (people excluded) + 100 images generated by Qwen-Image-2.1 from a BLIP caption of each real image (1024px). Ranked by AUROC; accuracy uses the 0.5 threshold.
+
+| Rank | Detector | AUROC | AP | acc@0.5 | fake detect % | real FP % |
+|---|---|---|---|---|---|---|
+| 1 | CO-SPY | 98.87 | 98.7 | 95.5 | 94.0 | 3.0 |
+| 2 | PGC | 98.5 | 98.3 | 60.0 | 20.0 | 0.0 |
+| 3 | VIB-Net | 98.35 | 98.51 | 91.5 | 98.0 | 15.0 |
+| 4 | SPAI | 96.71 | 96.1 | 92.0 | 92.0 | 8.0 |
+| 5 | CuRe | 92.68 | 94.28 | 82.0 | 64.0 | 0.0 |
+| 6 | SAFE | 90.85 | 81.07 | 76.5 | 63.0 | 10.0 |
+| 7 | NPR | 89.67 | 83.06 | 78.0 | 99.0 | 43.0 |
+| 8 | AIDE | 89.42 | 82.41 | 60.0 | 27.0 | 7.0 |
+| 9 | FatFormer | 76.74 | 78.17 | 67.5 | 39.0 | 4.0 |
+| 10 | DINOv3 probe | 53.2 | 51.63 | 49.5 | 12.0 | 13.0 |
+| 11 | CNNDetection (ResNet-50) | 52.52 | 52.58 | 50.0 | 1.0 | 1.0 |
+| 12 | UnivFD / CLIP | 50.11 | 50.43 | 48.5 | 4.0 | 7.0 |
+| 13 | Adapter-tuned DINOv2 | 49.15 | 49.17 | 50.0 | 0.0 | 0.0 |
+| 14 | DINOv2 probe | 44.69 | 46.44 | 47.0 | 7.0 | 13.0 |

benchmark/generator_runs/qwen_image_2_1/manifest.json

@@ -0,0 +1,611 @@
+{
+  "label": "Qwen-Image-2.1",
+  "generator": "qwen_image_2_1",
+  "n": 100,
+  "res": 1024,
+  "settings": "1024px \u00b7 text-to-image from a caption",
+  "created": "2026-10-04",
+  "reuse_from": "benchmark/generator_runs/reference/manifest.json",
+  "pairs": [
+    {
+      "idx": 0,
+      "caption": "sheep is white",
+      "prompt": "sheep is white, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG"
+    },
+    {
+      "idx": 1,
+      "caption": "a bird sitting on a branch in a tree",
+      "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG"
+    },
+    {
+      "idx": 2,
+      "caption": "a green grasshopper on a leaf",
+      "prompt": "a green grasshopper on a leaf, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG"
+    },
+    {
+      "idx": 3,
+      "caption": "the bell is black",
+      "prompt": "the bell is black, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG"
+    },
+    {
+      "idx": 4,
+      "caption": "a car driving down a city street in the middle of the day",
+      "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png"
+    },
+    {
+      "idx": 5,
+      "caption": "a dog is standing in the grass",
+      "prompt": "a dog is standing in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG"
+    },
+    {
+      "idx": 6,
+      "caption": "a small kitten walking across a rocky beach",
+      "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG"
+    },
+    {
+      "idx": 7,
+      "caption": "a mountain covered in snow",
+      "prompt": "a mountain covered in snow, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG"
+    },
+    {
+      "idx": 8,
+      "caption": "a sailer sailing in the ocean near a lighthouse",
+      "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG"
+    },
+    {
+      "idx": 9,
+      "caption": "a bowl of masoor",
+      "prompt": "a bowl of masoor, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG"
+    },
+    {
+      "idx": 10,
+      "caption": "a green bird house",
+      "prompt": "a green bird house, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG"
+    },
+    {
+      "idx": 11,
+      "caption": "a body of water",
+      "prompt": "a body of water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG"
+    },
+    {
+      "idx": 12,
+      "caption": "a dog in a pool",
+      "prompt": "a dog in a pool, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG"
+    },
+    {
+      "idx": 13,
+      "caption": "two tools, one with a wren and the other with a wren",
+      "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG"
+    },
+    {
+      "idx": 14,
+      "caption": "a black leather folder with a red trim",
+      "prompt": "a black leather folder with a red trim, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png"
+    },
+    {
+      "idx": 15,
+      "caption": "a small bird sitting on a wooden surface",
+      "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG"
+    },
+    {
+      "idx": 16,
+      "caption": "mouse mouse on mouse mouse mouse",
+      "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG"
+    },
+    {
+      "idx": 17,
+      "caption": "a small metal object on a wooden table",
+      "prompt": "a small metal object on a wooden table, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG"
+    },
+    {
+      "idx": 18,
+      "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour",
+      "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG"
+    },
+    {
+      "idx": 19,
+      "caption": "a white rabbit in a cage",
+      "prompt": "a white rabbit in a cage, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG"
+    },
+    {
+      "idx": 20,
+      "caption": "a grass bug on a green leaf",
+      "prompt": "a grass bug on a green leaf, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG"
+    },
+    {
+      "idx": 21,
+      "caption": "a field of grass",
+      "prompt": "a field of grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG"
+    },
+    {
+      "idx": 22,
+      "caption": "a black padlock with a key",
+      "prompt": "a black padlock with a key, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png"
+    },
+    {
+      "idx": 23,
+      "caption": "a group of balls",
+      "prompt": "a group of balls, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG"
+    },
+    {
+      "idx": 24,
+      "caption": "three dogs standing on a hill",
+      "prompt": "three dogs standing on a hill, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG"
+    },
+    {
+      "idx": 25,
+      "caption": "a dog standing in a field",
+      "prompt": "a dog standing in a field, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG"
+    },
+    {
+      "idx": 26,
+      "caption": "a pink toilet",
+      "prompt": "a pink toilet, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG"
+    },
+    {
+      "idx": 27,
+      "caption": "the sky is cloudy",
+      "prompt": "the sky is cloudy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG"
+    },
+    {
+      "idx": 28,
+      "caption": "a bird is standing on the ground",
+      "prompt": "a bird is standing on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG"
+    },
+    {
+      "idx": 29,
+      "caption": "a leopard sitting on a tree branch",
+      "prompt": "a leopard sitting on a tree branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG"
+    },
+    {
+      "idx": 30,
+      "caption": "an old barn with a broken roof",
+      "prompt": "an old barn with a broken roof, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG"
+    },
+    {
+      "idx": 31,
+      "caption": "a slug crawling on a concrete surface",
+      "prompt": "a slug crawling on a concrete surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG"
+    },
+    {
+      "idx": 32,
+      "caption": "the three bottles of the serums are shown in the image",
+      "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png"
+    },
+    {
+      "idx": 33,
+      "caption": "a bear cub",
+      "prompt": "a bear cub, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG"
+    },
+    {
+      "idx": 34,
+      "caption": "a street with a yellow line painted on the road",
+      "prompt": "a street with a yellow line painted on the road, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png"
+    },
+    {
+      "idx": 35,
+      "caption": "a lion fish swimming in the water",
+      "prompt": "a lion fish swimming in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG"
+    },
+    {
+      "idx": 36,
+      "caption": "a plastic container with food",
+      "prompt": "a plastic container with food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png"
+    },
+    {
+      "idx": 37,
+      "caption": "a car is driving down a street in a city",
+      "prompt": "a car is driving down a street in a city, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png"
+    },
+    {
+      "idx": 38,
+      "caption": "a wooden desk",
+      "prompt": "a wooden desk, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG"
+    },
+    {
+      "idx": 39,
+      "caption": "a bird standing in the water",
+      "prompt": "a bird standing in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG"
+    },
+    {
+      "idx": 40,
+      "caption": "two parking meters",
+      "prompt": "two parking meters, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG"
+    },
+    {
+      "idx": 41,
+      "caption": "a butterfly on a yellow flower",
+      "prompt": "a butterfly on a yellow flower, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG"
+    },
+    {
+      "idx": 42,
+      "caption": "a green snake on a rock",
+      "prompt": "a green snake on a rock, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG"
+    },
+    {
+      "idx": 43,
+      "caption": "a green snake in a tree",
+      "prompt": "a green snake in a tree, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG"
+    },
+    {
+      "idx": 44,
+      "caption": "purple and gold beaded bracelet",
+      "prompt": "purple and gold beaded bracelet, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG"
+    },
+    {
+      "idx": 45,
+      "caption": "a dog standing in the shade",
+      "prompt": "a dog standing in the shade, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG"
+    },
+    {
+      "idx": 46,
+      "caption": "a cliff with a cliff face",
+      "prompt": "a cliff with a cliff face, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG"
+    },
+    {
+      "idx": 47,
+      "caption": "a dog laying in the grass",
+      "prompt": "a dog laying in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG"
+    },
+    {
+      "idx": 48,
+      "caption": "a small dog sitting on a bench",
+      "prompt": "a small dog sitting on a bench, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG"
+    },
+    {
+      "idx": 49,
+      "caption": "a dog standing on a hill",
+      "prompt": "a dog standing on a hill, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG"
+    },
+    {
+      "idx": 50,
+      "caption": "a white plate with food on it",
+      "prompt": "a white plate with food on it, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG"
+    },
+    {
+      "idx": 51,
+      "caption": "a wooden chair",
+      "prompt": "a wooden chair, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG"
+    },
+    {
+      "idx": 52,
+      "caption": "a large church with a stained glass window",
+      "prompt": "a large church with a stained glass window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG"
+    },
+    {
+      "idx": 53,
+      "caption": "a dog running in the grass",
+      "prompt": "a dog running in the grass, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG"
+    },
+    {
+      "idx": 54,
+      "caption": "a hand holding a hot dog with lete and lete",
+      "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png"
+    },
+    {
+      "idx": 55,
+      "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish",
+      "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG"
+    },
+    {
+      "idx": 56,
+      "caption": "a brown dog with a long tongue",
+      "prompt": "a brown dog with a long tongue, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG"
+    },
+    {
+      "idx": 57,
+      "caption": "a wooden bench",
+      "prompt": "a wooden bench, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG"
+    },
+    {
+      "idx": 58,
+      "caption": "a white dog toy",
+      "prompt": "a white dog toy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG"
+    },
+    {
+      "idx": 59,
+      "caption": "a lake with trees and water in the background",
+      "prompt": "a lake with trees and water in the background, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG"
+    },
+    {
+      "idx": 60,
+      "caption": "a crab crawling on the ground",
+      "prompt": "a crab crawling on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG"
+    },
+    {
+      "idx": 61,
+      "caption": "a city street with a lot of buildings",
+      "prompt": "a city street with a lot of buildings, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png"
+    },
+    {
+      "idx": 62,
+      "caption": "old wooden barrels stock photo, old wooden barrels by andy",
+      "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG"
+    },
+    {
+      "idx": 63,
+      "caption": "a green beetle with red and green stripes",
+      "prompt": "a green beetle with red and green stripes, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG"
+    },
+    {
+      "idx": 64,
+      "caption": "a plate of food",
+      "prompt": "a plate of food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG"
+    },
+    {
+      "idx": 65,
+      "caption": "a fish in a tank",
+      "prompt": "a fish in a tank, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG"
+    },
+    {
+      "idx": 66,
+      "caption": "a street with a lot of luggage and palm trees",
+      "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png"
+    },
+    {
+      "idx": 67,
+      "caption": "a tree in the middle of a city",
+      "prompt": "a tree in the middle of a city, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png"
+    },
+    {
+      "idx": 68,
+      "caption": "a kitchen with a large white island and a large window",
+      "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png"
+    },
+    {
+      "idx": 69,
+      "caption": "a small brown and black dog standing on a white surface",
+      "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG"
+    },
+    {
+      "idx": 70,
+      "caption": "a train on the tracks",
+      "prompt": "a train on the tracks, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG"
+    },
+    {
+      "idx": 71,
+      "caption": "a large circular window",
+      "prompt": "a large circular window, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG"
+    },
+    {
+      "idx": 72,
+      "caption": "a machine in a factory",
+      "prompt": "a machine in a factory, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG"
+    },
+    {
+      "idx": 73,
+      "caption": "a red boat docked in a harbor",
+      "prompt": "a red boat docked in a harbor, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG"
+    },
+    {
+      "idx": 74,
+      "caption": "a bird perched on a branch",
+      "prompt": "a bird perched on a branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG"
+    },
+    {
+      "idx": 75,
+      "caption": "a large gray elephant",
+      "prompt": "a large gray elephant, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG"
+    },
+    {
+      "idx": 76,
+      "caption": "a street with a star painted on the sidewalk",
+      "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png"
+    },
+    {
+      "idx": 77,
+      "caption": "a fish in a tank with a fish inside",
+      "prompt": "a fish in a tank with a fish inside, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG"
+    },
+    {
+      "idx": 78,
+      "caption": "a pot of soup",
+      "prompt": "a pot of soup, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG"
+    },
+    {
+      "idx": 79,
+      "caption": "a bunch of bubbles",
+      "prompt": "a bunch of bubbles, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG"
+    },
+    {
+      "idx": 80,
+      "caption": "a vase on display",
+      "prompt": "a vase on display, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG"
+    },
+    {
+      "idx": 81,
+      "caption": "a toilet with a toilet seat and a toilet roll",
+      "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG"
+    },
+    {
+      "idx": 82,
+      "caption": "a piece of food",
+      "prompt": "a piece of food, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG"
+    },
+    {
+      "idx": 83,
+      "caption": "a car is parked in a street with a bus in the background",
+      "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png"
+    },
+    {
+      "idx": 84,
+      "caption": "a white car driving down a street next to a building",
+      "prompt": "a white car driving down a street next to a building, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png"
+    },
+    {
+      "idx": 85,
+      "caption": "a small canal in venice, italy",
+      "prompt": "a small canal in venice, italy, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG"
+    },
+    {
+      "idx": 86,
+      "caption": "a black velvet jacket",
+      "prompt": "a black velvet jacket, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG"
+    },
+    {
+      "idx": 87,
+      "caption": "a bear is sitting in a tree branch",
+      "prompt": "a bear is sitting in a tree branch, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG"
+    },
+    {
+      "idx": 88,
+      "caption": "a goat standing in the woods",
+      "prompt": "a goat standing in the woods, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG"
+    },
+    {
+      "idx": 89,
+      "caption": "a military vehicle",
+      "prompt": "a military vehicle, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG"
+    },
+    {
+      "idx": 90,
+      "caption": "a cup with a bunch of pencils in it",
+      "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG"
+    },
+    {
+      "idx": 91,
+      "caption": "a jaguar in the water",
+      "prompt": "a jaguar in the water, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG"
+    },
+    {
+      "idx": 92,
+      "caption": "a flower made out of felt and felt",
+      "prompt": "a flower made out of felt and felt, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG"
+    },
+    {
+      "idx": 93,
+      "caption": "a dog with a pink shirt on",
+      "prompt": "a dog with a pink shirt on, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG"
+    },
+    {
+      "idx": 94,
+      "caption": "a close up of a bug on a rock",
+      "prompt": "a close up of a bug on a rock, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG"
+    },
+    {
+      "idx": 95,
+      "caption": "a close up of a bird with a yellow eye",
+      "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG"
+    },
+    {
+      "idx": 96,
+      "caption": "a hyen walking down a dirt road",
+      "prompt": "a hyen walking down a dirt road, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG"
+    },
+    {
+      "idx": 97,
+      "caption": "a col of pictures of food including vegetables and vegetables",
+      "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG"
+    },
+    {
+      "idx": 98,
+      "caption": "a brown barn",
+      "prompt": "a brown barn, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG"
+    },
+    {
+      "idx": 99,
+      "caption": "a small black and white dog sitting on the ground",
+      "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail",
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG"
+    }
+  ]
+}

benchmark/generator_runs/qwen_image_2_1/results.json

@@ -0,0 +1,156 @@
+[
+  {
+    "detector": "co_spy",
+    "name": "CO-SPY",
+    "auroc": 98.87,
+    "ap": 98.7,
+    "acc": 95.5,
+    "fake_detect_rate": 94.0,
+    "real_false_pos": 3.0,
+    "mean_p_fake_on_fakes": 0.8087,
+    "mean_p_fake_on_reals": 0.116
+  },
+  {
+    "detector": "pgc",
+    "name": "PGC",
+    "auroc": 98.5,
+    "ap": 98.3,
+    "acc": 60.0,
+    "fake_detect_rate": 20.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.4666,
+    "mean_p_fake_on_reals": 0.2672
+  },
+  {
+    "detector": "vib_net",
+    "name": "VIB-Net",
+    "auroc": 98.35,
+    "ap": 98.51,
+    "acc": 91.5,
+    "fake_detect_rate": 98.0,
+    "real_false_pos": 15.0,
+    "mean_p_fake_on_fakes": 0.9181,
+    "mean_p_fake_on_reals": 0.2277
+  },
+  {
+    "detector": "spai",
+    "name": "SPAI",
+    "auroc": 96.71,
+    "ap": 96.1,
+    "acc": 92.0,
+    "fake_detect_rate": 92.0,
+    "real_false_pos": 8.0,
+    "mean_p_fake_on_fakes": 0.8983,
+    "mean_p_fake_on_reals": 0.0778
+  },
+  {
+    "detector": "cure",
+    "name": "CuRe",
+    "auroc": 92.68,
+    "ap": 94.28,
+    "acc": 82.0,
+    "fake_detect_rate": 64.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.6284,
+    "mean_p_fake_on_reals": 0.0129
+  },
+  {
+    "detector": "safe",
+    "name": "SAFE",
+    "auroc": 90.85,
+    "ap": 81.07,
+    "acc": 76.5,
+    "fake_detect_rate": 63.0,
+    "real_false_pos": 10.0,
+    "mean_p_fake_on_fakes": 0.6334,
+    "mean_p_fake_on_reals": 0.1425
+  },
+  {
+    "detector": "npr",
+    "name": "NPR",
+    "auroc": 89.67,
+    "ap": 83.06,
+    "acc": 78.0,
+    "fake_detect_rate": 99.0,
+    "real_false_pos": 43.0,
+    "mean_p_fake_on_fakes": 0.9944,
+    "mean_p_fake_on_reals": 0.4365
+  },
+  {
+    "detector": "aide",
+    "name": "AIDE",
+    "auroc": 89.42,
+    "ap": 82.41,
+    "acc": 60.0,
+    "fake_detect_rate": 27.0,
+    "real_false_pos": 7.0,
+    "mean_p_fake_on_fakes": 0.3517,
+    "mean_p_fake_on_reals": 0.1102
+  },
+  {
+    "detector": "fat_former",
+    "name": "FatFormer",
+    "auroc": 76.74,
+    "ap": 78.17,
+    "acc": 67.5,
+    "fake_detect_rate": 39.0,
+    "real_false_pos": 4.0,
+    "mean_p_fake_on_fakes": 0.3841,
+    "mean_p_fake_on_reals": 0.0412
+  },
+  {
+    "detector": "dinov3",
+    "name": "DINOv3 probe",
+    "auroc": 53.2,
+    "ap": 51.63,
+    "acc": 49.5,
+    "fake_detect_rate": 12.0,
+    "real_false_pos": 13.0,
+    "mean_p_fake_on_fakes": 0.2902,
+    "mean_p_fake_on_reals": 0.2792
+  },
+  {
+    "detector": "cnn",
+    "name": "CNNDetection (ResNet-50)",
+    "auroc": 52.52,
+    "ap": 52.58,
+    "acc": 50.0,
+    "fake_detect_rate": 1.0,
+    "real_false_pos": 1.0,
+    "mean_p_fake_on_fakes": 0.0126,
+    "mean_p_fake_on_reals": 0.0083
+  },
+  {
+    "detector": "clip",
+    "name": "UnivFD / CLIP",
+    "auroc": 50.11,
+    "ap": 50.43,
+    "acc": 48.5,
+    "fake_detect_rate": 4.0,
+    "real_false_pos": 7.0,
+    "mean_p_fake_on_fakes": 0.161,
+    "mean_p_fake_on_reals": 0.1656
+  },
+  {
+    "detector": "adapted_dino",
+    "name": "Adapter-tuned DINOv2",
+    "auroc": 49.15,
+    "ap": 49.17,
+    "acc": 50.0,
+    "fake_detect_rate": 0.0,
+    "real_false_pos": 0.0,
+    "mean_p_fake_on_fakes": 0.0,
+    "mean_p_fake_on_reals": 0.0
+  },
+  {
+    "detector": "dinov2",
+    "name": "DINOv2 probe",
+    "auroc": 44.69,
+    "ap": 46.44,
+    "acc": 47.0,
+    "fake_detect_rate": 7.0,
+    "real_false_pos": 13.0,
+    "mean_p_fake_on_fakes": 0.1279,
+    "mean_p_fake_on_reals": 0.1751
+  }
+]

benchmark/generator_runs/reference/manifest.json

@@ -0,0 +1,609 @@
+{
+  "n": 100,
+  "steps": 28,
+  "res": 512,
+  "attempts": 271,
+  "pairs": [
+    {
+      "idx": 0,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003443_ILSVRC2012_val_00028605.JPEG",
+      "caption": "sheep is white",
+      "prompt": "sheep is white, photorealistic, high detail"
+    },
+    {
+      "idx": 1,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000582_ILSVRC2012_val_00004554.JPEG",
+      "caption": "a bird sitting on a branch in a tree",
+      "prompt": "a bird sitting on a branch in a tree, photorealistic, high detail"
+    },
+    {
+      "idx": 2,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/002258_ILSVRC2012_val_00018535.JPEG",
+      "caption": "a green grasshopper on a leaf",
+      "prompt": "a green grasshopper on a leaf, photorealistic, high detail"
+    },
+    {
+      "idx": 3,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00028778.JPEG",
+      "caption": "the bell is black",
+      "prompt": "the bell is black, photorealistic, high detail"
+    },
+    {
+      "idx": 4,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103410.png",
+      "caption": "a car driving down a city street in the middle of the day",
+      "prompt": "a car driving down a city street in the middle of the day, photorealistic, high detail"
+    },
+    {
+      "idx": 5,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003845_ILSVRC2012_val_00031963.JPEG",
+      "caption": "a dog is standing in the grass",
+      "prompt": "a dog is standing in the grass, photorealistic, high detail"
+    },
+    {
+      "idx": 6,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000784_ILSVRC2012_val_00006407.JPEG",
+      "caption": "a small kitten walking across a rocky beach",
+      "prompt": "a small kitten walking across a rocky beach, photorealistic, high detail"
+    },
+    {
+      "idx": 7,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000414_ILSVRC2012_val_00003305.JPEG",
+      "caption": "a mountain covered in snow",
+      "prompt": "a mountain covered in snow, photorealistic, high detail"
+    },
+    {
+      "idx": 8,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00040977.JPEG",
+      "caption": "a sailer sailing in the ocean near a lighthouse",
+      "prompt": "a sailer sailing in the ocean near a lighthouse, photorealistic, high detail"
+    },
+    {
+      "idx": 9,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/001588_ILSVRC2012_val_00009675.JPEG",
+      "caption": "a bowl of masoor",
+      "prompt": "a bowl of masoor, photorealistic, high detail"
+    },
+    {
+      "idx": 10,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003296_ILSVRC2012_val_00026872.JPEG",
+      "caption": "a green bird house",
+      "prompt": "a green bird house, photorealistic, high detail"
+    },
+    {
+      "idx": 11,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001100_ILSVRC2012_val_00008791.JPEG",
+      "caption": "a body of water",
+      "prompt": "a body of water, photorealistic, high detail"
+    },
+    {
+      "idx": 12,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/003956_ILSVRC2012_val_00024961.JPEG",
+      "caption": "a dog in a pool",
+      "prompt": "a dog in a pool, photorealistic, high detail"
+    },
+    {
+      "idx": 13,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/002410_ILSVRC2012_val_00020055.JPEG",
+      "caption": "two tools, one with a wren and the other with a wren",
+      "prompt": "two tools, one with a wren and the other with a wren, photorealistic, high detail"
+    },
+    {
+      "idx": 14,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/446--n02840245_7225.png",
+      "caption": "a black leather folder with a red trim",
+      "prompt": "a black leather folder with a red trim, photorealistic, high detail"
+    },
+    {
+      "idx": 15,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000105_ILSVRC2012_val_00000837.JPEG",
+      "caption": "a small bird sitting on a wooden surface",
+      "prompt": "a small bird sitting on a wooden surface, photorealistic, high detail"
+    },
+    {
+      "idx": 16,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/000058_ILSVRC2012_val_00000430.JPEG",
+      "caption": "mouse mouse on mouse mouse mouse",
+      "prompt": "mouse mouse on mouse mouse mouse, photorealistic, high detail"
+    },
+    {
+      "idx": 17,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/003751_ILSVRC2012_val_00030855.JPEG",
+      "caption": "a small metal object on a wooden table",
+      "prompt": "a small metal object on a wooden table, photorealistic, high detail"
+    },
+    {
+      "idx": 18,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002367_ILSVRC2012_val_00019475.JPEG",
+      "caption": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour",
+      "prompt": "an hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour hour, photorealistic, high detail"
+    },
+    {
+      "idx": 19,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001872_ILSVRC2012_val_00015301.JPEG",
+      "caption": "a white rabbit in a cage",
+      "prompt": "a white rabbit in a cage, photorealistic, high detail"
+    },
+    {
+      "idx": 20,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00032265.JPEG",
+      "caption": "a grass bug on a green leaf",
+      "prompt": "a grass bug on a green leaf, photorealistic, high detail"
+    },
+    {
+      "idx": 21,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/001114_ILSVRC2012_val_00009176.JPEG",
+      "caption": "a field of grass",
+      "prompt": "a field of grass, photorealistic, high detail"
+    },
+    {
+      "idx": 22,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/695--n03874599_4507.png",
+      "caption": "a black padlock with a key",
+      "prompt": "a black padlock with a key, photorealistic, high detail"
+    },
+    {
+      "idx": 23,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002837_ILSVRC2012_val_00024198.JPEG",
+      "caption": "a group of balls",
+      "prompt": "a group of balls, photorealistic, high detail"
+    },
+    {
+      "idx": 24,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00022571.JPEG",
+      "caption": "three dogs standing on a hill",
+      "prompt": "three dogs standing on a hill, photorealistic, high detail"
+    },
+    {
+      "idx": 25,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004562_ILSVRC2012_val_00038455.JPEG",
+      "caption": "a dog standing in a field",
+      "prompt": "a dog standing in a field, photorealistic, high detail"
+    },
+    {
+      "idx": 26,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/005007_ILSVRC2012_val_00042247.JPEG",
+      "caption": "a pink toilet",
+      "prompt": "a pink toilet, photorealistic, high detail"
+    },
+    {
+      "idx": 27,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003771_ILSVRC2012_val_00031161.JPEG",
+      "caption": "the sky is cloudy",
+      "prompt": "the sky is cloudy, photorealistic, high detail"
+    },
+    {
+      "idx": 28,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004662_ILSVRC2012_val_00039216.JPEG",
+      "caption": "a bird is standing on the ground",
+      "prompt": "a bird is standing on the ground, photorealistic, high detail"
+    },
+    {
+      "idx": 29,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006884_ILSVRC2012_val_00043117.JPEG",
+      "caption": "a leopard sitting on a tree branch",
+      "prompt": "a leopard sitting on a tree branch, photorealistic, high detail"
+    },
+    {
+      "idx": 30,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/004995_ILSVRC2012_val_00041650.JPEG",
+      "caption": "an old barn with a broken roof",
+      "prompt": "an old barn with a broken roof, photorealistic, high detail"
+    },
+    {
+      "idx": 31,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/000580_ILSVRC2012_val_00004691.JPEG",
+      "caption": "a slug crawling on a concrete surface",
+      "prompt": "a slug crawling on a concrete surface, photorealistic, high detail"
+    },
+    {
+      "idx": 32,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/biggan/0_real/631--n03690938_23212.png",
+      "caption": "the three bottles of the serums are shown in the image",
+      "prompt": "the three bottles of the serums are shown in the image, photorealistic, high detail"
+    },
+    {
+      "idx": 33,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005149_ILSVRC2012_val_00043353.JPEG",
+      "caption": "a bear cub",
+      "prompt": "a bear cub, photorealistic, high detail"
+    },
+    {
+      "idx": 34,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00102007.png",
+      "caption": "a street with a yellow line painted on the road",
+      "prompt": "a street with a yellow line painted on the road, photorealistic, high detail"
+    },
+    {
+      "idx": 35,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000092_ILSVRC2012_val_00000699.JPEG",
+      "caption": "a lion fish swimming in the water",
+      "prompt": "a lion fish swimming in the water, photorealistic, high detail"
+    },
+    {
+      "idx": 36,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000113589.png",
+      "caption": "a plastic container with food",
+      "prompt": "a plastic container with food, photorealistic, high detail"
+    },
+    {
+      "idx": 37,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00103003.png",
+      "caption": "a car is driving down a street in a city",
+      "prompt": "a car is driving down a street in a city, photorealistic, high detail"
+    },
+    {
+      "idx": 38,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004470_ILSVRC2012_val_00037573.JPEG",
+      "caption": "a wooden desk",
+      "prompt": "a wooden desk, photorealistic, high detail"
+    },
+    {
+      "idx": 39,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001503_ILSVRC2012_val_00012618.JPEG",
+      "caption": "a bird standing in the water",
+      "prompt": "a bird standing in the water, photorealistic, high detail"
+    },
+    {
+      "idx": 40,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/000019_ILSVRC2012_val_00000201.JPEG",
+      "caption": "two parking meters",
+      "prompt": "two parking meters, photorealistic, high detail"
+    },
+    {
+      "idx": 41,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00006608.JPEG",
+      "caption": "a butterfly on a yellow flower",
+      "prompt": "a butterfly on a yellow flower, photorealistic, high detail"
+    },
+    {
+      "idx": 42,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000933_ILSVRC2012_val_00007877.JPEG",
+      "caption": "a green snake on a rock",
+      "prompt": "a green snake on a rock, photorealistic, high detail"
+    },
+    {
+      "idx": 43,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00042162.JPEG",
+      "caption": "a green snake in a tree",
+      "prompt": "a green snake in a tree, photorealistic, high detail"
+    },
+    {
+      "idx": 44,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002068_ILSVRC2012_val_00017972.JPEG",
+      "caption": "purple and gold beaded bracelet",
+      "prompt": "purple and gold beaded bracelet, photorealistic, high detail"
+    },
+    {
+      "idx": 45,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005084_ILSVRC2012_val_00042318.JPEG",
+      "caption": "a dog standing in the shade",
+      "prompt": "a dog standing in the shade, photorealistic, high detail"
+    },
+    {
+      "idx": 46,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00031528.JPEG",
+      "caption": "a cliff with a cliff face",
+      "prompt": "a cliff with a cliff face, photorealistic, high detail"
+    },
+    {
+      "idx": 47,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004359_ILSVRC2012_val_00027268.JPEG",
+      "caption": "a dog laying in the grass",
+      "prompt": "a dog laying in the grass, photorealistic, high detail"
+    },
+    {
+      "idx": 48,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/005226_ILSVRC2012_val_00043233.JPEG",
+      "caption": "a small dog sitting on a bench",
+      "prompt": "a small dog sitting on a bench, photorealistic, high detail"
+    },
+    {
+      "idx": 49,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001612_ILSVRC2012_val_00013323.JPEG",
+      "caption": "a dog standing on a hill",
+      "prompt": "a dog standing on a hill, photorealistic, high detail"
+    },
+    {
+      "idx": 50,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001277_ILSVRC2012_val_00010872.JPEG",
+      "caption": "a white plate with food on it",
+      "prompt": "a white plate with food on it, photorealistic, high detail"
+    },
+    {
+      "idx": 51,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/001537_ILSVRC2012_val_00012771.JPEG",
+      "caption": "a wooden chair",
+      "prompt": "a wooden chair, photorealistic, high detail"
+    },
+    {
+      "idx": 52,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/003670_ILSVRC2012_val_00030337.JPEG",
+      "caption": "a large church with a stained glass window",
+      "prompt": "a large church with a stained glass window, photorealistic, high detail"
+    },
+    {
+      "idx": 53,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004349_ILSVRC2012_val_00035935.JPEG",
+      "caption": "a dog running in the grass",
+      "prompt": "a dog running in the grass, photorealistic, high detail"
+    },
+    {
+      "idx": 54,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000118367.png",
+      "caption": "a hand holding a hot dog with lete and lete",
+      "prompt": "a hand holding a hot dog with lete and lete, photorealistic, high detail"
+    },
+    {
+      "idx": 55,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/wukong/0_real/004018_ILSVRC2012_val_00033065.JPEG",
+      "caption": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish",
+      "prompt": "a large satellite dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish dish, photorealistic, high detail"
+    },
+    {
+      "idx": 56,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002541_ILSVRC2012_val_00021171.JPEG",
+      "caption": "a brown dog with a long tongue",
+      "prompt": "a brown dog with a long tongue, photorealistic, high detail"
+    },
+    {
+      "idx": 57,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/002922_ILSVRC2012_val_00024245.JPEG",
+      "caption": "a wooden bench",
+      "prompt": "a wooden bench, photorealistic, high detail"
+    },
+    {
+      "idx": 58,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001637_ILSVRC2012_val_00013696.JPEG",
+      "caption": "a white dog toy",
+      "prompt": "a white dog toy, photorealistic, high detail"
+    },
+    {
+      "idx": 59,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/003749_ILSVRC2012_val_00031159.JPEG",
+      "caption": "a lake with trees and water in the background",
+      "prompt": "a lake with trees and water in the background, photorealistic, high detail"
+    },
+    {
+      "idx": 60,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002813_ILSVRC2012_val_00023550.JPEG",
+      "caption": "a crab crawling on the ground",
+      "prompt": "a crab crawling on the ground, photorealistic, high detail"
+    },
+    {
+      "idx": 61,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00103670.png",
+      "caption": "a city street with a lot of buildings",
+      "prompt": "a city street with a lot of buildings, photorealistic, high detail"
+    },
+    {
+      "idx": 62,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/000703_ILSVRC2012_val_00005570.JPEG",
+      "caption": "old wooden barrels stock photo, old wooden barrels by andy",
+      "prompt": "old wooden barrels stock photo, old wooden barrels by andy, photorealistic, high detail"
+    },
+    {
+      "idx": 63,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/006233_ILSVRC2012_val_00039157.JPEG",
+      "caption": "a green beetle with red and green stripes",
+      "prompt": "a green beetle with red and green stripes, photorealistic, high detail"
+    },
+    {
+      "idx": 64,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/002016_ILSVRC2012_val_00016730.JPEG",
+      "caption": "a plate of food",
+      "prompt": "a plate of food, photorealistic, high detail"
+    },
+    {
+      "idx": 65,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/001971_ILSVRC2012_val_00016470.JPEG",
+      "caption": "a fish in a tank",
+      "prompt": "a fish in a tank, photorealistic, high detail"
+    },
+    {
+      "idx": 66,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00105512.png",
+      "caption": "a street with a lot of luggage and palm trees",
+      "prompt": "a street with a lot of luggage and palm trees, photorealistic, high detail"
+    },
+    {
+      "idx": 67,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00100948.png",
+      "caption": "a tree in the middle of a city",
+      "prompt": "a tree in the middle of a city, photorealistic, high detail"
+    },
+    {
+      "idx": 68,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/gaugan/0_real/000000458768.png",
+      "caption": "a kitchen with a large white island and a large window",
+      "prompt": "a kitchen with a large white island and a large window, photorealistic, high detail"
+    },
+    {
+      "idx": 69,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003964_ILSVRC2012_val_00033278.JPEG",
+      "caption": "a small brown and black dog standing on a white surface",
+      "prompt": "a small brown and black dog standing on a white surface, photorealistic, high detail"
+    },
+    {
+      "idx": 70,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/005265_ILSVRC2012_val_00043551.JPEG",
+      "caption": "a train on the tracks",
+      "prompt": "a train on the tracks, photorealistic, high detail"
+    },
+    {
+      "idx": 71,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/003867_ILSVRC2012_val_00032479.JPEG",
+      "caption": "a large circular window",
+      "prompt": "a large circular window, photorealistic, high detail"
+    },
+    {
+      "idx": 72,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/004837_ILSVRC2012_val_00039836.JPEG",
+      "caption": "a machine in a factory",
+      "prompt": "a machine in a factory, photorealistic, high detail"
+    },
+    {
+      "idx": 73,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/001452_ILSVRC2012_val_00012691.JPEG",
+      "caption": "a red boat docked in a harbor",
+      "prompt": "a red boat docked in a harbor, photorealistic, high detail"
+    },
+    {
+      "idx": 74,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv5/0_real/004217_ILSVRC2012_val_00026362.JPEG",
+      "caption": "a bird perched on a branch",
+      "prompt": "a bird perched on a branch, photorealistic, high detail"
+    },
+    {
+      "idx": 75,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/001980_ILSVRC2012_val_00016437.JPEG",
+      "caption": "a large gray elephant",
+      "prompt": "a large gray elephant, photorealistic, high detail"
+    },
+    {
+      "idx": 76,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00101546.png",
+      "caption": "a street with a star painted on the sidewalk",
+      "prompt": "a street with a star painted on the sidewalk, photorealistic, high detail"
+    },
+    {
+      "idx": 77,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003951_ILSVRC2012_val_00033320.JPEG",
+      "caption": "a fish in a tank with a fish inside",
+      "prompt": "a fish in a tank with a fish inside, photorealistic, high detail"
+    },
+    {
+      "idx": 78,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002526_ILSVRC2012_val_00021615.JPEG",
+      "caption": "a pot of soup",
+      "prompt": "a pot of soup, photorealistic, high detail"
+    },
+    {
+      "idx": 79,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/005966_ILSVRC2012_val_00049716.JPEG",
+      "caption": "a bunch of bubbles",
+      "prompt": "a bunch of bubbles, photorealistic, high detail"
+    },
+    {
+      "idx": 80,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/005439_ILSVRC2012_val_00045714.JPEG",
+      "caption": "a vase on display",
+      "prompt": "a vase on display, photorealistic, high detail"
+    },
+    {
+      "idx": 81,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/003325_ILSVRC2012_val_00028223.JPEG",
+      "caption": "a toilet with a toilet seat and a toilet roll",
+      "prompt": "a toilet with a toilet seat and a toilet roll, photorealistic, high detail"
+    },
+    {
+      "idx": 82,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/001630_ILSVRC2012_val_00013254.JPEG",
+      "caption": "a piece of food",
+      "prompt": "a piece of food, photorealistic, high detail"
+    },
+    {
+      "idx": 83,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/crn/0_real/00104005.png",
+      "caption": "a car is parked in a street with a bus in the background",
+      "prompt": "a car is parked in a street with a bus in the background, photorealistic, high detail"
+    },
+    {
+      "idx": 84,
+      "real_src": "/hddstore/ssalehi/SynthScope/CNNDetection/test/imle/0_real/00105681.png",
+      "caption": "a white car driving down a street next to a building",
+      "prompt": "a white car driving down a street next to a building, photorealistic, high detail"
+    },
+    {
+      "idx": 85,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/sdv4/0_real/004284_ILSVRC2012_val_00035514.JPEG",
+      "caption": "a small canal in venice, italy",
+      "prompt": "a small canal in venice, italy, photorealistic, high detail"
+    },
+    {
+      "idx": 86,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003062_ILSVRC2012_val_00025342.JPEG",
+      "caption": "a black velvet jacket",
+      "prompt": "a black velvet jacket, photorealistic, high detail"
+    },
+    {
+      "idx": 87,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004938_ILSVRC2012_val_00041120.JPEG",
+      "caption": "a bear is sitting in a tree branch",
+      "prompt": "a bear is sitting in a tree branch, photorealistic, high detail"
+    },
+    {
+      "idx": 88,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/000197_ILSVRC2012_val_00001705.JPEG",
+      "caption": "a goat standing in the woods",
+      "prompt": "a goat standing in the woods, photorealistic, high detail"
+    },
+    {
+      "idx": 89,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/004633_ILSVRC2012_val_00039017.JPEG",
+      "caption": "a military vehicle",
+      "prompt": "a military vehicle, photorealistic, high detail"
+    },
+    {
+      "idx": 90,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/002918_ILSVRC2012_val_00024434.JPEG",
+      "caption": "a cup with a bunch of pencils in it",
+      "prompt": "a cup with a bunch of pencils in it, photorealistic, high detail"
+    },
+    {
+      "idx": 91,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/002525_ILSVRC2012_val_00021064.JPEG",
+      "caption": "a jaguar in the water",
+      "prompt": "a jaguar in the water, photorealistic, high detail"
+    },
+    {
+      "idx": 92,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/adm/0_real/003858_ILSVRC2012_val_00031978.JPEG",
+      "caption": "a flower made out of felt and felt",
+      "prompt": "a flower made out of felt and felt, photorealistic, high detail"
+    },
+    {
+      "idx": 93,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/001164_ILSVRC2012_val_00009888.JPEG",
+      "caption": "a dog with a pink shirt on",
+      "prompt": "a dog with a pink shirt on, photorealistic, high detail"
+    },
+    {
+      "idx": 94,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/midjourney/0_real/004231_ILSVRC2012_val_00035409.JPEG",
+      "caption": "a close up of a bug on a rock",
+      "prompt": "a close up of a bug on a rock, photorealistic, high detail"
+    },
+    {
+      "idx": 95,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/vqdm/0_real/000274_ILSVRC2012_val_00002228.JPEG",
+      "caption": "a close up of a bird with a yellow eye",
+      "prompt": "a close up of a bird with a yellow eye, photorealistic, high detail"
+    },
+    {
+      "idx": 96,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/biggan/0_real/002696_ILSVRC2012_val_00023004.JPEG",
+      "caption": "a hyen walking down a dirt road",
+      "prompt": "a hyen walking down a dirt road, photorealistic, high detail"
+    },
+    {
+      "idx": 97,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/005131_ILSVRC2012_val_00042927.JPEG",
+      "caption": "a col of pictures of food including vegetables and vegetables",
+      "prompt": "a col of pictures of food including vegetables and vegetables, photorealistic, high detail"
+    },
+    {
+      "idx": 98,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage/test/real/ILSVRC2012_val_00014180.JPEG",
+      "caption": "a brown barn",
+      "prompt": "a brown barn, photorealistic, high detail"
+    },
+    {
+      "idx": 99,
+      "real_src": "/hddstore/ssalehi/SynthScope/GenImage_full/test_full/glide/0_real/004154_ILSVRC2012_val_00034951.JPEG",
+      "caption": "a small black and white dog sitting on the ground",
+      "prompt": "a small black and white dog sitting on the ground, photorealistic, high detail"
+    }
+  ],
+  "label": "Ideogram 4.0 (512px)"
+}

benchmark/leaderboard.py

@@ -0,0 +1,160 @@
+"""Validate benchmark submissions and render the leaderboard.
+
+    python benchmark/leaderboard.py          # write LEADERBOARD.md
+    python benchmark/leaderboard.py --check   # CI: non-zero exit if invalid or stale
+
+Output is deterministic (no timestamps) so ``--check`` is a stable up-to-date test.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+BENCH_DIR = Path(__file__).resolve().parent
+SUBMISSIONS_DIR = BENCH_DIR / "submissions"
+LEADERBOARD_MD = BENCH_DIR.parent / "LEADERBOARD.md"
+
+# Track order, titles, and the metric each track ranks by.
+TRACKS = {
+    "forensynths": {
+        "title": "ForenSynths — image (ProGAN-trained, cross-generator)",
+        "primary": "map", "modality": "image",
+    },
+    "genimage-sdv1.4": {
+        "title": "GenImage SD-v1.4 — image", "primary": "auroc", "modality": "image",
+    },
+    "genvideo-val": {
+        "title": "GenVideo-Val — video", "primary": "ap", "modality": "video",
+    },
+}
+METRIC_COLS = [("acc", "Acc"), ("map", "mAP"), ("ap", "AP"), ("auroc", "AUROC")]
+EVALUATIONS = {"official-weights", "from-scratch", "frozen-probe", "training-free", "external"}
+MODALITIES = {"image", "video"}
+REQUIRED = ["method", "track", "modality", "evaluation", "metrics", "submitted_by"]
+
+
+def validate(sub: dict, where: str) -> list[str]:
+    """Return a list of human-readable errors for one submission (empty = valid)."""
+    errs = []
+    for k in REQUIRED:
+        if k not in sub:
+            errs.append(f"missing required field '{k}'")
+    if sub.get("track") not in TRACKS:
+        errs.append(f"track '{sub.get('track')}' not in {sorted(TRACKS)}")
+    if sub.get("modality") not in MODALITIES:
+        errs.append(f"modality '{sub.get('modality')}' not in {sorted(MODALITIES)}")
+    if sub.get("evaluation") not in EVALUATIONS:
+        errs.append(f"evaluation '{sub.get('evaluation')}' not in {sorted(EVALUATIONS)}")
+    metrics = sub.get("metrics")
+    if not isinstance(metrics, dict):
+        errs.append("'metrics' must be an object")
+    else:
+        vals = [metrics.get(m) for m, _ in METRIC_COLS]
+        if not any(isinstance(v, (int, float)) for v in vals):
+            errs.append("'metrics' needs at least one numeric value among acc/map/ap/auroc")
+    return [f"{where}: {e}" for e in errs]
+
+
+def load_submissions():
+    """Return (valid_submissions, errors)."""
+    subs, errors = [], []
+    for path in sorted(SUBMISSIONS_DIR.glob("*.json")):
+        try:
+            sub = json.loads(path.read_text())
+        except json.JSONDecodeError as exc:
+            errors.append(f"{path.name}: invalid JSON ({exc})")
+            continue
+        errs = validate(sub, path.name)
+        if errs:
+            errors.extend(errs)
+            continue
+        sub["_file"] = path.name
+        subs.append(sub)
+    return subs, errors
+
+
+def _fmt(v) -> str:
+    return f"{v:.2f}" if isinstance(v, (int, float)) else "—"
+
+
+def _sort_key(sub, primary):
+    v = sub.get("metrics", {}).get(primary)
+    # rank desc by primary; missing primary sorts last
+    return (0, -v) if isinstance(v, (int, float)) else (1, 0.0)
+
+
+def render() -> str:
+    subs, errors = load_submissions()
+    lines = [
+        "# SynthScope Leaderboard (v1)", "",
+        "_Auto-generated by `benchmark/leaderboard.py` — do not edit by hand._ "
+        "See [benchmark/README.md](benchmark/README.md) for the protocol and how to submit.",
+        "", "This is the **standardized, versioned** table: a few tracks with a fixed protocol that anyone can submit to. The web page's **Leaderboard** tab is broader (every paper benchmark, 30+ groups) and the repo supports many more datasets, which download and organize themselves: see [docs/BENCHMARKS.md](docs/BENCHMARKS.md).", "",
+        "Metrics are percentages. **mAP** = mean AP over generators. Each track is ranked by "
+        "its primary metric (in bold).", "",
+    ]
+    for track_id, cfg in TRACKS.items():
+        rows = [s for s in subs if s.get("track") == track_id]
+        primary = cfg["primary"]
+        header_cells = ["Rank", "Method", "Eval"]
+        for key, label in METRIC_COLS:
+            header_cells.append(f"**{label}**" if key == primary else label)
+        header_cells += ["Verified", "Notes"]
+        lines.append(f"## {cfg['title']}")
+        lines.append("")
+        if not rows:
+            lines += ["_No submissions yet._", ""]
+            continue
+        rows.sort(key=lambda s: _sort_key(s, primary))
+        lines.append("| " + " | ".join(header_cells) + " |")
+        lines.append("| " + " | ".join(["---"] * len(header_cells)) + " |")
+        for i, s in enumerate(rows, 1):
+            m = s.get("metrics", {})
+            cells = [str(i), s["method"], s["evaluation"]]
+            cells += [_fmt(m.get(key)) for key, _ in METRIC_COLS]
+            cells.append("✓" if s.get("verified") else "")
+            note = s.get("protocol_notes") or ""
+            cells.append(note.replace("|", "/"))
+            lines.append("| " + " | ".join(cells) + " |")
+        lines.append("")
+    if errors:
+        lines += ["## ⚠️ Invalid submissions", ""] + [f"- {e}" for e in errors] + [""]
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="Build/validate the SynthScope leaderboard.")
+    ap.add_argument("--check", action="store_true",
+                    help="validate submissions and verify LEADERBOARD.md is up to date (CI)")
+    args = ap.parse_args(argv)
+
+    subs, errors = load_submissions()
+    content = render()
+
+    if args.check:
+        ok = True
+        if errors:
+            print("Invalid submissions:", file=sys.stderr)
+            for e in errors:
+                print(f"  - {e}", file=sys.stderr)
+            ok = False
+        current = LEADERBOARD_MD.read_text() if LEADERBOARD_MD.exists() else ""
+        if current != content:
+            print("LEADERBOARD.md is stale — run `python benchmark/leaderboard.py`.", file=sys.stderr)
+            ok = False
+        if ok:
+            print(f"OK: {len(subs)} submissions, leaderboard up to date.")
+        return 0 if ok else 1
+
+    LEADERBOARD_MD.write_text(content)
+    print(f"Wrote {LEADERBOARD_MD} ({len(subs)} submissions"
+          + (f", {len(errors)} invalid" if errors else "") + ").")
+    for e in errors:
+        print(f"  invalid: {e}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmark/schema.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "SynthScope benchmark submission (v1)",
+  "description": "One result for one (method, track). Lives in benchmark/submissions/*.json.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["method", "track", "modality", "evaluation", "metrics", "submitted_by"],
+  "properties": {
+    "method": {"type": "string", "description": "Human-readable method name."},
+    "track": {"type": "string", "enum": ["forensynths", "genimage-sdv1.4", "genvideo-val"]},
+    "modality": {"type": "string", "enum": ["image", "video"]},
+    "model_type": {"type": ["string", "null"], "description": "SynthScope detector key, if applicable."},
+    "paper": {"type": ["string", "null"]},
+    "url": {"type": ["string", "null"], "description": "Paper or code URL."},
+    "evaluation": {
+      "type": "string",
+      "enum": ["official-weights", "from-scratch", "frozen-probe", "training-free", "external"]
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Percentages (0-100). At least one must be non-null.",
+      "properties": {
+        "acc": {"type": ["number", "null"]},
+        "map": {"type": ["number", "null"], "description": "mean AP over generators"},
+        "ap": {"type": ["number", "null"]},
+        "auroc": {"type": ["number", "null"]}
+      }
+    },
+    "per_generator": {
+      "type": ["object", "null"],
+      "description": "Optional per-generator metric breakdown."
+    },
+    "submitted_by": {"type": "string"},
+    "date": {"type": ["string", "null"], "description": "YYYY-MM-DD."},
+    "verified": {"type": "boolean", "description": "Traces to a logged SynthScope evaluation."},
+    "source": {"type": ["string", "null"], "description": "Log / run path or external reference."},
+    "protocol_notes": {"type": ["string", "null"]},
+    "synthscope_version": {"type": ["string", "null"]}
+  }
+}

benchmark/seed_reference_results.py

@@ -0,0 +1,70 @@
+"""Generate the reference (already-reproduced) leaderboard submissions.
+
+Every number here is **transcribed from the verified reproduction table in the repo README**
+(each traces to a logged `Test Metrics` evaluation; see `repro/FINAL_RESULTS.md`). No number is
+estimated. Re-run to regenerate `submissions/*.json`:
+
+    python benchmark/seed_reference_results.py
+"""
+import json
+from pathlib import Path
+
+SUB = Path(__file__).resolve().parent / "submissions"
+REPO = "https://github.com/SMSD75/SynthScope"
+SOURCE = "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md"
+DATE = "2026-06-02"
+
+# (slug, method, track, modality, model_type, paper, evaluation, metrics, protocol_notes)
+ROWS = [
+    # --- ForenSynths (image) ---
+    ("fat_former", "FatFormer", "forensynths", "image", "fat_former", "Liu et al., CVPR'24",
+     "official-weights", {"acc": 98.39, "map": 99.70}, "ForenSynths-8 subset; official released weights"),
+    ("co_spy_official", "CO-SPY (official weights)", "forensynths", "image", "co_spy", "Cheng et al., CVPR'25",
+     "official-weights", {"acc": 95.12, "map": 98.71}, "ForenSynths-8; vendored official architecture"),
+    ("npr", "NPR", "forensynths", "image", "npr", "Tan et al., CVPR'24",
+     "official-weights", {"acc": 91.83, "map": 87.09}, "ForenSynths-8; official ProGAN weights"),
+    ("aide", "AIDE", "forensynths", "image", "aide", "Yan et al., ICLR'25",
+     "official-weights", {"map": 92.08}, "ForenSynths-8; mAP reproduces, acc@0.5 miscalibrated (single-view vs official 5-view DCT)"),
+    ("clip", "UnivFD / CLIP", "forensynths", "image", "clip", "Ojha et al., CVPR'23",
+     "frozen-probe", {"map": 97.2, "auroc": 97.0}, "ForenSynths; frozen CLIP ViT-L/14 probe (ProGAN)"),
+    ("cnn", "CNNDetection", "forensynths", "image", "cnn", "Wang et al., CVPR'20",
+     "from-scratch", {"map": 96.8, "auroc": 96.6}, "ForenSynths; ResNet-50 fine-tune (ProGAN)"),
+    ("dinov2", "DINOv2 probe", "forensynths", "image", "dinov2", "Oquab et al., 2023",
+     "frozen-probe", {"map": 95.03, "auroc": 94.86}, "ForenSynths; frozen DINOv2 ViT-L/14 probe"),
+    ("safe", "SAFE", "forensynths", "image", "safe", "Li et al., KDD'25",
+     "from-scratch", {"map": 94.13, "auroc": 93.68}, "ForenSynths; trained from scratch"),
+    ("co_spy_scratch", "CO-SPY (from scratch)", "forensynths", "image", "co_spy", "Cheng et al., CVPR'25",
+     "from-scratch", {"map": 96.98, "auroc": 96.79}, "ForenSynths; SynthScope re-implementation trained on ProGAN"),
+    # --- GenImage SD-v1.4 (image) ---
+    ("spai_official", "SPAI (official weights)", "genimage-sdv1.4", "image", "spai", "Karageorgiou et al., CVPR'25",
+     "official-weights", {"auroc": 98.45, "ap": 94.01}, "GenImage SD-v1.4; vendored official architecture, fixed-res 2x2x224"),
+    ("vib_net", "VIB-Net", "genimage-sdv1.4", "image", "vib_net", "Zhang et al., CVPR'25",
+     "from-scratch", {"map": 99.33, "auroc": 99.33}, "GenImage SD-v1.4 single real/fake pool (mAP == AUROC by construction)"),
+    ("spai_scratch", "SPAI (from scratch)", "genimage-sdv1.4", "image", "spai", "Karageorgiou et al., CVPR'25",
+     "from-scratch", {"auroc": 97.66}, "DMID Latent-Diffusion vs COCO; SynthScope fixed-resolution variant"),
+    # --- GenVideo-Val (video) ---
+    ("d3", "D3", "genvideo-val", "video", "d3", "ICCV'25",
+     "training-free", {"ap": 95.6, "auroc": 93.6}, "Training-free frozen CLIP 2nd-order trajectory"),
+    ("restrav", "ReStraV", "genvideo-val", "video", "restrav", "2025",
+     "frozen-probe", {"ap": 96.67, "auroc": 97.06}, "Frozen DINOv2 trajectory geometry + MLP"),
+]
+
+METRIC_KEYS = ["acc", "map", "ap", "auroc"]
+
+
+def main() -> None:
+    SUB.mkdir(parents=True, exist_ok=True)
+    for slug, method, track, modality, model_type, paper, evaluation, metrics, notes in ROWS:
+        full_metrics = {k: metrics.get(k) for k in METRIC_KEYS}
+        sub = {
+            "method": method, "track": track, "modality": modality, "model_type": model_type,
+            "paper": paper, "url": REPO, "evaluation": evaluation, "metrics": full_metrics,
+            "per_generator": None, "submitted_by": "SynthScope", "date": DATE,
+            "verified": True, "source": SOURCE, "protocol_notes": notes, "synthscope_version": "0.1.0",
+        }
+        (SUB / f"{slug}.json").write_text(json.dumps(sub, indent=2) + "\n")
+    print(f"Wrote {len(ROWS)} reference submissions to {SUB}")
+
+
+if __name__ == "__main__":
+    main()

benchmark/submissions/aide.json

@@ -0,0 +1,22 @@
+{
+  "method": "AIDE",
+  "track": "forensynths",
+  "modality": "image",
+  "model_type": "aide",
+  "paper": "Yan et al., ICLR'25",
+  "url": "https://github.com/SMSD75/SynthScope",
+  "evaluation": "official-weights",
+  "metrics": {
+    "acc": null,
+    "map": 92.08,
+    "ap": null,
+    "auroc": null
+  },
+  "per_generator": null,
+  "submitted_by": "SynthScope",
+  "date": "2026-06-02",
+  "verified": true,
+  "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md",
+  "protocol_notes": "ForenSynths-8; mAP reproduces, acc@0.5 miscalibrated (single-view vs official 5-view DCT)",
+  "synthscope_version": "0.1.0"
+}

benchmark/submissions/clip.json

@@ -0,0 +1,22 @@
+{
+  "method": "UnivFD / CLIP",
+  "track": "forensynths",
+  "modality": "image",
+  "model_type": "clip",
+  "paper": "Ojha et al., CVPR'23",
+  "url": "https://github.com/SMSD75/SynthScope",
+  "evaluation": "frozen-probe",
+  "metrics": {
+    "acc": null,
+    "map": 97.2,
+    "ap": null,
+    "auroc": 97.0
+  },
+  "per_generator": null,
+  "submitted_by": "SynthScope",
+  "date": "2026-06-02",
+  "verified": true,
+  "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md",
+  "protocol_notes": "ForenSynths; frozen CLIP ViT-L/14 probe (ProGAN)",
+  "synthscope_version": "0.1.0"
+}

benchmark/submissions/cnn.json

@@ -0,0 +1,22 @@
+{
+  "method": "CNNDetection",
+  "track": "forensynths",
+  "modality": "image",
+  "model_type": "cnn",
+  "paper": "Wang et al., CVPR'20",
+  "url": "https://github.com/SMSD75/SynthScope",
+  "evaluation": "from-scratch",
+  "metrics": {
+    "acc": null,
+    "map": 96.8,
+    "ap": null,
+    "auroc": 96.6
+  },
+  "per_generator": null,
+  "submitted_by": "SynthScope",
+  "date": "2026-06-02",
+  "verified": true,
+  "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md",
+  "protocol_notes": "ForenSynths; ResNet-50 fine-tune (ProGAN)",
+  "synthscope_version": "0.1.0"
+}

benchmark/submissions/co_spy_official.json

@@ -0,0 +1,22 @@
+{
+  "method": "CO-SPY (official weights)",
+  "track": "forensynths",
+  "modality": "image",
+  "model_type": "co_spy",
+  "paper": "Cheng et al., CVPR'25",
+  "url": "https://github.com/SMSD75/SynthScope",
+  "evaluation": "official-weights",
+  "metrics": {
+    "acc": 95.12,
+    "map": 98.71,
+    "ap": null,
+    "auroc": null
+  },
+  "per_generator": null,
+  "submitted_by": "SynthScope",
+  "date": "2026-06-02",
+  "verified": true,
+  "source": "SynthScope reproduction (logged Test Metrics); see repro/FINAL_RESULTS.md",
+  "protocol_notes": "ForenSynths-8; vendored official architecture",
+  "synthscope_version": "0.1.0"
+}